diff options
67 files changed, 1649 insertions, 398 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java index 8b3d983d6c1..78a8c161c3b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java @@ -5,17 +5,14 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import org.w3c.dom.Element; -import java.util.Optional; import java.util.stream.Stream; /** * This class contains a context that is passed to a model builder, and can be used to retrieve the application package, * logger etc. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public final class ConfigModelContext { diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index 32cb1840e7c..5eb4afcc241 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -75,7 +75,11 @@ public class AdminModel extends ConfigModel { public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) { AbstractConfigProducer parent = modelContext.getParentProducer(); DeployProperties properties = modelContext.getDeployState().getProperties(); - DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), modelContext.getDeployState().getFileRegistry(), properties.multitenant(), properties.configServerSpecs()); + DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), + modelContext.getDeployState().getFileRegistry(), + properties.multitenant(), + properties.configServerSpecs(), + modelContext.getDeployState().disableFiledistributor()); model.admin = domBuilder.build(parent, adminElement); // TODO: Is required since other models depend on admin. if (parent instanceof ApplicationConfigProducerRoot) { @@ -101,7 +105,11 @@ public class AdminModel extends ConfigModel { public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) { AbstractConfigProducer parent = modelContext.getParentProducer(); DeployProperties properties = modelContext.getDeployState().getProperties(); - DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext, properties.multitenant(), properties.configServerSpecs(), model.getContainerModels()); + DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext, + properties.multitenant(), + properties.configServerSpecs(), + model.getContainerModels(), + modelContext.getDeployState().disableFiledistributor()); model.admin = domBuilder.build(parent, adminElement); // TODO: Is required since other models depend on admin. if (parent instanceof ApplicationConfigProducerRoot) { diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 25d45dee234..43d3fafdb78 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -68,8 +68,8 @@ public class DeployState implements ConfigDefinitionStore { private final ValidationOverrides validationOverrides; private final Version wantedNodeVespaVersion; private final Instant now; - private final HostProvisioner provisioner; + private final boolean disableFiledistributor; public static DeployState createTestState() { return new Builder().build(); @@ -82,8 +82,8 @@ public class DeployState implements ConfigDefinitionStore { private DeployState(ApplicationPackage applicationPackage, SearchDocumentModel searchDocumentModel, RankProfileRegistry rankProfileRegistry, FileRegistry fileRegistry, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, DeployProperties properties, Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo, - java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, - SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion) { + java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, + SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion, boolean disableFiledistributor) { this.logger = deployLogger; this.fileRegistry = fileRegistry; this.rankProfileRegistry = rankProfileRegistry; @@ -102,6 +102,7 @@ public class DeployState implements ConfigDefinitionStore { this.validationOverrides = applicationPackage.getValidationOverrides().map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); this.wantedNodeVespaVersion = wantedNodeVespaVersion; this.now = now; + this.disableFiledistributor = disableFiledistributor; } public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { @@ -169,6 +170,7 @@ public class DeployState implements ConfigDefinitionStore { public ApplicationPackage getApplicationPackage() { return applicationPackage; } + public List<SearchDefinition> getSearchDefinitions() { return searchDefinitions; } @@ -214,6 +216,8 @@ public class DeployState implements ConfigDefinitionStore { public Instant now() { return now; } + public boolean disableFiledistributor() { return disableFiledistributor; } + public static class Builder { private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty(); @@ -228,6 +232,7 @@ public class DeployState implements ConfigDefinitionStore { private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; + private boolean disableFiledistributor = false; public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; @@ -289,13 +294,19 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder disableFiledistributor(boolean disableFiledistributor) { + this.disableFiledistributor = disableFiledistributor; + return this; + } + public DeployState build() { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage); SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage); SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles); return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, fileRegistry, logger, hostProvisioner, - properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion); + properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, + zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion, disableFiledistributor); } private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, DeployLogger logger, QueryProfiles queryProfiles) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index 277e02e43e2..2a3ff1ca905 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -11,6 +11,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.config.provision.Zone; import com.yahoo.text.XML; import com.yahoo.vespa.model.ConfigProducer; import com.yahoo.vespa.model.HostSystem; @@ -147,7 +148,8 @@ public class MockRoot extends AbstractConfigProducerRoot { try { Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml))); - setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, deployState.getFileRegistry(), false, new ArrayList<>()). + setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, deployState.getFileRegistry(), + false, new ArrayList<>(), deployState.disableFiledistributor()). build(this, XML.getChildren(doc.getDocumentElement(), "admin").get(0))); } catch (SAXException | IOException e) { throw new RuntimeException(e); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java index b2cd8574076..c75864f81b7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java @@ -21,6 +21,8 @@ import java.util.Map; */ class ConstantTensorTransformer extends ExpressionTransformer { + public static final String CONSTANT = "constant"; + private final Map<String, Value> constants; private final Map<String, String> rankPropertiesOutput; @@ -64,7 +66,7 @@ class ConstantTensorTransformer extends ExpressionTransformer { return node; } TensorValue tensorValue = (TensorValue)value; - String featureName = "constant(" + node.getName() + ")"; + String featureName = CONSTANT + "(" + node.getName() + ")"; String tensorType = tensorValue.asTensor().type().toString(); rankPropertiesOutput.put(featureName + ".value", tensorValue.toString()); rankPropertiesOutput.put(featureName + ".type", tensorType); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index fd249956d5a..1021227b0e6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -706,6 +706,7 @@ public class RankProfile implements Serializable, Cloneable { expression = new ConstantTensorTransformer(constants, rankPropertiesOutput).transform(expression); expression = new MacroInliner(inlineMacros).transform(expression); expression = new MacroShadower(getMacros()).transform(expression); + expression = new TensorTransformer(this).transform(expression); expression = new Simplifier().transform(expression); for (Map.Entry<String, String> rankProperty : rankPropertiesOutput.entrySet()) { addRankProperty(rankProperty.getKey(), rankProperty.getValue()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/TensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/TensorTransformer.java new file mode 100644 index 00000000000..69e353ceb35 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/TensorTransformer.java @@ -0,0 +1,290 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchlib.rankingexpression.evaluation.Context; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; +import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.FunctionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.functions.Reduce; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Transforms and simplifies tensor expressions. + * + * Currently transforms min(tensor,dim) and max(tensor,dim) to + * reduce(tensor,min/max,dim). This is necessary as the backend does + * not recognize these forms of min and max. + * + * @author lesters + */ +public class TensorTransformer extends ExpressionTransformer { + + private Search search; + private RankProfile rankprofile; + private Map<String, RankProfile.Macro> macros; + + public TensorTransformer(RankProfile rankprofile) { + this.rankprofile = rankprofile; + this.search = rankprofile.getSearch(); + this.macros = rankprofile.getMacros(); + } + + @Override + public ExpressionNode transform(ExpressionNode node) { + if (node instanceof CompositeNode) { + node = transformChildren((CompositeNode) node); + } + if (node instanceof FunctionNode) { + node = transformFunctionNode((FunctionNode) node); + } + return node; + } + + private ExpressionNode transformFunctionNode(FunctionNode node) { + switch (node.getFunction()) { + case min: + case max: + return transformMaxAndMinFunctionNode(node); + } + return node; + } + + /** + * Transforms max and min functions if it can be proven that the first + * argument resolves to a tensor and the second argument is a valid + * dimension in the tensor. If these do not hold, the node will not + * be transformed. + * + * The test for whether or not the first argument resolves to a tensor + * is to evaluate that expression. All values used in the expression + * is bound to a context with dummy values with enough information to + * deduce tensor types. + * + * There is currently no guarantee that all cases will be found. For + * instance, if-statements are problematic. + */ + private ExpressionNode transformMaxAndMinFunctionNode(FunctionNode node) { + if (node.children().size() != 2) { + return node; + } + ExpressionNode arg1 = node.children().get(0); + Optional<String> dimension = dimensionName(node.children().get(1)); + if (dimension.isPresent()) { + try { + Context context = buildContext(arg1); + Value value = arg1.evaluate(context); + if (isTensorWithDimension(value, dimension.get())) { + return replaceMaxAndMinFunction(node); + } + } catch (IllegalArgumentException e) { + // Thrown from evaluate if some variables are not bound, for + // instance for a backend rank feature. Means we don't have + // enough information to replace expression. + } + } + return node; + } + + private Optional<String> dimensionName(ExpressionNode arg) { + if (arg instanceof ReferenceNode && ((ReferenceNode)arg).children().size() == 0) { + return Optional.of(((ReferenceNode) arg).getName()); + } + return Optional.empty(); + } + + private boolean isTensorWithDimension(Value value, String dimension) { + if (value instanceof TensorValue) { + Tensor tensor = ((TensorValue) value).asTensor(); + TensorType type = tensor.type(); + return type.dimensionNames().contains(dimension); + } + return false; + } + + private ExpressionNode replaceMaxAndMinFunction(FunctionNode node) { + ExpressionNode arg1 = node.children().get(0); + ExpressionNode arg2 = node.children().get(1); + + TensorFunctionNode.TensorFunctionExpressionNode expression = TensorFunctionNode.wrapArgument(arg1); + Reduce.Aggregator aggregator = Reduce.Aggregator.valueOf(node.getFunction().name()); + String dimension = ((ReferenceNode) arg2).getName(); + + return new TensorFunctionNode(new Reduce(expression, aggregator, dimension)); + } + + /** + * Creates an evaluation context by iterating through the expression tree, and + * adding dummy values with correct types to the context. + */ + private Context buildContext(ExpressionNode node) { + Context context = new MapContext(); + addRoot(node, context); + return context; + } + + private Value emptyStringValue() { + return new StringValue(""); + } + + private Value emptyDoubleValue() { + return new DoubleValue(0.0); + } + + private Value emptyTensorValue(TensorType type) { + Tensor empty = Tensor.Builder.of(type).build(); + return new TensorValue(empty); + } + + private void addRoot(ExpressionNode node, Context context) { + addChildren(node, context); + if (node instanceof ReferenceNode) { + ReferenceNode referenceNode = (ReferenceNode) node; + addIfAttribute(referenceNode, context); + addIfConstant(referenceNode, context); + addIfQuery(referenceNode, context); + addIfTensorFrom(referenceNode, context); + addIfMacro(referenceNode, context); + } + } + + private void addChildren(ExpressionNode node, Context context) { + if (node instanceof CompositeNode) { + List<ExpressionNode> children = ((CompositeNode) node).children(); + for (ExpressionNode child : children) { + addRoot(child, context); + } + } + } + + private void addIfAttribute(ReferenceNode node, Context context) { + if (!node.getName().equals("attribute")) { + return; + } + if (node.children().size() == 0) { + return; + } + String attribute = node.children().get(0).toString(); + Attribute a = search.getAttribute(attribute); + if (a == null) { + return; + } + Value v; + if (a.getType() == Attribute.Type.STRING) { + v = emptyStringValue(); + } else if (a.getType() == Attribute.Type.TENSOR) { + v = emptyTensorValue(a.tensorType().orElseThrow(RuntimeException::new)); + } else { + v = emptyDoubleValue(); + } + context.put(node.toString(), v); + } + + private void addIfConstant(ReferenceNode node, Context context) { + if (!node.getName().equals(ConstantTensorTransformer.CONSTANT)) { + return; + } + if (node.children().size() != 1) { + return; + } + ExpressionNode child = node.children().get(0); + while (child instanceof CompositeNode && ((CompositeNode) child).children().size() > 0) { + child = ((CompositeNode) child).children().get(0); + } + String name = child.toString(); + addIfConstantInRankProfile(name, node, context); + addIfConstantInRankingConstants(name, node, context); + } + + private void addIfConstantInRankProfile(String name, ReferenceNode node, Context context) { + if (rankprofile.getConstants().containsKey(name)) { + context.put(node.toString(), rankprofile.getConstants().get(name)); + } + } + + private void addIfConstantInRankingConstants(String name, ReferenceNode node, Context context) { + for (RankingConstant rankingConstant : search.getRankingConstants()) { + if (rankingConstant.getName().equals(name)) { + context.put(node.toString(), emptyTensorValue(rankingConstant.getTensorType())); + } + } + } + + private void addIfQuery(ReferenceNode node, Context context) { + if (!node.getName().equals("query")) { + return; + } + if (node.children().size() != 1) { + return; + } + String name = node.children().get(0).toString(); + if (rankprofile.getQueryFeatureTypes().containsKey(name)) { + String type = rankprofile.getQueryFeatureTypes().get(name); + Value v; + if (type.contains("tensor")) { + v = emptyTensorValue(TensorType.fromSpec(type)); + } else if (type.equalsIgnoreCase("string")) { + v = emptyStringValue(); + } else { + v = emptyDoubleValue(); + } + context.put(node.toString(), v); + } + } + + private void addIfTensorFrom(ReferenceNode node, Context context) { + if (!node.getName().startsWith("tensorFrom")) { + return; + } + if (node.children().size() < 1 || node.children().size() > 2) { + return; + } + ExpressionNode source = node.children().get(0); + if (source instanceof CompositeNode && ((CompositeNode) source).children().size() > 0) { + source = ((CompositeNode) source).children().get(0); + } + String dimension = source.toString(); + if (node.children().size() == 2) { + dimension = node.children().get(1).toString(); + } + TensorType type = (new TensorType.Builder()).mapped(dimension).build(); + context.put(node.toString(), emptyTensorValue(type)); + } + + private void addIfMacro(ReferenceNode node, Context context) { + RankProfile.Macro macro = macros.get(node.getName()); + if (macro == null) { + return; + } + ExpressionNode root = macro.getRankingExpression().getRoot(); + Context macroContext = buildContext(root); + addMacroArguments(node, context, macro, macroContext); + Value value = root.evaluate(macroContext); + context.put(node.toString(), value); + } + + private void addMacroArguments(ReferenceNode node, Context context, RankProfile.Macro macro, Context macroContext) { + if (macro.getFormalParams().size() > 0 && node.children().size() > 0) { + for (int i = 0; i < macro.getFormalParams().size() && i < node.children().size(); ++i) { + String param = macro.getFormalParams().get(i); + ExpressionNode argumentExpression = node.children().get(i); + Value arg = argumentExpression.evaluate(context); + macroContext.put(param, arg); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index e6ed91165ca..5e74a2ebc8a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -7,9 +7,7 @@ import com.yahoo.config.provision.Flavor; import javax.annotation.Nullable; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -266,30 +264,4 @@ public class HostResource implements Comparable<HostResource> { return this.host.compareTo(other.host); } - /** - * Picks hosts by some mixture of host name and index - * (where the mix of one or the other is decided by the last parameter). - */ - public static List<HostResource> pickHosts(Collection<HostResource> hosts, int count, int targetHostsSelectedByIndex) { - targetHostsSelectedByIndex = Math.min(Math.min(targetHostsSelectedByIndex, count), hosts.size()); - - List<HostResource> hostsSortedByName = new ArrayList<>(hosts); - Collections.sort(hostsSortedByName); - - List<HostResource> hostsSortedByIndex = new ArrayList<>(hosts); - hostsSortedByIndex.sort(Comparator.comparingInt(host -> host.primaryClusterMembership().get().index())); - return pickHosts(hostsSortedByName, hostsSortedByIndex, count, targetHostsSelectedByIndex); - } - public static List<HostResource> pickHosts(List<HostResource> hostsSelectedByName, List<HostResource> hostsSelectedByIndex, - int count, int targetHostsSelectedByIndex) { - hostsSelectedByName = hostsSelectedByName.subList(0, Math.min(count - targetHostsSelectedByIndex, hostsSelectedByName.size())); - hostsSelectedByIndex.removeAll(hostsSelectedByName); - hostsSelectedByIndex = hostsSelectedByIndex.subList(0, Math.min(targetHostsSelectedByIndex, hostsSelectedByIndex.size())); - - List<HostResource> finalHosts = new ArrayList<>(); - finalHosts.addAll(hostsSelectedByName); - finalHosts.addAll(hostsSelectedByIndex); - return finalHosts; - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index a3a1a0fbebc..3e825c3912a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -37,16 +37,19 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu private final ApplicationType applicationType; private final List<ConfigServerSpec> configServerSpecs; private final FileRegistry fileRegistry; + private final boolean disableFiledistributor; protected final boolean multitenant; - public DomAdminBuilderBase(ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { + DomAdminBuilderBase(ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant, + List<ConfigServerSpec> configServerSpecs, boolean disableFiledistributor) { this.applicationType = applicationType; this.fileRegistry = fileRegistry; this.multitenant = multitenant; this.configServerSpecs = configServerSpecs; + this.disableFiledistributor = disableFiledistributor; } - protected List<Configserver> getConfigServersFromSpec(AbstractConfigProducer parent) { + List<Configserver> getConfigServersFromSpec(AbstractConfigProducer parent) { List<Configserver> configservers = new ArrayList<>(); for (ConfigServerSpec spec : configServerSpecs) { HostSystem hostSystem = parent.getHostSystem(); @@ -76,7 +79,9 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu new ModelConfigProvider(admin); - FileDistributionOptions fileDistributionOptions = new DomFileDistributionOptionsBuilder().build(XML.getChild(adminElement, "filedistribution")); + FileDistributionOptions fileDistributionOptions = FileDistributionOptions.defaultOptions(); + fileDistributionOptions.disabled(disableFiledistributor); + fileDistributionOptions = new DomFileDistributionOptionsBuilder(fileDistributionOptions).build(XML.getChild(adminElement, "filedistribution")); admin.setFileDistribution(new FileDistributionConfigProducer.Builder(fileDistributionOptions).build(parent, fileRegistry)); return admin; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index dd1d4e36255..d966f3b49f6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -18,7 +19,6 @@ import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBu import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; -import com.yahoo.config.application.api.FileRegistry; import org.w3c.dom.Element; import java.util.List; @@ -37,8 +37,9 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { public DomAdminV2Builder(ConfigModelContext.ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant, - List<ConfigServerSpec> configServerSpecs) { - super(applicationType, fileRegistry, multitenant, configServerSpecs); + List<ConfigServerSpec> configServerSpecs, + boolean disableFiledistributor) { + super(applicationType, fileRegistry, multitenant, configServerSpecs, disableFiledistributor); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index f33c86134cb..a7a785cea43 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -19,7 +19,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; /** * Builds the admin model from a version 4 XML tag, or as a default when an admin 3 tag or no admin tag is used. @@ -31,8 +30,10 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { private final Collection<ContainerModel> containerModels; private final ConfigModelContext context; - public DomAdminV4Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs, Collection<ContainerModel> containerModels) { - super(context.getApplicationType(), context.getDeployState().getFileRegistry(), multitenant, configServerSpecs); + public DomAdminV4Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs, + Collection<ContainerModel> containerModels, boolean disableFiledistributor) { + super(context.getApplicationType(), context.getDeployState().getFileRegistry(), multitenant, + configServerSpecs, disableFiledistributor); this.containerModels = containerModels; this.context = context; } @@ -118,11 +119,12 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { /** Returns the count first containers in the current model having isRetired set to the given value */ private List<HostResource> sortedContainerHostsFrom(ContainerModel model, int count, boolean retired) { - List<HostResource> hosts = model.getCluster().getContainers().stream() - .filter(container -> retired == container.isRetired()) - .map(Container::getHostResource) - .collect(Collectors.toList()); - return HostResource.pickHosts(hosts, count, 1); + List<HostResource> hosts = new ArrayList<>(); + for (Container container : model.getCluster().getContainers()) + if (retired == container.isRetired()) + hosts.add(container.getHostResource()); + Collections.sort(hosts); + return hosts.subList(0, Math.min(count, hosts.size())); } private void createLogserver(Admin admin, Collection<HostResource> hosts) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java index 8a5d6846a64..90d71f186ae 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java @@ -15,6 +15,11 @@ import java.util.Optional; * @author hmusum */ public class DomFileDistributionOptionsBuilder { + private final FileDistributionOptions fileDistributionOptions; + + public DomFileDistributionOptionsBuilder(FileDistributionOptions fileDistributionOptions) { + this.fileDistributionOptions = fileDistributionOptions; + } private static void throwExceptionForElementInFileDistribution(String subElement, String reason) { throw new RuntimeException("In element '" + subElement + "' contained in 'filedistribution': " + reason); @@ -34,15 +39,14 @@ public class DomFileDistributionOptionsBuilder { } public FileDistributionOptions build(Element fileDistributionElement) { - FileDistributionOptions options = FileDistributionOptions.defaultOptions(); if (fileDistributionElement != null) { - getAmount("uploadbitrate", fileDistributionElement).ifPresent(options::uploadBitRate); - getAmount("downloadbitrate", fileDistributionElement).ifPresent(options::downloadBitRate); + getAmount("uploadbitrate", fileDistributionElement).ifPresent(fileDistributionOptions::uploadBitRate); + getAmount("downloadbitrate", fileDistributionElement).ifPresent(fileDistributionOptions::downloadBitRate); Element disable = XML.getChild(fileDistributionElement, "disabled"); if (disable != null) { - options.disabled(Boolean.valueOf(XML.getValue(disable))); + fileDistributionOptions.disabled(Boolean.valueOf(XML.getValue(disable))); } } - return options; + return fileDistributionOptions; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 62828b314d0..bce78017bdd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -144,6 +144,9 @@ public class ConfigserverCluster extends AbstractConfigProducer if (options.loadBalancerAddress().isPresent()) { builder.loadBalancerAddress(options.loadBalancerAddress().get()); } + if (options.disableFiledistributor().isPresent()) { + builder.disableFiledistributor(options.disableFiledistributor().get()); + } } private String[] getConfigModelPluginDirs() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index aeb86ae9d59..866bae6666a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.container.configserver.option; import java.util.Optional; /** - * @author tonytv + * @author Tony Vaagenes */ public interface CloudConfigOptions { @@ -44,4 +44,5 @@ public interface CloudConfigOptions { Optional<String> dockerRegistry(); Optional<String> dockerVespaBaseImage(); Optional<String> loadBalancerAddress(); + Optional<Boolean> disableFiledistributor(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 2bfa8fa93a0..81aca977400 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.document.select.DocumentSelector; @@ -18,11 +20,15 @@ import java.util.logging.Logger; /** * Generates distributor-specific configuration. */ -public class DistributorCluster extends AbstractConfigProducer<Distributor> - implements StorDistributormanagerConfig.Producer, StorServerConfig.Producer, MetricsmanagerConfig.Producer { +public class DistributorCluster extends AbstractConfigProducer<Distributor> implements + StorDistributormanagerConfig.Producer, + StorServerConfig.Producer, + MetricsmanagerConfig.Producer, + BucketspacesConfig.Producer { public static final Logger log = Logger.getLogger(DistributorCluster.class.getPackage().toString()); + private static class GcOptions { public final int interval; public final String selection; @@ -145,6 +151,20 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> builder.is_distributor(true); } + private static final String DEFAULT_BUCKET_SPACE = "default"; + private static final String GLOBAL_BUCKET_SPACE = "global"; + + @Override + public void getConfig(BucketspacesConfig.Builder builder) { + for (NewDocumentType docType : parent.getDocumentDefinitions().values()) { + BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder(); + docTypeBuilder.name(docType.getName()); + String bucketSpace = (parent.isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); + docTypeBuilder.bucketspace(bucketSpace); + builder.documenttype(docTypeBuilder); + } + } + public String getClusterName() { return parent.getName(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 6e82379d00b..7889b857fff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; -import com.yahoo.lang.MutableInteger; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; @@ -329,11 +328,9 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri } private List<HostResource> drawControllerHosts(int count, StorageGroup rootGroup, Collection<ContainerModel> containers) { - List<HostResource> hostsByName = drawContentHostsRecursively(count, false, rootGroup); - List<HostResource> hostsByIndex = drawContentHostsRecursively(count, true, rootGroup); + List<HostResource> hosts = drawContentHostsRecursively(count, rootGroup); // if (hosts.size() < count) // supply with containers TODO: Currently disabled due to leading to topology change problems // hosts.addAll(drawContainerHosts(count - hosts.size(), containers, new HashSet<>(hosts))); - List<HostResource> hosts = HostResource.pickHosts(hostsByName, hostsByIndex, count, 1); if (hosts.size() % 2 == 0) // ZK clusters of even sizes are less available (even in the size=2 case) hosts = hosts.subList(0, hosts.size()-1); return hosts; @@ -406,24 +403,20 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri */ // Note: This method cannot be changed to draw different nodes without ensuring that it will draw nodes // which overlaps with previously drawn nodes as this will prevent rolling upgrade - private List<HostResource> drawContentHostsRecursively(int count, boolean byIndex, StorageGroup group) { + private List<HostResource> drawContentHostsRecursively(int count, StorageGroup group) { Set<HostResource> hosts = new HashSet<>(); if (group.getNodes().isEmpty()) { int hostsPerSubgroup = (int)Math.ceil((double)count / group.getSubgroups().size()); for (StorageGroup subgroup : group.getSubgroups()) - hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, byIndex, subgroup)); + hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, subgroup)); } else { hosts.addAll(group.getNodes().stream() - .filter(node -> ! node.isRetired()) // Avoid retired controllers to avoid surprises on expiry - .map(StorageNode::getHostResource).collect(Collectors.toList())); + .filter(node -> ! node.isRetired()) // Avoid retired controllers to avoid surprises on expiry + .map(StorageNode::getHostResource).collect(Collectors.toList())); } - List<HostResource> sortedHosts = new ArrayList<>(hosts); - if (byIndex) - sortedHosts.sort(Comparator.comparingInt(host -> host.primaryClusterMembership().get().index())); - else // by name - Collections.sort(sortedHosts); + Collections.sort(sortedHosts); sortedHosts = sortedHosts.subList(0, Math.min(count, hosts.size())); return sortedHosts; } @@ -531,6 +524,10 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri */ public Map<String, NewDocumentType> getDocumentDefinitions() { return documentDefinitions; } + public boolean isGloballyDistributed(NewDocumentType docType) { + return globallyDistributedDocuments.contains(docType); + } + public final ContentSearchCluster getSearch() { return search; } public Redundancy redundancy() { return redundancy; } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 6ea15788cf3..63d5d37598b 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -391,16 +391,16 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default28", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default31", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default54", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default10", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default13", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default16", clusterControllers.getContainers().get(2).getHostName()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(9, cluster.getRootGroup().getSubgroups().size()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(3)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0")); - assertEquals("default54", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("default10", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getConfigId(), is("bar/storage/1")); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(2).getDistributionKey(), is(2)); @@ -409,13 +409,13 @@ public class ModelProvisioningTest { assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(3)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(3)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/3")); - assertEquals("default51", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("default13", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getDistributionKey(), is(4)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getConfigId(), is("bar/storage/4")); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getDistributionKey(), is(5)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getConfigId(), is("bar/storage/5")); // ... - assertEquals("default48", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); + assertEquals("default16", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); // ... assertThat(cluster.getRootGroup().getSubgroups().get(8).getIndex(), is("8")); assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().size(), is(3)); @@ -430,23 +430,23 @@ public class ModelProvisioningTest { clusterControllers = cluster.getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("baz-controllers", clusterControllers.getName()); - assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default02", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default27", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default37", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default38", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default39", clusterControllers.getContainers().get(2).getHostName()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(27, cluster.getRootGroup().getSubgroups().size()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("baz/storage/0")); - assertEquals("default27", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("default37", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1")); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("baz/storage/1")); - assertEquals("default26", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("default38", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); // ... - assertEquals("default25", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); + assertEquals("default39", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); // ... assertThat(cluster.getRootGroup().getSubgroups().get(26).getIndex(), is("26")); assertThat(cluster.getRootGroup().getSubgroups().get(26).getNodes().size(), is(1)); @@ -483,9 +483,9 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default02", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default08", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default10", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default11", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default12", clusterControllers.getContainers().get(2).getHostName()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(8, cluster.getRootGroup().getSubgroups().size()); assertEquals(8, cluster.distributionBits()); @@ -494,19 +494,19 @@ public class ModelProvisioningTest { assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0")); - assertEquals("default08", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("default10", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); // second group assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1")); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/1")); - assertEquals("default07", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("default11", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); // ... last group assertThat(cluster.getRootGroup().getSubgroups().get(7).getIndex(), is("7")); assertThat(cluster.getRootGroup().getSubgroups().get(7).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getDistributionKey(), is(7)); assertThat(cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getConfigId(), is("bar/storage/7")); - assertEquals("default01", cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getHostName()); + assertEquals("default17", cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getHostName()); } @Test @@ -538,51 +538,16 @@ public class ModelProvisioningTest { ContentCluster cluster = model.getContentClusters().get("bar"); ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals( 8, cluster.distributionBits()); - assertEquals("We get the closest odd number", 5, clusterControllers.getContainers().size()); + assertEquals("We get the closest odd numer", 5, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default02", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default04", clusterControllers.getContainers().get(2).getHostName()); - assertEquals("default05", clusterControllers.getContainers().get(3).getHostName()); - assertEquals("default07", clusterControllers.getContainers().get(4).getHostName()); - assertEquals("default09", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); - assertEquals("default08", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getHostName()); - assertEquals("default06", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); - assertEquals("default03", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); - } - - @Test - public void testClusterControllersWithGroupSize2() { - String services = - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<services>" + - " <admin version='4.0'/>" + - " <container version='1.0' id='foo'>" + - " <nodes count='10'/>" + - " </container>" + - " <content version='1.0' id='bar'>" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document type='type1' mode='index'/>" + - " </documents>" + - " <nodes count='8' groups='4'/>" + - " </content>" + - "</services>"; - - int numberOfHosts = 18; - VespaModelTester tester = new VespaModelTester(); - tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); - - // Check content clusters - ContentCluster cluster = model.getContentClusters().get("bar"); - ContainerCluster clusterControllers = cluster.getClusterControllers(); - assertEquals("We get the closest odd number", 3, clusterControllers.getContainers().size()); - assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default03", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default08", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default10", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default11", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default13", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default14", clusterControllers.getContainers().get(3).getHostName()); // Should be 16 for perfect distribution ... + assertEquals("default10", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("default11", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getHostName()); + assertEquals("default13", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("default16", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); } @Test @@ -639,7 +604,7 @@ public class ModelProvisioningTest { int numberOfHosts = 19; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "default09", "default06", "default03"); + VespaModel model = tester.createModel(services, true, "default10", "default13", "default16"); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); // Check content clusters @@ -647,9 +612,9 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("Skipping retired default09", "default01", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("Skipping retired default03", "default04", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("Skipping retired default06", "default08", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("Skipping retired default10", "default11", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("Skipping retired default13", "default14", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("Skipping retired default16", "default17", clusterControllers.getContainers().get(2).getHostName()); } @Test @@ -666,15 +631,15 @@ public class ModelProvisioningTest { int numberOfHosts = 10; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "default09"); + VespaModel model = tester.createModel(services, true, "default0"); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); // Check slobroks clusters assertEquals("Includes retired node", 1+3, model.getAdmin().getSlobroks().size()); - assertEquals("default01", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("default02", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("default10", model.getAdmin().getSlobroks().get(2).getHostName()); - assertEquals("Included in addition because it is retired", "default09", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("default1", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("default2", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("default3", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("Included in addition because it is retired", "default0", model.getAdmin().getSlobroks().get(3).getHostName()); } @Test @@ -691,16 +656,16 @@ public class ModelProvisioningTest { int numberOfHosts = 10; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "default09", "default08"); + VespaModel model = tester.createModel(services, true, "default3", "default4"); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); // Check slobroks clusters assertEquals("Includes retired node", 3+2, model.getAdmin().getSlobroks().size()); - assertEquals("default01", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("default02", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("default10", model.getAdmin().getSlobroks().get(2).getHostName()); - assertEquals("Included in addition because it is retired", "default08", model.getAdmin().getSlobroks().get(3).getHostName()); - assertEquals("Included in addition because it is retired", "default09", model.getAdmin().getSlobroks().get(4).getHostName()); + assertEquals("default0", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("default1", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("default2", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("Included in addition because it is retired", "default3", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("Included in addition because it is retired", "default4", model.getAdmin().getSlobroks().get(4).getHostName()); } @Test @@ -720,19 +685,19 @@ public class ModelProvisioningTest { int numberOfHosts = 13; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "default12", "default03", "default02"); + VespaModel model = tester.createModel(services, true, "default0", "default10", "default11"); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); // Check slobroks clusters // ... from cluster default assertEquals("Includes retired node", 3+3, model.getAdmin().getSlobroks().size()); - assertEquals("default04", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("default13", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("Included in addition because it is retired", "default12", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("default1", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("default2", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("Included in addition because it is retired", "default0", model.getAdmin().getSlobroks().get(2).getHostName()); // ... from cluster bar - assertEquals("default01", model.getAdmin().getSlobroks().get(3).getHostName()); - assertEquals("Included in addition because it is retired", "default02", model.getAdmin().getSlobroks().get(4).getHostName()); - assertEquals("Included in addition because it is retired", "default03", model.getAdmin().getSlobroks().get(5).getHostName()); + assertEquals("default12", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("Included in addition because it is retired", "default10", model.getAdmin().getSlobroks().get(4).getHostName()); + assertEquals("Included in addition because it is retired", "default11", model.getAdmin().getSlobroks().get(5).getHostName()); } @Test @@ -863,10 +828,10 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(4, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default04", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default03", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default02", clusterControllers.getContainers().get(2).getHostName()); - assertEquals("default01", clusterControllers.getContainers().get(3).getHostName()); + assertEquals("default19", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default20", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default21", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default22", clusterControllers.getContainers().get(3).getHostName()); } @Test @@ -992,7 +957,7 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(1, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default0", clusterControllers.getContainers().get(0).getHostName()); assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3 assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4 assertEquals(1, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3 diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java new file mode 100644 index 00000000000..12bdd8d2b5c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -0,0 +1,206 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.collections.Pair; +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.derived.RawRankProfile; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertTrue; + +public class TensorTransformTestCase extends SearchDefinitionTestCase { + + @Test + public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException { + assertContainsExpression("max(1.0,2.0)", "max(1.0,2.0)"); + assertContainsExpression("min(attribute(double_field),x)", "min(attribute(double_field),x)"); + assertContainsExpression("max(attribute(double_field),attribute(double_array_field))", "max(attribute(double_field),attribute(double_array_field))"); + assertContainsExpression("min(attribute(tensor_field_1),attribute(double_field))", "min(attribute(tensor_field_1),attribute(double_field))"); + assertContainsExpression("max(attribute(tensor_field_1),attribute(tensor_field_2))", "max(attribute(tensor_field_1),attribute(tensor_field_2))"); + assertContainsExpression("min(test_constant_tensor,1.0)", "min(constant(test_constant_tensor),1.0)"); + assertContainsExpression("max(base_constant_tensor,1.0)", "max(constant(base_constant_tensor),1.0)"); + assertContainsExpression("min(constant(file_constant_tensor),1.0)", "min(constant(file_constant_tensor),1.0)"); + assertContainsExpression("max(query(q),1.0)", "max(query(q),1.0)"); + assertContainsExpression("max(query(n),1.0)", "max(query(n),1.0)"); + } + + @Test + public void requireThatMaxAndMinWithTensorAttributesAreReplaced() throws ParseException { + assertContainsExpression("max(attribute(tensor_field_1),x)", "reduce(attribute(tensor_field_1),max,x)"); + assertContainsExpression("1 + max(attribute(tensor_field_1),x)", "1+reduce(attribute(tensor_field_1),max,x)"); + assertContainsExpression("if(attribute(double_field),1 + max(attribute(tensor_field_1),x),0)", "if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),0)"); + assertContainsExpression("max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)", "reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)"); + assertContainsExpression("max(if(attribute(double_field),attribute(tensor_field_1),attribute(tensor_field_2)),x)", "reduce(if(attribute(double_field),attribute(tensor_field_1),attribute(tensor_field_2)),max,x)"); + assertContainsExpression("max(max(attribute(tensor_field_1),x),x)", "max(reduce(attribute(tensor_field_1),max,x),x)"); // will result in deploy error. + assertContainsExpression("max(max(attribute(tensor_field_2),x),y)", "reduce(reduce(attribute(tensor_field_2),max,x),max,y)"); + } + + @Test + public void requireThatMaxAndMinWithConstantTensorsAreReplaced() throws ParseException { + assertContainsExpression("max(test_constant_tensor,x)", "reduce(constant(test_constant_tensor),max,x)"); + assertContainsExpression("max(base_constant_tensor,x)", "reduce(constant(base_constant_tensor),max,x)"); + assertContainsExpression("min(constant(file_constant_tensor),x)", "reduce(constant(file_constant_tensor),min,x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorExpressionsAreReplaced() throws ParseException { + assertContainsExpression("min(attribute(double_field) + attribute(tensor_field_1),x)", "reduce(attribute(double_field)+attribute(tensor_field_1),min,x)"); + assertContainsExpression("min(attribute(tensor_field_1) * attribute(tensor_field_2),x)", "reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)"); + assertContainsExpression("min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)", "reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)"); + assertContainsExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) + assertContainsExpression("min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)", "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorFromIsReplaced() throws ParseException { + assertContainsExpression("max(tensorFromLabels(attribute(double_array_field)),double_array_field)", "reduce(tensorFromLabels(attribute(double_array_field)),max,double_array_field)"); + assertContainsExpression("max(tensorFromLabels(attribute(double_array_field),x),x)", "reduce(tensorFromLabels(attribute(double_array_field),x),max,x)"); + assertContainsExpression("max(tensorFromWeightedSet(attribute(weightedset_field)),weightedset_field)", "reduce(tensorFromWeightedSet(attribute(weightedset_field)),max,weightedset_field)"); + assertContainsExpression("max(tensorFromWeightedSet(attribute(weightedset_field),x),x)", "reduce(tensorFromWeightedSet(attribute(weightedset_field),x),max,x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorInQueryIsReplaced() throws ParseException { + assertContainsExpression("max(query(q),x)", "reduce(query(q),max,x)"); + assertContainsExpression("max(query(n),x)", "max(query(n),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensoresReturnedFromMacrosAreReplaced() throws ParseException { + assertContainsExpression("max(returns_tensor,x)", "reduce(rankingExpression(returns_tensor),max,x)"); + assertContainsExpression("max(wraps_returns_tensor,x)", "reduce(rankingExpression(wraps_returns_tensor),max,x)"); + assertContainsExpression("max(tensor_inheriting,x)", "reduce(rankingExpression(tensor_inheriting),max,x)"); + assertContainsExpression("max(returns_tensor_with_arg(attribute(tensor_field_1)),x)", "reduce(rankingExpression(returns_tensor_with_arg@),max,x)"); + } + + + private void assertContainsExpression(String expr, String transformedExpression) throws ParseException { + assertTrue("Expected expression '" + transformedExpression + "' not found", + containsExpression(expr, transformedExpression)); + } + + private boolean containsExpression(String expr, String transformedExpression) throws ParseException { + for (Pair<String, String> rankPropertyExpression : buildSearch(expr)) { + String rankProperty = rankPropertyExpression.getFirst(); + if (rankProperty.equals("rankingExpression(firstphase).rankingScript")) { + String rankExpression = censorBindingHash(rankPropertyExpression.getSecond().replace(" ","")); + return rankExpression.equals(transformedExpression); + } + } + return false; + } + + private List<Pair<String, String>> buildSearch(String expression) throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field double_field type double { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field double_array_field type array<double> { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field weightedset_field type weightedset<double> { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field tensor_field_1 type tensor(x{}) { \n" + + " indexing: summary | attribute \n" + + " attribute: tensor(x{}) \n" + + " }\n" + + " field tensor_field_2 type tensor(x[3],y[3]) { \n" + + " indexing: summary | attribute \n" + + " attribute: tensor(x[3],y[3]) \n" + + " }\n" + + " }\n" + + " constant file_constant_tensor {\n" + + " file: constants/tensor.json\n" + + " type: tensor(x{})\n" + + " }\n" + + " rank-profile base {\n" + + " constants {\n" + + " base_constant_tensor {\n" + + " value: { {x:0}:0 }\n" + + " }\n" + + " }\n" + + " macro base_tensor() {\n" + + " expression: constant(base_constant_tensor)\n" + + " }\n" + + " }\n" + + " rank-profile test inherits base {\n" + + " constants {\n" + + " test_constant_tensor {\n" + + " value: { {x:0}:1 }\n" + + " }\n" + + " }\n" + + " macro returns_tensor_with_arg(arg1) {\n" + + " expression: 2.0 * arg1\n" + + " }\n" + + " macro wraps_returns_tensor() {\n" + + " expression: returns_tensor\n" + + " }\n" + + " macro returns_tensor() {\n" + + " expression: attribute(tensor_field_2)\n" + + " }\n" + + " macro tensor_inheriting() {\n" + + " expression: base_tensor\n" + + " }\n" + + " first-phase {\n" + + " expression: " + expression + "\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(new BaseDeployLogger(), setupQueryProfileTypes()); + Search s = builder.getSearch(); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); + for (Object o : testRankProperties) + System.out.println(o); + return testRankProperties; + } + + private static QueryProfiles setupQueryProfileTypes() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); + QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); + type.addField(new FieldDescription("ranking.features.query(q)", + FieldType.fromString("tensor(x{})", typeRegistry)), typeRegistry); + type.addField(new FieldDescription("ranking.features.query(n)", + FieldType.fromString("integer", typeRegistry)), typeRegistry); + typeRegistry.register(type); + return new QueryProfiles(registry); + } + + private String censorBindingHash(String s) { + StringBuilder b = new StringBuilder(); + boolean areInHash = false; + for (int i = 0; i < s.length() ; i++) { + char current = s.charAt(i); + if ( ! Character.isLetterOrDigit(current)) // end of hash + areInHash = false; + if ( ! areInHash) + b.append(current); + if (current == '@') // start of hash + areInHash = true; + } + return b.toString(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 8bd62789d52..91f6dd65bda 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -207,7 +207,10 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { } private Admin buildAdmin(Element xml, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { - final DomAdminV2Builder domAdminBuilder = new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, root.getDeployState().getFileRegistry(), multitenant, configServerSpecs); + final DomAdminV2Builder domAdminBuilder = + new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, + root.getDeployState().getFileRegistry(), multitenant, + configServerSpecs, root.getDeployState().disableFiledistributor()); Admin admin = domAdminBuilder.build(root, xml); admin.addPerHostServices(root.getHostSystem().getHosts(), new DeployProperties.Builder().build()); return admin; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index e2c8f2e2c52..1784fe0e974 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -6,8 +6,7 @@ import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import java.util.Optional; /** - * @author lulf - * @since 5. + * @author Ulf Lilleengen */ public class TestOptions implements CloudConfigOptions { private Optional<Integer> rpcPort = Optional.empty(); @@ -20,6 +19,7 @@ public class TestOptions implements CloudConfigOptions { private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); private Optional<Integer> numParallelTenantLoaders = Optional.empty(); + private Optional<Boolean> disableFiledistributor = Optional.empty(); @Override public Optional<Integer> rpcPort() { @@ -118,6 +118,9 @@ public class TestOptions implements CloudConfigOptions { @Override public Optional<String> loadBalancerAddress() { return Optional.empty(); } + @Override + public Optional<Boolean> disableFiledistributor() { return disableFiledistributor; } + public TestOptions numParallelTenantLoaders(int numLoaders) { this.numParallelTenantLoaders = Optional.of(numLoaders); return this; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index 624d3e8ded8..d4e804d3f95 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; @@ -305,22 +306,28 @@ public class DistributorTest { private static class DocDef { public final String type; public final String mode; + public final boolean global; - private DocDef(String type, String mode) { + private DocDef(String type, String mode, boolean global) { this.type = type; this.mode = mode; + this.global = global; } public static DocDef storeOnly(String type) { - return new DocDef(type, "store-only"); + return new DocDef(type, "store-only", false); } public static DocDef index(String type) { - return new DocDef(type, "index"); + return new DocDef(type, "index", false); + } + + public static DocDef indexGlobal(String type) { + return new DocDef(type, "index", true); } public static DocDef streaming(String type) { - return new DocDef(type, "streaming"); + return new DocDef(type, "streaming", false); } } @@ -328,7 +335,8 @@ public class DistributorTest { return "<content id='storage'>\n" + " <documents>\n" + Arrays.stream(defs) - .map(def -> String.format(" <document type='%s' mode='%s'/>", def.type, def.mode)) + .map(def -> String.format(" <document type='%s' mode='%s' global='%s'/>", + def.type, def.mode, (def.global ? "true" : "false"))) .collect(Collectors.joining("\n")) + "\n </documents>\n" + "</content>"; @@ -371,4 +379,24 @@ public class DistributorTest { generateXmlForDocDefs(DocDef.streaming("music"))); assertThat(config.disable_bucket_activation(), is(true)); } + + private BucketspacesConfig clusterXmlToBucketspacesConfig(String xml) { + BucketspacesConfig.Builder builder = new BucketspacesConfig.Builder(); + parse(xml).getConfig(builder); + return new BucketspacesConfig(builder); + } + + private void assertDocumentType(String expName, String expBucketSpace, BucketspacesConfig.Documenttype docType) { + assertEquals(expName, docType.name()); + assertEquals(expBucketSpace, docType.bucketspace()); + } + + @Test + public void bucket_spaces_config_is_produced_for_distributor_cluster() { + BucketspacesConfig config = clusterXmlToBucketspacesConfig( + generateXmlForDocDefs(DocDef.index("music"), DocDef.indexGlobal("movies"))); + assertEquals(2, config.documenttype().size()); + assertDocumentType("movies", "global", config.documenttype(0)); + assertDocumentType("music", "default", config.documenttype(1)); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index b88ba0276c4..901bd7f55b5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -28,7 +29,7 @@ import java.util.Optional; * Helper class which sets up a system with multiple hosts. * Usage: * <code> - * VespaModelteser tester = new VespaModelTester(); + * VespaModelteser teser = new VespaModelTester(); * tester.addHosts(count, flavor); * ... add more nodes * VesoaModel model = tester.createModel(servicesString); @@ -42,7 +43,7 @@ public class VespaModelTester { private final ConfigModelRegistry configModelRegistry; private boolean hosted = true; - private Map<String, Collection<Host>> hostsByFlavor = new HashMap<>(); + private Map<String, Collection<Host>> hosts = new HashMap<>(); public VespaModelTester() { this(new NullConfigModelRegistry()); @@ -54,31 +55,20 @@ public class VespaModelTester { /** Adds some hosts of the 'default' flavor to this system */ public Hosts addHosts(int count) { return addHosts("default", count); } - /** Adds some hosts to this system */ public Hosts addHosts(String flavor, int count) { - return addHosts(Optional.empty(), flavor, count); + List<Host> hosts = new ArrayList<>(); + for (int i = 0; i < count; i++) + hosts.add(new com.yahoo.config.model.provision.Host(flavor + i)); + this.hosts.put(flavor.isEmpty() ? "default" : flavor, hosts); + return new Hosts(hosts); } - public void addHosts(Flavor flavor, int count) { - addHosts(Optional.of(flavor), flavor.name(), count); - } - - private Hosts addHosts(Optional<Flavor> flavor, String flavorName, int count) { List<Host> hosts = new ArrayList<>(); - for (int i = 0; i < count; ++i) { - // Let host names sort in the opposite order of the order the hosts are added - // This allows us to test index vs. name order selection when subsets of hosts are selected from a cluster - // (for e.g cluster controllers and slobrok nodes) - String hostname = String.format("%s%02d", flavorName, count - i); - hosts.add(new Host(hostname, ImmutableList.of(), flavor)); + hosts.add(new Host(flavor.name() + i, ImmutableList.of(), Optional.of(flavor))); } - this.hostsByFlavor.put(flavorName, hosts); - - if (hosts.size() > 100) - throw new IllegalStateException("The host naming scheme is nameNN. To test more than 100 hosts, change to nameNNN"); - return new Hosts(hosts); + this.hosts.put(flavor.name(), hosts); } /** Sets whether this sets up a model for a hosted system. Default: true */ @@ -105,7 +95,7 @@ public class VespaModelTester { ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; HostProvisioner provisioner = hosted ? - new InMemoryProvisioner(hostsByFlavor, failOnOutOfCapacity, startIndexForClusters, retiredHostNames) : + new InMemoryProvisioner(hosts, failOnOutOfCapacity, startIndexForClusters, retiredHostNames) : new SingleNodeProvisioner(); DeployState deployState = new DeployState.Builder() diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java index 2aa7c66ce87..8acab56d838 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java @@ -130,4 +130,32 @@ public class ConfigSetSubscriptionTest { assertEquals(hS.getConfig().stringVal(), "new StringVal"); } + @Test + public void requireThatWeGetLatestConfigWhenTwoUpdatesBeforeClientChecks() { + ConfigSet myConfigs = new ConfigSet(); + AppConfig.Builder a0builder = new AppConfig.Builder().message("A message, 1"); + myConfigs.addBuilder("app/0", a0builder); + ConfigSubscriber subscriber = new ConfigSubscriber(myConfigs); + ConfigHandle<AppConfig> hA0 = subscriber.subscribe(AppConfig.class, "app/0"); + + assertTrue(subscriber.nextConfig(0)); + assertTrue(hA0.isChanged()); + assertEquals(hA0.getConfig().message(), "A message, 1"); + + assertFalse(subscriber.nextConfig(10)); + assertFalse(hA0.isChanged()); + assertEquals(hA0.getConfig().message(), "A message, 1"); + + //Reconfigure two times in a row + a0builder.message("A new message, 2"); + subscriber.reload(1); + a0builder.message("An even newer message, 3"); + subscriber.reload(2); + + // Should pick up the last one + assertTrue(subscriber.nextConfig(0)); + assertTrue(hA0.isChanged()); + assertEquals(hA0.getConfig().message(), "An even newer message, 3"); + } + } diff --git a/config/src/vespa/config/subscription/configsubscriptionset.cpp b/config/src/vespa/config/subscription/configsubscriptionset.cpp index c26d2b74fda..f6268c8a84a 100644 --- a/config/src/vespa/config/subscription/configsubscriptionset.cpp +++ b/config/src/vespa/config/subscription/configsubscriptionset.cpp @@ -88,8 +88,9 @@ ConfigSubscriptionSet::acquireSnapshot(uint64_t timeoutInMillis, bool ignoreChan _state = CONFIGURED; for (const auto & subscription : _subscriptionList) { const ConfigKey & key(subscription->getKey()); - LOG(debug, "Updated config id(%s), has changed: %s, lastGenerationChanged: %ld", + LOG(debug, "Updated config id(%s), defname(%s), has changed: %s, lastGenerationChanged: %ld", key.getConfigId().c_str(), + key.getDefName().c_str(), (subscription->hasChanged() ? "true" : "false"), subscription->getLastGenerationChanged()); subscription->flip(); diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index c13665342ef..3c99875f978 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -1,5 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=cloud.config + rpcport int default=19070 httpport int default=19071 numthreads int default=16 @@ -43,4 +44,7 @@ dockerRegistry string default="" dockerVespaBaseImage string default="" # Athenz config -loadBalancerAddress string default=""
\ No newline at end of file +loadBalancerAddress string default="" + +# File distributions +disableFiledistributor bool default=false diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 7d0ba6cd9bd..d8a560c6159 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -5,16 +5,21 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; import com.yahoo.text.Utf8; import net.jpountz.xxhash.XXHash64; import net.jpountz.xxhash.XXHashFactory; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; import java.util.logging.Logger; public class FileDirectory { @@ -94,6 +99,7 @@ public class FileDirectory { public FileReference addFile(File source, FileReference reference) { ensureRootExist(); try { + logfileInfo(source); File destinationDir = new File(root, reference.value()); if (!destinationDir.exists()) { destinationDir.mkdir(); @@ -102,7 +108,7 @@ public class FileDirectory { if (source.isDirectory()) IOUtils.copyDirectory(source, destination); else - IOUtils.copy(source, destination); + copyFile(source, destination); if (!destinationDir.exists()) { if ( ! tempDestinationDir.toFile().renameTo(destinationDir)) { log.warning("Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'."); @@ -117,4 +123,18 @@ public class FileDirectory { throw new IllegalArgumentException(e); } } + + private void logfileInfo(File file ) throws IOException { + BasicFileAttributes basicFileAttributes = Files.readAttributes(file.toPath(), BasicFileAttributes.class); + log.log(LogLevel.DEBUG, "Adding file " + file.getAbsolutePath() + " (created " + basicFileAttributes.creationTime() + + ", modified " + basicFileAttributes.lastModifiedTime() + + ", size " + basicFileAttributes.size() + ")"); + } + + private static void copyFile(File source, File dest) throws IOException { + try (FileChannel sourceChannel = new FileInputStream(source).getChannel(); + FileChannel destChannel = new FileOutputStream(dest).getChannel()) { + destChannel.transferFrom(sourceChannel, 0, sourceChannel.size()); + } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 9dc94c9fe93..9316a9a5c8e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -2,24 +2,33 @@ package com.yahoo.vespa.config.server.filedistribution; import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.io.IOUtils; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.net.HostName; +import com.yahoo.vespa.config.Connection; +import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.JRTConnectionPool; +import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.filedistribution.FileDownloader; import java.io.File; import java.io.IOException; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Logger; +import java.util.stream.Collectors; public class FileServer { private static final Logger log = Logger.getLogger(FileServer.class.getName()); private final FileDirectory root; private final ExecutorService executor; - private final FileDownloader downloader = new FileDownloader(new JRTConnectionPool(ConfigSourceSet.createDefault())); + private final FileDownloader downloader; public static class ReplayStatus { private final int code; @@ -38,18 +47,21 @@ public class FileServer { } @Inject - public FileServer() { - this(FileDistribution.getDefaultFileDBPath()); + public FileServer(ConfigserverConfig configserverConfig) { + this(createConnectionPool(configserverConfig), FileDistribution.getDefaultFileDBPath()); } + // For testing only public FileServer(File rootDir) { - this(rootDir, Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())); + this(new EmptyConnectionPool(), rootDir); } - public FileServer(File rootDir, ExecutorService executor) { + private FileServer(ConnectionPool connectionPool, File rootDir) { + this.downloader = new FileDownloader(connectionPool); this.root = new FileDirectory(rootDir); - this.executor = executor; + this.executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); } + public boolean hasFile(String fileName) { return hasFile(new FileReference(fileName)); } @@ -94,4 +106,40 @@ public class FileServer { public void download(FileReference fileReference) { downloader.getFile(fileReference); } + + public FileDownloader downloader() { + return downloader; + } + + // Connection pool with all config servers except this one (might be an empty pool if there is only one config server) + private static ConnectionPool createConnectionPool(ConfigserverConfig configserverConfig) { + List<String> configServers = ConfigServerSpec.fromConfig(configserverConfig) + .stream() + .filter(spec -> !spec.getHostName().equals(HostName.getLocalhost())) + .map(spec -> "tcp/" + spec.getHostName() + ":" + spec.getConfigServerPort()) + .collect(Collectors.toList()); + + return configServers.size() > 0 ? new JRTConnectionPool(new ConfigSourceSet(configServers)) : new EmptyConnectionPool(); + } + + private static class EmptyConnectionPool implements ConnectionPool { + + @Override + public void close() {} + + @Override + public void setError(Connection connection, int i) {} + + @Override + public Connection getCurrent() { return null; } + + @Override + public Connection setNewCurrentConnection() { return null; } + + @Override + public int getSize() { return 0; } + + @Override + public Supervisor getSupervisor() { return new Supervisor(new Transport()); } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index 1806414f510..85249d4e87d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -75,7 +75,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer { serviceInfo.getServiceType().equals("qrserver")). findAny(); if (container.isPresent()) { - activeRotation |= Boolean.valueOf(container.get().getProperty("activeRotation").get()); + activeRotation |= Boolean.valueOf(container.get().getProperty("activeRotation").orElse("false")); } } return activeRotation; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 8f2cc04fad7..48732814919 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.NoSuchElementException; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -99,7 +100,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { catch (RuntimeException e) { boolean isOldestMajor = i == majorVersions.size() - 1; if (isOldestMajor) { - if (e instanceof NullPointerException) { + if (e instanceof NullPointerException || e instanceof NoSuchElementException) { log.log(LogLevel.WARNING, "Unexpected error when building model ", e); throw new InternalServerException(applicationId + ": Error loading model", e); } else { diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 635ce07e727..fbab854ae9e 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -34,6 +34,7 @@ <component id="com.yahoo.config.provision.Zone" bundle="config-provisioning" /> <component id="com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.application.HttpProxy" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.filedistribution.FileServer" bundle="configserver" /> <component id="com.yahoo.vespa.serviceview.ConfigServerLocation" bundle="configserver" /> diff --git a/configserver/src/main/sh/start-filedistribution b/configserver/src/main/sh/start-filedistribution index bb8599f2bc9..a0cf1971215 100755 --- a/configserver/src/main/sh/start-filedistribution +++ b/configserver/src/main/sh/start-filedistribution @@ -63,7 +63,7 @@ ROOT=${VESPA_HOME%/} VESPA_CONFIG_ID="dir:${ROOT}/conf/filedistributor" export VESPA_CONFIG_ID -if [ "$multitenant" = "true" ]; then +if [ "$multitenant" = "true" -a "$disable_filedistributor" = "false" ]; then foo=`${ROOT}/libexec/vespa/vespa-config.pl -mkfiledistributorconfig` PIDFILE_FILEDISTRIBUTOR=var/run/filedistributor.pid LOGFILE="${ROOT}/logs/vespa/vespa.log" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java index 4913798e5ad..09260987ac0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java @@ -1,12 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.filedistribution; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.FileReference; import com.yahoo.io.IOUtils; +import com.yahoo.net.HostName; import org.junit.Test; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -57,6 +60,7 @@ public class FileServerTest { this.content.complete(content); } } + @Test public void requireThatWeCanReplayFile() throws IOException, InterruptedException, ExecutionException { createCleanDir("12y"); @@ -67,6 +71,33 @@ public class FileServerTest { cleanup(); } + @Test + public void requireThatDifferentNumberOfConfigServersWork() throws IOException { + // Empty connection pool in tests etc. + ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); + FileServer fileServer = new FileServer(new ConfigserverConfig(builder)); + assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); + + // Empty connection pool when only one server, no use in downloading from yourself + List<ConfigserverConfig.Zookeeperserver.Builder> servers = new ArrayList<>(); + ConfigserverConfig.Zookeeperserver.Builder serverBuilder = new ConfigserverConfig.Zookeeperserver.Builder(); + serverBuilder.hostname(HostName.getLocalhost()); + serverBuilder.port(123456); + servers.add(serverBuilder); + builder.zookeeperserver(servers); + fileServer = new FileServer(new ConfigserverConfig(builder)); + assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); + + // connection pool of size 1 when 2 servers + ConfigserverConfig.Zookeeperserver.Builder serverBuilder2 = new ConfigserverConfig.Zookeeperserver.Builder(); + serverBuilder2.hostname("bar"); + serverBuilder2.port(123456); + servers.add(serverBuilder2); + builder.zookeeperserver(servers); + fileServer = new FileServer(new ConfigserverConfig(builder)); + assertEquals(1, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); + } + private void cleanup() { created.forEach((file) -> IOUtils.recursiveDeleteDir(file)); created.clear(); diff --git a/container-search/src/main/java/com/yahoo/fs4/MapEncoder.java b/container-search/src/main/java/com/yahoo/fs4/MapEncoder.java index 4245f51ace8..2f915257938 100644 --- a/container-search/src/main/java/com/yahoo/fs4/MapEncoder.java +++ b/container-search/src/main/java/com/yahoo/fs4/MapEncoder.java @@ -6,10 +6,13 @@ import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.text.Utf8; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; /** * A static utility for encoding values to the binary map representation used in fs4 packets. diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java index df182b56fd8..37356f2c2a6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java @@ -19,8 +19,9 @@ public class Issue { private final String label; private final User assignee; private final PropertyId propertyId; + private final Type type; - private Issue(String summary, String description, String label, User assignee, PropertyId propertyId) { + private Issue(String summary, String description, String label, User assignee, PropertyId propertyId, Type type) { if (summary.isEmpty()) throw new IllegalArgumentException("Issue summary can not be empty!"); if (description.isEmpty()) throw new IllegalArgumentException("Issue description can not be empty!"); Objects.requireNonNull(propertyId, "An issue must belong to a property!"); @@ -30,26 +31,31 @@ public class Issue { this.label = label; this.assignee = assignee; this.propertyId = propertyId; + this.type = type; } public Issue(String summary, String description, PropertyId propertyId) { - this(summary, description, null, null, propertyId); + this(summary, description, null, null, propertyId, Type.defect); } public Issue append(String appendage) { - return new Issue(summary, description + appendage, label, assignee, propertyId); + return new Issue(summary, description + appendage, label, assignee, propertyId, type); } - public Issue withLabel(String label) { - return new Issue(summary, description, label, assignee, propertyId); + public Issue with(String label) { + return new Issue(summary, description, label, assignee, propertyId, type); } - public Issue withAssignee(User assignee) { - return new Issue(summary, description, label, assignee, propertyId); + public Issue with(User assignee) { + return new Issue(summary, description, label, assignee, propertyId, type); } - public Issue withPropertyId(PropertyId propertyId) { - return new Issue(summary, description, label, assignee, propertyId); + public Issue with(PropertyId propertyId) { + return new Issue(summary, description, label, assignee, propertyId, type); + } + + public Issue with(Type type) { + return new Issue(summary, description, label, assignee, propertyId, type); } public String summary() { @@ -72,4 +78,16 @@ public class Issue { return propertyId; } + public Type type() { + return type; + } + + + public enum Type { + + defect, // A defect which needs fixing. + task // A task the humans must perform. + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java index 9eef1dac70b..363a2ea19cd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java @@ -4,13 +4,12 @@ package com.yahoo.vespa.hosted.rotation; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; +import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; -import com.yahoo.metrics.simple.Gauge; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; import com.yahoo.vespa.hosted.controller.api.ApplicationAlias; -import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; +import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; +import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.jetbrains.annotations.NotNull; @@ -31,17 +30,16 @@ import java.util.stream.Collectors; public class ControllerRotationRepository implements RotationRepository { private static final Logger log = Logger.getLogger(ControllerRotationRepository.class.getName()); - - private static final String REMAINING_ROTATIONS_METRIC_NAME = "remaining_rotations"; - private final Gauge remainingRotations; + public static final String REMAINING_ROTATIONS_METRIC_NAME = "remaining_rotations"; private final ControllerDb controllerDb; private final Map<RotationId, Rotation> rotationsMap; + private final Metric metric; - public ControllerRotationRepository(RotationsConfig rotationConfig, ControllerDb controllerDb, MetricReceiver metricReceiver) { + public ControllerRotationRepository(RotationsConfig rotationConfig, ControllerDb controllerDb, Metric metric) { this.controllerDb = controllerDb; this.rotationsMap = buildRotationsMap(rotationConfig); - this.remainingRotations = metricReceiver.declareGauge(REMAINING_ROTATIONS_METRIC_NAME); + this.metric = metric; } private static Map<RotationId, Rotation> buildRotationsMap(RotationsConfig rotationConfig) { @@ -73,7 +71,7 @@ public class ControllerRotationRepository implements RotationRepository { .collect(Collectors.toSet()); } - if( ! deploymentSpec.globalServiceId().isPresent()) { + if (!deploymentSpec.globalServiceId().isPresent()) { return Collections.emptySet(); } @@ -84,13 +82,12 @@ public class ControllerRotationRepository implements RotationRepository { if (productionZoneCount >= 2) { return assignRotation(applicationId); - } - else { + } else { throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined"); } } - private boolean isCorp(DeploymentSpec.DeclaredZone zone) { + private static boolean isCorp(DeploymentSpec.DeclaredZone zone) { return zone.region().isPresent() && zone.region().get().value().contains("corp"); } @@ -139,7 +136,8 @@ public class ControllerRotationRepository implements RotationRepository { try { int freeRotationsCount = availableRotations().size(); log.log(LogLevel.INFO, "Rotation: {0} global rotations remaining", freeRotationsCount); - remainingRotations.sample(freeRotationsCount); + metric.set(REMAINING_ROTATIONS_METRIC_NAME, freeRotationsCount, + metric.createContext(Collections.emptyMap())); } catch (Exception e) { log.log(LogLevel.INFO, "Failed to report rotations metric", e); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java index 561799529f9..b4074fc1944 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.rotation; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; @@ -22,6 +22,10 @@ import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; /** * @author Oyvind Gronnesby @@ -100,12 +104,13 @@ public class ControllerRotationRepositoryTest { private ControllerRotationRepository repository; private ControllerRotationRepository repositoryWhitespaces; - + private Metric metric; @Before public void setup_repository() { - repository = new ControllerRotationRepository(rotationsConfig, controllerDb, MetricReceiver.nullImplementation); - repositoryWhitespaces = new ControllerRotationRepository(rotationsConfigWhitespaces, controllerDb, MetricReceiver.nullImplementation); + metric = mock(Metric.class); + repository = new ControllerRotationRepository(rotationsConfig, controllerDb, metric); + repositoryWhitespaces = new ControllerRotationRepository(rotationsConfigWhitespaces, controllerDb, metric); controllerDb.assignRotation(new RotationId("foo-1"), applicationId); } @@ -129,6 +134,7 @@ public class ControllerRotationRepositoryTest { Set<Rotation> rotations = repository.getOrAssignRotation(other, deploymentSpec); Rotation assignedRotation = new Rotation(new RotationId("foo-2"), "foo-2.com"); assertContainsOnly(assignedRotation, rotations); + verify(metric).set(eq(ControllerRotationRepository.REMAINING_ROTATIONS_METRIC_NAME), eq(1), any()); } @Test @@ -140,6 +146,7 @@ public class ControllerRotationRepositoryTest { thrown.expectMessage("no rotations available"); repository.getOrAssignRotation(third, deploymentSpec); + verify(metric).set(eq(ControllerRotationRepository.REMAINING_ROTATIONS_METRIC_NAME), eq(0), any()); } @Test diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 41622792e43..10b77b540e7 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -19,11 +19,6 @@ <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>config-lib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> </dependency> diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index cd4b3afb9b5..fde410bc8d7 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -146,8 +146,7 @@ public class FileDownloader { fileReferenceDownloader.addToDownloadQueue(fileReferenceDownload); } - Set<FileReference> queuedDownloads() { - return fileReferenceDownloader.queuedDownloads(); + public FileReferenceDownloader fileReferenceDownloader() { + return fileReferenceDownloader; } - } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 08595662f36..4c9c37dd6da 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -37,7 +37,7 @@ import java.util.stream.Collectors; * @author hmusum */ // TODO: Handle shutdown of executors -class FileReferenceDownloader { +public class FileReferenceDownloader { private final static Logger log = Logger.getLogger(FileReferenceDownloader.class.getName()); private final static Duration rpcTimeout = Duration.ofSeconds(10); @@ -107,8 +107,7 @@ class FileReferenceDownloader { Thread.sleep(10); } catch (InterruptedException e) { /* ignore for now */} } else { - log.log(LogLevel.INFO, "Polling queue, found file reference '" + - fileReferenceDownload.fileReference().value() + "' to download"); + log.log(LogLevel.DEBUG, "Will download file reference '" + fileReferenceDownload.fileReference().value() + "'"); downloadExecutor.submit(() -> startDownload(fileReferenceDownload.fileReference(), downloadTimeout, fileReferenceDownload)); } } while (true); @@ -133,12 +132,16 @@ class FileReferenceDownloader { return true; } else { log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found for " + connection.getAddress()); + connectionPool.setNewCurrentConnection(); return false; } } else { - log.log(LogLevel.WARNING, "Request failed. Req: " + request + "\nSpec: " + connection.getAddress()); - if (request.isError() && request.errorCode() == ErrorCode.CONNECTION) - connection.setError(request.errorCode()); + log.log(LogLevel.WARNING, "Request failed. Req: " + request + "\nSpec: " + connection.getAddress() + + ", error code: " + request.errorCode()); + if (request.isError() && request.errorCode() == ErrorCode.CONNECTION || request.errorCode() == ErrorCode.TIMEOUT) { + log.log(LogLevel.WARNING, "Setting error for connection " + connection.getAddress()); + connectionPool.setError(connection, request.errorCode()); + } return false; } } @@ -181,4 +184,7 @@ class FileReferenceDownloader { return ImmutableMap.copyOf(downloadStatus); } + public ConnectionPool connectionPool() { + return connectionPool; + } } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java new file mode 100644 index 00000000000..28935c203fe --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java @@ -0,0 +1,98 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.io.IOUtils; +import com.yahoo.jrt.DataValue; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.Int64Value; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Target; +import com.yahoo.jrt.Transport; +import com.yahoo.log.LogLevel; +import net.jpountz.xxhash.XXHash64; +import net.jpountz.xxhash.XXHashFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.logging.Logger; + +public class RpcTester { + + private static final Logger log = Logger.getLogger(RpcTester.class.getName()); + + private final Target target; + + private RpcTester(Target target) { + this.target = target; + } + + private void call(String fileReference, String filename, byte[] blob) { + new FileReceiver(target).receive(new FileReference(fileReference), filename, blob); + } + + public static void main(String[] args) { + //String fileReference = args[0]; + String fileReference = "59f93f445438c9db7ccbf1629f583c2aa004a68b"; + String filename = "com.yahoo.vespatest.ExtraHitSearcher-1.0.0-deploy.jar"; + File file = new File(String.format("/tmp/%s/%s", fileReference, filename)); + byte[] blob = null; + + try { + blob = IOUtils.readFileBytes(file); + } catch (IOException e) { + e.printStackTrace(); + } + + log.log(LogLevel.INFO, "Read blob from " + file.getAbsolutePath()); + + + Supervisor supervisor = new Supervisor(new Transport()); + + Spec spec = new Spec("tcp/localhost:19090"); + log.log(LogLevel.INFO, "Connecting to " + spec); + Target target = supervisor.connect(spec); + if (! target.isValid()) { + log.log(LogLevel.INFO, "Could not connect"); + System.exit(1); + } else { + log.log(LogLevel.INFO, "Connected to " + spec); + } + + new RpcTester(target).call(fileReference, filename, blob); + } + + class FileReceiver { + + Target target; + + FileReceiver(Target target) { + this.target = target; + } + + void receive(FileReference reference, String filename, byte[] content) { + + log.log(LogLevel.INFO, "Preparing receive call for " + reference.value() + " and file " + filename); + + XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); + Request fileBlob = new Request("filedistribution.receiveFile"); + + log.log(LogLevel.INFO, "Calling " + fileBlob.methodName() + " with target " + target); + + fileBlob.parameters().add(new StringValue(reference.value())); + fileBlob.parameters().add(new StringValue(filename)); + fileBlob.parameters().add(new DataValue(content)); + fileBlob.parameters().add(new Int64Value(hasher.hash(ByteBuffer.wrap(content), 0))); + fileBlob.parameters().add(new Int32Value(0)); + fileBlob.parameters().add(new StringValue("OK")); + log.log(LogLevel.INFO, "Doing invokeSync"); + target.invokeSync(fileBlob, 5); + log.log(LogLevel.INFO, "Done with invokeSync"); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index a3a647e1d14..763291bf86b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -139,7 +139,7 @@ public class StorageMaintainer { Process duCommand = new ProcessBuilder().command(command).start(); if (!duCommand.waitFor(60, TimeUnit.SECONDS)) { duCommand.destroy(); - throw new RuntimeException("Disk usage command timedout, aborting."); + throw new RuntimeException("Disk usage command timed out, aborting."); } String output = IOUtils.readAll(new InputStreamReader(duCommand.getInputStream())); String[] results = output.split("\t"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index bcc18cfd876..7de138fa954 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -45,8 +45,12 @@ public class MockNodeRepository extends NodeRepository { */ public MockNodeRepository(MockCurator curator, NodeFlavors flavors) throws Exception { super(flavors, curator, Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(), - new MockNameResolver().mockAnyLookup(), new DockerImage("docker-registry.domain.tld:8080/dist/vespa")); + new MockNameResolver() + .addRecord("test-container-1", "::2") + .mockAnyLookup(), + new DockerImage("docker-registry.domain.tld:8080/dist/vespa")); this.flavors = flavors; + curator.setConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234"); populate(); } @@ -133,4 +137,4 @@ public class MockNodeRepository extends NodeRepository { provisioner.activate(transaction, application, hosts); transaction.commit(); } -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 37b2f54da4d..54e0c59ff00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -40,9 +39,20 @@ public class RestApiTest { private final static String responsesPath = "src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/"; + private JDisc container; + + @Before + public void startContainer() { + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); + } + + @After + public void stopContainer() { + container.close(); + } + /** This test gives examples of all the requests that can be made to nodes/v2 */ @Test - @Ignore /** TODO re-enable this and verify correctness */ public void test_requests() throws Exception { // GET assertFile(new Request("http://localhost:8080/nodes/v2/"), "root.json"); @@ -54,28 +64,28 @@ public class RestApiTest { assertFile(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "node2.json"); // GET with filters - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&hostname=host2.yahoo.com%20host1.yahoo.com"), "application2-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterType=content"), "active-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&hostname=host6.yahoo.com%20host2.yahoo.com"), "application2-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterType=content"), "content-nodes.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterId=id2"), "application2-nodes.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&application=tenant2.application2.instance2"), "application2-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&parentHost=parent1.yahoo.com,parent.host.yahoo.com"), "parent-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&parentHost=dockerhost1.yahoo.com"), "child-nodes.json"); // POST restart command assertRestart(1, new Request("http://localhost:8080/nodes/v2/command/restart?hostname=host2.yahoo.com", new byte[0], Request.Method.POST)); assertRestart(2, new Request("http://localhost:8080/nodes/v2/command/restart?application=tenant2.application2.instance2", new byte[0], Request.Method.POST)); - assertRestart(4, new Request("http://localhost:8080/nodes/v2/command/restart", + assertRestart(9, new Request("http://localhost:8080/nodes/v2/command/restart", new byte[0], Request.Method.POST)); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "\"restartGeneration\":3"); // POST reboot command - assertReboot(5, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", + assertReboot(10, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", new byte[0], Request.Method.POST)); assertReboot(2, new Request("http://localhost:8080/nodes/v2/command/reboot?application=tenant2.application2.instance2", new byte[0], Request.Method.POST)); - assertReboot(10, new Request("http://localhost:8080/nodes/v2/command/reboot", + assertReboot(15, new Request("http://localhost:8080/nodes/v2/command/reboot", new byte[0], Request.Method.POST)); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "\"rebootGeneration\":4"); @@ -106,9 +116,9 @@ public class RestApiTest { assertFile(new Request("http://localhost:8080/nodes/v2/node/parent2.yahoo.com"), "parent2.json"); // DELETE a provisioned node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host11.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com", new byte[0], Request.Method.DELETE), - "{\"message\":\"Removed host11.yahoo.com\"}"); + "{\"message\":\"Removed host9.yahoo.com\"}"); // PUT nodes ready assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host8.yahoo.com", @@ -125,15 +135,15 @@ public class RestApiTest { "{\"message\":\"Moved host8.yahoo.com to ready\"}"); // PUT a node in failed ... - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host3.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host2.yahoo.com", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host3.yahoo.com to failed\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host3.yahoo.com"), + "{\"message\":\"Moved host2.yahoo.com to failed\"}"); + assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "\"state\":\"failed\""); // ... and put it back in active (after fixing). This is useful to restore data when multiple nodes fail. - assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host3.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host3.yahoo.com to active\"}"); + "{\"message\":\"Moved host2.yahoo.com to active\"}"); // PUT a node in parked ... assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host8.yahoo.com", @@ -147,30 +157,29 @@ public class RestApiTest { "{\"message\":\"Removed host8.yahoo.com\"}"); // or, PUT a node in failed ... - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/test-container-1", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host6.yahoo.com to failed\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), + "{\"message\":\"Moved test-container-1 to failed\"}"); + assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-container-1"), "\"state\":\"failed\""); // ... and deallocate it such that it moves to dirty and is recycled - assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/test-container-1", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host6.yahoo.com to dirty\"}"); + "{\"message\":\"Moved test-container-1 to dirty\"}"); // ... and set it back to ready as if this was from the node-admin with the temporary state rest api - assertResponse(new Request("http://localhost:8080/nodes/v2/state/availablefornewallocations/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/availablefornewallocations/test-container-1", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host6.yahoo.com to ready\"}"); + "{\"message\":\"Marked following nodes as available for new allocation: test-container-1\"}"); // Put a host in failed and make sure it's children are also failed - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/parent1.yahoo.com", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host10.yahoo.com, host5.yahoo.com, parent1.yahoo.com to failed\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/dockerhost1.yahoo.com", new byte[0], Request.Method.PUT), + "{\"message\":\"Moved dockerhost1.yahoo.com, host4.yahoo.com to failed\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed"), "{\"nodes\":[" + - "{\"url\":\"http://localhost:8080/nodes/v2/node/parent1.yahoo.com\"}," + + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}," + "{\"url\":\"http://localhost:8080/nodes/v2/node/host5.yahoo.com\"}," + - "{\"url\":\"http://localhost:8080/nodes/v2/node/host10.yahoo.com\"}]}"); - + "{\"url\":\"http://localhost:8080/nodes/v2/node/host4.yahoo.com\"}]}"); // Update (PATCH) a node (multiple fields can also be sent in one request body) assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", @@ -514,13 +523,6 @@ public class RestApiTest { } } - private JDisc container; - @Before - public void startContainer() { - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); } - @After - public void stopContainer() { container.close(); } - private String asDockerNodeJson(String hostname, String parentHostname, int additionalIpCount, String... ipAddress) { return "{\"hostname\":\"" + hostname + "\", \"parentHostname\":\"" + parentHostname + "\"," + createIpAddresses(ipAddress) + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json index d1df5b83f24..c67ba904f9a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json @@ -1,8 +1,13 @@ { "nodes": [ + @include(docker-node4.json), + @include(docker-node5.json), + @include(docker-node2.json), + @include(docker-node1.json), + @include(docker-node3.json), @include(node6.json), - @include(node3.json), @include(node2.json), - @include(node1.json) + @include(docker-container1.json), + @include(node4.json) ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json index 1d4d97315cd..4581ecba73d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json @@ -1,6 +1,6 @@ { "nodes": [ - @include(node2.json), - @include(node1.json) + @include(node6.json), + @include(node2.json) ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json new file mode 100644 index 00000000000..dae92aae091 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json @@ -0,0 +1,5 @@ +{ + "nodes": [ + @include(node4.json) + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json new file mode 100644 index 00000000000..47a2c012b17 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json @@ -0,0 +1,8 @@ +{ + "nodes": [ + @include(node6.json), + @include(node2.json), + @include(docker-container1.json), + @include(node4.json) + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json new file mode 100644 index 00000000000..7823ed0431d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json @@ -0,0 +1,56 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/test-container-1", + "id": "test-container-1", + "state": "active", + "type": "tenant", + "hostname": "test-container-1", + "parentHostname": "dockerhost3.yahoo.com", + "openStackId": "fake-test-container-1", + "flavor": "docker", + "canonicalFlavor": "docker", + "minDiskAvailableGb": 100.0, + "minMainMemoryAvailableGb": 0.5, + "description": "Flavor-name-is-docker", + "minCpuCores": 0.2, + "fastDisk": true, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" + }, + "membership": { + "clustertype": "content", + "clusterid": "id3", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "reserved", + "at": 123, + "agent": "system" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::2" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json new file mode 100644 index 00000000000..a13dfae927f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json new file mode 100644 index 00000000000..f7a1d6ab9a9 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com", + "id": "dockerhost2.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost2.yahoo.com", + "openStackId": "dockerhost2", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json new file mode 100644 index 00000000000..f877d33672f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com", + "id": "dockerhost3.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost3.yahoo.com", + "openStackId": "dockerhost3", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 2, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json new file mode 100644 index 00000000000..913cf9852aa --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost4.yahoo.com", + "id": "dockerhost4.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost4.yahoo.com", + "openStackId": "dockerhost4", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 3, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json new file mode 100644 index 00000000000..685b0a52b15 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost5.yahoo.com", + "id": "dockerhost5.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost5.yahoo.com", + "openStackId": "dockerhost5", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 4, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json index fea4fb8d4d2..c09829e7f85 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json @@ -28,6 +28,9 @@ "name":"OperatorChangeApplicationMaintainer" }, { + "name":"ProvisionedExpirer" + }, + { "name":"RetiredEarlyExpirer" }, { @@ -43,4 +46,4 @@ "inactive":[ "NodeFailer" ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index f311c240b1d..cb250e2033b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -1,7 +1,7 @@ { "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", "id": "host4.yahoo.com", - "state": "reserved", + "state": "failed", "type": "tenant", "hostname": "host4.yahoo.com", "parentHostname": "parent.yahoo.com", @@ -12,36 +12,36 @@ "minMainMemoryAvailableGb": 12.0, "description": "Flavor-name-is-medium-disk", "minCpuCores": 6.0, - "fastDisk":true, + "fastDisk": true, "environment": "BARE_METAL", "owner": { - "tenant": "tenant1", - "application": "application1", - "instance": "instance1" + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" }, "membership": { - "clustertype": "container", - "clusterid": "id1", + "clustertype": "content", + "clusterid": "id3", "group": "0", - "index": 1, + "index": 0, "retired": false }, - "restartGeneration": 0, + "restartGeneration": 1, "currentRestartGeneration": 1, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", "allowedToBeDown": false, - "rebootGeneration": 2, + "rebootGeneration": 3, "currentRebootGeneration": 1, "vespaVersion": "6.43.0", "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.43.0", "hostedVersion": "6.43.0", "convergedStateVersion": "6.43.0", - "failCount": 0, + "failCount": 1, "hardwareFailure": true, "hardwareFailureDescription": "memory_mcelog", - "wantToRetire" : true, - "wantToDeprovision" : true, + "wantToRetire": true, + "wantToDeprovision": true, "history": [ { "event": "provisioned", @@ -51,7 +51,7 @@ { "event": "readied", "at": 123, - "agent": "system" + "agent": "system" }, { "event": "reserved", @@ -59,11 +59,24 @@ "agent": "application" }, { + "event": "activated", + "at": 123, + "agent": "application" + }, + { + "event": "failed", + "at": 123, + "agent": "operator" + }, + { "event": "rebooted", "at": 123, "agent": "system" } ], - "ipAddresses":["127.0.0.1", "::1"], - "additionalIpAddresses":[] + "ipAddresses": [ + "127.0.0.1", + "::1" + ], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json index a9feed81674..475b914989b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json @@ -1,14 +1,19 @@ { "nodes": [ @include(node7.json), - @include(parent1.json), + @include(node3.json), @include(node10.json), - @include(node4.json), + @include(node1.json), + @include(docker-node4.json), @include(node6.json), - @include(node3.json), + @include(docker-node5.json), + @include(docker-node2.json), @include(node2.json), - @include(node1.json), + @include(docker-node1.json), + @include(docker-node3.json), + @include(docker-container1.json), + @include(node4.json), @include(node55.json), @include(node5.json) ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json index 67b65259f8a..3bfaa95d5ee 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json @@ -4,31 +4,46 @@ "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/parent1.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host10.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host4.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host6.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost4.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host3.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host2.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost5.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host1.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host55.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host5.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/test-container-1" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com" } ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json deleted file mode 100644 index 81ca0465c4b..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "nodes": [ - @include(node10.json), - @include(node5.json) - ] -}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json index 4ee1d5ed9b9..183f81ee3e1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json @@ -9,23 +9,28 @@ "ready": { "url": "http://localhost:8080/nodes/v2/state/ready", "nodes": [ - @include(parent1.json) + @include(node3.json) ] }, "reserved": { "url": "http://localhost:8080/nodes/v2/state/reserved", "nodes": [ @include(node10.json), - @include(node4.json) + @include(node1.json) ] }, "active": { "url": "http://localhost:8080/nodes/v2/state/active", "nodes": [ + @include(docker-node4.json), + @include(docker-node5.json), + @include(docker-node2.json), + @include(docker-node1.json), + @include(docker-node3.json), @include(node6.json), - @include(node3.json), @include(node2.json), - @include(node1.json) + @include(docker-container1.json), + @include(node4.json) ] }, "inactive": { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/Simplifier.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/Simplifier.java index ede7c861d98..ebad0d5c21f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/Simplifier.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/Simplifier.java @@ -94,7 +94,7 @@ public class Simplifier extends ExpressionTransformer { private ExpressionNode transformIf(IfNode node) { if ( ! isConstant(node.getCondition())) return node; - if (((BooleanValue)node.getCondition().evaluate(null)).asBoolean()) + if ((node.getCondition().evaluate(null)).asBoolean()) return node.getTrueExpression(); else return node.getFalseExpression(); diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala index fe1af676b54..41e6b66b986 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala @@ -10,8 +10,8 @@ import scala.language.implicitConversions import scala.util.Try /** - * @author tonytv - */ + * @author Tony Vaagenes + */ class CloudConfigYinstVariables extends CloudConfigOptions { import CloudConfigYinstVariables._ @@ -39,6 +39,7 @@ class CloudConfigYinstVariables extends CloudConfigOptions { override val dockerRegistry = optionalYinstVar[java.lang.String]("docker_registry") override val dockerVespaBaseImage = optionalYinstVar[java.lang.String]("docker_vespa_base_image") override val loadBalancerAddress = optionalYinstVar[java.lang.String]("load_balancer_address") + override val disableFiledistributor = optionalYinstVar[java.lang.Boolean]("disable_filedistributor") } object CloudConfigYinstVariables { diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.h b/storage/src/vespa/storage/bucketdb/lockablemap.h index 03d94b27f0b..a4382ceb683 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.h +++ b/storage/src/vespa/storage/bucketdb/lockablemap.h @@ -16,10 +16,12 @@ #include <map> #include <vespa/vespalib/util/printable.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/document/bucket/bucketid.h> +#include <mutex> +#include <condition_variable> +#include <cassert> namespace storage { @@ -238,7 +240,8 @@ private: }; Map _map; - vespalib::Monitor _lock; + mutable std::mutex _lock; + std::condition_variable _cond; LockIdSet _lockedKeys; LockWaiters _lockWaiters; @@ -247,9 +250,9 @@ private: const char* clientId, bool haslock, bool& preExisted); void unlock(const key_type& key); bool findNextKey(key_type& key, mapped_type& val, const char* clientId, - vespalib::MonitorGuard& guard); + std::unique_lock<std::mutex> &guard); bool handleDecision(key_type& key, mapped_type& val, Decision decision); - void ackquireKey(const LockId & lid, vespalib::MonitorGuard & guard); + void acquireKey(const LockId & lid, std::unique_lock<std::mutex> &guard); /** * Process up to `chunkSize` bucket database entries from--and possibly @@ -304,7 +307,7 @@ private: void addAndLockResults(const std::vector<BucketId::Type> keys, const char* clientId, std::map<BucketId, WrappedEntry>& results, - vespalib::MonitorGuard& guard); + std::unique_lock<std::mutex> &guard); }; } // storage diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.hpp b/storage/src/vespa/storage/bucketdb/lockablemap.hpp index f5d692139be..f370a792145 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.hpp +++ b/storage/src/vespa/storage/bucketdb/lockablemap.hpp @@ -69,6 +69,7 @@ template<typename Map> LockableMap<Map>::LockableMap() : _map(), _lock(), + _cond(), _lockedKeys(), _lockWaiters() {} @@ -80,8 +81,8 @@ template<typename Map> bool LockableMap<Map>::operator==(const LockableMap<Map>& other) const { - vespalib::LockGuard guard(_lock); - vespalib::LockGuard guard2(other._lock); + std::lock_guard<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard2(other._lock); return (_map == other._map); } @@ -89,8 +90,8 @@ template<typename Map> bool LockableMap<Map>::operator<(const LockableMap<Map>& other) const { - vespalib::LockGuard guard(_lock); - vespalib::LockGuard guard2(other._lock); + std::lock_guard<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard2(other._lock); return (_map < other._map); } @@ -98,7 +99,7 @@ template<typename Map> typename Map::size_type LockableMap<Map>::size() const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _map.size(); } @@ -106,17 +107,16 @@ template<typename Map> typename Map::size_type LockableMap<Map>::getMemoryUsage() const { - vespalib::MonitorGuard guard(_lock); - return _map.getMemoryUsage() - + _lockedKeys.getMemoryUsage() - + sizeof(vespalib::Monitor); + std::lock_guard<std::mutex> guard(_lock); + return _map.getMemoryUsage() + _lockedKeys.getMemoryUsage() + + sizeof(std::mutex) + sizeof(std::condition_variable); } template<typename Map> bool LockableMap<Map>::empty() const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _map.empty(); } @@ -124,18 +124,18 @@ template<typename Map> void LockableMap<Map>::swap(LockableMap<Map>& other) { - vespalib::LockGuard guard(_lock); - vespalib::LockGuard guard2(other._lock); + std::lock_guard<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard2(other._lock); return _map.swap(other._map); } template<typename Map> -void LockableMap<Map>::ackquireKey(const LockId & lid, vespalib::MonitorGuard & guard) +void LockableMap<Map>::acquireKey(const LockId & lid, std::unique_lock<std::mutex> &guard) { if (_lockedKeys.exist(lid)) { typename LockWaiters::Key waitId(_lockWaiters.insert(lid)); while (_lockedKeys.exist(lid)) { - guard.wait(); + _cond.wait(guard); } _lockWaiters.erase(waitId); } @@ -148,8 +148,8 @@ LockableMap<Map>::get(const key_type& key, const char* clientId, bool lockIfNonExistingAndNotCreating) { LockId lid(key, clientId); - vespalib::MonitorGuard guard(_lock); - ackquireKey(lid, guard); + std::unique_lock<std::mutex> guard(_lock); + acquireKey(lid, guard); bool preExisted = false; typename Map::iterator it = _map.find(key, createIfNonExisting, preExisted); @@ -197,9 +197,9 @@ bool LockableMap<Map>::erase(const key_type& key, const char* clientId, bool haslock) { LockId lid(key, clientId); - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); if (!haslock) { - ackquireKey(lid, guard); + acquireKey(lid, guard); } #ifdef ENABLE_BUCKET_OPERATION_LOGGING debug::logBucketDbErase(key, debug::TypeTag<mapped_type>()); @@ -213,9 +213,9 @@ LockableMap<Map>::insert(const key_type& key, const mapped_type& value, const char* clientId, bool haslock, bool& preExisted) { LockId lid(key, clientId); - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); if (!haslock) { - ackquireKey(lid, guard); + acquireKey(lid, guard); } #ifdef ENABLE_BUCKET_OPERATION_LOGGING debug::logBucketDbInsert(key, value); @@ -227,7 +227,7 @@ template<typename Map> void LockableMap<Map>::clear() { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _map.clear(); } @@ -235,13 +235,13 @@ template<typename Map> bool LockableMap<Map>::findNextKey(key_type& key, mapped_type& val, const char* clientId, - vespalib::MonitorGuard& guard) + std::unique_lock<std::mutex> &guard) { // Wait for next value to unlock. typename Map::iterator it(_map.lower_bound(key)); while (it != _map.end() && _lockedKeys.exist(LockId(it->first, ""))) { typename LockWaiters::Key waitId(_lockWaiters.insert(LockId(it->first, clientId))); - guard.wait(); + _cond.wait(guard); _lockWaiters.erase(waitId); it = _map.lower_bound(key); } @@ -279,16 +279,16 @@ LockableMap<Map>::each(Functor& functor, const char* clientId, mapped_type val; Decision decision; { - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); if (findNextKey(key, val, clientId, guard) || key > last) return; _lockedKeys.insert(LockId(key, clientId)); } try{ while (true) { decision = functor(const_cast<const key_type&>(key), val); - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); _lockedKeys.erase(LockId(key, clientId)); - guard.broadcast(); + _cond.notify_all(); if (handleDecision(key, val, decision)) return; ++key; if (findNextKey(key, val, clientId, guard) || key > last) return; @@ -297,9 +297,9 @@ LockableMap<Map>::each(Functor& functor, const char* clientId, } catch (...) { // Assuming only the functor call can throw exceptions, we need // to unlock the current key before exiting - vespalib::MonitorGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _lockedKeys.erase(LockId(key, clientId)); - guard.broadcast(); + _cond.notify_all(); throw; } } @@ -314,16 +314,16 @@ LockableMap<Map>::each(const Functor& functor, const char* clientId, mapped_type val; Decision decision; { - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); if (findNextKey(key, val, clientId, guard) || key > last) return; _lockedKeys.insert(LockId(key, clientId)); } try{ while (true) { decision = functor(const_cast<const key_type&>(key), val); - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); _lockedKeys.erase(LockId(key, clientId)); - guard.broadcast(); + _cond.notify_all(); if (handleDecision(key, val, decision)) return; ++key; if (findNextKey(key, val, clientId, guard) || key > last) return; @@ -332,9 +332,9 @@ LockableMap<Map>::each(const Functor& functor, const char* clientId, } catch (...) { // Assuming only the functor call can throw exceptions, we need // to unlock the current key before exiting - vespalib::MonitorGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _lockedKeys.erase(LockId(key, clientId)); - guard.broadcast(); + _cond.notify_all(); throw; } } @@ -347,7 +347,7 @@ LockableMap<Map>::all(Functor& functor, const char* clientId, { key_type key = first; mapped_type val; - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); while (true) { if (findNextKey(key, val, clientId, guard) || key > last) return; Decision d(functor(const_cast<const key_type&>(key), val)); @@ -364,7 +364,7 @@ LockableMap<Map>::all(const Functor& functor, const char* clientId, { key_type key = first; mapped_type val; - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); while (true) { if (findNextKey(key, val, clientId, guard) || key > last) return; Decision d(functor(const_cast<const key_type&>(key), val)); @@ -383,7 +383,7 @@ LockableMap<Map>::processNextChunk(Functor& functor, const uint32_t chunkSize) { mapped_type val; - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); for (uint32_t processed = 0; processed < chunkSize; ++processed) { if (findNextKey(key, val, clientId, guard)) { return false; @@ -422,7 +422,7 @@ void LockableMap<Map>::print(std::ostream& out, bool verbose, const std::string& indent) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); out << "LockableMap {\n" << indent << " "; if (verbose) { @@ -462,9 +462,9 @@ template<typename Map> void LockableMap<Map>::unlock(const key_type& key) { - vespalib::MonitorGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _lockedKeys.erase(LockId(key, "")); - guard.broadcast(); + _cond.notify_all(); } /** @@ -550,7 +550,7 @@ LockableMap<Map>::addAndLockResults( const std::vector<BucketId::Type> keys, const char* clientId, std::map<BucketId, WrappedEntry>& results, - vespalib::MonitorGuard& guard) + std::unique_lock<std::mutex> &guard) { // Wait until all buckets are free to be added, then add them all. while (true) { @@ -567,7 +567,7 @@ LockableMap<Map>::addAndLockResults( if (!allOk) { typename LockWaiters::Key waitId(_lockWaiters.insert(LockId(waitingFor, clientId))); - guard.wait(); + _cond.wait(guard); _lockWaiters.erase(waitId); } else { for (uint32_t i=0; i<keys.size(); i++) { @@ -593,7 +593,7 @@ LockableMap<Map>::createAppropriateBucket( const char* clientId, const BucketId& bucket) { - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); typename Map::const_iterator iter = _map.lower_bound(bucket.toKey()); // Find the two buckets around the possible new bucket. The new @@ -613,7 +613,7 @@ LockableMap<Map>::createAppropriateBucket( BucketId::Type key = newBucket.stripUnused().toKey(); LockId lid(key, clientId); - ackquireKey(lid, guard); + acquireKey(lid, guard); bool preExisted; typename Map::iterator it = _map.find(key, true, preExisted); _lockedKeys.insert(LockId(key, clientId)); @@ -625,7 +625,7 @@ std::map<document::BucketId, typename LockableMap<Map>::WrappedEntry> LockableMap<Map>::getContained(const BucketId& bucket, const char* clientId) { - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); std::map<BucketId, WrappedEntry> results; BucketId result; @@ -718,7 +718,7 @@ std::map<document::BucketId, typename LockableMap<Map>::WrappedEntry> LockableMap<Map>::getAll(const BucketId& bucket, const char* clientId, const BucketId& sibling) { - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); std::map<BucketId, WrappedEntry> results; std::vector<BucketId::Type> keys; @@ -734,7 +734,7 @@ template<typename Map> bool LockableMap<Map>::isConsistent(const typename LockableMap<Map>::WrappedEntry& entry) { - vespalib::MonitorGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); BucketId sibling(0); std::vector<BucketId::Type> keys; @@ -750,7 +750,7 @@ template<typename Map> void LockableMap<Map>::showLockClients(vespalib::asciistream & out) const { - vespalib::MonitorGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); out << "Currently grabbed locks:"; for (typename LockIdSet::const_iterator it = _lockedKeys.begin(); it != _lockedKeys.end(); ++it) diff --git a/storage/src/vespa/storage/config/CMakeLists.txt b/storage/src/vespa/storage/config/CMakeLists.txt index 4a20d510043..65eeeaf3221 100644 --- a/storage/src/vespa/storage/config/CMakeLists.txt +++ b/storage/src/vespa/storage/config/CMakeLists.txt @@ -28,3 +28,5 @@ vespa_generate_config(storage_storageconfig stor-prioritymapping.def) install_config_definition(stor-prioritymapping.def vespa.config.content.core.stor-prioritymapping.def) vespa_generate_config(storage_storageconfig rpc-provider.def) install_config_definition(rpc-provider.def vespa.config.content.core.rpc-provider.def) +vespa_generate_config(storage_storageconfig bucketspaces.def) +install_config_definition(bucketspaces.def vespa.config.content.core.bucketspaces.def) diff --git a/storage/src/vespa/storage/config/bucketspaces.def b/storage/src/vespa/storage/config/bucketspaces.def new file mode 100644 index 00000000000..3ed1abba0b4 --- /dev/null +++ b/storage/src/vespa/storage/config/bucketspaces.def @@ -0,0 +1,11 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +## This config contains the document types handled by a given content cluster +## and the bucket space they belong to. + +## The name of a document type. +documenttype[].name string + +## The bucket space this document type belongs to. +documenttype[].bucketspace string diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp b/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp index 6d562510f23..f7e207d7b8c 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp @@ -77,7 +77,7 @@ pairAsRange(Pair pair) std::vector<uint64_t> PendingMessageTracker::clearMessagesForNode(uint16_t node) { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); MessagesByNodeAndBucket& idx(boost::multi_index::get<1>(_messages)); auto range = pairAsRange(idx.equal_range(boost::make_tuple(node))); @@ -95,7 +95,7 @@ void PendingMessageTracker::insert( const std::shared_ptr<api::StorageMessage>& msg) { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); if (msg->getAddress()) { _messages.insert( MessageEntry(currentTime(), @@ -118,7 +118,7 @@ PendingMessageTracker::insert( document::Bucket PendingMessageTracker::reply(const api::StorageReply& r) { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); document::Bucket bucket; LOG(debug, "Got reply: %s", r.toString().c_str()); @@ -171,7 +171,7 @@ PendingMessageTracker::updateOperationStats(OperationStats& opStats, NodeStatsSnapshot PendingMessageTracker::getLatencyStatistics() const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); NodeStatsSnapshot snapshot; // Conveniently, snapshot data structure is exactly the same as our own. snapshot.nodeToStats = _nodeIndexToStats; @@ -205,7 +205,7 @@ PendingMessageTracker::checkPendingMessages(uint16_t node, const document::Bucket &bucket, Checker& checker) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); const MessagesByNodeAndBucket& msgs(boost::multi_index::get<1>(_messages)); auto range = pairAsRange(msgs.equal_range(boost::make_tuple(node, bucket))); @@ -216,7 +216,7 @@ void PendingMessageTracker::checkPendingMessages(const document::Bucket &bucket, Checker& checker) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); const MessagesByBucketAndType& msgs(boost::multi_index::get<2>(_messages)); auto range = pairAsRange(msgs.equal_range(boost::make_tuple(bucket))); @@ -228,7 +228,7 @@ PendingMessageTracker::hasPendingMessage(uint16_t node, const document::Bucket &bucket, uint32_t messageType) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); const MessagesByNodeAndBucket& msgs(boost::multi_index::get<1>(_messages)); auto range = msgs.equal_range(boost::make_tuple(node, bucket, messageType)); @@ -247,7 +247,7 @@ PendingMessageTracker::getStatusStartPage(std::ostream& out) const void PendingMessageTracker::getStatusPerBucket(std::ostream& out) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); const MessagesByNodeAndBucket& msgs = boost::multi_index::get<1>(_messages); using BucketMap = std::map<document::Bucket, std::vector<vespalib::string>>; @@ -285,7 +285,7 @@ PendingMessageTracker::getStatusPerBucket(std::ostream& out) const void PendingMessageTracker::getStatusPerNode(std::ostream& out) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); const MessagesByNodeAndBucket& msgs = boost::multi_index::get<1>(_messages); int lastNode = -1; for (MessagesByNodeAndBucket::const_iterator iter = @@ -337,7 +337,7 @@ PendingMessageTracker::print(std::ostream& /*out*/, NodeStats PendingMessageTracker::getNodeStats(uint16_t node) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); auto nodeIter = _nodeIndexToStats.find(node); return (nodeIter != _nodeIndexToStats.end() ? nodeIter->second : NodeStats()); diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.h b/storage/src/vespa/storage/distributor/pendingmessagetracker.h index 48999cc837b..e7bcf85a38d 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.h +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.h @@ -10,7 +10,6 @@ #include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/vespalib/stllike/hash_set.h> -#include <vespa/vespalib/util/sync.h> #include <boost/multi_index_container.hpp> #include <boost/multi_index/identity.hpp> #include <boost/multi_index/member.hpp> @@ -22,6 +21,7 @@ #include <set> #include <unordered_map> #include <chrono> +#include <mutex> namespace storage { namespace distributor { @@ -220,7 +220,7 @@ private: // Since distributor is currently single-threaded, this will only // contend when status page is being accessed. It is, however, required // to be present for that exact purpose. - vespalib::Lock _lock; + mutable std::mutex _lock; /** * Increment latency and operation count stats for the node the message |