diff options
Diffstat (limited to 'config-model')
20 files changed, 834 insertions, 234 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java index b2cd8574076..c75864f81b7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java @@ -21,6 +21,8 @@ import java.util.Map; */ class ConstantTensorTransformer extends ExpressionTransformer { + public static final String CONSTANT = "constant"; + private final Map<String, Value> constants; private final Map<String, String> rankPropertiesOutput; @@ -64,7 +66,7 @@ class ConstantTensorTransformer extends ExpressionTransformer { return node; } TensorValue tensorValue = (TensorValue)value; - String featureName = "constant(" + node.getName() + ")"; + String featureName = CONSTANT + "(" + node.getName() + ")"; String tensorType = tensorValue.asTensor().type().toString(); rankPropertiesOutput.put(featureName + ".value", tensorValue.toString()); rankPropertiesOutput.put(featureName + ".type", tensorType); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index fd249956d5a..1021227b0e6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -706,6 +706,7 @@ public class RankProfile implements Serializable, Cloneable { expression = new ConstantTensorTransformer(constants, rankPropertiesOutput).transform(expression); expression = new MacroInliner(inlineMacros).transform(expression); expression = new MacroShadower(getMacros()).transform(expression); + expression = new TensorTransformer(this).transform(expression); expression = new Simplifier().transform(expression); for (Map.Entry<String, String> rankProperty : rankPropertiesOutput.entrySet()) { addRankProperty(rankProperty.getKey(), rankProperty.getValue()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/TensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/TensorTransformer.java new file mode 100644 index 00000000000..69e353ceb35 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/TensorTransformer.java @@ -0,0 +1,290 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchlib.rankingexpression.evaluation.Context; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; +import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.FunctionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.functions.Reduce; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Transforms and simplifies tensor expressions. + * + * Currently transforms min(tensor,dim) and max(tensor,dim) to + * reduce(tensor,min/max,dim). This is necessary as the backend does + * not recognize these forms of min and max. + * + * @author lesters + */ +public class TensorTransformer extends ExpressionTransformer { + + private Search search; + private RankProfile rankprofile; + private Map<String, RankProfile.Macro> macros; + + public TensorTransformer(RankProfile rankprofile) { + this.rankprofile = rankprofile; + this.search = rankprofile.getSearch(); + this.macros = rankprofile.getMacros(); + } + + @Override + public ExpressionNode transform(ExpressionNode node) { + if (node instanceof CompositeNode) { + node = transformChildren((CompositeNode) node); + } + if (node instanceof FunctionNode) { + node = transformFunctionNode((FunctionNode) node); + } + return node; + } + + private ExpressionNode transformFunctionNode(FunctionNode node) { + switch (node.getFunction()) { + case min: + case max: + return transformMaxAndMinFunctionNode(node); + } + return node; + } + + /** + * Transforms max and min functions if it can be proven that the first + * argument resolves to a tensor and the second argument is a valid + * dimension in the tensor. If these do not hold, the node will not + * be transformed. + * + * The test for whether or not the first argument resolves to a tensor + * is to evaluate that expression. All values used in the expression + * is bound to a context with dummy values with enough information to + * deduce tensor types. + * + * There is currently no guarantee that all cases will be found. For + * instance, if-statements are problematic. + */ + private ExpressionNode transformMaxAndMinFunctionNode(FunctionNode node) { + if (node.children().size() != 2) { + return node; + } + ExpressionNode arg1 = node.children().get(0); + Optional<String> dimension = dimensionName(node.children().get(1)); + if (dimension.isPresent()) { + try { + Context context = buildContext(arg1); + Value value = arg1.evaluate(context); + if (isTensorWithDimension(value, dimension.get())) { + return replaceMaxAndMinFunction(node); + } + } catch (IllegalArgumentException e) { + // Thrown from evaluate if some variables are not bound, for + // instance for a backend rank feature. Means we don't have + // enough information to replace expression. + } + } + return node; + } + + private Optional<String> dimensionName(ExpressionNode arg) { + if (arg instanceof ReferenceNode && ((ReferenceNode)arg).children().size() == 0) { + return Optional.of(((ReferenceNode) arg).getName()); + } + return Optional.empty(); + } + + private boolean isTensorWithDimension(Value value, String dimension) { + if (value instanceof TensorValue) { + Tensor tensor = ((TensorValue) value).asTensor(); + TensorType type = tensor.type(); + return type.dimensionNames().contains(dimension); + } + return false; + } + + private ExpressionNode replaceMaxAndMinFunction(FunctionNode node) { + ExpressionNode arg1 = node.children().get(0); + ExpressionNode arg2 = node.children().get(1); + + TensorFunctionNode.TensorFunctionExpressionNode expression = TensorFunctionNode.wrapArgument(arg1); + Reduce.Aggregator aggregator = Reduce.Aggregator.valueOf(node.getFunction().name()); + String dimension = ((ReferenceNode) arg2).getName(); + + return new TensorFunctionNode(new Reduce(expression, aggregator, dimension)); + } + + /** + * Creates an evaluation context by iterating through the expression tree, and + * adding dummy values with correct types to the context. + */ + private Context buildContext(ExpressionNode node) { + Context context = new MapContext(); + addRoot(node, context); + return context; + } + + private Value emptyStringValue() { + return new StringValue(""); + } + + private Value emptyDoubleValue() { + return new DoubleValue(0.0); + } + + private Value emptyTensorValue(TensorType type) { + Tensor empty = Tensor.Builder.of(type).build(); + return new TensorValue(empty); + } + + private void addRoot(ExpressionNode node, Context context) { + addChildren(node, context); + if (node instanceof ReferenceNode) { + ReferenceNode referenceNode = (ReferenceNode) node; + addIfAttribute(referenceNode, context); + addIfConstant(referenceNode, context); + addIfQuery(referenceNode, context); + addIfTensorFrom(referenceNode, context); + addIfMacro(referenceNode, context); + } + } + + private void addChildren(ExpressionNode node, Context context) { + if (node instanceof CompositeNode) { + List<ExpressionNode> children = ((CompositeNode) node).children(); + for (ExpressionNode child : children) { + addRoot(child, context); + } + } + } + + private void addIfAttribute(ReferenceNode node, Context context) { + if (!node.getName().equals("attribute")) { + return; + } + if (node.children().size() == 0) { + return; + } + String attribute = node.children().get(0).toString(); + Attribute a = search.getAttribute(attribute); + if (a == null) { + return; + } + Value v; + if (a.getType() == Attribute.Type.STRING) { + v = emptyStringValue(); + } else if (a.getType() == Attribute.Type.TENSOR) { + v = emptyTensorValue(a.tensorType().orElseThrow(RuntimeException::new)); + } else { + v = emptyDoubleValue(); + } + context.put(node.toString(), v); + } + + private void addIfConstant(ReferenceNode node, Context context) { + if (!node.getName().equals(ConstantTensorTransformer.CONSTANT)) { + return; + } + if (node.children().size() != 1) { + return; + } + ExpressionNode child = node.children().get(0); + while (child instanceof CompositeNode && ((CompositeNode) child).children().size() > 0) { + child = ((CompositeNode) child).children().get(0); + } + String name = child.toString(); + addIfConstantInRankProfile(name, node, context); + addIfConstantInRankingConstants(name, node, context); + } + + private void addIfConstantInRankProfile(String name, ReferenceNode node, Context context) { + if (rankprofile.getConstants().containsKey(name)) { + context.put(node.toString(), rankprofile.getConstants().get(name)); + } + } + + private void addIfConstantInRankingConstants(String name, ReferenceNode node, Context context) { + for (RankingConstant rankingConstant : search.getRankingConstants()) { + if (rankingConstant.getName().equals(name)) { + context.put(node.toString(), emptyTensorValue(rankingConstant.getTensorType())); + } + } + } + + private void addIfQuery(ReferenceNode node, Context context) { + if (!node.getName().equals("query")) { + return; + } + if (node.children().size() != 1) { + return; + } + String name = node.children().get(0).toString(); + if (rankprofile.getQueryFeatureTypes().containsKey(name)) { + String type = rankprofile.getQueryFeatureTypes().get(name); + Value v; + if (type.contains("tensor")) { + v = emptyTensorValue(TensorType.fromSpec(type)); + } else if (type.equalsIgnoreCase("string")) { + v = emptyStringValue(); + } else { + v = emptyDoubleValue(); + } + context.put(node.toString(), v); + } + } + + private void addIfTensorFrom(ReferenceNode node, Context context) { + if (!node.getName().startsWith("tensorFrom")) { + return; + } + if (node.children().size() < 1 || node.children().size() > 2) { + return; + } + ExpressionNode source = node.children().get(0); + if (source instanceof CompositeNode && ((CompositeNode) source).children().size() > 0) { + source = ((CompositeNode) source).children().get(0); + } + String dimension = source.toString(); + if (node.children().size() == 2) { + dimension = node.children().get(1).toString(); + } + TensorType type = (new TensorType.Builder()).mapped(dimension).build(); + context.put(node.toString(), emptyTensorValue(type)); + } + + private void addIfMacro(ReferenceNode node, Context context) { + RankProfile.Macro macro = macros.get(node.getName()); + if (macro == null) { + return; + } + ExpressionNode root = macro.getRankingExpression().getRoot(); + Context macroContext = buildContext(root); + addMacroArguments(node, context, macro, macroContext); + Value value = root.evaluate(macroContext); + context.put(node.toString(), value); + } + + private void addMacroArguments(ReferenceNode node, Context context, RankProfile.Macro macro, Context macroContext) { + if (macro.getFormalParams().size() > 0 && node.children().size() > 0) { + for (int i = 0; i < macro.getFormalParams().size() && i < node.children().size(); ++i) { + String param = macro.getFormalParams().get(i); + ExpressionNode argumentExpression = node.children().get(i); + Value arg = argumentExpression.evaluate(context); + macroContext.put(param, arg); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 5e74a2ebc8a..e6ed91165ca 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -7,7 +7,9 @@ import com.yahoo.config.provision.Flavor; import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -264,4 +266,30 @@ public class HostResource implements Comparable<HostResource> { return this.host.compareTo(other.host); } + /** + * Picks hosts by some mixture of host name and index + * (where the mix of one or the other is decided by the last parameter). + */ + public static List<HostResource> pickHosts(Collection<HostResource> hosts, int count, int targetHostsSelectedByIndex) { + targetHostsSelectedByIndex = Math.min(Math.min(targetHostsSelectedByIndex, count), hosts.size()); + + List<HostResource> hostsSortedByName = new ArrayList<>(hosts); + Collections.sort(hostsSortedByName); + + List<HostResource> hostsSortedByIndex = new ArrayList<>(hosts); + hostsSortedByIndex.sort(Comparator.comparingInt(host -> host.primaryClusterMembership().get().index())); + return pickHosts(hostsSortedByName, hostsSortedByIndex, count, targetHostsSelectedByIndex); + } + public static List<HostResource> pickHosts(List<HostResource> hostsSelectedByName, List<HostResource> hostsSelectedByIndex, + int count, int targetHostsSelectedByIndex) { + hostsSelectedByName = hostsSelectedByName.subList(0, Math.min(count - targetHostsSelectedByIndex, hostsSelectedByName.size())); + hostsSelectedByIndex.removeAll(hostsSelectedByName); + hostsSelectedByIndex = hostsSelectedByIndex.subList(0, Math.min(targetHostsSelectedByIndex, hostsSelectedByIndex.size())); + + List<HostResource> finalHosts = new ArrayList<>(); + finalHosts.addAll(hostsSelectedByName); + finalHosts.addAll(hostsSelectedByIndex); + return finalHosts; + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index cb8ec205395..f33c86134cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -19,6 +19,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * Builds the admin model from a version 4 XML tag, or as a default when an admin 3 tag or no admin tag is used. @@ -117,12 +118,11 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { /** Returns the count first containers in the current model having isRetired set to the given value */ private List<HostResource> sortedContainerHostsFrom(ContainerModel model, int count, boolean retired) { - List<HostResource> hosts = new ArrayList<>(); - for (Container container : model.getCluster().getContainers()) - if (retired == container.isRetired()) - hosts.add(container.getHostResource()); - Collections.sort(hosts); - return hosts.subList(0, Math.min(count, hosts.size())); + List<HostResource> hosts = model.getCluster().getContainers().stream() + .filter(container -> retired == container.isRetired()) + .map(Container::getHostResource) + .collect(Collectors.toList()); + return HostResource.pickHosts(hosts, count, 1); } private void createLogserver(Admin admin, Collection<HostResource> hosts) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 91d5b7fe267..4383e55e45d 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -9,13 +9,10 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ComponentInfo; -import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.docproc.DocprocConfig; import com.yahoo.config.docproc.SchemamappingConfig; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.model.producer.AbstractConfigProducerRoot; -import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; import com.yahoo.container.BundlesConfig; import com.yahoo.container.ComponentsConfig; @@ -85,7 +82,6 @@ import com.yahoo.vespaclient.config.FeederConfig; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; -import java.io.Reader; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; @@ -382,8 +378,6 @@ public final class ContainerCluster container.setClusterName(name); container.setProp("clustername", name) .setProp("index", this.containers.size()); - setRotations(container, getRotations(), getGlobalServiceId(), name); - container.setProp("activeRotation", Boolean.toString(getActiveRotation())); containers.add(container); } @@ -393,55 +387,6 @@ public final class ContainerCluster } } - private Optional<String> getGlobalServiceId() { - Optional<DeploymentSpec> deploymentSpec = getDeploymentSpec(); - if (deploymentSpec.isPresent()) return deploymentSpec.get().globalServiceId(); - return Optional.empty(); - } - - private Set<Rotation> getRotations() { - return Optional.ofNullable(getRoot()) - .map(root -> root.getDeployState().getRotations()) - .orElse(Collections.emptySet()); - } - - private boolean getActiveRotation() { - return Optional.ofNullable(getRoot()) - .map(root -> root.getDeployState().getProperties().zone()) - .map(this::zoneHasActiveRotation) - .orElse(false); - } - - private boolean zoneHasActiveRotation(Zone zone) { - Optional<DeploymentSpec> spec = getDeploymentSpec(); - if (!spec.isPresent()) { - return false; - } - return spec.get().zones().stream() - .anyMatch(declaredZone -> declaredZone.deploysTo(zone.environment(), Optional.of(zone.region())) && - declaredZone.active()); - } - - private Optional<DeploymentSpec> getDeploymentSpec() { - Optional<DeploymentSpec> deploymentSpec = Optional.empty(); - AbstractConfigProducerRoot root = getRoot(); - if (root != null) { - final Optional<Reader> deployment = root.getDeployState().getApplicationPackage().getDeployment(); - if (deployment.isPresent()) { - deploymentSpec = Optional.of(DeploymentSpec.fromXml(deployment.get())); - } - } - return deploymentSpec; - } - - private void setRotations(Container container, Set<Rotation> rotations, Optional<String> globalServiceId, String containerClusterName) { - if ( ! rotations.isEmpty() && globalServiceId.isPresent()) { - if (containerClusterName.equals(globalServiceId.get())) { - container.setProp("rotations", rotations.stream().map(Rotation::getId).collect(Collectors.joining(","))); - } - } - } - public void setProcessingChains(ProcessingChains processingChains, String... serverBindings) { if (this.processingChains != null) throw new IllegalStateException("ProcessingChains should only be set once."); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Identity.java b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java index bc7a6e20361..0368f7eaf3e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Identity.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.HostName; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl; @@ -9,14 +11,14 @@ import com.yahoo.vespa.model.container.component.SimpleComponent; /** * @author mortent */ -public class Identity extends SimpleComponent implements IdentityConfig.Producer { +public class IdentityProvider extends SimpleComponent implements IdentityConfig.Producer { public static final String CLASS = AthenzIdentityProviderImpl.class.getName(); - private final String domain; - private final String service; + private final AthenzDomain domain; + private final AthenzService service; private final HostName loadBalancerName; - public Identity(String domain, String service, HostName loadBalancerName) { + public IdentityProvider(AthenzDomain domain, AthenzService service, HostName loadBalancerName) { super(CLASS); this.domain = domain; this.service = service; @@ -25,8 +27,8 @@ public class Identity extends SimpleComponent implements IdentityConfig.Producer @Override public void getConfig(IdentityConfig.Builder builder) { - builder.domain(domain); - builder.service(service); + builder.domain(domain.value()); + builder.service(service.value()); // Current interpretation of loadbalancer address is: hostname. // Config should be renamed or send the uri builder.loadBalancerAddress(loadBalancerName.value()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index d59846cd5e2..81fc464327e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -6,18 +6,22 @@ import com.yahoo.component.Version; import com.yahoo.config.application.Xml; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.application.provider.IncludeDirs; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.config.MetricDefaultsConfig; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.text.XML; @@ -39,7 +43,7 @@ import com.yahoo.vespa.model.clients.ContainerDocumentApi; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; -import com.yahoo.vespa.model.container.Identity; +import com.yahoo.vespa.model.container.IdentityProvider; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; @@ -64,6 +68,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Consumer; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -167,14 +172,42 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Athenz copper argos // NOTE: Must be done after addNodes() - addIdentity(spec, - cluster, - context.getDeployState().getProperties().configServerSpecs(), - context.getDeployState().getProperties().loadBalancerName()); + app.getDeployment().map(DeploymentSpec::fromXml) + .ifPresent(deploymentSpec -> { + addIdentityProvider(cluster, + context.getDeployState().getProperties().configServerSpecs(), + context.getDeployState().getProperties().loadBalancerName(), + context.getDeployState().zone(), + deploymentSpec); + + addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getRotations(), deploymentSpec); + }); //TODO: overview handler, see DomQrserverClusterBuilder } + private void addRotationProperties(ContainerCluster cluster, Zone zone, Set<Rotation> rotations, DeploymentSpec spec) { + cluster.getContainers().forEach(container -> { + setRotations(container, rotations, spec.globalServiceId(), cluster.getName()); + container.setProp("activeRotation", Boolean.toString(zoneHasActiveRotation(zone, spec))); + }); + } + + private boolean zoneHasActiveRotation(Zone zone, DeploymentSpec spec) { + return spec.zones().stream() + .anyMatch(declaredZone -> declaredZone.deploysTo(zone.environment(), Optional.of(zone.region())) && + declaredZone.active()); + } + + private void setRotations(Container container, Set<Rotation> rotations, Optional<String> globalServiceId, String containerClusterName) { + + if ( ! rotations.isEmpty() && globalServiceId.isPresent()) { + if (containerClusterName.equals(globalServiceId.get())) { + container.setProp("rotations", rotations.stream().map(Rotation::getId).collect(Collectors.joining(","))); + } + } + } + private void addRoutingAliases(ContainerCluster cluster, Element spec, Environment environment) { if (environment != Environment.prod) return; @@ -698,31 +731,33 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } - private void addIdentity(Element element, ContainerCluster cluster, List<ConfigServerSpec> configServerSpecs, HostName loadBalancerName) { - Element identityElement = XML.getChild(element, "identity"); - if(identityElement != null) { - String domain = XML.getValue(XML.getChild(identityElement, "domain")); - String service = XML.getValue(XML.getChild(identityElement, "service")); - - // Set lbaddress, or use first hostname if not specified. - HostName lbName = Optional.ofNullable(loadBalancerName) - .orElseGet( - () -> HostName.from(configServerSpecs.stream() - .findFirst() - .map(ConfigServerSpec::getHostName) - .orElse("unknown") // Currently unable to test this, hence the unknown - )); - - Identity identity = new Identity(domain.trim(), service.trim(), lbName); - cluster.addComponent(identity); + private void addIdentityProvider(ContainerCluster cluster, List<ConfigServerSpec> configServerSpecs, HostName loadBalancerName, Zone zone, DeploymentSpec spec) { + spec.athenzDomain().ifPresent(domain -> { + AthenzService service = spec.athenzService(zone.environment(), zone.region()) + .orElseThrow(() -> new RuntimeException("Missing Athenz service configuration")); + IdentityProvider identityProvider = new IdentityProvider(domain, service, getLoadBalancerName(loadBalancerName, configServerSpecs)); + cluster.addComponent(identityProvider); cluster.getContainers().forEach(container -> { - container.setProp("identity.domain", domain); - container.setProp("identity.service", service); + container.setProp("identity.domain", domain.value()); + container.setProp("identity.service", service.value()); }); - } + }); + } + + private HostName getLoadBalancerName(HostName loadbalancerName, List<ConfigServerSpec> configServerSpecs) { + // Set lbaddress, or use first hostname if not specified. + // TODO: Remove this method and use the loadbalancerName directly + return Optional.ofNullable(loadbalancerName) + .orElseGet( + () -> HostName.from(configServerSpecs.stream() + .findFirst() + .map(ConfigServerSpec::getHostName) + .orElse("unknown") // Currently unable to test this, hence the unknown + )); } + /** * Disallow renderers named "DefaultRenderer" or "JsonRenderer" */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 7889b857fff..16b0674fc14 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.lang.MutableInteger; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; @@ -328,9 +329,11 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri } private List<HostResource> drawControllerHosts(int count, StorageGroup rootGroup, Collection<ContainerModel> containers) { - List<HostResource> hosts = drawContentHostsRecursively(count, rootGroup); + List<HostResource> hostsByName = drawContentHostsRecursively(count, false, rootGroup); + List<HostResource> hostsByIndex = drawContentHostsRecursively(count, true, rootGroup); // if (hosts.size() < count) // supply with containers TODO: Currently disabled due to leading to topology change problems // hosts.addAll(drawContainerHosts(count - hosts.size(), containers, new HashSet<>(hosts))); + List<HostResource> hosts = HostResource.pickHosts(hostsByName, hostsByIndex, count, 1); if (hosts.size() % 2 == 0) // ZK clusters of even sizes are less available (even in the size=2 case) hosts = hosts.subList(0, hosts.size()-1); return hosts; @@ -403,20 +406,24 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri */ // Note: This method cannot be changed to draw different nodes without ensuring that it will draw nodes // which overlaps with previously drawn nodes as this will prevent rolling upgrade - private List<HostResource> drawContentHostsRecursively(int count, StorageGroup group) { + private List<HostResource> drawContentHostsRecursively(int count, boolean byIndex, StorageGroup group) { Set<HostResource> hosts = new HashSet<>(); if (group.getNodes().isEmpty()) { int hostsPerSubgroup = (int)Math.ceil((double)count / group.getSubgroups().size()); for (StorageGroup subgroup : group.getSubgroups()) - hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, subgroup)); + hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, byIndex, subgroup)); } else { hosts.addAll(group.getNodes().stream() - .filter(node -> ! node.isRetired()) // Avoid retired controllers to avoid surprises on expiry - .map(StorageNode::getHostResource).collect(Collectors.toList())); + .filter(node -> ! node.isRetired()) // Avoid retired controllers to avoid surprises on expiry + .map(StorageNode::getHostResource).collect(Collectors.toList())); } + List<HostResource> sortedHosts = new ArrayList<>(hosts); - Collections.sort(sortedHosts); + if (byIndex) + sortedHosts.sort(Comparator.comparingInt(host -> host.primaryClusterMembership().get().index())); + else // by name + Collections.sort(sortedHosts); sortedHosts = sortedHosts.subList(0, Math.min(count, hosts.size())); return sortedHosts; } diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 47cf0638d72..6a90bef7bb2 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -7,8 +7,7 @@ ContainerCluster = element container | jdisc { ContainerServices & DocumentBinding* & Aliases? & - NodesOfContainerCluster? & - Identity? + NodesOfContainerCluster? } ContainerServices = @@ -226,8 +225,3 @@ DocumentBinding = element document { attribute class { xsd:NCName } & attribute bundle { xsd:NCName } } - -Identity = element identity { - element domain { xsd:NCName } & - element service { xsd:NCName } -} diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 90bff8e31b3..2ecfa781876 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -4,6 +4,8 @@ start = element deployment { attribute version { "1.0" } & + attribute athenz-domain { xsd:string }? & + attribute athenz-service { xsd:string }? & Upgrade? & BlockChange* & Test? & @@ -31,15 +33,18 @@ BlockUpgrade = element block-upgrade { # Legacy name - remove on Vespa 7 } Test = element test { - text + attribute athenz-service { xsd:string }? & + text } Staging = element staging { - text + attribute athenz-service { xsd:string }? & + text } Prod = element prod { attribute global-service-id { text }? & + attribute athenz-service { xsd:string }? & Region* & Delay* & Parallel* diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 63d5d37598b..6ea15788cf3 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -391,16 +391,16 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default10", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default13", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default16", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default28", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default31", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default54", clusterControllers.getContainers().get(2).getHostName()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(9, cluster.getRootGroup().getSubgroups().size()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(3)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0")); - assertEquals("default10", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("default54", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getConfigId(), is("bar/storage/1")); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(2).getDistributionKey(), is(2)); @@ -409,13 +409,13 @@ public class ModelProvisioningTest { assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(3)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(3)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/3")); - assertEquals("default13", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("default51", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getDistributionKey(), is(4)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getConfigId(), is("bar/storage/4")); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getDistributionKey(), is(5)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getConfigId(), is("bar/storage/5")); // ... - assertEquals("default16", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); + assertEquals("default48", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); // ... assertThat(cluster.getRootGroup().getSubgroups().get(8).getIndex(), is("8")); assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().size(), is(3)); @@ -430,23 +430,23 @@ public class ModelProvisioningTest { clusterControllers = cluster.getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("baz-controllers", clusterControllers.getName()); - assertEquals("default37", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default38", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default39", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default02", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default27", clusterControllers.getContainers().get(2).getHostName()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(27, cluster.getRootGroup().getSubgroups().size()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("baz/storage/0")); - assertEquals("default37", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("default27", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1")); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("baz/storage/1")); - assertEquals("default38", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("default26", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); // ... - assertEquals("default39", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); + assertEquals("default25", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); // ... assertThat(cluster.getRootGroup().getSubgroups().get(26).getIndex(), is("26")); assertThat(cluster.getRootGroup().getSubgroups().get(26).getNodes().size(), is(1)); @@ -483,9 +483,9 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default10", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default11", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default12", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default02", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default08", clusterControllers.getContainers().get(2).getHostName()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(8, cluster.getRootGroup().getSubgroups().size()); assertEquals(8, cluster.distributionBits()); @@ -494,19 +494,19 @@ public class ModelProvisioningTest { assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0")); - assertEquals("default10", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("default08", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); // second group assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1")); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/1")); - assertEquals("default11", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("default07", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); // ... last group assertThat(cluster.getRootGroup().getSubgroups().get(7).getIndex(), is("7")); assertThat(cluster.getRootGroup().getSubgroups().get(7).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getDistributionKey(), is(7)); assertThat(cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getConfigId(), is("bar/storage/7")); - assertEquals("default17", cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getHostName()); + assertEquals("default01", cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getHostName()); } @Test @@ -538,16 +538,51 @@ public class ModelProvisioningTest { ContentCluster cluster = model.getContentClusters().get("bar"); ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals( 8, cluster.distributionBits()); - assertEquals("We get the closest odd numer", 5, clusterControllers.getContainers().size()); + assertEquals("We get the closest odd number", 5, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default10", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default11", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default13", clusterControllers.getContainers().get(2).getHostName()); - assertEquals("default14", clusterControllers.getContainers().get(3).getHostName()); // Should be 16 for perfect distribution ... - assertEquals("default10", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); - assertEquals("default11", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getHostName()); - assertEquals("default13", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); - assertEquals("default16", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); + assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default02", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default04", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default05", clusterControllers.getContainers().get(3).getHostName()); + assertEquals("default07", clusterControllers.getContainers().get(4).getHostName()); + assertEquals("default09", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("default08", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getHostName()); + assertEquals("default06", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("default03", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); + } + + @Test + public void testClusterControllersWithGroupSize2() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'/>" + + " <container version='1.0' id='foo'>" + + " <nodes count='10'/>" + + " </container>" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='8' groups='4'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 18; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(numberOfHosts); + VespaModel model = tester.createModel(services, true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + // Check content clusters + ContentCluster cluster = model.getContentClusters().get("bar"); + ContainerCluster clusterControllers = cluster.getClusterControllers(); + assertEquals("We get the closest odd number", 3, clusterControllers.getContainers().size()); + assertEquals("bar-controllers", clusterControllers.getName()); + assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default03", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default08", clusterControllers.getContainers().get(2).getHostName()); } @Test @@ -604,7 +639,7 @@ public class ModelProvisioningTest { int numberOfHosts = 19; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "default10", "default13", "default16"); + VespaModel model = tester.createModel(services, true, "default09", "default06", "default03"); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); // Check content clusters @@ -612,9 +647,9 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("Skipping retired default10", "default11", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("Skipping retired default13", "default14", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("Skipping retired default16", "default17", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("Skipping retired default09", "default01", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("Skipping retired default03", "default04", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("Skipping retired default06", "default08", clusterControllers.getContainers().get(2).getHostName()); } @Test @@ -631,15 +666,15 @@ public class ModelProvisioningTest { int numberOfHosts = 10; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "default0"); + VespaModel model = tester.createModel(services, true, "default09"); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); // Check slobroks clusters assertEquals("Includes retired node", 1+3, model.getAdmin().getSlobroks().size()); - assertEquals("default1", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("default2", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("default3", model.getAdmin().getSlobroks().get(2).getHostName()); - assertEquals("Included in addition because it is retired", "default0", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("default01", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("default02", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("default10", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("Included in addition because it is retired", "default09", model.getAdmin().getSlobroks().get(3).getHostName()); } @Test @@ -656,16 +691,16 @@ public class ModelProvisioningTest { int numberOfHosts = 10; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "default3", "default4"); + VespaModel model = tester.createModel(services, true, "default09", "default08"); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); // Check slobroks clusters assertEquals("Includes retired node", 3+2, model.getAdmin().getSlobroks().size()); - assertEquals("default0", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("default1", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("default2", model.getAdmin().getSlobroks().get(2).getHostName()); - assertEquals("Included in addition because it is retired", "default3", model.getAdmin().getSlobroks().get(3).getHostName()); - assertEquals("Included in addition because it is retired", "default4", model.getAdmin().getSlobroks().get(4).getHostName()); + assertEquals("default01", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("default02", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("default10", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("Included in addition because it is retired", "default08", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("Included in addition because it is retired", "default09", model.getAdmin().getSlobroks().get(4).getHostName()); } @Test @@ -685,19 +720,19 @@ public class ModelProvisioningTest { int numberOfHosts = 13; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "default0", "default10", "default11"); + VespaModel model = tester.createModel(services, true, "default12", "default03", "default02"); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); // Check slobroks clusters // ... from cluster default assertEquals("Includes retired node", 3+3, model.getAdmin().getSlobroks().size()); - assertEquals("default1", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("default2", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("Included in addition because it is retired", "default0", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("default04", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("default13", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("Included in addition because it is retired", "default12", model.getAdmin().getSlobroks().get(2).getHostName()); // ... from cluster bar - assertEquals("default12", model.getAdmin().getSlobroks().get(3).getHostName()); - assertEquals("Included in addition because it is retired", "default10", model.getAdmin().getSlobroks().get(4).getHostName()); - assertEquals("Included in addition because it is retired", "default11", model.getAdmin().getSlobroks().get(5).getHostName()); + assertEquals("default01", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("Included in addition because it is retired", "default02", model.getAdmin().getSlobroks().get(4).getHostName()); + assertEquals("Included in addition because it is retired", "default03", model.getAdmin().getSlobroks().get(5).getHostName()); } @Test @@ -828,10 +863,10 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(4, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default19", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default20", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default21", clusterControllers.getContainers().get(2).getHostName()); - assertEquals("default22", clusterControllers.getContainers().get(3).getHostName()); + assertEquals("default04", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default03", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default02", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default01", clusterControllers.getContainers().get(3).getHostName()); } @Test @@ -957,7 +992,7 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(1, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default0", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3 assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4 assertEquals(1, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3 diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java new file mode 100644 index 00000000000..12bdd8d2b5c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -0,0 +1,206 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.collections.Pair; +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.derived.RawRankProfile; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertTrue; + +public class TensorTransformTestCase extends SearchDefinitionTestCase { + + @Test + public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException { + assertContainsExpression("max(1.0,2.0)", "max(1.0,2.0)"); + assertContainsExpression("min(attribute(double_field),x)", "min(attribute(double_field),x)"); + assertContainsExpression("max(attribute(double_field),attribute(double_array_field))", "max(attribute(double_field),attribute(double_array_field))"); + assertContainsExpression("min(attribute(tensor_field_1),attribute(double_field))", "min(attribute(tensor_field_1),attribute(double_field))"); + assertContainsExpression("max(attribute(tensor_field_1),attribute(tensor_field_2))", "max(attribute(tensor_field_1),attribute(tensor_field_2))"); + assertContainsExpression("min(test_constant_tensor,1.0)", "min(constant(test_constant_tensor),1.0)"); + assertContainsExpression("max(base_constant_tensor,1.0)", "max(constant(base_constant_tensor),1.0)"); + assertContainsExpression("min(constant(file_constant_tensor),1.0)", "min(constant(file_constant_tensor),1.0)"); + assertContainsExpression("max(query(q),1.0)", "max(query(q),1.0)"); + assertContainsExpression("max(query(n),1.0)", "max(query(n),1.0)"); + } + + @Test + public void requireThatMaxAndMinWithTensorAttributesAreReplaced() throws ParseException { + assertContainsExpression("max(attribute(tensor_field_1),x)", "reduce(attribute(tensor_field_1),max,x)"); + assertContainsExpression("1 + max(attribute(tensor_field_1),x)", "1+reduce(attribute(tensor_field_1),max,x)"); + assertContainsExpression("if(attribute(double_field),1 + max(attribute(tensor_field_1),x),0)", "if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),0)"); + assertContainsExpression("max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)", "reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)"); + assertContainsExpression("max(if(attribute(double_field),attribute(tensor_field_1),attribute(tensor_field_2)),x)", "reduce(if(attribute(double_field),attribute(tensor_field_1),attribute(tensor_field_2)),max,x)"); + assertContainsExpression("max(max(attribute(tensor_field_1),x),x)", "max(reduce(attribute(tensor_field_1),max,x),x)"); // will result in deploy error. + assertContainsExpression("max(max(attribute(tensor_field_2),x),y)", "reduce(reduce(attribute(tensor_field_2),max,x),max,y)"); + } + + @Test + public void requireThatMaxAndMinWithConstantTensorsAreReplaced() throws ParseException { + assertContainsExpression("max(test_constant_tensor,x)", "reduce(constant(test_constant_tensor),max,x)"); + assertContainsExpression("max(base_constant_tensor,x)", "reduce(constant(base_constant_tensor),max,x)"); + assertContainsExpression("min(constant(file_constant_tensor),x)", "reduce(constant(file_constant_tensor),min,x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorExpressionsAreReplaced() throws ParseException { + assertContainsExpression("min(attribute(double_field) + attribute(tensor_field_1),x)", "reduce(attribute(double_field)+attribute(tensor_field_1),min,x)"); + assertContainsExpression("min(attribute(tensor_field_1) * attribute(tensor_field_2),x)", "reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)"); + assertContainsExpression("min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)", "reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)"); + assertContainsExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) + assertContainsExpression("min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)", "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorFromIsReplaced() throws ParseException { + assertContainsExpression("max(tensorFromLabels(attribute(double_array_field)),double_array_field)", "reduce(tensorFromLabels(attribute(double_array_field)),max,double_array_field)"); + assertContainsExpression("max(tensorFromLabels(attribute(double_array_field),x),x)", "reduce(tensorFromLabels(attribute(double_array_field),x),max,x)"); + assertContainsExpression("max(tensorFromWeightedSet(attribute(weightedset_field)),weightedset_field)", "reduce(tensorFromWeightedSet(attribute(weightedset_field)),max,weightedset_field)"); + assertContainsExpression("max(tensorFromWeightedSet(attribute(weightedset_field),x),x)", "reduce(tensorFromWeightedSet(attribute(weightedset_field),x),max,x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorInQueryIsReplaced() throws ParseException { + assertContainsExpression("max(query(q),x)", "reduce(query(q),max,x)"); + assertContainsExpression("max(query(n),x)", "max(query(n),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensoresReturnedFromMacrosAreReplaced() throws ParseException { + assertContainsExpression("max(returns_tensor,x)", "reduce(rankingExpression(returns_tensor),max,x)"); + assertContainsExpression("max(wraps_returns_tensor,x)", "reduce(rankingExpression(wraps_returns_tensor),max,x)"); + assertContainsExpression("max(tensor_inheriting,x)", "reduce(rankingExpression(tensor_inheriting),max,x)"); + assertContainsExpression("max(returns_tensor_with_arg(attribute(tensor_field_1)),x)", "reduce(rankingExpression(returns_tensor_with_arg@),max,x)"); + } + + + private void assertContainsExpression(String expr, String transformedExpression) throws ParseException { + assertTrue("Expected expression '" + transformedExpression + "' not found", + containsExpression(expr, transformedExpression)); + } + + private boolean containsExpression(String expr, String transformedExpression) throws ParseException { + for (Pair<String, String> rankPropertyExpression : buildSearch(expr)) { + String rankProperty = rankPropertyExpression.getFirst(); + if (rankProperty.equals("rankingExpression(firstphase).rankingScript")) { + String rankExpression = censorBindingHash(rankPropertyExpression.getSecond().replace(" ","")); + return rankExpression.equals(transformedExpression); + } + } + return false; + } + + private List<Pair<String, String>> buildSearch(String expression) throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field double_field type double { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field double_array_field type array<double> { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field weightedset_field type weightedset<double> { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field tensor_field_1 type tensor(x{}) { \n" + + " indexing: summary | attribute \n" + + " attribute: tensor(x{}) \n" + + " }\n" + + " field tensor_field_2 type tensor(x[3],y[3]) { \n" + + " indexing: summary | attribute \n" + + " attribute: tensor(x[3],y[3]) \n" + + " }\n" + + " }\n" + + " constant file_constant_tensor {\n" + + " file: constants/tensor.json\n" + + " type: tensor(x{})\n" + + " }\n" + + " rank-profile base {\n" + + " constants {\n" + + " base_constant_tensor {\n" + + " value: { {x:0}:0 }\n" + + " }\n" + + " }\n" + + " macro base_tensor() {\n" + + " expression: constant(base_constant_tensor)\n" + + " }\n" + + " }\n" + + " rank-profile test inherits base {\n" + + " constants {\n" + + " test_constant_tensor {\n" + + " value: { {x:0}:1 }\n" + + " }\n" + + " }\n" + + " macro returns_tensor_with_arg(arg1) {\n" + + " expression: 2.0 * arg1\n" + + " }\n" + + " macro wraps_returns_tensor() {\n" + + " expression: returns_tensor\n" + + " }\n" + + " macro returns_tensor() {\n" + + " expression: attribute(tensor_field_2)\n" + + " }\n" + + " macro tensor_inheriting() {\n" + + " expression: base_tensor\n" + + " }\n" + + " first-phase {\n" + + " expression: " + expression + "\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(new BaseDeployLogger(), setupQueryProfileTypes()); + Search s = builder.getSearch(); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); + for (Object o : testRankProperties) + System.out.println(o); + return testRankProperties; + } + + private static QueryProfiles setupQueryProfileTypes() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); + QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); + type.addField(new FieldDescription("ranking.features.query(q)", + FieldType.fromString("tensor(x{})", typeRegistry)), typeRegistry); + type.addField(new FieldDescription("ranking.features.query(n)", + FieldType.fromString("integer", typeRegistry)), typeRegistry); + typeRegistry.register(type); + return new QueryProfiles(registry); + } + + private String censorBindingHash(String s) { + StringBuilder b = new StringBuilder(); + boolean areInHash = false; + for (int i = 0; i < s.length() ; i++) { + char current = s.charAt(i); + if ( ! Character.isLetterOrDigit(current)) // end of hash + areInHash = false; + if ( ! areInHash) + b.append(current); + if (current == '@') // start of hash + areInHash = true; + } + return b.toString(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index af846f10ffe..d9c151480fe 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -18,15 +18,14 @@ import com.yahoo.container.jdisc.config.MetricDefaultsConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; -import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerClusterVerifier; +import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.search.ContainerSearch; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import org.junit.Test; -import java.util.Arrays; import java.util.Collection; import java.util.Iterator; import java.util.Optional; @@ -266,48 +265,6 @@ public class ContainerClusterTest { assertEquals(0, cluster.getAllComponents().stream().map(c -> c.getClassId().getName()).filter(c -> c.equals("com.yahoo.jdisc.http.filter.security.RoutingConfigProvider")).count()); } - @Test - public void setsRotationActiveAccordingToDeploymentSpec() { - String deploymentSpec = "<deployment>\n" + - " <prod> \n" + - " <region active='true'>us-north-1</region>\n" + - " <parallel>\n" + - " <region active='false'>us-north-2</region>\n" + - " <region active='true'>us-north-3</region>\n" + - " </parallel>\n" + - " <region active='false'>us-north-4</region>\n" + - " </prod>\n" + - "</deployment>"; - for (String region : Arrays.asList("us-north-1", "us-north-3")) { - Container container = containerIn(region, deploymentSpec); - assertEquals("Region " + region + " is active", "true", - container.getServicePropertyString("activeRotation")); - } - for (String region : Arrays.asList("us-north-2", "us-north-4")) { - Container container = containerIn(region, deploymentSpec); - assertEquals("Region " + region + " is inactive", "false", - container.getServicePropertyString("activeRotation")); - } - Container container = containerIn("unknown", deploymentSpec); - assertEquals("Unknown region is inactive", "false", - container.getServicePropertyString("activeRotation")); - } - - private static Container containerIn(String regionName, String deploymentSpec) { - ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() - .withDeploymentSpec(deploymentSpec) - .build(); - DeployState state = new DeployState.Builder() - .applicationPackage(applicationPackage) - .properties(new DeployProperties.Builder().hostedVespa(true).zone( - new Zone(Environment.prod, RegionName.from(regionName))).build() - ) - .build(); - MockRoot root = new MockRoot("foo", state); - ContainerCluster cluster = new ContainerCluster(root, "container0", "container1"); - addContainer(cluster, "c1", "c1.domain"); - return cluster.getContainers().get(0); - } private static void addContainer(ContainerCluster cluster, String name, String hostName) { Container container = new Container(cluster, name, 0); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 0e6bce73867..d09211aea45 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -26,7 +26,6 @@ import com.yahoo.container.servlet.ServletConfigConfig; import com.yahoo.container.usability.BindingsOverviewHandler; import com.yahoo.jdisc.http.ServletPathsConfig; import com.yahoo.prelude.cluster.QrMonitorConfig; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.Container; @@ -46,6 +45,7 @@ import java.util.Map; import java.util.logging.Level; import static com.yahoo.test.LinePatternMatcher.containsLineWithPattern; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/IdentityBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/IdentityBuilderTest.java index df118b0e349..d3ad2ccc721 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/IdentityBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/IdentityBuilderTest.java @@ -1,9 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.container.core.identity.IdentityConfig; -import com.yahoo.vespa.model.container.Identity; +import com.yahoo.vespa.model.container.IdentityProvider; import org.junit.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; @@ -17,17 +20,23 @@ import static org.junit.Assert.assertEquals; */ public class IdentityBuilderTest extends ContainerModelBuilderTestBase { @Test - public void identity_config_produced() throws IOException, SAXException { + public void identity_config_produced_from_deployment_spec() throws IOException, SAXException { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>", - " <identity>", - " <domain>domain</domain>", - " <service>service</service>", - " </identity>", - "</jdisc>"); + "<jdisc id='default' version='1.0'><search /></jdisc>"); + String deploymentXml = "<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" + + " <test/>\n" + + " <prod>\n" + + " <region active='true'>default</region>\n" + + " </prod>\n" + + "</deployment>\n"; - createModel(root, clusterElem); - IdentityConfig identityConfig = root.getConfig(IdentityConfig.class, "default/component/" + Identity.CLASS); + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() + .withDeploymentSpec(deploymentXml) + .build(); + + createModel(root, DeployState.createTestState(applicationPackage), clusterElem); + + IdentityConfig identityConfig = root.getConfig(IdentityConfig.class, "default/component/" + IdentityProvider.CLASS); assertEquals("domain", identityConfig.domain()); assertEquals("service", identityConfig.service()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/RoutingBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/RoutingBuilderTest.java new file mode 100644 index 00000000000..a2f32694340 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/RoutingBuilderTest.java @@ -0,0 +1,78 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import org.junit.Test; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +/** + * @author mortent + */ +public class RoutingBuilderTest extends ContainerModelBuilderTestBase { + + @Test + public void setsRotationActiveAccordingToDeploymentSpec() throws IOException, SAXException { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'><search /></jdisc>"); + + String deploymentSpec = "<deployment>\n" + + " <prod> \n" + + " <region active='true'>us-north-1</region>\n" + + " <parallel>\n" + + " <region active='false'>us-north-2</region>\n" + + " <region active='true'>us-north-3</region>\n" + + " </parallel>\n" + + " <region active='false'>us-north-4</region>\n" + + " </prod>\n" + + "</deployment>"; + + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() + .withDeploymentSpec(deploymentSpec) + .build(); + //root = new MockRoot("root", applicationPackage); + for (String region : Arrays.asList("us-north-1", "us-north-3")) { + Container container = getContainer(applicationPackage, region, clusterElem); + + assertEquals("Region " + region + " is active", "true", + container.getServicePropertyString("activeRotation")); + } + for (String region : Arrays.asList("us-north-2", "us-north-4")) { + Container container = getContainer(applicationPackage, region, clusterElem); + + assertEquals("Region " + region + " is inactive", "false", + container.getServicePropertyString("activeRotation")); + } + Container container = getContainer(applicationPackage, "unknown", clusterElem); + assertEquals("Unknown region is inactive", "false", + container.getServicePropertyString("activeRotation")); + } + + + private Container getContainer(ApplicationPackage applicationPackage, String region, Element clusterElem) throws IOException, SAXException { + DeployState deployState = new DeployState.Builder() + .applicationPackage(applicationPackage) + .zone(new Zone(Environment.prod, RegionName.from(region))) + .build(); + + root = new MockRoot("root", deployState); + createModel(root, deployState, clusterElem); + ContainerCluster cluster = getContainerCluster("default"); + return cluster.getContainers().get(0); + + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 901bd7f55b5..b88ba0276c4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -18,7 +18,6 @@ import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -29,7 +28,7 @@ import java.util.Optional; * Helper class which sets up a system with multiple hosts. * Usage: * <code> - * VespaModelteser teser = new VespaModelTester(); + * VespaModelteser tester = new VespaModelTester(); * tester.addHosts(count, flavor); * ... add more nodes * VesoaModel model = tester.createModel(servicesString); @@ -43,7 +42,7 @@ public class VespaModelTester { private final ConfigModelRegistry configModelRegistry; private boolean hosted = true; - private Map<String, Collection<Host>> hosts = new HashMap<>(); + private Map<String, Collection<Host>> hostsByFlavor = new HashMap<>(); public VespaModelTester() { this(new NullConfigModelRegistry()); @@ -55,20 +54,31 @@ public class VespaModelTester { /** Adds some hosts of the 'default' flavor to this system */ public Hosts addHosts(int count) { return addHosts("default", count); } + /** Adds some hosts to this system */ public Hosts addHosts(String flavor, int count) { - List<Host> hosts = new ArrayList<>(); - for (int i = 0; i < count; i++) - hosts.add(new com.yahoo.config.model.provision.Host(flavor + i)); - this.hosts.put(flavor.isEmpty() ? "default" : flavor, hosts); - return new Hosts(hosts); + return addHosts(Optional.empty(), flavor, count); } + public void addHosts(Flavor flavor, int count) { + addHosts(Optional.of(flavor), flavor.name(), count); + } + + private Hosts addHosts(Optional<Flavor> flavor, String flavorName, int count) { List<Host> hosts = new ArrayList<>(); + for (int i = 0; i < count; ++i) { - hosts.add(new Host(flavor.name() + i, ImmutableList.of(), Optional.of(flavor))); + // Let host names sort in the opposite order of the order the hosts are added + // This allows us to test index vs. name order selection when subsets of hosts are selected from a cluster + // (for e.g cluster controllers and slobrok nodes) + String hostname = String.format("%s%02d", flavorName, count - i); + hosts.add(new Host(hostname, ImmutableList.of(), flavor)); } - this.hosts.put(flavor.name(), hosts); + this.hostsByFlavor.put(flavorName, hosts); + + if (hosts.size() > 100) + throw new IllegalStateException("The host naming scheme is nameNN. To test more than 100 hosts, change to nameNNN"); + return new Hosts(hosts); } /** Sets whether this sets up a model for a hosted system. Default: true */ @@ -95,7 +105,7 @@ public class VespaModelTester { ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; HostProvisioner provisioner = hosted ? - new InMemoryProvisioner(hosts, failOnOutOfCapacity, startIndexForClusters, retiredHostNames) : + new InMemoryProvisioner(hostsByFlavor, failOnOutOfCapacity, startIndexForClusters, retiredHostNames) : new SingleNodeProvisioner(); DeployState deployState = new DeployState.Builder() diff --git a/config-model/src/test/schema-test-files/deployment.xml b/config-model/src/test/schema-test-files/deployment.xml index f469d22b6f0..f9a62eb648f 100644 --- a/config-model/src/test/schema-test-files/deployment.xml +++ b/config-model/src/test/schema-test-files/deployment.xml @@ -1,12 +1,12 @@ <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<deployment version='1.0'> +<deployment version='1.0' athenz-domain='vespa' athenz-service='service'> <upgrade policy='canary'/> <test/> <staging/> <block-change revision='true' version='false' days="mon,tue" hours="14,15"/> <block-change days="mon,tue" hours="14,15" time-zone="CET"/> <block-upgrade days="wed" hours="16" time-zone="CET"/><!-- Tests legacy name. Remove in Vespa 7 --> - <prod global-service-id='qrs'> + <prod global-service-id='qrs' athenz-service='other-service'> <region active='true'>us-west-1</region> <delay hours='3'/> <region active='true'>us-central-1</region> diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 98637c03020..a02346193cc 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -35,10 +35,6 @@ </config> <jdisc id='qrsCluster_1' version='1.0'> - <identity> - <domain>mydomain</domain> - <service>myservice</service> - </identity> <rest-api path="jersey1"> <components bundle="my-bundle" /> <components bundle="other-bundle"> |