diff options
| author | Jon Bratseth <bratseth@verizonmedia.com> | 2019-05-02 21:29:29 +0200 |
|---|---|---|
| committer | Jon Bratseth <bratseth@verizonmedia.com> | 2019-05-02 21:29:29 +0200 |
| commit | 8457cd74f0ac3d876d1f7fd6cd7ea7b503cae491 (patch) | |
| tree | 4a2eb5ed891f5da85397797fc4c943c33ade8b65 | |
| parent | 75e2698805c454d54afb4b5a8bc62b046c4e3246 (diff) | |
Allow continuous node resource specs
61 files changed, 732 insertions, 505 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 3a9a5d59b1a..bae9c52b06e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Configserver; import com.yahoo.vespa.model.admin.LogForwarder; import com.yahoo.vespa.model.admin.ModelConfigProvider; -import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.DefaultMonitoring; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; @@ -26,10 +25,8 @@ import org.w3c.dom.Element; import java.util.ArrayList; import java.util.List; -import java.util.Map; import static com.yahoo.vespa.model.admin.monitoring.builder.PredefinedMetricSets.predefinedMetricSets; -import static java.util.logging.Level.WARNING; /** * A base class for admin model builders, to support common functionality across versions. @@ -116,11 +113,11 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu void addLogForwarders(ModelElement logForwardingElement, Admin admin) { if (logForwardingElement == null) return; - for (ModelElement e : logForwardingElement.getChildren("splunk")) { + for (ModelElement e : logForwardingElement.children("splunk")) { LogForwarder.Config cfg = LogForwarder.cfg() - .withSplunkHome(e.getStringAttribute("splunk-home")) - .withDeploymentServer(e.getStringAttribute("deployment-server")) - .withClientName(e.getStringAttribute("client-name")); + .withSplunkHome(e.stringAttribute("splunk-home")) + .withDeploymentServer(e.stringAttribute("deployment-server")) + .withClientName(e.stringAttribute("client-name")); admin.setLogForwarderConfig(cfg); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 3a16264b148..1b0e04b50a8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilder; -import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; import org.w3c.dom.Element; import java.util.List; @@ -51,9 +50,9 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { admin.setClusterControllers(addConfiguredClusterControllers(deployState, admin, adminE)); ModelElement adminElement = new ModelElement(adminE); - addLogForwarders(adminElement.getChild("logforwarding"), admin); + addLogForwarders(adminElement.child("logforwarding"), admin); - if (adminElement.getChild("filedistribution") != null) { + if (adminElement.child("filedistribution") != null) { deployState.getDeployLogger().log(LogLevel.WARNING, "'filedistribution' element is deprecated and ignored"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 8980c7db93d..71ce6492b09 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -54,14 +54,14 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { // Note: These two elements only exists in admin version 4.0 // This build handles admin version 3.0 by ignoring its content (as the content is not useful) Optional<NodesSpecification> requestedSlobroks = - NodesSpecification.optionalDedicatedFromParent(adminElement.getChild("slobroks"), context); + NodesSpecification.optionalDedicatedFromParent(adminElement.child("slobroks"), context); Optional<NodesSpecification> requestedLogservers = - NodesSpecification.optionalDedicatedFromParent(adminElement.getChild("logservers"), context); + NodesSpecification.optionalDedicatedFromParent(adminElement.child("logservers"), context); assignSlobroks(deployState.getDeployLogger(), requestedSlobroks.orElse(NodesSpecification.nonDedicated(3, context)), admin); assignLogserver(deployState, requestedLogservers.orElse(createNodesSpecificationForLogserver()), admin); - addLogForwarders(adminElement.getChild("logforwarding"), admin); + addLogForwarders(adminElement.child("logforwarding"), admin); } private void assignSlobroks(DeployLogger deployLogger, NodesSpecification nodesSpecification, Admin admin) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java index 1fc70e343f3..1f649b122fa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java @@ -20,54 +20,38 @@ public class ModelElement { private final Element xml; public ModelElement(Element xml) { + if (xml == null) throw new NullPointerException("Can not create ModelElement with null element"); + if (xml.getNodeName() == null) throw new NullPointerException("Can not create ModelElement with unnamed element"); this.xml = xml; - if (xml == null) { - throw new NullPointerException("Can not create ModelElement with null element"); - } - if (xml.getNodeName() == null) { - throw new NullPointerException("Can not create ModelElement with unnamed element"); - } } - public Element getXml() { - return xml; - } + public Element getXml() { return xml; } - /** - * If not found, return null. - */ - public ModelElement getChild(String name) { + /** Returns the child with the given name, or null if none. */ + public ModelElement child(String name) { Element e = XML.getChild(xml, name); - - if (e != null) { - return new ModelElement(e); - } - - return null; + if (e == null) return null; + return new ModelElement(e); } - /** - * If not found, return empty list - */ - public List<ModelElement> getChildren(String name) { + /** If not found, return empty list. */ + public List<ModelElement> children(String name) { List<Element> e = XML.getChildren(xml, name); List<ModelElement> list = new ArrayList<>(); e.forEach(element -> list.add(new ModelElement(element))); - return list; } - public ModelElement getChildByPath(String path) { + public ModelElement childByPath(String path) { StringTokenizer tokenizer = new StringTokenizer(path, "."); ModelElement curElem = this; while (tokenizer.hasMoreTokens() && curElem != null) { String pathElem = tokenizer.nextToken(); - ModelElement child = curElem.getChild(pathElem); + ModelElement child = curElem.child(pathElem); if (!tokenizer.hasMoreTokens()) { - if (child != null) { + if (child != null) return child; - } } curElem = child; } @@ -79,9 +63,9 @@ public class ModelElement { ModelElement curElem = this; while (tokenizer.hasMoreTokens() && curElem != null) { String pathElem = tokenizer.nextToken(); - ModelElement child = curElem.getChild(pathElem); + ModelElement child = curElem.child(pathElem); if (!tokenizer.hasMoreTokens()) { - String attr = curElem.getStringAttribute(pathElem); + String attr = curElem.stringAttribute(pathElem); if (attr != null) { return attr; } else if (child != null) { @@ -111,97 +95,84 @@ public class ModelElement { public Long childAsLong(String path) { String child = childAsString(path); - if (child == null) { - return null; - } + if (child == null) return null; return Long.parseLong(child.trim()); } public Integer childAsInteger(String path) { String child = childAsString(path); - if (child == null) { - return null; - } + if (child == null) return null; return Integer.parseInt(child.trim()); } public Double childAsDouble(String path) { String child = childAsString(path); - if (child == null) { - return null; - } + if (child == null) return null; return Double.parseDouble(child.trim()); } public Boolean childAsBoolean(String path) { String child = childAsString(path); - if (child == null) { - return null; - } + if (child == null) return null; return Boolean.parseBoolean(child.trim()); } public Duration childAsDuration(String path) { String child = childAsString(path); - if (child == null) { - return null; - } + if (child == null) return null; return new Duration(child); } /** Returns the given attribute or throws IllegalArgumentException if not present */ public int requiredIntegerAttribute(String name) { - if (getStringAttribute(name) == null) + if (stringAttribute(name) == null) throw new IllegalArgumentException("Required attribute '" + name + "' is missing"); - return getIntegerAttribute(name, null); + return integerAttribute(name, null); } /** Returns the value of this attribute or null if not present */ - public Integer getIntegerAttribute(String name) { - return getIntegerAttribute(name, null); + public Integer integerAttribute(String name) { + return integerAttribute(name, null); } - public Integer getIntegerAttribute(String name, Integer defaultValue) { - String value = getStringAttribute(name); - if (value == null) { - return defaultValue; - } + public Integer integerAttribute(String name, Integer defaultValue) { + String value = stringAttribute(name); + if (value == null) return defaultValue; return (int) BinaryUnit.valueOf(value); } - public boolean getBooleanAttribute(String name) { - return getBooleanAttribute(name, false); + public boolean booleanAttribute(String name) { + return booleanAttribute(name, false); } - public boolean getBooleanAttribute(String name, boolean defaultValue) { - String value = getStringAttribute(name); - if (value == null) { - return defaultValue; - } + public boolean booleanAttribute(String name, boolean defaultValue) { + String value = stringAttribute(name); + if (value == null) return defaultValue; return Boolean.parseBoolean(value); } - public Long getLongAttribute(String name) { - String value = getStringAttribute(name); - if (value == null) { - return null; - } + public Long longAttribute(String name) { + String value = stringAttribute(name); + if (value == null) return null; return (long) BinaryUnit.valueOf(value); } - public Double getDoubleAttribute(String name) { - String value = getStringAttribute(name); - if (value == null) { - return null; - } + public Double doubleAttribute(String name) { + String value = stringAttribute(name); + if (value == null) return null; return Double.parseDouble(value); } - public String getStringAttribute(String name) { - if (!xml.hasAttribute(name)) { - return null; - } + /** Returns the given attribute or throws IllegalArgumentException if not present */ + public double requiredDoubleAttribute(String name) { + if (stringAttribute(name) == null) + throw new IllegalArgumentException("Required attribute '" + name + "' is missing"); + return doubleAttribute(name); + } + /** Returns the content of the attribute with the given name, or null if none */ + public String stringAttribute(String name) { + if ( ! xml.hasAttribute(name)) return null; return xml.getAttribute(name); } @@ -209,10 +180,8 @@ public class ModelElement { List<Element> elements = XML.getChildren(xml, name); List<ModelElement> helpers = new ArrayList<>(); - for (Element e : elements) { + for (Element e : elements) helpers.add(new ModelElement(e)); - } - return helpers; } @@ -220,4 +189,5 @@ public class ModelElement { public String toString() { return xml.getNodeName(); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index 84fdbf0a75c..1031ae7b787 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.RotationName; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; @@ -44,14 +45,14 @@ public class NodesSpecification { private final boolean exclusive; /** The flavor the nodes should have, or empty to use the default */ - private final Optional<String> flavor; + private final Optional<FlavorSpec> flavor; /** The identifier of the custom docker image layer to use (not supported yet) */ private final Optional<String> dockerImage; private NodesSpecification(boolean dedicated, int count, int groups, Version version, boolean required, boolean canFail, boolean exclusive, - Optional<String> flavor, Optional<String> dockerImage) { + Optional<FlavorSpec> flavor, Optional<String> dockerImage) { this.dedicated = dedicated; this.count = count; this.groups = groups; @@ -66,18 +67,16 @@ public class NodesSpecification { private NodesSpecification(boolean dedicated, boolean canFail, Version version, ModelElement nodesElement) { this(dedicated, nodesElement.requiredIntegerAttribute("count"), - nodesElement.getIntegerAttribute("groups", 1), + nodesElement.integerAttribute("groups", 1), version, - nodesElement.getBooleanAttribute("required", false), + nodesElement.booleanAttribute("required", false), canFail, - nodesElement.getBooleanAttribute("exclusive", false), - Optional.ofNullable(nodesElement.getStringAttribute("flavor")), - Optional.ofNullable(nodesElement.getStringAttribute("docker-image"))); + nodesElement.booleanAttribute("exclusive", false), + getFlavor(nodesElement), + Optional.ofNullable(nodesElement.stringAttribute("docker-image"))); } - /** - * Returns a requirement for dedicated nodes taken from the given <code>nodes</code> element - */ + /** Returns a requirement for dedicated nodes taken from the given <code>nodes</code> element */ public static NodesSpecification from(ModelElement nodesElement, ConfigModelContext context) { return new NodesSpecification(true, ! context.getDeployState().getProperties().isBootstrap(), @@ -92,7 +91,7 @@ public class NodesSpecification { */ public static Optional<NodesSpecification> fromParent(ModelElement parentElement, ConfigModelContext context) { if (parentElement == null) return Optional.empty(); - ModelElement nodesElement = parentElement.getChild("nodes"); + ModelElement nodesElement = parentElement.child("nodes"); if (nodesElement == null) return Optional.empty(); return Optional.of(from(nodesElement, context)); } @@ -105,9 +104,9 @@ public class NodesSpecification { public static Optional<NodesSpecification> optionalDedicatedFromParent(ModelElement parentElement, ConfigModelContext context) { if (parentElement == null) return Optional.empty(); - ModelElement nodesElement = parentElement.getChild("nodes"); + ModelElement nodesElement = parentElement.child("nodes"); if (nodesElement == null) return Optional.empty(); - return Optional.of(new NodesSpecification(nodesElement.getBooleanAttribute("dedicated", false), + return Optional.of(new NodesSpecification(nodesElement.booleanAttribute("dedicated", false), ! context.getDeployState().getProperties().isBootstrap(), context.getDeployState().getWantedNodeVespaVersion(), nodesElement)); @@ -171,7 +170,22 @@ public class NodesSpecification { DeployLogger logger, Set<RotationName> rotations) { ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId, version, exclusive, rotations); - return hostSystem.allocateHosts(cluster, Capacity.fromNodeCount(count, flavor, required, canFail), groups, logger); + return hostSystem.allocateHosts(cluster, Capacity.fromCount(count, flavor, required, canFail), groups, logger); + } + + private static Optional<FlavorSpec> getFlavor(ModelElement nodesElement) { + ModelElement flavor = nodesElement.child("flavor"); + if (flavor != null) { + return Optional.of(new FlavorSpec(flavor.requiredDoubleAttribute("cpus"), + flavor.requiredDoubleAttribute("memory"), + flavor.requiredDoubleAttribute("disk"))); + } + else if (nodesElement.stringAttribute("flavor") != null) { // legacy fallback + return Optional.of(FlavorSpec.fromLegacyFlavorName(nodesElement.stringAttribute("flavor"))); + } + else { // Get the default + return Optional.empty(); + } } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java b/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java index 9f7494f470d..cb588f20f0e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; -import com.yahoo.vespa.model.content.cluster.ContentCluster; /** * Represents configuration for bucket splitting. @@ -15,16 +14,16 @@ public class BucketSplitting implements StorDistributormanagerConfig.Producer { public static class Builder { public BucketSplitting build(ModelElement clusterElem) { - ModelElement tuning = clusterElem.getChild("tuning"); + ModelElement tuning = clusterElem.child("tuning"); if (tuning == null) { return new BucketSplitting(null, null, null); } - ModelElement bucketSplitting = tuning.getChild("bucket-splitting"); + ModelElement bucketSplitting = tuning.child("bucket-splitting"); if (bucketSplitting != null) { - Integer maxDocuments = bucketSplitting.getIntegerAttribute("max-documents"); - Integer splitSize = bucketSplitting.getIntegerAttribute("max-size"); - Integer minSplitCount = bucketSplitting.getIntegerAttribute("minimum-bits"); + Integer maxDocuments = bucketSplitting.integerAttribute("max-documents"); + Integer splitSize = bucketSplitting.integerAttribute("max-size"); + Integer minSplitCount = bucketSplitting.integerAttribute("minimum-bits"); return new BucketSplitting(maxDocuments, splitSize, minSplitCount); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java index 176ba31857b..1b0af3e9046 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -31,11 +31,11 @@ public class ClusterControllerConfig extends AbstractConfigProducer implements F protected ClusterControllerConfig doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { ModelElement tuning = null; - ModelElement clusterTuning = clusterElement.getChild("tuning"); + ModelElement clusterTuning = clusterElement.child("tuning"); Integer bucketSplittingMinimumBits = null; Double minNodeRatioPerGroup = null; if (clusterTuning != null) { - tuning = clusterTuning.getChild("cluster-controller"); + tuning = clusterTuning.child("cluster-controller"); minNodeRatioPerGroup = clusterTuning.childAsDouble("min-node-ratio-per-group"); bucketSplittingMinimumBits = clusterTuning.childAsInteger("bucket-splitting.minimum-bits"); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 219c8a01afe..db7b4a88721 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -86,11 +86,11 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot ContentSearchCluster search = new ContentSearchCluster(ancestor, clusterName, documentDefinitions, globallyDistributedDocuments, getFlushOnShutdown(flushOnShutdownElem, deployState)); - ModelElement tuning = clusterElem.getChildByPath("engine.proton.tuning"); + ModelElement tuning = clusterElem.childByPath("engine.proton.tuning"); if (tuning != null) { search.setTuning(new DomSearchTuningBuilder().build(deployState, search, tuning.getXml())); } - ModelElement protonElem = clusterElem.getChildByPath("engine.proton"); + ModelElement protonElem = clusterElem.childByPath("engine.proton"); if (protonElem != null) { search.setResourceLimits(DomResourceLimitsBuilder.build(protonElem)); } @@ -112,14 +112,14 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot } private void buildAllStreamingSearchClusters(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) { - ModelElement docElem = clusterElem.getChild("documents"); + ModelElement docElem = clusterElem.child("documents"); if (docElem == null) { return; } for (ModelElement docType : docElem.subElements("document")) { - String mode = docType.getStringAttribute("mode"); + String mode = docType.stringAttribute("mode"); if ("streaming".equals(mode)) { buildStreamingSearchCluster(deployState, clusterElem, clusterName, search, docType); } @@ -128,7 +128,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot private void buildStreamingSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search, ModelElement docType) { - String docTypeName = docType.getStringAttribute("type"); + String docTypeName = docType.stringAttribute("type"); StreamingSearchCluster cluster = new StreamingSearchCluster(search, clusterName + "." + docTypeName, 0, docTypeName, clusterName); search.addSearchCluster(deployState, cluster, getQueryTimeout(clusterElem), Arrays.asList(docType)); } @@ -151,13 +151,13 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot private List<ModelElement> getIndexedSearchDefinitions(ModelElement clusterElem) { List<ModelElement> indexedDefs = new ArrayList<>(); - ModelElement docElem = clusterElem.getChild("documents"); + ModelElement docElem = clusterElem.child("documents"); if (docElem == null) { return indexedDefs; } for (ModelElement docType : docElem.subElements("document")) { - String mode = docType.getStringAttribute("mode"); + String mode = docType.stringAttribute("mode"); if ("index".equals(mode)) { indexedDefs.add(docType); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java index ddc816d2c17..c55548cfd2b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java @@ -27,8 +27,8 @@ public class Distributor extends ContentNode { @Override protected Distributor doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { - return new Distributor((DistributorCluster)ancestor, new ModelElement(producerSpec).getIntegerAttribute("distribution-key"), - clusterXml.getIntegerAttribute("distributor-base-port"), persistenceProvider); + return new Distributor((DistributorCluster)ancestor, new ModelElement(producerSpec).integerAttribute("distribution-key"), + clusterXml.integerAttribute("distributor-base-port"), persistenceProvider); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index ad067d6faed..cd790a67742 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -52,7 +52,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl DocumentSelector s = new DocumentSelector(selStr); boolean enableGC = false; if (documentNode != null) { - enableGC = documentNode.getBooleanAttribute("garbage-collection", false); + enableGC = documentNode.booleanAttribute("garbage-collection", false); } if (!enableGC) { return null; @@ -64,7 +64,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private int getGCInterval(ModelElement documentNode) { int gcInterval = 3600; if (documentNode != null) { - gcInterval = documentNode.getIntegerAttribute("garbage-collection-interval", gcInterval); + gcInterval = documentNode.integerAttribute("garbage-collection-interval", gcInterval); } return gcInterval; } @@ -90,13 +90,13 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private boolean clusterContainsIndexedDocumentType(ModelElement documentsNode) { return documentsNode != null && documentsNode.subElements("document").stream() - .anyMatch(node -> documentModeImpliesIndexing(node.getStringAttribute("mode"))); + .anyMatch(node -> documentModeImpliesIndexing(node.stringAttribute("mode"))); } @Override protected DistributorCluster doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { final ModelElement clusterElement = new ModelElement(producerSpec); - final ModelElement documentsNode = clusterElement.getChild("documents"); + final ModelElement documentsNode = clusterElement.child("documents"); final GcOptions gc = parseGcOptions(documentsNode); final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 17ff15d3a77..84132427427 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -204,12 +204,12 @@ public class StorageGroup { } public StorageGroup buildRootGroup(DeployState deployState) { - Optional<ModelElement> group = Optional.ofNullable(clusterElement.getChild("group")); + Optional<ModelElement> group = Optional.ofNullable(clusterElement.child("group")); Optional<ModelElement> nodes = getNodes(clusterElement); if (group.isPresent() && nodes.isPresent()) throw new IllegalStateException("Both group and nodes exists, only one of these tags is legal"); - if (group.isPresent() && (group.get().getStringAttribute("name") != null || group.get().getIntegerAttribute("distribution-key") != null)) + if (group.isPresent() && (group.get().stringAttribute("name") != null || group.get().integerAttribute("distribution-key") != null)) deployState.getDeployLogger().log(LogLevel.INFO, "'distribution-key' attribute on a content cluster's root group is ignored"); GroupBuilder groupBuilder = collectGroup(group, nodes, null, null); @@ -409,7 +409,7 @@ public class StorageGroup { throw new IllegalArgumentException("A group can contain either explicit subgroups or a nodes specification, but not both."); Optional<NodesSpecification> nodeRequirement; - if (nodesElement.isPresent() && nodesElement.get().getStringAttribute("count") != null ) // request these nodes + if (nodesElement.isPresent() && nodesElement.get().stringAttribute("count") != null ) // request these nodes nodeRequirement = Optional.of(NodesSpecification.from(nodesElement.get(), context)); else if (! nodesElement.isPresent() && subGroups.isEmpty() && context.getDeployState().isHosted()) // request one node nodeRequirement = Optional.of(NodesSpecification.nonDedicated(1, context)); @@ -434,12 +434,12 @@ public class StorageGroup { private boolean booleanAttributeOr(Optional<ModelElement> element, String attributeName, boolean defaultValue) { if ( ! element.isPresent()) return defaultValue; - return element.get().getBooleanAttribute(attributeName, defaultValue); + return element.get().booleanAttribute(attributeName, defaultValue); } private Optional<ModelElement> getNodes(ModelElement groupOrNodesElement) { if (groupOrNodesElement.getXml().getTagName().equals("nodes")) return Optional.of(groupOrNodesElement); - return Optional.ofNullable(groupOrNodesElement.getChild("nodes")); + return Optional.ofNullable(groupOrNodesElement.child("nodes")); } private List<XmlNodeBuilder> collectExplicitNodes(Optional<ModelElement> groupOrNodesElement) { @@ -461,8 +461,8 @@ public class StorageGroup { indexPrefix = parentGroup.index + "."; } for (ModelElement g : subGroupElements) { - subGroups.add(collectGroup(Optional.of(g), Optional.ofNullable(g.getChild("nodes")), g.getStringAttribute("name"), - indexPrefix + g.getIntegerAttribute("distribution-key"))); + subGroups.add(collectGroup(Optional.of(g), Optional.ofNullable(g.child("nodes")), g.stringAttribute("name"), + indexPrefix + g.integerAttribute("distribution-key"))); } return subGroups; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java index 9ea92048f3b..c8220071373 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java @@ -32,7 +32,7 @@ public class StorageNode extends ContentNode implements StorServerConfig.Produce @Override protected StorageNode doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { ModelElement e = new ModelElement(producerSpec); - return new StorageNode((StorageCluster)ancestor, e.getDoubleAttribute("capacity"), e.getIntegerAttribute("distribution-key"), false); + return new StorageNode((StorageCluster)ancestor, e.doubleAttribute("capacity"), e.integerAttribute("distribution-key"), false); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 79675febe2f..833afb67f58 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -119,7 +119,7 @@ public class ContentCluster extends AbstractConfigProducer implements ModelElement contentElement = new ModelElement(w3cContentElement); DeployState deployState = context.getDeployState(); - ModelElement documentsElement = contentElement.getChild("documents"); + ModelElement documentsElement = contentElement.child("documents"); Map<String, NewDocumentType> documentDefinitions = new SearchDefinitionBuilder().build(deployState.getDocumentModel().getDocumentManager(), documentsElement); @@ -150,7 +150,7 @@ public class ContentCluster extends AbstractConfigProducer implements } if (documentsElement != null) { - ModelElement e = documentsElement.getChild("document-processing"); + ModelElement e = documentsElement.child("document-processing"); if (e != null) { setupDocumentProcessing(c, e); } @@ -158,11 +158,11 @@ public class ContentCluster extends AbstractConfigProducer implements throw new IllegalArgumentException("The specified content engine requires the <documents> element to be specified."); } - ModelElement tuning = contentElement.getChild("tuning"); + ModelElement tuning = contentElement.child("tuning"); if (tuning != null) { setupTuning(c, tuning); } - ModelElement experimental = contentElement.getChild("experimental"); + ModelElement experimental = contentElement.child("experimental"); if (experimental != null) { setupExperimental(c, experimental); } @@ -217,7 +217,7 @@ public class ContentCluster extends AbstractConfigProducer implements } private void setupDocumentProcessing(ContentCluster c, ModelElement e) { - String docprocCluster = e.getStringAttribute("cluster"); + String docprocCluster = e.stringAttribute("cluster"); if (docprocCluster != null) { docprocCluster = docprocCluster.trim(); } @@ -227,7 +227,7 @@ public class ContentCluster extends AbstractConfigProducer implements } } - String docprocChain = e.getStringAttribute("chain"); + String docprocChain = e.stringAttribute("chain"); if (docprocChain != null) { docprocChain = docprocChain.trim(); } @@ -239,9 +239,9 @@ public class ContentCluster extends AbstractConfigProducer implements } private void setupTuning(ContentCluster c, ModelElement tuning) { - ModelElement distribution = tuning.getChild("distribution"); + ModelElement distribution = tuning.child("distribution"); if (distribution != null) { - String attr = distribution.getStringAttribute("type"); + String attr = distribution.stringAttribute("type"); if (attr != null) { if (attr.toLowerCase().equals("strict")) { c.distributionMode = DistributionMode.STRICT; @@ -254,9 +254,9 @@ public class ContentCluster extends AbstractConfigProducer implements } } } - ModelElement merges = tuning.getChild("merges"); + ModelElement merges = tuning.child("merges"); if (merges != null) { - Integer attr = merges.getIntegerAttribute("max-nodes-per-merge"); + Integer attr = merges.integerAttribute("max-nodes-per-merge"); if (attr != null) { c.maxNodesPerMerge = attr; } @@ -307,7 +307,7 @@ public class ContentCluster extends AbstractConfigProducer implements else if (admin.multitenant()) { String clusterName = contentClusterName + "-controllers"; NodesSpecification nodesSpecification = - NodesSpecification.optionalDedicatedFromParent(contentElement.getChild("controllers"), context) + NodesSpecification.optionalDedicatedFromParent(contentElement.child("controllers"), context) .orElse(NodesSpecification.nonDedicated(3, context)); Collection<HostResource> hosts = nodesSpecification.isDedicated() ? getControllerHosts(nodesSpecification, admin, clusterName, context) : @@ -520,7 +520,7 @@ public class ContentCluster extends AbstractConfigProducer implements } public static String getClusterName(ModelElement clusterElem) { - String clusterName = clusterElem.getStringAttribute("id"); + String clusterName = clusterElem.stringAttribute("id"); if (clusterName == null) { clusterName = "content"; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java index c6008ef4084..07b87a41b2f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java @@ -54,9 +54,9 @@ public class DocumentSelectionBuilder { sb.append(" OR "); } sb.append('('); - String type = e.getStringAttribute("type"); + String type = e.stringAttribute("type"); sb.append(type); - String selection = e.getStringAttribute("selection"); + String selection = e.stringAttribute("selection"); if (selection != null) { validateSelectionExpression(selection, type); sb.append(" AND ("); @@ -66,7 +66,7 @@ public class DocumentSelectionBuilder { sb.append(')'); } - String globalSelection = elem.getStringAttribute("selection"); + String globalSelection = elem.stringAttribute("selection"); if (globalSelection != null) { validateSelectionExpression(globalSelection, null); StringBuilder global = new StringBuilder(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilder.java index 9ef64e0b288..40bfa3241e2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilder.java @@ -11,7 +11,7 @@ public class DomContentSearchBuilder { public static ContentSearch build(ModelElement contentXml) { ContentSearch.Builder builder = new ContentSearch.Builder(); - ModelElement searchElement = contentXml.getChild("search"); + ModelElement searchElement = contentXml.child("search"); if (searchElement == null) { return builder.build(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomDispatchBuilder.java index 3d06d120b09..ffe07a00775 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomDispatchBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomDispatchBuilder.java @@ -17,7 +17,7 @@ public class DomDispatchBuilder { public static DispatchSpec build(ModelElement contentXml) { DispatchSpec.Builder builder = new DispatchSpec.Builder(); - ModelElement dispatchElement = contentXml.getChild("dispatch"); + ModelElement dispatchElement = contentXml.child("dispatch"); if (dispatchElement == null) { return builder.build(); } @@ -48,6 +48,6 @@ public class DomDispatchBuilder { } private static DispatchSpec.Node buildNode(ModelElement nodeElement) { - return new DispatchSpec.Node(nodeElement.getIntegerAttribute("distribution-key")); + return new DispatchSpec.Node(nodeElement.integerAttribute("distribution-key")); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java index 87412ad3596..61bf42af379 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java @@ -13,14 +13,14 @@ public class DomResourceLimitsBuilder { public static ResourceLimits build(ModelElement contentXml) { ResourceLimits.Builder builder = new ResourceLimits.Builder(); - ModelElement resourceLimits = contentXml.getChild("resource-limits"); + ModelElement resourceLimits = contentXml.child("resource-limits"); if (resourceLimits == null) { return builder.build(); } - if (resourceLimits.getChild("disk") != null) { + if (resourceLimits.child("disk") != null) { builder.setDiskLimit(resourceLimits.childAsDouble("disk")); } - if (resourceLimits.getChild("memory") != null) { + if (resourceLimits.child("memory") != null) { builder.setMemoryLimit(resourceLimits.childAsDouble("memory")); } return builder.build(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilder.java index cfc110d7a13..64953deb36e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilder.java @@ -11,11 +11,11 @@ public class DomSearchCoverageBuilder { public static SearchCoverage build(ModelElement contentXml) { SearchCoverage.Builder builder = new SearchCoverage.Builder(); - ModelElement searchElement = contentXml.getChild("search"); + ModelElement searchElement = contentXml.child("search"); if (searchElement == null) { return builder.build(); } - ModelElement coverageElement = searchElement.getChild("coverage"); + ModelElement coverageElement = searchElement.child("coverage"); if (coverageElement == null) { return builder.build(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java index f470e75abf0..5f26b0628c2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java @@ -11,11 +11,11 @@ public class DomTuningDispatchBuilder { public static TuningDispatch build(ModelElement contentXml) { TuningDispatch.Builder builder = new TuningDispatch.Builder(); - ModelElement tuningElement = contentXml.getChild("tuning"); + ModelElement tuningElement = contentXml.child("tuning"); if (tuningElement == null) { return builder.build(); } - ModelElement dispatchElement = tuningElement.getChild("dispatch"); + ModelElement dispatchElement = tuningElement.child("dispatch"); if (dispatchElement == null) { return builder.build(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java index 83e9553ebd5..e31e9703f11 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java @@ -9,15 +9,15 @@ import com.yahoo.vespa.model.content.engines.*; */ public class EngineFactoryBuilder { public PersistenceEngine.PersistenceFactory build(ModelElement clusterElem, ContentCluster c) { - ModelElement persistence = clusterElem.getChild("engine"); + ModelElement persistence = clusterElem.child("engine"); if (persistence != null) { - if (c.getSearch().hasIndexedCluster() && persistence.getChild("proton") == null) { + if (c.getSearch().hasIndexedCluster() && persistence.child("proton") == null) { throw new IllegalArgumentException("Persistence engine does not allow for indexed search. Please use <proton> as your engine."); } - if (persistence.getChild("proton") != null) { + if (persistence.child("proton") != null) { return new ProtonEngine.Factory(c.getSearch()); - } else if (persistence.getChild("dummy") != null) { + } else if (persistence.child("dummy") != null) { return new com.yahoo.vespa.model.content.engines.DummyPersistence.Factory(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java index bdc15074258..1adb5d6d2d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java @@ -36,11 +36,11 @@ public class GlobalDistributionBuilder { } private static boolean isGloballyDistributed(ModelElement e) { - return e.getBooleanAttribute("global", false); + return e.booleanAttribute("global", false); } private static String getDocumentName(ModelElement e) { - return e.getStringAttribute("type"); + return e.stringAttribute("type"); } private NewDocumentType getDocumentType(String name) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java index e1675007bbc..fe73fcc904b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java @@ -14,9 +14,9 @@ public class RedundancyBuilder { Integer finalRedundancy = 3; Integer readyCopies = 2; - ModelElement redundancyElement = clusterXml.getChild("redundancy"); + ModelElement redundancyElement = clusterXml.child("redundancy"); if (redundancyElement != null) { - initialRedundancy = redundancyElement.getIntegerAttribute("reply-after"); + initialRedundancy = redundancyElement.integerAttribute("reply-after"); finalRedundancy = (int)redundancyElement.asLong(); if (initialRedundancy == null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java index 53220268bf8..44cb9cb96dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java @@ -21,7 +21,7 @@ public class SearchDefinitionBuilder { if (elem != null) { for (ModelElement e : elem.subElements("document")) { - String name = e.getStringAttribute("type"); // Schema-guaranteed presence + String name = e.stringAttribute("type"); // Schema-guaranteed presence NewDocumentType documentType = repo.getDocumentType(name); if (documentType != null) { docTypes.put(documentType.getName(), documentType); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index 0cc62fb6680..785088e96fc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -16,16 +16,16 @@ public class FileStorProducer implements StorFilestorConfig.Producer { } private Integer getThreads(ModelElement clusterElem) { - ModelElement tuning = clusterElem.getChild("tuning"); + ModelElement tuning = clusterElem.child("tuning"); if (tuning == null) { return null; } - ModelElement threads = tuning.getChild("persistence-threads"); + ModelElement threads = tuning.child("persistence-threads"); if (threads == null) { return null; } - Integer count = threads.getIntegerAttribute("count"); + Integer count = threads.integerAttribute("count"); if (count != null) { return count; } @@ -33,7 +33,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { // Backward compatible fallback int numThreads = 0; for (ModelElement thread : threads.subElements("thread")) { - count = thread.getIntegerAttribute("count"); + count = thread.integerAttribute("count"); numThreads += (count == null) ? 1 : count; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/PersistenceProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/PersistenceProducer.java index dabfbd4f75d..749cf8f374e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/PersistenceProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/PersistenceProducer.java @@ -12,7 +12,7 @@ public class PersistenceProducer implements PersistenceConfig.Producer { public static class Builder { public PersistenceProducer build(ModelElement element) { - ModelElement persistence = element.getChild("engine"); + ModelElement persistence = element.child("engine"); if (persistence == null) { return new PersistenceProducer(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index baebbcd3f65..51fc610bf28 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -11,20 +11,20 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; public class StorServerProducer implements StorServerConfig.Producer { public static class Builder { StorServerProducer build(ModelElement element) { - ModelElement tuning = element.getChild("tuning"); + ModelElement tuning = element.child("tuning"); if (tuning == null) { return new StorServerProducer(ContentCluster.getClusterName(element), null, null); } - ModelElement merges = tuning.getChild("merges"); + ModelElement merges = tuning.child("merges"); if (merges == null) { return new StorServerProducer(ContentCluster.getClusterName(element), null, null); } return new StorServerProducer(ContentCluster.getClusterName(element), - merges.getIntegerAttribute("max-per-node"), - merges.getIntegerAttribute("max-queue-size")); + merges.integerAttribute("max-per-node"), + merges.integerAttribute("max-queue-size")); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorVisitorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorVisitorProducer.java index a831d3866d0..ec93881aae7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorVisitorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorVisitorProducer.java @@ -10,18 +10,18 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; public class StorVisitorProducer implements StorVisitorConfig.Producer { public static class Builder { public StorVisitorProducer build(ModelElement element) { - ModelElement tuning = element.getChild("tuning"); + ModelElement tuning = element.child("tuning"); if (tuning == null) { return new StorVisitorProducer(); } - ModelElement visitors = tuning.getChild("visitors"); + ModelElement visitors = tuning.child("visitors"); if (visitors == null) { return new StorVisitorProducer(); } - return new StorVisitorProducer(visitors.getIntegerAttribute("thread-count"), - visitors.getIntegerAttribute("max-queue-size"), + return new StorVisitorProducer(visitors.integerAttribute("thread-count"), + visitors.integerAttribute("max-queue-size"), visitors.childAsInteger("max-concurrent.fixed"), visitors.childAsInteger("max-concurrent.variable")); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java index 11758284a31..1054253e3f0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.search; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; -import org.w3c.dom.Element; import java.io.Serializable; import java.util.List; @@ -17,9 +16,9 @@ public class SearchDefinitionXMLHandler implements Serializable { private String sdName; public SearchDefinitionXMLHandler(ModelElement elem) { - sdName = elem.getStringAttribute("name"); + sdName = elem.stringAttribute("name"); if (sdName == null) { - sdName = elem.getStringAttribute("type"); + sdName = elem.stringAttribute("type"); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index 6df617ea335..f635b986558 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -17,11 +17,11 @@ public final class Capacity { private final boolean canFail; - private final Optional<String> flavor; + private final Optional<FlavorSpec> flavor; private final NodeType type; - private Capacity(int nodeCount, Optional<String> flavor, boolean required, boolean canFail, NodeType type) { + private Capacity(int nodeCount, Optional<FlavorSpec> flavor, boolean required, boolean canFail, NodeType type) { this.nodeCount = nodeCount; this.required = required; this.canFail = canFail; @@ -36,7 +36,10 @@ public final class Capacity { * The node flavor requested, or empty if no particular flavor is specified. * This may be satisfied by the requested flavor or a suitable replacement */ - public Optional<String> flavor() { return flavor; } + public Optional<String> flavor() { return flavor.map(FlavorSpec::legacyFlavorName); } + + /** Returns the capacity specified for each node, or empty to leave this decision to provisioning */ + public Optional<FlavorSpec> flavorSpec() { return flavor; } /** Returns whether the requested number of nodes must be met exactly for a request for this to succeed */ public boolean isRequired() { return required; } @@ -65,10 +68,18 @@ public final class Capacity { return fromNodeCount(capacity, Optional.empty(), false, true); } - public static Capacity fromNodeCount(int nodeCount, Optional<String> flavor, boolean required, boolean canFail) { + public static Capacity fromCount(int nodeCount, FlavorSpec flavor, boolean required, boolean canFail) { + return new Capacity(nodeCount, Optional.of(flavor), required, canFail, NodeType.tenant); + } + + public static Capacity fromCount(int nodeCount, Optional<FlavorSpec> flavor, boolean required, boolean canFail) { return new Capacity(nodeCount, flavor, required, canFail, NodeType.tenant); } + public static Capacity fromNodeCount(int nodeCount, Optional<String> flavor, boolean required, boolean canFail) { + return new Capacity(nodeCount, flavor.map(FlavorSpec::fromLegacyFlavorName), required, canFail, NodeType.tenant); + } + /** Creates this from a node type */ public static Capacity fromRequiredNodeType(NodeType type) { return new Capacity(0, Optional.empty(), true, false, type); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index 79a17c23dd7..8b6fa863af6 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.config.provisioning.FlavorsConfig; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -31,6 +32,7 @@ public class Flavor { /** * Creates a Flavor, but does not set the replacesFlavors. + * * @param flavorConfig config to be used for Flavor. */ public Flavor(FlavorsConfig.Flavor flavorConfig) { @@ -49,6 +51,25 @@ public class Flavor { this.idealHeadroom = flavorConfig.idealHeadroom(); } + /** Create a Flavor from a Flavor spec and all other fields set to Docker defaults */ + public Flavor(FlavorSpec spec) { + if (spec.allocateByLegacyName()) + throw new IllegalArgumentException("Can not create flavor '" + spec.legacyFlavorName() + "' from a spec: " + + "Non-docker flavors must be of a configured flavor"); + this.name = spec.legacyFlavorName(); + this.cost = 0; + this.isStock = true; + this.type = Type.DOCKER_CONTAINER; + this.minCpuCores = spec.cpuCores(); + this.minMainMemoryAvailableGb = spec.memoryGb(); + this.minDiskAvailableGb = spec.diskGb(); + this.fastDisk = true; + this.bandwidth = 1; + this.description = ""; + this.retired = false; + this.replacesFlavors = Collections.emptyList(); + } + /** Returns the unique identity of this flavor */ public String name() { return name; } @@ -147,6 +168,13 @@ public class Flavor { this.fastDisk || ! other.fastDisk; } + public FlavorSpec asSpec() { + if (isDocker()) + return new FlavorSpec(minCpuCores, minMainMemoryAvailableGb, minDiskAvailableGb); + else + return FlavorSpec.fromLegacyFlavorName(name); + } + @Override public int hashCode() { return name.hashCode(); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/FlavorSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/FlavorSpec.java new file mode 100644 index 00000000000..62cfb59c51c --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/FlavorSpec.java @@ -0,0 +1,107 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import java.util.Objects; + +/** + * The node capacity specified by an application, which is matched to an actual flavor during provisioning. + * + * @author bratseth + */ +public class FlavorSpec { + + private final double cpuCores; + private final double memoryGb; + private final double diskGb; + + private final boolean allocateByLegacyName; + private final String legacyFlavorName; + + public FlavorSpec(double cpuCores, double memoryGb, double diskGb) { + this.cpuCores = cpuCores; + this.memoryGb = memoryGb; + this.diskGb = diskGb; + this.allocateByLegacyName = false; + this.legacyFlavorName = null; + } + + private FlavorSpec(double cpuCores, double memoryGb, double diskGb, boolean allocateByLegacyName, String legacyFlavorName) { + this.cpuCores = cpuCores; + this.memoryGb = memoryGb; + this.diskGb = diskGb; + this.allocateByLegacyName = allocateByLegacyName; + this.legacyFlavorName = legacyFlavorName; + } + + public double cpuCores() { return cpuCores; } + public double memoryGb() { return memoryGb; } + public double diskGb() { return diskGb; } + + /** + * If this is true, a non-docker legacy name was used to specify this and we'll respect that by mapping directly. + * The other getters of this will return 0. + */ + public boolean allocateByLegacyName() { return allocateByLegacyName; } + + /** Returns the legacy flavor string of this. This is never null. */ + public String legacyFlavorName() { + if (legacyFlavorName != null) + return legacyFlavorName; + else + return "d-" + (int)Math.ceil(cpuCores) + "-" + (int)Math.ceil(memoryGb) + "-" + (int)Math.ceil(diskGb); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof FlavorSpec)) return false; + FlavorSpec other = (FlavorSpec)o; + if (allocateByLegacyName) { + return this.legacyFlavorName.equals(other.legacyFlavorName); + } + else { + if (this.cpuCores != other.cpuCores) return false; + if (this.memoryGb != other.memoryGb) return false; + if (this.diskGb != other.diskGb) return false; + return true; + } + } + + @Override + public int hashCode() { + if (allocateByLegacyName) + return legacyFlavorName.hashCode(); + else + return (int)(2503 * cpuCores + 22123 * memoryGb + 26987 * diskGb); + } + + @Override + public String toString() { + if (allocateByLegacyName) + return "flavor '" + legacyFlavorName + "'"; + else + return "cpu cores: " + cpuCores + ", memory: " + memoryGb + " Gb, disk " + diskGb + " Gb"; + } + + /** + * Create this from a legacy flavor string. + * + * @throws IllegalArgumentException if the given string does not map to a legacy flavor + */ + public static FlavorSpec fromLegacyFlavorName(String flavorString) { + if (flavorString.startsWith("d-")) { // A docker flavor + String[] parts = flavorString.split("-"); + double cpu = Integer.parseInt(parts[1]); + double mem = Integer.parseInt(parts[2]); + double dsk = Integer.parseInt(parts[3]); + if (cpu == 0) cpu = 0.5; + if (cpu == 2 && mem == 8 ) cpu = 1.5; + if (cpu == 2 && mem == 12 ) cpu = 2.3; + return new FlavorSpec(cpu, mem, dsk, false, flavorString); + } + else { + return new FlavorSpec(0, 0, 0, true, flavorString); + } + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java index e64028e216f..b87f6eeec31 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java @@ -20,34 +20,49 @@ import java.util.stream.Collectors; */ public class NodeFlavors { - /** Flavors <b>which are configured</b> in this zone */ - private final ImmutableMap<String, Flavor> flavors; + /** Flavors which are configured in this zone */ + private final ImmutableMap<String, Flavor> configuredFlavors; @Inject public NodeFlavors(FlavorsConfig config) { ImmutableMap.Builder<String, Flavor> b = new ImmutableMap.Builder<>(); for (Flavor flavor : toFlavors(config)) b.put(flavor.name(), flavor); - this.flavors = b.build(); + this.configuredFlavors = b.build(); } public List<Flavor> getFlavors() { - return new ArrayList<>(flavors.values()); + return new ArrayList<>(configuredFlavors.values()); } - /** Returns a flavor by name, or empty if there is no flavor with this name. */ + /** Returns a flavor by name, or empty if there is no flavor with this name and it cannot be created on the fly. */ public Optional<Flavor> getFlavor(String name) { - return Optional.ofNullable(flavors.get(name)); + if (configuredFlavors.containsKey(name)) + return Optional.of(configuredFlavors.get(name)); + + FlavorSpec flavorSpec = FlavorSpec.fromLegacyFlavorName(name); + if (flavorSpec.allocateByLegacyName()) + return Optional.empty(); + else + return Optional.of(new Flavor(flavorSpec)); } - /** Returns the flavor with the given name or throws an IllegalArgumentException if it does not exist */ + /** + * Returns the flavor with the given name or throws an IllegalArgumentException if it does not exist + * and cannot be created on the fly. + */ public Flavor getFlavorOrThrow(String flavorName) { return getFlavor(flavorName).orElseThrow(() -> new IllegalArgumentException("Unknown flavor '" + flavorName + - "'. Flavors are " + canonicalFlavorNames())); + "'. Flavors are " + canonicalFlavorNames())); + } + + /** Returns true if this flavor is configured or can be created on the fly */ + public boolean exists(String flavorName) { + return getFlavor(flavorName).isPresent(); } private List<String> canonicalFlavorNames() { - return flavors.values().stream().map(Flavor::canonicalName).distinct().sorted().collect(Collectors.toList()); + return configuredFlavors.values().stream().map(Flavor::canonicalName).distinct().sorted().collect(Collectors.toList()); } private static Collection<Flavor> toFlavors(FlavorsConfig config) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 5ba56dfa8ed..3831bfc55c1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NetworkPortsSerializer; import com.yahoo.config.provision.NodeFlavors; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index e437badf0dc..654a73ee3cd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; @@ -41,19 +42,23 @@ public class CapacityPolicies { } } - public Flavor decideFlavor(Capacity requestedCapacity, ClusterSpec cluster) { - // for now, always use the requested flavor if a docker flavor is requested - Optional<String> requestedFlavor = requestedCapacity.flavor(); - if (requestedFlavor.isPresent() && - flavors.getFlavorOrThrow(requestedFlavor.get()).getType() == Flavor.Type.DOCKER_CONTAINER) - return flavors.getFlavorOrThrow(requestedFlavor.get()); + public FlavorSpec decideFlavor(Capacity requestedCapacity, ClusterSpec cluster) { + Optional<FlavorSpec> requestedFlavor = requestedCapacity.flavorSpec(); + if (requestedFlavor.isPresent() && ! requestedFlavor.get().allocateByLegacyName()) + return requestedFlavor.get(); - String defaultFlavorName = zone.defaultFlavor(cluster.type()); + FlavorSpec defaultFlavor = FlavorSpec.fromLegacyFlavorName(zone.defaultFlavor(cluster.type())); + if ( requestedFlavor.isEmpty()) + return defaultFlavor; + + // Flavor is specified and is allocateByLegacyName: Handle legacy flavor specs if (zone.system() == SystemName.cd) - return flavors.getFlavorOrThrow(requestedFlavor.orElse(defaultFlavorName)); - switch(zone.environment()) { - case dev : case test : case staging : return flavors.getFlavorOrThrow(defaultFlavorName); - default : return flavors.getFlavorOrThrow(requestedFlavor.orElse(defaultFlavorName)); + return flavors.exists(requestedFlavor.get().legacyFlavorName()) ? requestedFlavor.get() : defaultFlavor; + else { + switch (zone.environment()) { + case dev: case test: case staging: return defaultFlavor; + default: return flavors.getFlavorOrThrow(requestedFlavor.get().legacyFlavorName()).asSpec(); + } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index a8b48751d23..5db9eaf3d08 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.lang.MutableInteger; import com.yahoo.transaction.Mutex; @@ -64,7 +65,8 @@ public class GroupPreparer { // Create a prioritized set of nodes NodeList nodeList = nodeRepository.list(); NodePrioritizer prioritizer = new NodePrioritizer( - nodeList, application, cluster, requestedNodes, spareCount, nodeRepository.nameResolver()); + nodeList, application, cluster, requestedNodes, spareCount, nodeRepository.nameResolver(), + nodeRepository.getAvailableFlavors()); prioritizer.addApplicationNodes(); prioritizer.addSurplusNodes(surplusActiveNodes); @@ -73,14 +75,14 @@ public class GroupPreparer { // Allocate from the prioritized list NodeAllocation allocation = new NodeAllocation(nodeList, application, cluster, requestedNodes, - highestIndex, nodeRepository.zone(), nodeRepository.clock()); + highestIndex, nodeRepository.getAvailableFlavors(), + nodeRepository.zone(), nodeRepository.clock()); allocation.offer(prioritizer.prioritize()); if (dynamicProvisioningEnabled) { List<ProvisionedHost> provisionedHosts = allocation.getFulfilledDockerDeficit() - .map(deficit -> hostProvisioner.get().provisionHosts( - nodeRepository.database().getProvisionIndexes(deficit.getCount()), - deficit.getFlavor())) + .map(deficit -> hostProvisioner.get().provisionHosts(nodeRepository.database().getProvisionIndexes(deficit.getCount()), + deficit.getFlavor())) .orElseGet(List::of); // At this point we have started provisioning of the hosts, the first priority is to make sure that diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 79296e58045..4defaaef57c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.vespa.hosted.provision.Node; import java.util.List; @@ -19,11 +20,11 @@ public interface HostProvisioner { * * @param provisionIndexes List of unique provision indexes which will be used to generate the host hostnames * on the form of <code>[prefix][index].[domain]</code> - * @param nodeFlavor Vespa flavor of the node that will run on this host. The resulting provisioned host - * will be of a flavor that is at least as big or bigger than this. + * @param flavor the spec of the flavor (capacity) to provision. The resulting provisioned host + * will be of a flavor that is at least as big or bigger than this. * @return list of {@link ProvisionedHost} describing the provisioned hosts and nodes on them. */ - List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, Flavor nodeFlavor); + List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, FlavorSpec flavor); /** * Continue provisioning of given list of Nodes. @@ -47,4 +48,5 @@ public interface HostProvisioner { * @param host host to deprovision. */ void deprovision(Node host); + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index dcc3c4a0ef8..219ba759e24 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -5,6 +5,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -67,17 +69,18 @@ class NodeAllocation { /** The next membership index to assign to a new node */ private final MutableInteger highestIndex; + private final NodeFlavors flavors; private final Zone zone; - private final Clock clock; NodeAllocation(NodeList allNodes, ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, - MutableInteger highestIndex, Zone zone, Clock clock) { + MutableInteger highestIndex, NodeFlavors flavors, Zone zone, Clock clock) { this.allNodes = allNodes; this.application = application; this.cluster = cluster; this.requestedNodes = requestedNodes; this.highestIndex = highestIndex; + this.flavors = flavors; this.zone = zone; this.clock = clock; } @@ -217,7 +220,7 @@ class NodeAllocation { } private boolean hasCompatibleFlavor(Node node) { - return requestedNodes.isCompatible(node.flavor()); + return requestedNodes.isCompatible(node.flavor().asSpec(), flavors); } private Node acceptNode(PrioritizableNode prioritizableNode, boolean wantToRetire) { @@ -285,7 +288,7 @@ class NodeAllocation { .filter(NodeSpec.CountNodeSpec.class::isInstance) .map(NodeSpec.CountNodeSpec.class::cast) .map(spec -> new FlavorCount(spec.getFlavor(), spec.fulfilledDeficitCount(acceptedOfRequestedFlavor))) - .filter(flavorCount -> flavorCount.getFlavor().getType() == Flavor.Type.DOCKER_CONTAINER) + .filter(flavorCount -> ! flavorCount.getFlavor().allocateByLegacyName()) .filter(flavorCount -> flavorCount.getCount() > 0); } @@ -364,15 +367,16 @@ class NodeAllocation { } static class FlavorCount { - private final Flavor flavor; + + private final FlavorSpec flavor; private final int count; - private FlavorCount(Flavor flavor, int count) { + private FlavorCount(FlavorSpec flavor, int count) { this.flavor = flavor; this.count = count; } - Flavor getFlavor() { + FlavorSpec getFlavor() { return flavor; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 5a3f0380fa2..13006dd6ef7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.log.LogLevel; import com.yahoo.transaction.Mutex; @@ -44,21 +46,22 @@ class NodePrioritizer { private final ApplicationId appId; private final ClusterSpec clusterSpec; private final NameResolver nameResolver; + private final NodeFlavors flavors; private final boolean isDocker; private final boolean isAllocatingForReplacement; private final Set<Node> spareHosts; NodePrioritizer(NodeList allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec, - int spares, NameResolver nameResolver) { + int spares, NameResolver nameResolver, NodeFlavors flavors) { this.allNodes = allNodes; this.capacity = new DockerHostCapacity(allNodes); this.requestedNodes = nodeSpec; this.clusterSpec = clusterSpec; this.appId = appId; this.nameResolver = nameResolver; + this.flavors = flavors; this.spareHosts = findSpareHosts(allNodes, capacity, spares); - int nofFailedNodes = (int) allNodes.asList().stream() .filter(node -> node.state().equals(Node.State.failed)) .filter(node -> node.allocation().isPresent()) @@ -143,7 +146,7 @@ class NodePrioritizer { } void addNewDockerNodesOn(Mutex allocationLock, NodeList candidates) { - if (!isDocker) return; + if ( ! isDocker) return; ResourceCapacity wantedResourceCapacity = ResourceCapacity.of(getFlavor(requestedNodes)); for (Node node : candidates) { @@ -171,10 +174,10 @@ class NodePrioritizer { Collections.emptySet(), allocation.get().hostname(), Optional.of(node.hostname()), - getFlavor(requestedNodes), + new Flavor(getFlavor(requestedNodes)), NodeType.tenant); PrioritizableNode nodePri = toNodePriority(newNode, false, true); - if (!nodePri.violatesSpares || isAllocatingForReplacement) { + if ( ! nodePri.violatesSpares || isAllocatingForReplacement) { log.log(LogLevel.DEBUG, "Adding new Docker node " + newNode); nodes.put(newNode, nodePri); } @@ -215,8 +218,7 @@ class NodePrioritizer { PrioritizableNode.Builder builder = new PrioritizableNode.Builder(node) .withSurplusNode(isSurplusNode) .withNewNode(isNewNode) - .withPreferredOnFlavor( - requestedNodes.specifiesNonStockFlavor() && node.flavor().equals(getFlavor(requestedNodes))); + .withPreferredOnFlavor(preferredOnFlavor(node)); allNodes.parentOf(node).ifPresent(parent -> { builder.withParent(parent).withFreeParentCapacity(capacity.freeCapacityOf(parent, false)); @@ -229,6 +231,18 @@ class NodePrioritizer { return builder.build(); } + /** Needed to handle requests for legacy non-docker nodes only */ + private boolean preferredOnFlavor(Node node) { + if (requestedNodes instanceof NodeSpec.CountNodeSpec) { + FlavorSpec requestedFlavorSpec = ((NodeSpec.CountNodeSpec)requestedNodes).getFlavor(); + if (requestedFlavorSpec.allocateByLegacyName()) { + Flavor requestedFlavor = flavors.getFlavorOrThrow(requestedFlavorSpec.legacyFlavorName()); + return ! requestedFlavor.isStock() && node.flavor().equals(requestedFlavor); + } + } + return false; + } + static boolean isPreferredNodeToBeRelocated(List<Node> nodes, Node node, Node parent) { NodeList list = new NodeList(nodes); return list.childrenOf(parent).asList().stream() @@ -243,7 +257,7 @@ class NodePrioritizer { return requestedNodes.fulfilledBy(nofNodesInCluster - nodeFailedNodes); } - private static Flavor getFlavor(NodeSpec requestedNodes) { + private static FlavorSpec getFlavor(NodeSpec requestedNodes) { if (requestedNodes instanceof NodeSpec.CountNodeSpec) { NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) requestedNodes; return countSpec.getFlavor(); @@ -252,8 +266,8 @@ class NodePrioritizer { } private boolean isDocker() { - Flavor flavor = getFlavor(requestedNodes); - return (flavor != null) && flavor.getType().equals(Flavor.Type.DOCKER_CONTAINER); + FlavorSpec flavor = getFlavor(requestedNodes); + return (flavor != null) && ! flavor.allocateByLegacyName(); } private static int compareForRelocation(Node a, Node b) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index c91d28e17ce..44ca50a29a6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; @@ -86,7 +87,7 @@ public class NodeRepositoryProvisioner implements Provisioner { log.log(zone.system() == SystemName.cd ? Level.INFO : LogLevel.DEBUG, () -> "Received deploy prepare request for " + requestedCapacity + " in " + - wantedGroups + " groups for application " + application + ", cluster " + cluster); + wantedGroups + " groups for application " + application + ", cluster " + cluster); int effectiveGroups; NodeSpec requestedNodes; @@ -96,7 +97,7 @@ public class NodeRepositoryProvisioner implements Provisioner { if (zone.environment().isManuallyDeployed() && nodeCount < requestedCapacity.nodeCount()) logger.log(Level.INFO, "Requested " + requestedCapacity.nodeCount() + " nodes for " + cluster + ", downscaling to " + nodeCount + " nodes in " + zone.environment()); - Flavor flavor = capacityPolicies.decideFlavor(requestedCapacity, cluster); + FlavorSpec flavor = capacityPolicies.decideFlavor(requestedCapacity, cluster); log.log(LogLevel.DEBUG, () -> "Decided flavor for requested tenant nodes: " + flavor); boolean exclusive = capacityPolicies.decideExclusivity(cluster.isExclusive()); effectiveGroups = wantedGroups > nodeCount ? nodeCount : wantedGroups; // cannot have more groups than nodes diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index e033d994f24..ed95a76a997 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.config.provision.FlavorSpec; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Flavor; import com.yahoo.vespa.hosted.provision.Node; @@ -26,13 +28,7 @@ public interface NodeSpec { boolean isExclusive(); /** Returns whether the given flavor is compatible with this spec */ - boolean isCompatible(Flavor flavor); - - /** Returns whether the given flavor is exactly specified by this node spec */ - boolean matchesExactly(Flavor flavor); - - /** Returns whether this requests a non-stock flavor */ - boolean specifiesNonStockFlavor(); + boolean isCompatible(FlavorSpec flavor, NodeFlavors flavors); /** Returns whether the given node count is sufficient to consider this spec fulfilled to the maximum amount */ boolean saturatedBy(int count); @@ -61,7 +57,7 @@ public interface NodeSpec { */ Node assignRequestedFlavor(Node node); - static NodeSpec from(int nodeCount, Flavor flavor, boolean exclusive, boolean canFail) { + static NodeSpec from(int nodeCount, FlavorSpec flavor, boolean exclusive, boolean canFail) { return new CountNodeSpec(nodeCount, flavor, exclusive, canFail); } @@ -73,20 +69,20 @@ public interface NodeSpec { class CountNodeSpec implements NodeSpec { private final int count; - private final Flavor requestedFlavor; + private final FlavorSpec requestedFlavorSpec; private final boolean exclusive; private final boolean canFail; - CountNodeSpec(int count, Flavor flavor, boolean exclusive, boolean canFail) { + CountNodeSpec(int count, FlavorSpec flavor, boolean exclusive, boolean canFail) { this.count = count; - this.requestedFlavor = Objects.requireNonNull(flavor, "A flavor must be specified"); + this.requestedFlavorSpec = Objects.requireNonNull(flavor, "A flavor must be specified"); this.exclusive = exclusive; this.canFail = canFail; } // TODO: Remove usage of this - public Flavor getFlavor() { - return requestedFlavor; + public FlavorSpec getFlavor() { + return requestedFlavorSpec; } @Override @@ -96,16 +92,18 @@ public interface NodeSpec { public NodeType type() { return NodeType.tenant; } @Override - public boolean isCompatible(Flavor flavor) { - if (flavor.satisfies(requestedFlavor)) return true; - return requestedFlavorCanBeAchievedByResizing(flavor); - } - - @Override - public boolean matchesExactly(Flavor flavor) { return flavor.equals(this.requestedFlavor); } + public boolean isCompatible(FlavorSpec flavorSpec, NodeFlavors flavors) { + if (flavorSpec.allocateByLegacyName()) { + Flavor flavor = flavors.getFlavorOrThrow(flavorSpec.legacyFlavorName()); + Flavor requestedFlavor = flavors.getFlavorOrThrow(requestedFlavorSpec.legacyFlavorName()); + if (flavor.satisfies(requestedFlavor)) return true; + } + else { + if (flavorSpec.equals(requestedFlavorSpec)) return true; + } - @Override - public boolean specifiesNonStockFlavor() { return ! requestedFlavor.isStock(); } + return requestedFlavorCanBeAchievedByResizing(flavorSpec); + } @Override public boolean saturatedBy(int count) { return fulfilledBy(count); } // min=max for count specs @@ -123,23 +121,22 @@ public interface NodeSpec { @Override public NodeSpec fraction(int divisor) { - return new CountNodeSpec(count/divisor, requestedFlavor, exclusive, canFail); + return new CountNodeSpec(count/divisor, requestedFlavorSpec, exclusive, canFail); } @Override public Node assignRequestedFlavor(Node node) { - // Docker nodes can change flavor in place - if (requestedFlavorCanBeAchievedByResizing(node.flavor())) - return node.with(requestedFlavor); - + // Docker nodes can change flavor in place - disabled - see below + // if (requestedFlavorCanBeAchievedByResizing(node.flavor())) + // return node.with(requestedFlavor); return node; } @Override - public String toString() { return "request for " + count + " nodes of " + requestedFlavor; } + public String toString() { return "request for " + count + " nodes with " + requestedFlavorSpec; } /** Docker nodes can be downsized in place */ - private boolean requestedFlavorCanBeAchievedByResizing(Flavor flavor) { + private boolean requestedFlavorCanBeAchievedByResizing(FlavorSpec flavor) { // TODO: Enable this when we can do it safely // Then also re-enable ProvisioningTest.application_deployment_with_inplace_downsize() // return flavor.isDocker() && requestedFlavor.isDocker() && flavor.isLargerThan(requestedFlavor); @@ -164,13 +161,7 @@ public interface NodeSpec { public boolean isExclusive() { return false; } @Override - public boolean isCompatible(Flavor flavor) { return true; } - - @Override - public boolean matchesExactly(Flavor flavor) { return false; } - - @Override - public boolean specifiesNonStockFlavor() { return false; } + public boolean isCompatible(FlavorSpec flavor, NodeFlavors flavors) { return true; } @Override public boolean saturatedBy(int count) { return false; } @@ -180,11 +171,9 @@ public interface NodeSpec { @Override public int idealRetiredCount(int acceptedCount, int currentRetiredCount) { - /* - * All nodes marked with wantToRetire get marked as retired just before this function is called, - * the job of this function is to throttle the retired count. If no nodes are marked as retired - * then continue this way, otherwise allow only 1 node to be retired - */ + // All nodes marked with wantToRetire get marked as retired just before this function is called, + // the job of this function is to throttle the retired count. If no nodes are marked as retired + // then continue this way, otherwise allow only 1 node to be retired return Math.min(1, currentRetiredCount); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index e0cf882fffa..4f729bdd03b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -15,6 +15,7 @@ import java.util.Set; * @author freva */ public class ProvisionedHost { + private final String id; private final String hostHostname; private final Flavor hostFlavor; @@ -86,4 +87,5 @@ public class ProvisionedHost { ", nodeFlavor=" + nodeFlavor + '}'; } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java index 903951ef93b..4c52f739c40 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.vespa.hosted.provision.Node; /** @@ -31,6 +32,10 @@ public class ResourceCapacity { flavor.getMinMainMemoryAvailableGb(), flavor.getMinCpuCores(), flavor.getMinDiskAvailableGb()); } + static ResourceCapacity of(FlavorSpec flavor) { + return new ResourceCapacity(flavor.memoryGb(), flavor.cpuCores(), flavor.diskGb()); + } + static ResourceCapacity of(Node node) { return ResourceCapacity.of(node.flavor()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index b9fb88e900e..b8b17011a8e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -8,6 +8,8 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeFlavors; @@ -35,7 +37,7 @@ import java.util.Set; /** * A mock repository prepopulated with some applications. - * Instantiated by DI from application package above. + * Instantiated by DI. */ public class MockNodeRepository extends NodeRepository { @@ -59,36 +61,47 @@ public class MockNodeRepository extends NodeRepository { } private void populate() { - NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, flavors, Zone.defaultZone(), - new MockProvisionServiceProvider(), new InMemoryFlagSource()); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, + flavors, + Zone.defaultZone(), + new MockProvisionServiceProvider(), + new InMemoryFlagSource()); List<Node> nodes = new ArrayList<>(); // Regular nodes Set<String> ipAddresses = ImmutableSet.of("::1", "127.0.0.1"); Set<String> ipAddressPool = ImmutableSet.of("::2", "::3", "::4"); - nodes.add(createNode("node1", "host1.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant)); - nodes.add(createNode("node2", "host2.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant)); - nodes.add(createNode("node3", "host3.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("expensive"), NodeType.tenant)); + nodes.add(createNode("node1", "host1.yahoo.com", ipAddresses, Optional.empty(), + flavors.getFlavorOrThrow("default"), NodeType.tenant)); + nodes.add(createNode("node2", "host2.yahoo.com", ipAddresses, Optional.empty(), + flavors.getFlavorOrThrow("default"), NodeType.tenant)); + nodes.add(createNode("node3", "host3.yahoo.com", ipAddresses, Optional.empty(), + flavors.getFlavorOrThrow("expensive"), NodeType.tenant)); - Node node4 = createNode("node4", "host4.yahoo.com", ipAddresses, Optional.of("dockerhost1.yahoo.com"), flavors.getFlavorOrThrow("docker"), NodeType.tenant); + Node node4 = createNode("node4", "host4.yahoo.com", ipAddresses, Optional.of("dockerhost1.yahoo.com"), + new Flavor(new FlavorSpec(1, 1, 100)), NodeType.tenant); node4 = node4.with(node4.status() .withVespaVersion(new Version("6.41.0")) .withDockerImage(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.41.0"))); nodes.add(node4); - Node node5 = createNode("node5", "host5.yahoo.com", ipAddresses, Optional.of("dockerhost2.yahoo.com"), flavors.getFlavorOrThrow("docker"), NodeType.tenant); + Node node5 = createNode("node5", "host5.yahoo.com", ipAddresses, Optional.of("dockerhost2.yahoo.com"), + new Flavor(new FlavorSpec(1, 1, 100)), NodeType.tenant); nodes.add(node5.with(node5.status() .withVespaVersion(new Version("1.2.3")) .withDockerImage(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:1.2.3")))); - nodes.add(createNode("node6", "host6.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant)); - Node node7 = createNode("node7", "host7.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant); + nodes.add(createNode("node6", "host6.yahoo.com", ipAddresses, Optional.empty(), + flavors.getFlavorOrThrow("default"), NodeType.tenant)); + Node node7 = createNode("node7", "host7.yahoo.com", ipAddresses, Optional.empty(), + flavors.getFlavorOrThrow("default"), NodeType.tenant); nodes.add(node7); // 8, 9, 11 and 12 are added by web service calls - Node node10 = createNode("node10", "host10.yahoo.com", ipAddresses, Optional.of("parent1.yahoo.com"), flavors.getFlavorOrThrow("default"), NodeType.tenant); + Node node10 = createNode("node10", "host10.yahoo.com", ipAddresses, Optional.of("parent1.yahoo.com"), + flavors.getFlavorOrThrow("default"), NodeType.tenant); Status node10newStatus = node10.status(); node10newStatus = node10newStatus .withVespaVersion(Version.fromString("5.104.142")) @@ -96,24 +109,34 @@ public class MockNodeRepository extends NodeRepository { node10 = node10.with(node10newStatus); nodes.add(node10); - Node node13 = createNode("node13", "host13.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.tenant); - Node node14 = createNode("node14", "host14.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.tenant); + Node node13 = createNode("node13", "host13.yahoo.com", ipAddresses, Optional.empty(), + flavors.getFlavorOrThrow("large"), NodeType.tenant); + Node node14 = createNode("node14", "host14.yahoo.com", ipAddresses, Optional.empty(), + flavors.getFlavorOrThrow("large"), NodeType.tenant); nodes.add(node13); nodes.add(node14); - Node node55 = createNode("node55", "host55.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant); + Node node55 = createNode("node55", "host55.yahoo.com", ipAddresses, Optional.empty(), + flavors.getFlavorOrThrow("default"), NodeType.tenant); nodes.add(node55.with(node55.status().withWantToRetire(true).withWantToDeprovision(true))); /* Setup docker hosts (two of these will be reserved for spares */ - nodes.add(createNode("dockerhost1", "dockerhost1.yahoo.com", ipAddresses, ipAddressPool, Optional.empty(), Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host)); - nodes.add(createNode("dockerhost2", "dockerhost2.yahoo.com", ipAddresses, ipAddressPool, Optional.empty(), Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host)); - nodes.add(createNode("dockerhost3", "dockerhost3.yahoo.com", ipAddresses, ipAddressPool, Optional.empty(), Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host)); - nodes.add(createNode("dockerhost4", "dockerhost4.yahoo.com", ipAddresses, ipAddressPool, Optional.empty(), Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host)); - nodes.add(createNode("dockerhost5", "dockerhost5.yahoo.com", ipAddresses, ipAddressPool, Optional.empty(), Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host)); + nodes.add(createNode("dockerhost1", "dockerhost1.yahoo.com", ipAddresses, ipAddressPool, Optional.empty(), Optional.empty(), + flavors.getFlavorOrThrow("large"), NodeType.host)); + nodes.add(createNode("dockerhost2", "dockerhost2.yahoo.com", ipAddresses, ipAddressPool, Optional.empty(), Optional.empty(), + flavors.getFlavorOrThrow("large"), NodeType.host)); + nodes.add(createNode("dockerhost3", "dockerhost3.yahoo.com", ipAddresses, ipAddressPool, Optional.empty(), Optional.empty(), + flavors.getFlavorOrThrow("large"), NodeType.host)); + nodes.add(createNode("dockerhost4", "dockerhost4.yahoo.com", ipAddresses, ipAddressPool, Optional.empty(), Optional.empty(), + flavors.getFlavorOrThrow("large"), NodeType.host)); + nodes.add(createNode("dockerhost5", "dockerhost5.yahoo.com", ipAddresses, ipAddressPool, Optional.empty(), Optional.empty(), + flavors.getFlavorOrThrow("large"), NodeType.host)); // Config servers - nodes.add(createNode("cfg1", "cfg1.yahoo.com", Collections.singleton("127.0.1.1"), Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.config)); - nodes.add(createNode("cfg2", "cfg2.yahoo.com", Collections.singleton("127.0.1.2"), Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.config)); + nodes.add(createNode("cfg1", "cfg1.yahoo.com", Collections.singleton("127.0.1.1"), Optional.empty(), + flavors.getFlavorOrThrow("default"), NodeType.config)); + nodes.add(createNode("cfg2", "cfg2.yahoo.com", Collections.singleton("127.0.1.2"), Optional.empty(), + flavors.getFlavorOrThrow("default"), NodeType.config)); // Ready all nodes, except 7 and 55 nodes = addNodes(nodes); @@ -152,7 +175,7 @@ public class MockNodeRepository extends NodeRepository { ClusterSpec.Id.from("id3"), Version.fromString("6.42"), false, Collections.emptySet()); - activate(provisioner.prepare(app3, cluster3, Capacity.fromNodeCount(2, Optional.of("docker"), false, true), 1, null), app3, provisioner); + activate(provisioner.prepare(app3, cluster3, Capacity.fromCount(2, new FlavorSpec(1, 1, 100), false, true), 1, null), app3, provisioner); ApplicationId app4 = ApplicationId.from(TenantName.from("tenant4"), ApplicationName.from("application4"), InstanceName.from("instance4")); ClusterSpec cluster4 = ClusterSpec.request(ClusterSpec.Type.container, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index a1a959b6438..d727ad68425 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeFlavors; @@ -89,7 +90,7 @@ public class FailedExpirerTest { scenario.clock().advance(Duration.ofDays(2)); scenario.expirer().run(); - scenario.assertNodesIn(Node.State.failed, "node1"); + scenario.assertNodesIn(Node.State.dirty, "node1"); scenario.assertNodesIn(Node.State.parked, "node2", "node3"); } @@ -125,7 +126,7 @@ public class FailedExpirerTest { scenario.clock().advance(Duration.ofHours(2)); scenario.expirer().run(); - scenario.assertNodesIn(Node.State.failed, "node1"); + scenario.assertNodesIn(Node.State.dirty, "node1"); scenario.assertNodesIn(Node.State.parked, "node2", "node3"); } @@ -232,8 +233,8 @@ public class FailedExpirerTest { private static class FailureScenario { private static final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", "docker"); - public static final Flavor defaultFlavor = nodeFlavors.getFlavorOrThrow("default"); - public static final Flavor dockerFlavor = nodeFlavors.getFlavorOrThrow("docker"); + public static final FlavorSpec defaultFlavor = new FlavorSpec(2, 2, 2); + public static final FlavorSpec dockerFlavor = new FlavorSpec(1, 1, 1); private final MockCurator curator = new MockCurator(); private final ManualClock clock = new ManualClock(); @@ -269,15 +270,15 @@ public class FailedExpirerTest { .orElseThrow(() -> new IllegalArgumentException("No such node: " + hostname)); } - public FailureScenario withNode(NodeType type, Flavor flavor, String hostname, String parentHostname) { + public FailureScenario withNode(NodeType type, FlavorSpec flavor, String hostname, String parentHostname) { nodeRepository.addNodes(Collections.singletonList( nodeRepository.createNode(UUID.randomUUID().toString(), hostname, - Optional.ofNullable(parentHostname), flavor, type) + Optional.ofNullable(parentHostname), new Flavor(flavor), type) )); return this; } - public FailureScenario withNode(NodeType type, Flavor flavor, String hostname) { + public FailureScenario withNode(NodeType type, FlavorSpec flavor, String hostname) { return withNode(type, flavor, hostname, null); } @@ -317,13 +318,13 @@ public class FailedExpirerTest { return allocate(clusterType, defaultFlavor, hostname); } - public FailureScenario allocate(ClusterSpec.Type clusterType, Flavor flavor, String... hostname) { + public FailureScenario allocate(ClusterSpec.Type clusterType, FlavorSpec flavor, String... hostname) { ClusterSpec clusterSpec = ClusterSpec.request(clusterType, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Collections.emptySet()); - Capacity capacity = Capacity.fromNodeCount(hostname.length, Optional.of(flavor.name()), false, true); + Capacity capacity = Capacity.fromCount(hostname.length, Optional.of(flavor), false, true); return allocate(applicationId, clusterSpec, capacity); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index 52d297232de..3b37c46add5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; @@ -77,11 +78,11 @@ public class LoadBalancerExpirerTest { } private void deployApplication(ApplicationId application, ClusterSpec.Id cluster) { - tester.makeReadyNodes(10, "default"); + tester.makeReadyNodes(10, "d-1-1-1"); List<HostSpec> hosts = tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster, Vtag.currentVersion, false, Collections.emptySet()), 2, 1, - "default"); + new FlavorSpec(1, 1, 1)); tester.activate(application, hosts); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 77f6a801d04..8b69fb63aed 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeFlavors; @@ -30,6 +31,7 @@ import com.yahoo.vespa.hosted.provision.monitoring.MetricsReporterTest; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.NodeSpec; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider; @@ -129,7 +131,7 @@ public class NodeFailTester { List<Node> hosts = tester.createHostNodes(numberOfHosts); for (int i = 0; i < hosts.size(); i++) { tester.createReadyNodes(nodesPerHost, i * nodesPerHost, Optional.of("parent" + i), - nodeFlavors.getFlavorOrThrow("docker"), NodeType.tenant); + nodeFlavors.getFlavorOrThrow("d-1-1-1"), NodeType.tenant); } // Create applications @@ -137,8 +139,8 @@ public class NodeFailTester { ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false, Collections.emptySet()); ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false, Collections.emptySet()); Capacity allHosts = Capacity.fromRequiredNodeType(NodeType.host); - Capacity capacity1 = Capacity.fromNodeCount(3, Optional.of("docker"), false, true); - Capacity capacity2 = Capacity.fromNodeCount(5, Optional.of("docker"), false, true); + Capacity capacity1 = Capacity.fromCount(3, new FlavorSpec(1, 1, 1), false, true); + Capacity capacity2 = Capacity.fromCount(5, new FlavorSpec(1, 1, 1), false, true); tester.activate(nodeAdminApp, clusterNodeAdminApp, allHosts); tester.activate(app1, clusterApp1, capacity1); tester.activate(app2, clusterApp2, capacity2); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index a55211a112a..04014b3fa46 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; @@ -15,6 +16,7 @@ import org.junit.Test; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -36,26 +38,26 @@ public class AclProvisioningTest { @Test public void trusted_nodes_for_allocated_node() { - List<Node> configServers = tester.makeConfigServers(3, "default", Version.fromString("6.123.456")); + List<Node> configServers = tester.makeConfigServers(3, "d-1-1-1", Version.fromString("6.123.456")); // Populate repo - tester.makeReadyNodes(10, "default"); - List<Node> dockerHost = tester.makeReadyNodes(1, "default", NodeType.host); + tester.makeReadyNodes(10, "d-1-1-1"); + List<Node> dockerHost = tester.makeReadyNodes(1, "d-1-1-1", NodeType.host); ApplicationId zoneApplication = tester.makeApplicationId(); deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.host)); - tester.makeReadyVirtualDockerNodes(1, "default", dockerHost.get(0).hostname()); - List<Node> proxyNodes = tester.makeReadyNodes(3, "default", NodeType.proxy); + tester.makeReadyVirtualDockerNodes(1, FlavorSpec.fromLegacyFlavorName("d-1-1-1"), dockerHost.get(0).hostname()); + List<Node> proxyNodes = tester.makeReadyNodes(3, "d-1-1-1", NodeType.proxy); // Allocate 2 nodes ApplicationId application = tester.makeApplicationId(); - List<Node> activeNodes = deploy(application, 2); + List<Node> activeNodes = deploy(application, Capacity.fromCount(2, FlavorSpec.fromLegacyFlavorName("d-1-1-1"), false, true)); assertEquals(2, activeNodes.size()); // Get trusted nodes for the first active node Node node = activeNodes.get(0); Supplier<List<NodeAcl>> nodeAcls = () -> tester.nodeRepository().getNodeAcls(node, false); - // Trusted nodes is active nodes in same application, proxy nodes and config servers + // Trusted nodes are active nodes in same application, proxy nodes and config servers assertAcls(Arrays.asList(activeNodes, proxyNodes, configServers, dockerHost), ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"), nodeAcls.get()); @@ -130,7 +132,7 @@ public class AclProvisioningTest { // Populate repo List<Node> dockerHostNodes = tester.makeReadyNodes(2, "default", NodeType.host); Node dockerHostNodeUnderTest = dockerHostNodes.get(0); - List<Node> dockerNodes = tester.makeReadyVirtualDockerNodes(5, "dockerSmall", + List<Node> dockerNodes = tester.makeReadyVirtualDockerNodes(5, new FlavorSpec(1, 1, 1), dockerHostNodeUnderTest.hostname()); List<NodeAcl> acls = tester.nodeRepository().getNodeAcls(dockerHostNodeUnderTest, true); @@ -211,12 +213,16 @@ public class AclProvisioningTest { } private static void assertAcls(List<List<Node>> expectedNodes, Set<String> expectedNetworks, List<NodeAcl> actual) { - Set<Node> expectedTrustedNodes = expectedNodes.stream() + List<Node> expectedTrustedNodes = expectedNodes.stream() .flatMap(Collection::stream) - .collect(Collectors.toSet()); - Set<Node> actualTrustedNodes = actual.stream() + .distinct() + .sorted(Comparator.comparing(Node::hostname)) + .collect(Collectors.toList()); + List<Node> actualTrustedNodes = actual.stream() .flatMap(acl -> acl.trustedNodes().stream()) - .collect(Collectors.toSet()); + .distinct() + .sorted(Comparator.comparing(Node::hostname)) + .collect(Collectors.toList()); assertEquals(expectedTrustedNodes, actualTrustedNodes); Set<String> actualTrustedNetworks = actual.stream() diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java index 78fbca554f0..242bb7df146 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -118,7 +118,6 @@ public class AllocationSimulator { public void addCluster(String task, int count, Flavor flavor, String id) { // TODO: Implement - NodeSpec.CountNodeSpec nodeSpec = new NodeSpec.CountNodeSpec(count, flavor, false, true); nodes = new NodeList(nodes.asList()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index 0fdf857e97f..da32939dfd6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeType; @@ -36,7 +37,7 @@ import static org.junit.Assert.fail; */ public class DockerProvisioningTest { - private static final String dockerFlavor = "dockerSmall"; + private static final FlavorSpec dockerFlavor = new FlavorSpec(1, 1, 1); @Test public void docker_application_deployment() { @@ -55,7 +56,7 @@ public class DockerProvisioningTest { NodeList nodes = tester.getNodes(application1, Node.State.active); assertEquals(nodeCount, nodes.size()); - assertEquals(dockerFlavor, nodes.asList().get(0).flavor().canonicalName()); + assertEquals(dockerFlavor, nodes.asList().get(0).flavor().asSpec()); // Upgrade Vespa version on nodes Version upgradedWantedVespaVersion = Version.fromString("6.40"); @@ -65,7 +66,7 @@ public class DockerProvisioningTest { tester.activate(application1, new HashSet<>(upgradedHosts)); NodeList upgradedNodes = tester.getNodes(application1, Node.State.active); assertEquals(nodeCount, upgradedNodes.size()); - assertEquals(dockerFlavor, upgradedNodes.asList().get(0).flavor().canonicalName()); + assertEquals(dockerFlavor, upgradedNodes.asList().get(0).flavor().asSpec()); assertEquals(hosts, upgradedHosts); } @@ -74,7 +75,7 @@ public class DockerProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId zoneApplication = tester.makeApplicationId(); - List<Node> parents = tester.makeReadyVirtualDockerHosts(10, "large"); + List<Node> parents = tester.makeReadyVirtualDockerHosts(10, new FlavorSpec(2, 2, 2)); for (Node parent : parents) tester.makeReadyVirtualDockerNodes(1, dockerFlavor, parent.hostname()); @@ -204,7 +205,7 @@ public class DockerProvisioningTest { } catch (Exception e) { assertEquals("No room for 3 nodes as 2 of 4 hosts are exclusive", - "Could not satisfy request for 3 nodes of flavor 'dockerSmall' for container cluster 'myContainer' group 0 6.39 in tenant1.app1: Not enough nodes available due to host exclusivity constraints.", + "Could not satisfy request for 3 nodes with cpu cores: 1.0, memory: 1.0 Gb, disk 1.0 Gb for container cluster 'myContainer' group 0 6.39 in tenant1.app1: Not enough nodes available due to host exclusivity constraints.", e.getMessage()); } @@ -225,7 +226,7 @@ public class DockerProvisioningTest { NodeList nodes = tester.getNodes(application1, Node.State.active); assertEquals(1, nodes.size()); - assertEquals(dockerFlavor, nodes.asList().get(0).flavor().canonicalName()); + assertEquals(dockerFlavor.legacyFlavorName(), nodes.asList().get(0).flavor().canonicalName()); } private Set<String> hostsOf(NodeList nodes) { @@ -235,7 +236,7 @@ public class DockerProvisioningTest { private void prepareAndActivate(ApplicationId application, int nodeCount, boolean exclusive, ProvisioningTester tester) { Set<HostSpec> hosts = new HashSet<>(tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.39"), exclusive, Collections.emptySet()), - Capacity.fromNodeCount(nodeCount, Optional.of(dockerFlavor), false, true), + Capacity.fromCount(nodeCount, Optional.of(dockerFlavor), false, true), 1)); tester.activate(application, hosts); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 74541677714..3e72b332077 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.OutOfCapacityException; @@ -64,7 +65,7 @@ public class DynamicDockerAllocationTest { tester.makeReadyNodes(4, "host-small", NodeType.host, 32); deployZoneApp(tester); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); + FlavorSpec flavor = new FlavorSpec(1, 1, 1); // Application 1 ApplicationId application1 = makeApplicationId("t1", "a1"); @@ -107,7 +108,7 @@ public class DynamicDockerAllocationTest { tester.makeReadyNodes(5, "host-small", NodeType.host, 32); deployZoneApp(tester); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); + FlavorSpec flavor = new FlavorSpec(1, 1, 1); // Application 1 ApplicationId application1 = makeApplicationId("t1", "a1"); @@ -164,7 +165,7 @@ public class DynamicDockerAllocationTest { tester.makeReadyNodes(2, "host-small", NodeType.host, 32); deployZoneApp(tester); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); + FlavorSpec flavor = new FlavorSpec(1, 1, 1); // Application 1 ApplicationId application1 = makeApplicationId("t1", "a1"); @@ -190,11 +191,10 @@ public class DynamicDockerAllocationTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); deployZoneApp(tester); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); //Deploy an application having 6 nodes (3 nodes in 2 groups). We only have 5 docker hosts available ApplicationId application1 = tester.makeApplicationId(); - tester.prepare(application1, clusterSpec("myContent.t1.a1"), 6, 2, flavor.canonicalName()); + tester.prepare(application1, clusterSpec("myContent.t1.a1"), 6, 2, new FlavorSpec(1, 1, 1)); fail("Two groups have been allocated to the same parent host"); } @@ -212,27 +212,27 @@ public class DynamicDockerAllocationTest { ApplicationId application1 = tester.makeApplicationId(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); deployZoneApp(tester); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-3"); + FlavorSpec flavor = new FlavorSpec(1, 1, 1); // Deploy initial state (can max deploy 3 nodes due to redundancy requirements) ClusterSpec clusterSpec = clusterSpec("myContent.t1.a1"); - List<HostSpec> hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor.canonicalName()); + List<HostSpec> hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor); tester.activate(application1, ImmutableSet.copyOf(hosts)); DockerHostCapacity capacity = new DockerHostCapacity(tester.nodeRepository().getNodes(Node.State.values())); - assertThat(capacity.freeCapacityInFlavorEquivalence(flavor), greaterThan(0)); + assertThat(capacity.freeCapacityInFlavorEquivalence(new Flavor(flavor)), greaterThan(0)); List<Node> initialSpareCapacity = findSpareCapacity(tester); assertThat(initialSpareCapacity.size(), is(2)); try { - hosts = tester.prepare(application1, clusterSpec, 4, 1, flavor.canonicalName()); + hosts = tester.prepare(application1, clusterSpec, 4, 1, flavor); fail("Was able to deploy with 4 nodes, should not be able to use spare capacity"); } catch (OutOfCapacityException e) { } tester.fail(hosts.get(0)); - hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor.canonicalName()); + hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor); tester.activate(application1, ImmutableSet.copyOf(hosts)); List<Node> finalSpareCapacity = findSpareCapacity(tester); @@ -245,10 +245,8 @@ public class DynamicDockerAllocationTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.perf, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(3, "host-small", NodeType.host, 32); deployZoneApp(tester); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-3"); - ApplicationId application1 = tester.makeApplicationId(); - List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, flavor.canonicalName()); + List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new FlavorSpec(1, 1, 1)); tester.activate(application1, ImmutableSet.copyOf(hosts)); List<Node> initialSpareCapacity = findSpareCapacity(tester); @@ -262,8 +260,7 @@ public class DynamicDockerAllocationTest { deployZoneApp(tester); ApplicationId application = tester.makeApplicationId(); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-3"); - tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor.canonicalName()); + tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, new FlavorSpec(1, 1, 1)); } @Test @@ -273,8 +270,7 @@ public class DynamicDockerAllocationTest { deployZoneApp(tester); ApplicationId application = tester.makeApplicationId(); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-3"); - List<HostSpec> hosts = tester.prepare(application, clusterSpec("myContent.t1.a1"), 2, 1, flavor.canonicalName()); + List<HostSpec> hosts = tester.prepare(application, clusterSpec("myContent.t1.a1"), 2, 1, new FlavorSpec(1, 1, 1)); tester.activate(application, hosts); List<Node> activeNodes = tester.nodeRepository().getNodes(application); @@ -286,13 +282,13 @@ public class DynamicDockerAllocationTest { return ApplicationId.from(tenant, appName, "default"); } - private void deployApp(ApplicationId id, ClusterSpec spec, Flavor flavor, ProvisioningTester tester, int nodeCount) { - List<HostSpec> hostSpec = tester.prepare(id, spec, nodeCount, 1, flavor.canonicalName()); + private void deployApp(ApplicationId id, ClusterSpec spec, FlavorSpec flavor, ProvisioningTester tester, int nodeCount) { + List<HostSpec> hostSpec = tester.prepare(id, spec, nodeCount, 1, flavor); tester.activate(id, new HashSet<>(hostSpec)); } - private void addAndAssignNode(ApplicationId id, String hostname, String parentHostname, ClusterSpec clusterSpec, Flavor flavor, int index, ProvisioningTester tester) { - Node node1a = Node.create("open1", Collections.singleton("127.0.0.100"), new HashSet<>(), hostname, Optional.of(parentHostname), Optional.empty(), flavor, NodeType.tenant); + private void addAndAssignNode(ApplicationId id, String hostname, String parentHostname, ClusterSpec clusterSpec, FlavorSpec flavor, int index, ProvisioningTester tester) { + Node node1a = Node.create("open1", Collections.singleton("127.0.0.100"), new HashSet<>(), hostname, Optional.of(parentHostname), Optional.empty(), new Flavor(flavor), NodeType.tenant); ClusterMembership clusterMembership1 = ClusterMembership.from( clusterSpec.with(Optional.of(ClusterSpec.Group.from(0))), index); // Need to add group here so that group is serialized in node allocation Node node1aAllocation = node1a.allocate(id, clusterMembership1, Instant.now()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 60e9289b9bf..7c7892055e2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.flags.Flags; @@ -44,10 +45,10 @@ public class DynamicDockerProvisionTest { assertEquals(0, tester.nodeRepository().list().size()); ApplicationId application1 = tester.makeApplicationId(); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("dockerSmall"); + FlavorSpec flavor = new FlavorSpec(1, 1, 1); mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("small")); - List<HostSpec> hostSpec = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 4, 1, flavor.canonicalName()); + List<HostSpec> hostSpec = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 4, 1, flavor); verify(hostProvisioner).provisionHosts(List.of(100, 101, 102, 103), flavor); // Total of 8 nodes should now be in node-repo, 4 hosts in state provisioned, and 4 reserved nodes @@ -64,21 +65,21 @@ public class DynamicDockerProvisionTest { deployZoneApp(tester); ApplicationId application = tester.makeApplicationId(); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("dockerSmall"); + FlavorSpec flavor = new FlavorSpec(1, 1, 1); mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("small")); - tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor.canonicalName()); + tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor); verify(hostProvisioner).provisionHosts(List.of(100, 101), flavor); } @Test public void allocates_to_hosts_already_hosting_nodes_by_this_tenant() { ApplicationId application = tester.makeApplicationId(); - Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("dockerSmall"); + FlavorSpec flavor = new FlavorSpec(1, 1, 1); List<Integer> expectedProvisionIndexes = List.of(100, 101); mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("large")); - tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor.canonicalName()); + tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor); verify(hostProvisioner).provisionHosts(expectedProvisionIndexes, flavor); // Ready the provisioned hosts, add an IP addreses to pool and activate them @@ -92,7 +93,7 @@ public class DynamicDockerProvisionTest { deployZoneApp(tester); mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("small")); - tester.prepare(application, clusterSpec("another-id"), 2, 1, flavor.canonicalName()); + tester.prepare(application, clusterSpec("another-id"), 2, 1, flavor); // Verify there was only 1 call to provision hosts (during the first prepare) verify(hostProvisioner).provisionHosts(any(), any()); @@ -124,9 +125,9 @@ public class DynamicDockerProvisionTest { private static void mockHostProvisioner(HostProvisioner hostProvisioner, Flavor hostFlavor) { doAnswer(invocation -> { List<Integer> provisionIndexes = (List<Integer>) invocation.getArguments()[0]; - Flavor nodeFlavor = (Flavor) invocation.getArguments()[1]; + FlavorSpec nodeFlavor = (FlavorSpec) invocation.getArguments()[1]; return provisionIndexes.stream() - .map(i -> new ProvisionedHost("id-" + i, "host-" + i, hostFlavor, "host-" + i + "-1", nodeFlavor)) + .map(i -> new ProvisionedHost("id-" + i, "host-" + i, hostFlavor, "host-" + i + "-1", new Flavor(nodeFlavor))) .collect(Collectors.toList()); }).when(hostProvisioner).provisionHosts(any(), any()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 4f7e09d0bd7..d982669413e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -5,6 +5,7 @@ import com.google.common.collect.Iterators; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.RotationName; @@ -136,10 +137,10 @@ public class LoadBalancerProvisionerTest { } private Set<HostSpec> prepare(ApplicationId application, ClusterSpec... specs) { - tester.makeReadyNodes(specs.length * 2, "default"); + tester.makeReadyNodes(specs.length * 2, "d-1-1-1"); Set<HostSpec> allNodes = new LinkedHashSet<>(); for (ClusterSpec spec : specs) { - allNodes.addAll(tester.prepare(application, spec, 2, 1, "default")); + allNodes.addAll(tester.prepare(application, spec, 2, 1, new FlavorSpec(1, 1, 1))); } return allNodes; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 0eca5e25d85..953bce6b6b2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -10,10 +10,12 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; @@ -66,37 +68,37 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); ApplicationId application2 = tester.makeApplicationId(); - tester.makeReadyNodes(21, "default"); + tester.makeReadyNodes(21, "d-1-1-1"); // deploy - SystemState state1 = prepare(application1, 2, 2, 3, 3, "default", tester); + SystemState state1 = prepare(application1, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state1.allHosts); // redeploy - SystemState state2 = prepare(application1, 2, 2, 3, 3, "default", tester); + SystemState state2 = prepare(application1, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), tester); state2.assertEquals(state1); tester.activate(application1, state2.allHosts); // deploy another application - SystemState state1App2 = prepare(application2, 2, 2, 3, 3, "default", tester); + SystemState state1App2 = prepare(application2, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), tester); assertFalse("Hosts to different apps are disjunct", state1App2.allHosts.removeAll(state1.allHosts)); tester.activate(application2, state1App2.allHosts); // prepare twice - SystemState state3 = prepare(application1, 2, 2, 3, 3, "default", tester); - SystemState state4 = prepare(application1, 2, 2, 3, 3, "default", tester); + SystemState state3 = prepare(application1, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), tester); + SystemState state4 = prepare(application1, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), tester); state3.assertEquals(state2); state4.assertEquals(state3); tester.activate(application1, state4.allHosts); // remove nodes before deploying - SystemState state5 = prepare(application1, 2, 2, 3, 3, "default", tester); + SystemState state5 = prepare(application1, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), tester); HostSpec removed = tester.removeOne(state5.allHosts); tester.activate(application1, state5.allHosts); assertEquals(removed.hostname(), tester.nodeRepository().getNodes(application1, Node.State.inactive).get(0).hostname()); // remove some of the clusters - SystemState state6 = prepare(application1, 0, 2, 0, 3, "default", tester); + SystemState state6 = prepare(application1, 0, 2, 0, 3, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state6.allHosts); assertEquals(5, tester.getNodes(application1, Node.State.active).size()); assertEquals(5, tester.getNodes(application1, Node.State.inactive).size()); @@ -115,14 +117,14 @@ public class ProvisioningTest { HostSpec failed = tester.removeOne(state1App2.allHosts); tester.fail(failed); assertEquals(9, tester.getNodes(application2, Node.State.active).size()); - SystemState state2App2 = prepare(application2, 2, 2, 3, 3, "default", tester); + SystemState state2App2 = prepare(application2, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), tester); assertFalse("Hosts to different apps are disjunct", state2App2.allHosts.removeAll(state1.allHosts)); assertEquals("A new node was reserved to replace the failed one", 10, state2App2.allHosts.size()); assertFalse("The new host is not the failed one", state2App2.allHosts.contains(failed)); tester.activate(application2, state2App2.allHosts); // deploy first app again - SystemState state7 = prepare(application1, 2, 2, 3, 3, "default", tester); + SystemState state7 = prepare(application1, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), tester); state7.assertEquals(state1); tester.activate(application1, state7.allHosts); assertEquals(0, tester.getNodes(application1, Node.State.inactive).size()); @@ -146,10 +148,10 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(4, "default"); + tester.makeReadyNodes(4, "d-1-1-1"); // deploy - SystemState state1 = prepare(application1, 1, 1, 1, 1, "default", tester); + SystemState state1 = prepare(application1, 1, 1, 1, 1, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state1.allHosts); HostSpec host1 = state1.container0.iterator().next(); @@ -158,7 +160,7 @@ public class ProvisioningTest { tester.nodeRepository().write(node1.with(node1.status().withVespaVersion(Version.fromString("1.2.3")))); // redeploy - SystemState state2 = prepare(application1, 1, 1, 1, 1, "default", tester); + SystemState state2 = prepare(application1, 1, 1, 1, 1, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state2.allHosts); host1 = state2.container0.iterator().next(); @@ -171,20 +173,20 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(24, "default"); + tester.makeReadyNodes(24, "d-1-1-1"); // deploy - SystemState state1 = prepare(application1, 2, 2, 3, 3, "default", tester); + SystemState state1 = prepare(application1, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state1.allHosts); // redeploy with increased sizes - SystemState state2 = prepare(application1, 3, 4, 4, 5, "default", tester); + SystemState state2 = prepare(application1, 3, 4, 4, 5, new FlavorSpec(1, 1, 1), tester); state2.assertExtends(state1); assertEquals("New nodes are reserved", 6, tester.getNodes(application1, Node.State.reserved).size()); tester.activate(application1, state2.allHosts); // decrease again - SystemState state3 = prepare(application1, 2, 2, 3, 3, "default", tester); + SystemState state3 = prepare(application1, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state3.allHosts); assertEquals("Superfluous container nodes are deactivated", 3-2 + 4-2, tester.getNodes(application1, Node.State.inactive).size()); @@ -192,7 +194,7 @@ public class ProvisioningTest { 4-3 + 5-3, tester.getNodes(application1, Node.State.active).retired().size()); // increase even more, and remove one node before deploying - SystemState state4 = prepare(application1, 4, 5, 5, 6, "default", tester); + SystemState state4 = prepare(application1, 4, 5, 5, 6, new FlavorSpec(1, 1, 1), tester); assertEquals("Inactive nodes are reused", 0, tester.getNodes(application1, Node.State.inactive).size()); assertEquals("Earlier retired nodes are not unretired before activate", 4-3 + 5-3, tester.getNodes(application1, Node.State.active).retired().size()); @@ -208,7 +210,7 @@ public class ProvisioningTest { 0, tester.getNodes(application1, Node.State.active).retired().size()); // decrease again - SystemState state5 = prepare(application1, 2, 2, 3, 3, "default", tester); + SystemState state5 = prepare(application1, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state5.allHosts); assertEquals("Superfluous container nodes are also deactivated", 4-2 + 5-2 + 1, tester.getNodes(application1, Node.State.inactive).size()); // @@ -216,13 +218,13 @@ public class ProvisioningTest { 5-3 + 6-3 - 1, tester.getNodes(application1, Node.State.active).retired().size()); // increase content slightly - SystemState state6 = prepare(application1, 2, 2, 4, 3, "default", tester); + SystemState state6 = prepare(application1, 2, 2, 4, 3, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state6.allHosts); assertEquals("One content node is unretired", 5-4 + 6-3 - 1, tester.getNodes(application1, Node.State.active).retired().size()); // Then reserve more - SystemState state7 = prepare(application1, 8, 2, 2, 2, "default", tester); + SystemState state7 = prepare(application1, 8, 2, 2, 2, new FlavorSpec(1, 1, 1), tester); // delete app NestedTransaction removeTransaction = new NestedTransaction(); @@ -238,27 +240,27 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(12, "small"); - tester.makeReadyNodes(16, "large"); + tester.makeReadyNodes(12, "d-1-1-1"); + tester.makeReadyNodes(16, "d-2-2-2"); // deploy - SystemState state1 = prepare(application1, 2, 2, 4, 4, "small", tester); + SystemState state1 = prepare(application1, 2, 2, 4, 4, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state1.allHosts); // redeploy with reduced size (to cause us to have retired nodes before switching flavor) - SystemState state2 = prepare(application1, 2, 2, 3, 3, "small", tester); + SystemState state2 = prepare(application1, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state2.allHosts); // redeploy with increased sizes and new flavor - SystemState state3 = prepare(application1, 3, 4, 4, 5, "large", tester); + SystemState state3 = prepare(application1, 3, 4, 4, 5, new FlavorSpec(2, 2, 2), tester); assertEquals("New nodes are reserved", 16, tester.nodeRepository().getNodes(application1, Node.State.reserved).size()); tester.activate(application1, state3.allHosts); assertEquals("'small' container nodes are retired because we are swapping the entire cluster", - 2 + 2, tester.getNodes(application1, Node.State.active).retired().type(ClusterSpec.Type.container).flavor("small").size()); + 2 + 2, tester.getNodes(application1, Node.State.active).retired().type(ClusterSpec.Type.container).flavor("d-1-1-1").size()); assertEquals("'small' content nodes are retired", - 4 + 4, tester.getNodes(application1, Node.State.active).retired().type(ClusterSpec.Type.content).flavor("small").size()); + 4 + 4, tester.getNodes(application1, Node.State.active).retired().type(ClusterSpec.Type.content).flavor("d-1-1-1").size()); assertEquals("No 'large' content nodes are retired", - 0, tester.getNodes(application1, Node.State.active).retired().flavor("large").size()); + 0, tester.getNodes(application1, Node.State.active).retired().flavor("d-2-2-2").size()); } // TODO: Enable when this feature is re-enabled @@ -269,14 +271,14 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(14, "dockerLarge"); + tester.makeReadyNodes(14, "d-2-2-2", NodeType.host); // deploy - SystemState state1 = prepare(application1, 2, 2, 4, 4, "dockerLarge", tester); + SystemState state1 = prepare(application1, 2, 2, 4, 4, new FlavorSpec(2, 2, 2), tester); tester.activate(application1, state1.allHosts); // redeploy with smaller docker flavor - causes in-place flavor change - SystemState state2 = prepare(application1, 2, 2, 4, 4, "dockerSmall", tester); + SystemState state2 = prepare(application1, 2, 2, 4, 4, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state2.allHosts); assertEquals(12, tester.getNodes(application1, Node.State.active).size()); @@ -321,11 +323,13 @@ public class ProvisioningTest { tester.makeReadyNodes(8, "large-variant"); // deploy with flavor which will be fulfilled by some old and new nodes - SystemState state1 = prepare(application1, 2, 2, 4, 4, "old-large1", tester); + SystemState state1 = prepare(application1, 2, 2, 4, 4, + FlavorSpec.fromLegacyFlavorName("old-large1"), tester); tester.activate(application1, state1.allHosts); // redeploy with increased sizes, this will map to the remaining old/new nodes - SystemState state2 = prepare(application1, 3, 4, 4, 5, "old-large2", tester); + SystemState state2 = prepare(application1, 3, 4, 4, 5, + FlavorSpec.fromLegacyFlavorName("old-large2"), tester); assertEquals("New nodes are reserved", 4, tester.getNodes(application1, Node.State.reserved).size()); tester.activate(application1, state2.allHosts); assertEquals("All nodes are used", @@ -334,12 +338,14 @@ public class ProvisioningTest { 0, tester.getNodes(application1, Node.State.active).retired().size()); // This is a noop as we are already using large nodes and nodes which replace large - SystemState state3 = prepare(application1, 3, 4, 4, 5, "large", tester); + SystemState state3 = prepare(application1, 3, 4, 4, 5, + FlavorSpec.fromLegacyFlavorName("large"), tester); assertEquals("Noop", 0, tester.getNodes(application1, Node.State.reserved).size()); tester.activate(application1, state3.allHosts); try { - SystemState state4 = prepare(application1, 3, 4, 4, 5, "large-variant", tester); + SystemState state4 = prepare(application1, 3, 4, 4, 5, + FlavorSpec.fromLegacyFlavorName("large-variant"), tester); fail("Should fail as we don't have that many large-variant nodes"); } catch (OutOfCapacityException expected) { @@ -347,7 +353,8 @@ public class ProvisioningTest { // make enough nodes to complete the switch to large-variant tester.makeReadyNodes(8, "large-variant"); - SystemState state4 = prepare(application1, 3, 4, 4, 5, "large-variant", tester); + SystemState state4 = prepare(application1, 3, 4, 4, 5, + FlavorSpec.fromLegacyFlavorName("large-variant"), tester); assertEquals("New 'large-variant' nodes are reserved", 8, tester.getNodes(application1, Node.State.reserved).size()); tester.activate(application1, state4.allHosts); // (we can not check for the precise state here without carrying over from earlier as the distribution of @@ -360,22 +367,25 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(5, "default"); + tester.makeReadyNodes(5, "d-1-1-1"); // deploy - SystemState state1 = prepare(application1, 2, 0, 3, 0, "default", tester); + SystemState state1 = prepare(application1, 2, 0, 3, 0, + new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state1.allHosts); // redeploy a too large application try { - SystemState state2 = prepare(application1, 3, 0, 3, 0, "default", tester); + SystemState state2 = prepare(application1, 3, 0, 3, 0, + new FlavorSpec(1, 1, 1), tester); fail("Expected out of capacity exception"); } catch (OutOfCapacityException expected) { } // deploy first state again - SystemState state3 = prepare(application1, 2, 0, 3, 0, "default", tester); + SystemState state3 = prepare(application1, 2, 0, 3, 0, + new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state3.allHosts); } @@ -384,8 +394,9 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(4, "default"); - SystemState state = prepare(application, 2, 2, 3, 3, "default", tester); + tester.makeReadyNodes(4, "d-1-1-1"); + SystemState state = prepare(application, 2, 2, 3, 3, + new FlavorSpec(1, 1, 1), tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } @@ -395,8 +406,8 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(4, "default"); - SystemState state = prepare(application, 2, 2, 3, 3, "default", Version.fromString("6.91"), tester); + tester.makeReadyNodes(4, "d-1-1-1"); + SystemState state = prepare(application, 2, 2, 3, 3, new FlavorSpec(1, 1, 1), Version.fromString("6.91"), tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } @@ -406,8 +417,9 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(4, "default"); - SystemState state = prepare(application, 2, 2, 3, 3, "default", tester); + tester.makeReadyNodes(4, "d-1-1-1"); + SystemState state = prepare(application, 2, 2, 3, 3, + new FlavorSpec(1, 1, 1), tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } @@ -417,8 +429,9 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(10, "default"); - prepare(application, 1, 2, 3, 3, "default", tester); + tester.makeReadyNodes(10, "d-1-1-1"); + prepare(application, 1, 2, 3, 3, + new FlavorSpec(1, 1, 1), tester); } /** Dev always uses the zone default flavor */ @@ -427,8 +440,9 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(4, "default"); - SystemState state = prepare(application, 2, 2, 3, 3, "large", tester); + tester.makeReadyNodes(4, "d-2-2-2"); + SystemState state = prepare(application, 2, 2, 3, 3, + new FlavorSpec(2, 2, 2), tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } @@ -439,8 +453,9 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(4, "default"); - SystemState state = prepare(application, 2, 2, 3, 3, "large", tester); + tester.makeReadyNodes(4, "d-2-2-2"); + SystemState state = prepare(application, 2, 2, 3, 3, + new FlavorSpec(2, 2, 2), tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } @@ -450,8 +465,9 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.staging, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(14, "default"); - SystemState state = prepare(application, 1, 1, 1, 64, "default", tester); // becomes 1, 1, 1, 6 + tester.makeReadyNodes(14, "d-1-1-1"); + SystemState state = prepare(application, 1, 1, 1, 64, + new FlavorSpec(1, 1, 1), tester); // becomes 1, 1, 1, 6 assertEquals(9, state.allHosts.size()); tester.activate(application, state.allHosts); } @@ -460,9 +476,10 @@ public class ProvisioningTest { public void activate_after_reservation_timeout() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(10, "default"); + tester.makeReadyNodes(10, "d-1-1-1"); ApplicationId application = tester.makeApplicationId(); - SystemState state = prepare(application, 2, 2, 3, 3, "default", tester); + SystemState state = prepare(application, 2, 2, 3, 3, + new FlavorSpec(1, 1, 1), tester); // Simulate expiry NestedTransaction deactivateTransaction = new NestedTransaction(); @@ -482,10 +499,11 @@ public class ProvisioningTest { public void out_of_capacity() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(9, "default"); // need 2+2+3+3=10 + tester.makeReadyNodes(9, "d-1-1-1"); // need 2+2+3+3=10 ApplicationId application = tester.makeApplicationId(); try { - prepare(application, 2, 2, 3, 3, "default", tester); + prepare(application, 2, 2, 3, 3, + new FlavorSpec(1, 1, 1), tester); fail("Expected exception"); } catch (OutOfCapacityException e) { @@ -496,7 +514,7 @@ public class ProvisioningTest { @Test public void out_of_capacity_but_cannot_fail() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, "default"); + tester.makeReadyNodes(4, "d-1-1-1"); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music"), @@ -514,11 +532,12 @@ public class ProvisioningTest { tester.makeReadyNodes( 9, "large"); // need 2+2+3+3=10 ApplicationId application = tester.makeApplicationId(); try { - prepare(application, 2, 2, 3, 3, "large", tester); + prepare(application, 2, 2, 3, 3, + FlavorSpec.fromLegacyFlavorName("large"), tester); fail("Expected exception"); } catch (OutOfCapacityException e) { - assertTrue(e.getMessage().startsWith("Could not satisfy request for 3 nodes of flavor 'large'")); + assertTrue(e.getMessage().startsWith("Could not satisfy request for 3 nodes with flavor 'large'")); } } @@ -538,8 +557,8 @@ public class ProvisioningTest { ApplicationId application = tester.makeApplicationId(); try { - prepare(application, 2, 0, 2, 0, flavorToRetire, - tester); + prepare(application, 2, 0, 2, 0, + FlavorSpec.fromLegacyFlavorName(flavorToRetire), tester); fail("Expected exception"); } catch (OutOfCapacityException e) { assertTrue(e.getMessage().startsWith("Could not satisfy request")); @@ -552,11 +571,12 @@ public class ProvisioningTest { ApplicationId application = tester.makeApplicationId(); // Flag all nodes for retirement - List<Node> readyNodes = tester.makeReadyNodes(5, "default"); + List<Node> readyNodes = tester.makeReadyNodes(5, "d-1-1-1"); readyNodes.forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true)))); try { - prepare(application, 2, 0, 2, 0, "default", tester); + prepare(application, 2, 0, 2, 0, + new FlavorSpec(1, 1, 1), tester); fail("Expected exception"); } catch (OutOfCapacityException e) { assertTrue(e.getMessage().startsWith("Could not satisfy request")); @@ -569,7 +589,8 @@ public class ProvisioningTest { ApplicationId application = tester.makeApplicationId(); try { - prepare(application, 2, 2, 3, 3, "nonexisting", tester); + prepare(application, 2, 2, 3, 3, + FlavorSpec.fromLegacyFlavorName("nonexisting"), tester); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -583,14 +604,14 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(14, "default"); + tester.makeReadyNodes(14, "d-1-1-1"); // deploy - SystemState state1 = prepare(application1, 3, 3, 4, 4, "default", tester); + SystemState state1 = prepare(application1, 3, 3, 4, 4, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state1.allHosts); // decrease cluster sizes - SystemState state2 = prepare(application1, 2, 2, 2, 2, "default", tester); + SystemState state2 = prepare(application1, 2, 2, 2, 2, new FlavorSpec(1, 1, 1), tester); tester.activate(application1, state2.allHosts); // content0 @@ -608,12 +629,12 @@ public class ProvisioningTest { @Test public void application_deployment_prefers_cheapest_stock_nodes() { - assertCorrectFlavorPreferences(true); + assertCorrectBareMetalFlavorPreferences(true); } @Test public void application_deployment_prefers_exact_nonstock_nodes() { - assertCorrectFlavorPreferences(false); + assertCorrectBareMetalFlavorPreferences(false); } @Test @@ -636,7 +657,7 @@ public class ProvisioningTest { .flavorsConfig(b.build()).curator(curator).nameResolver(nameResolver).build(); tester.makeReadyNodes(4, flavorToRetire); SystemState state = prepare(application, 2, 0, 2, 0, - flavorToRetire, tester); + FlavorSpec.fromLegacyFlavorName(flavorToRetire), tester); tester.activate(application, state.allHosts); } @@ -656,7 +677,7 @@ public class ProvisioningTest { tester.makeReadyNodes(4, replacementFlavor); SystemState state = prepare(application, 2, 0, 2, 0, - flavorToRetire, tester); + FlavorSpec.fromLegacyFlavorName(flavorToRetire), tester); tester.activate(application, state.allHosts); @@ -687,7 +708,7 @@ public class ProvisioningTest { tester.makeReadyNodes(4, replacementFlavor); SystemState state = prepare(application, 2, 0, 2, 0, - flavorToRetire, tester); + FlavorSpec.fromLegacyFlavorName(flavorToRetire), tester); tester.activate(application, state.allHosts); @@ -701,12 +722,12 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(10, "default"); + tester.makeReadyNodes(10, "d-1-1-1"); // Deploy application { SystemState state = prepare(application, 2, 0, 2, 0, - "default", tester); + new FlavorSpec(1, 1, 1), tester); tester.activate(application, state.allHosts); assertEquals(4, tester.getNodes(application, Node.State.active).size()); } @@ -716,7 +737,7 @@ public class ProvisioningTest { List<Node> nodesToRetire = tester.getNodes(application, Node.State.active).asList().subList(0, 2); nodesToRetire.forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true)))); - SystemState state = prepare(application, 2, 0, 2, 0, "default", tester); + SystemState state = prepare(application, 2, 0, 2, 0, new FlavorSpec(1, 1, 1), tester); tester.activate(application, state.allHosts); List<Node> retiredNodes = tester.getNodes(application).retired().asList(); @@ -730,24 +751,24 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(2, "default"); + tester.makeReadyNodes(2, "d-1-1-1"); // Deploy fails with out of capacity try { prepare(application, 2, 0, 2, 0, - "default", tester); + new FlavorSpec(1, 1, 1), tester); fail("Expected exception"); } catch (OutOfCapacityException ignored) {} assertEquals("Reserved a subset of required nodes", 2, tester.getNodes(application, Node.State.reserved).size()); // Enough nodes become available - tester.makeReadyNodes(2, "default"); + tester.makeReadyNodes(2, "d-1-1-1"); // Deploy is retried after a few minutes tester.clock().advance(Duration.ofMinutes(2)); SystemState state = prepare(application, 2, 0, 2, 0, - "default", tester); + new FlavorSpec(1, 1, 1), tester); List<Node> reserved = tester.getNodes(application, Node.State.reserved).asList(); assertEquals("Reserved required nodes", 4, reserved.size()); assertTrue("Time of event is updated for all nodes", @@ -774,12 +795,12 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); try { - prepare(application, 1, 0, 1, 0, true, "default", Version.fromString("6.42"), tester); + prepare(application, 1, 0, 1, 0, true, new FlavorSpec(1, 1, 1), Version.fromString("6.42"), tester); fail("Expected exception"); } catch (IllegalArgumentException ignored) {} } - private void assertCorrectFlavorPreferences(boolean largeIsStock) { + private void assertCorrectBareMetalFlavorPreferences(boolean largeIsStock) { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("large", 4., 8., 100, Flavor.Type.BARE_METAL).cost(10).stock(largeIsStock); FlavorsConfig.Flavor.Builder largeVariant = b.addFlavor("large-variant", 3., 9., 101, Flavor.Type.BARE_METAL).cost(9); @@ -797,8 +818,10 @@ public class ProvisioningTest { ClusterSpec contentClusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.42"), false, Collections.emptySet()); ClusterSpec containerClusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.42"), false, Collections.emptySet()); - List<HostSpec> containerNodes = tester.prepare(applicationId, containerClusterSpec, 5, 1, "large"); - List<HostSpec> contentNodes = tester.prepare(applicationId, contentClusterSpec, 10, 1, "large"); + List<HostSpec> containerNodes = tester.prepare(applicationId, containerClusterSpec, 5, 1, + FlavorSpec.fromLegacyFlavorName("large")); + List<HostSpec> contentNodes = tester.prepare(applicationId, contentClusterSpec, 10, 1, + FlavorSpec.fromLegacyFlavorName("large")); if (largeIsStock) { // 'large' is replaced by 'large-variant' when possible, as it is cheaper tester.assertNumberOfNodesWithFlavor(containerNodes, "large-variant", 5); @@ -815,19 +838,19 @@ public class ProvisioningTest { } private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size, - int content1Size, String flavor, ProvisioningTester tester) { + int content1Size, FlavorSpec flavor, ProvisioningTester tester) { return prepare(application, container0Size, container1Size, content0Size, content1Size, flavor, Version.fromString("6.42"), tester); } private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size, - int content1Size, String flavor, Version wantedVersion, ProvisioningTester tester) { + int content1Size, FlavorSpec flavor, Version wantedVersion, ProvisioningTester tester) { return prepare(application, container0Size, container1Size, content0Size, content1Size, false, flavor, wantedVersion, tester); } private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size, - int content1Size, boolean required, String flavor, Version wantedVersion, + int content1Size, boolean required, FlavorSpec flavor, Version wantedVersion, ProvisioningTester tester) { // "deploy prepare" with a two container clusters and a storage cluster having of two groups ClusterSpec containerCluster0 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container0"), wantedVersion, false, Collections.emptySet()); @@ -870,7 +893,7 @@ public class ProvisioningTest { } private Set<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups, - boolean required, String flavor, ProvisioningTester tester) { + boolean required, FlavorSpec flavor, ProvisioningTester tester) { if (nodeCount == 0) return Collections.emptySet(); // this is a shady practice return new HashSet<>(tester.prepare(application, cluster, nodeCount, groups, required, flavor)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 8e41ddc0c0c..6f026181510 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; @@ -129,12 +130,12 @@ public class ProvisioningTester { public void patchNode(Node node) { nodeRepository.write(node); } - public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups, String flavor) { + public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups, FlavorSpec flavor) { return prepare(application, cluster, nodeCount, groups, false, flavor); } - public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups, boolean required, String flavor) { - return prepare(application, cluster, Capacity.fromNodeCount(nodeCount, Optional.ofNullable(flavor), required, true), groups); + public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups, boolean required, FlavorSpec flavor) { + return prepare(application, cluster, Capacity.fromCount(nodeCount, Optional.ofNullable(flavor), required, true), groups); } public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, int groups) { @@ -234,7 +235,7 @@ public class ProvisioningTester { return makeProvisionedNodes(count, flavor, type, ipAddressPoolSize, false); } - public List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { + public List<Node> makeProvisionedNodes(int n, String flavorName, NodeType type, int ipAddressPoolSize, boolean dualStack) { List<Node> nodes = new ArrayList<>(n); for (int i = 0; i < n; i++) { @@ -271,6 +272,13 @@ public class ProvisioningTester { nameResolver.addRecord(String.format("node-%d-of-%s", poolIp, hostname), ipv4Addr); } } + Optional<Flavor> flavor = nodeFlavors.getFlavor(flavorName); + if (flavor.isEmpty()) { + if (type == NodeType.tenant) // Tenant nodes can have any (docker) flavor + flavor = Optional.of(new Flavor(FlavorSpec.fromLegacyFlavorName(flavorName))); + else + throw new IllegalArgumentException("No flavor '" + flavorName + "'"); + } nodes.add(nodeRepository.createNode(hostname, hostname, @@ -278,7 +286,7 @@ public class ProvisioningTester { ipAddressPool, Optional.empty(), Optional.empty(), - nodeFlavors.getFlavorOrThrow(flavor), + flavor.get(), type)); } nodes = nodeRepository.addNodes(nodes); @@ -328,37 +336,37 @@ public class ProvisioningTester { } /** Creates a set of virtual docker hosts */ - public List<Node> makeReadyVirtualDockerHosts(int n, String flavor) { + public List<Node> makeReadyVirtualDockerHosts(int n, FlavorSpec flavor) { return makeReadyVirtualNodes(n, 1, flavor, Optional.empty(), i -> "dockerHost" + i, NodeType.host); } /** Creates a set of virtual docker nodes on a single docker host starting with index 1 and increasing */ - public List<Node> makeReadyVirtualDockerNodes(int n, String flavor, String dockerHostId) { + public List<Node> makeReadyVirtualDockerNodes(int n, FlavorSpec flavor, String dockerHostId) { return makeReadyVirtualNodes(n, 1, flavor, Optional.of(dockerHostId), i -> String.format("%s-%03d", dockerHostId, i), NodeType.tenant); } /** Creates a single of virtual docker node on a single parent host */ - public List<Node> makeReadyVirtualDockerNode(int index, String flavor, String dockerHostId) { + public List<Node> makeReadyVirtualDockerNode(int index, FlavorSpec flavor, String dockerHostId) { return makeReadyVirtualNodes(1, index, flavor, Optional.of(dockerHostId), i -> String.format("%s-%03d", dockerHostId, i), NodeType.tenant); } /** Creates a set of virtual nodes without a parent host */ - public List<Node> makeReadyVirtualNodes(int n, String flavor) { + public List<Node> makeReadyVirtualNodes(int n, FlavorSpec flavor) { return makeReadyVirtualNodes(n, 0, flavor, Optional.empty(), i -> UUID.randomUUID().toString(), NodeType.tenant); } /** Creates a set of virtual nodes on a single parent host */ - private List<Node> makeReadyVirtualNodes(int count, int startIndex, String flavor, Optional<String> parentHostId, + private List<Node> makeReadyVirtualNodes(int count, int startIndex, FlavorSpec flavor, Optional<String> parentHostId, Function<Integer, String> nodeNamer, NodeType nodeType) { List<Node> nodes = new ArrayList<>(count); for (int i = startIndex; i < count + startIndex; i++) { String hostname = nodeNamer.apply(i); nodes.add(nodeRepository.createNode("openstack-id", hostname, parentHostId, - nodeFlavors.getFlavorOrThrow(flavor), nodeType)); + new Flavor(flavor), nodeType)); } nodes = nodeRepository.addNodes(nodes); nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index 0d9ce179d5c..5482731bfb6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.FlavorSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.RegionName; @@ -34,7 +35,7 @@ import static org.junit.Assert.assertNotNull; // to remove these tests public class VirtualNodeProvisioningTest { - private static final String flavor = "v-4-8-100"; + private static final FlavorSpec flavor = new FlavorSpec(4, 8, 100); private static final ClusterSpec contentClusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.42"), false, Collections.emptySet()); private static final ClusterSpec containerClusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.42"), false, Collections.emptySet()); @@ -63,13 +64,13 @@ public class VirtualNodeProvisioningTest { // Go down to 3 nodes in container cluster List<HostSpec> containerHosts2 = prepare(containerClusterSpec, containerNodeCount - 1, groups); activate(containerHosts2); - final List<Node> nodes2 = getNodes(applicationId); + List<Node> nodes2 = getNodes(applicationId); assertDistinctParentHosts(nodes2, ClusterSpec.Type.container, containerNodeCount - 1); // Go up to 4 nodes again in container cluster List<HostSpec> containerHosts3 = prepare(containerClusterSpec, containerNodeCount, groups); activate(containerHosts3); - final List<Node> nodes3 = getNodes(applicationId); + List<Node> nodes3 = getNodes(applicationId); assertDistinctParentHosts(nodes3, ClusterSpec.Type.container, containerNodeCount); } @@ -81,11 +82,12 @@ public class VirtualNodeProvisioningTest { // Allowed to use same parent host for several nodes in same cluster in dev { + FlavorSpec flavor = new FlavorSpec(1, 1, 1); tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyVirtualDockerNodes(4, "default", "parentHost1"); + tester.makeReadyVirtualDockerNodes(4, flavor, "parentHost1"); - List<HostSpec> containerHosts = prepare(containerClusterSpec, containerNodeCount, groups); - List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); + List<HostSpec> containerHosts = prepare(containerClusterSpec, containerNodeCount, groups, flavor); + List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups, flavor); activate(containerHosts, contentHosts); // downscaled to 1 node per cluster in dev, so 2 in total @@ -251,9 +253,9 @@ public class VirtualNodeProvisioningTest { public void unknown_distribution_with_known_and_unknown_ready_nodes() { tester.makeReadyVirtualNodes(3, flavor); - final int contentNodeCount = 3; - final int groups = 1; - final List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); + int contentNodeCount = 3; + int groups = 1; + List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); activate(contentHosts); assertEquals(3, getNodes(applicationId).size()); @@ -293,6 +295,10 @@ public class VirtualNodeProvisioningTest { return tester.prepare(applicationId, clusterSpec, nodeCount, groups, flavor); } + private List<HostSpec> prepare(ClusterSpec clusterSpec, int nodeCount, int groups, FlavorSpec flavor) { + return tester.prepare(applicationId, clusterSpec, nodeCount, groups, flavor); + } + @SafeVarargs private final void activate(List<HostSpec>... hostLists) { HashSet<HostSpec> hosts = new HashSet<>(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 75995245274..d75cdaa3a2b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -798,7 +798,7 @@ public class RestApiTest { return "{\"hostname\":\"" + hostname + "\", \"parentHostname\":\"" + parentHostname + "\"," + createIpAddresses(ipAddress) + createAdditionalIpAddresses(additionalIpCount) + - "\"openStackId\":\"" + hostname + "\",\"flavor\":\"docker\"}"; + "\"openStackId\":\"" + hostname + "\",\"flavor\":\"d-1-1-100\"}"; } private String asNodeJson(String hostname, String flavor, String... ipAddress) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json index c98fbb46ff8..86565780cfe 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json @@ -6,14 +6,13 @@ "hostname": "test-container-1", "parentHostname": "dockerhost3.yahoo.com", "openStackId": "fake-test-container-1", - "flavor": "docker", - "canonicalFlavor": "docker", + "flavor": "d-1-1-100", + "canonicalFlavor": "d-1-1-100", "minDiskAvailableGb": 100.0, - "minMainMemoryAvailableGb": 0.5, - "description": "Flavor-name-is-docker", - "minCpuCores": 0.2, + "minMainMemoryAvailableGb": 1.0, + "minCpuCores": 1.0, "fastDisk": true, - "bandwidth":0.0, + "bandwidth":1.0, "environment": "DOCKER_CONTAINER", "owner": { "tenant": "tenant3", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json index b1329eebb2d..40e34891545 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json @@ -6,14 +6,13 @@ "hostname": "host11.yahoo.com", "parentHostname": "parent.host.yahoo.com", "openStackId": "host11.yahoo.com", - "flavor": "docker", - "canonicalFlavor": "docker", + "flavor": "d-1-1-100", + "canonicalFlavor": "d-1-1-100", "minDiskAvailableGb": 100.0, - "minMainMemoryAvailableGb": 0.5, - "description": "Flavor-name-is-docker", - "minCpuCores": 0.2, + "minMainMemoryAvailableGb": 1.0, + "minCpuCores": 1.0, "fastDisk": true, - "bandwidth":0.0, + "bandwidth":1.0, "environment": "DOCKER_CONTAINER", "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json index a02035efd88..9a3c45c709f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json @@ -6,14 +6,13 @@ "hostname": "host4.yahoo.com", "parentHostname": "dockerhost1.yahoo.com", "openStackId": "node4", - "flavor": "docker", - "canonicalFlavor": "docker", + "flavor": "d-1-1-100", + "canonicalFlavor": "d-1-1-100", "minDiskAvailableGb": 100.0, - "minMainMemoryAvailableGb": 0.5, - "description": "Flavor-name-is-docker", - "minCpuCores": 0.2, + "minMainMemoryAvailableGb": 1.0, + "minCpuCores": 1.0, "fastDisk": true, - "bandwidth":0.0, + "bandwidth":1.0, "environment": "DOCKER_CONTAINER", "owner": { "tenant": "tenant3", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json index 8eb3a74ce2a..f7b920490a6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json @@ -6,14 +6,13 @@ "hostname": "host5.yahoo.com", "parentHostname": "dockerhost2.yahoo.com", "openStackId": "node5", - "flavor": "docker", - "canonicalFlavor": "docker", + "flavor": "d-1-1-100", + "canonicalFlavor": "d-1-1-100", "minDiskAvailableGb": 100.0, - "minMainMemoryAvailableGb": 0.5, - "description": "Flavor-name-is-docker", - "minCpuCores": 0.2, + "minMainMemoryAvailableGb": 1.0, + "minCpuCores": 1.0, "fastDisk": true, - "bandwidth":0.0, + "bandwidth": 1.0, "environment": "DOCKER_CONTAINER", "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json index a14443e096d..4376b27cd5a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json @@ -6,14 +6,13 @@ "hostname": "host5.yahoo.com", "parentHostname": "dockerhost2.yahoo.com", "openStackId": "node5", - "flavor": "docker", - "canonicalFlavor": "docker", + "flavor": "d-1-1-100", + "canonicalFlavor": "d-1-1-100", "minDiskAvailableGb": 100.0, - "minMainMemoryAvailableGb": 0.5, - "description": "Flavor-name-is-docker", - "minCpuCores": 0.2, + "minMainMemoryAvailableGb": 1.0, + "minCpuCores": 1.0, "fastDisk": true, - "bandwidth":0.0, + "bandwidth": 1.0, "environment": "DOCKER_CONTAINER", "rebootGeneration": 1, "currentRebootGeneration": 0, |