diff options
author | Jon Bratseth <bratseth@vespa.ai> | 2023-05-15 20:47:16 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@vespa.ai> | 2023-05-15 20:47:16 +0200 |
commit | 9406d9e50a3b0d183ec2d76595679eea41f0129f (patch) | |
tree | c75ae3cd85aa29522043ceb1f3f49e80fe6f8f5f /config-model | |
parent | c2a220bed85e7af09f62d34de339b168d9507b87 (diff) |
Make all node resource elements optional
Diffstat (limited to 'config-model')
6 files changed, 60 insertions, 23 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 41697e61bf2..4b993f8e244 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -232,9 +232,14 @@ public class InMemoryProvisioner implements HostProvisioner { } // Minimal capacity policies - private NodeResources decideResources(NodeResources requestedResources) { - if (requestedResources.isUnspecified()) return defaultNodeResources; - return requestedResources; + private NodeResources decideResources(NodeResources resources) { + if (resources.vcpuIsUnspecified()) + resources = resources.withVcpu(defaultNodeResources.vcpu()); + if (resources.memoryGbIsUnspecified()) + resources = resources.withMemoryGb(defaultNodeResources.memoryGb()); + if (resources.diskGbIsUnspecified()) + resources = resources.withDiskGb(defaultNodeResources.diskGb()); + return resources; } private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, NodeResources requestedResourcesOrUnspecified, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index c968e31325a..e70c555a366 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -307,9 +307,9 @@ public class NodesSpecification { } private static Pair<NodeResources, NodeResources> nodeResourcesFromResourcesElement(ModelElement element) { - Pair<Double, Double> vcpu = toRange(element.requiredStringAttribute("vcpu"), .0, Double::parseDouble); - Pair<Double, Double> memory = toRange(element.requiredStringAttribute("memory"), .0, s -> parseGbAmount(s, "B")); - Pair<Double, Double> disk = toRange(element.requiredStringAttribute("disk"), .0, s -> parseGbAmount(s, "B")); + Pair<Double, Double> vcpu = toRange(element.stringAttribute("vcpu"), .0, Double::parseDouble); + Pair<Double, Double> memory = toRange(element.stringAttribute("memory"), .0, s -> parseGbAmount(s, "B")); + Pair<Double, Double> disk = toRange(element.stringAttribute("disk"), .0, s -> parseGbAmount(s, "B")); Pair<Double, Double> bandwith = toRange(element.stringAttribute("bandwidth"), .3, s -> parseGbAmount(s, "BPS")); NodeResources.DiskSpeed diskSpeed = parseOptionalDiskSpeed(element.stringAttribute("disk-speed")); NodeResources.StorageType storageType = parseOptionalStorageType(element.stringAttribute("storage-type")); diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index 538a8f069f5..21f3399a027 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -23,9 +23,9 @@ Nodes = element nodes { } Resources = element resources { - attribute vcpu { xsd:double { minExclusive = "0.0" } | xsd:string } & - attribute memory { xsd:string } & - attribute disk { xsd:string } & + attribute vcpu { xsd:double { minExclusive = "0.0" } | xsd:string }? & + attribute memory { xsd:string }? & + attribute disk { xsd:string }? & attribute disk-speed { xsd:string }? & attribute storage-type { xsd:string }? & attribute architecture { xsd:string }? & diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index f1dffe53ad7..5472ea2ca82 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1542,7 +1542,7 @@ public class ModelProvisioningTest { tester.addHosts(new NodeResources(8, 200, 1000000, 0.3), 5); // Content-foo tester.addHosts(new NodeResources(10, 64, 200, 0.3), 6); // Content-bar tester.addHosts(new NodeResources(0.5, 2, 10, 0.3), 6); // Cluster-controller - VespaModel model = tester.createModel(services, true, 0); + VespaModel model = tester.createModel(services, true, NodeResources.unspecified(), 0); assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size()); } @@ -2425,7 +2425,7 @@ public class ModelProvisioningTest { assertTrue(config.build().server().stream().noneMatch(ZookeeperServerConfig.Server::joining), "Initial servers are not joining"); } { - VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(3), true, false, false, 0, Optional.of(model), new DeployState.Builder(), "node-1-3-50-04", "node-1-3-50-03"); + VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(3), true, false, false, NodeResources.unspecified(), 0, Optional.of(model), new DeployState.Builder(), "node-1-3-50-04", "node-1-3-50-03"); ApplicationContainerCluster cluster = nextModel.getContainerClusters().get("zk"); ZookeeperServerConfig.Builder config = new ZookeeperServerConfig.Builder(); cluster.getContainers().forEach(c -> c.getConfig(config)); @@ -2491,13 +2491,35 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(new NodeResources(1, 3, 10, 5, NodeResources.DiskSpeed.slow), 5); - VespaModel model = tester.createModel(services, true, 0); + VespaModel model = tester.createModel(services, true, NodeResources.unspecified(), 0); ContentSearchCluster cluster = model.getContentClusters().get("test").getSearch(); assertEquals(2, cluster.getSearchNodes().size()); assertEquals(40, getProtonConfig(cluster, 0).hwinfo().disk().writespeed(), 0.001); assertEquals(40, getProtonConfig(cluster, 1).hwinfo().disk().writespeed(), 0.001); } + @Test + public void require_that_resources_can_be_partially_specified() { + String services = joinLines("<?xml version='1.0' encoding='utf-8' ?>", + "<services>", + " <content version='1.0' id='test'>", + " <redundancy>2</redundancy>" + + " <documents>", + " <document type='type1' mode='index'/>", + " </documents>", + " <nodes count='2'>", + " <resources vcpu='1'/>", + " </nodes>", + " </content>", + "</services>"); + + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(new NodeResources(1, 3, 10, 5), 5); + VespaModel model = tester.createModel(services, true, new NodeResources(1.0, 3.0, 9.0, 1.0), 0); + ContentSearchCluster cluster = model.getContentClusters().get("test").getSearch(); + assertEquals(2, cluster.getSearchNodes().size()); + } + private static ProtonConfig getProtonConfig(ContentSearchCluster cluster, int searchNodeIdx) { ProtonConfig.Builder builder = new ProtonConfig.Builder(); List<SearchNode> searchNodes = cluster.getSearchNodes(); @@ -2542,7 +2564,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(new NodeResources(1, 3, 10, 1), 4); tester.addHosts(new NodeResources(1, 128, 100, 0.3), 1); - VespaModel model = tester.createModel(services, true, 0); + VespaModel model = tester.createModel(services, true, NodeResources.unspecified(), 0); ContentSearchCluster cluster = model.getContentClusters().get("test").getSearch(); ProtonConfig cfg = getProtonConfig(model, cluster.getSearchNodes().get(0).getConfigId()); assertEquals(2000, cfg.flush().memory().maxtlssize()); // from config override diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 500fb0838e1..e7d46e1c009 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -130,51 +130,60 @@ public class VespaModelTester { /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, String hosts, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, hosts, failOnOutOfCapacity, false, false, 0, + return createModel(Zone.defaultZone(), services, hosts, failOnOutOfCapacity, false, false, + NodeResources.unspecified(), 0, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, DeployState.Builder builder) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, false, 0, Optional.empty(), builder); + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, false, + NodeResources.unspecified(), 0, Optional.empty(), builder); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, boolean useMaxResources, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, false, 0, + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, false, + NodeResources.unspecified(), 0, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, boolean useMaxResources, boolean alwaysReturnOneNode, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, alwaysReturnOneNode, 0, + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, alwaysReturnOneNode, + NodeResources.unspecified(), 0, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ - public VespaModel createModel(String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, false, startIndexForClusters, + public VespaModel createModel(String services, boolean failOnOutOfCapacity, NodeResources defaultResources, + int startIndexForClusters, String ... retiredHostNames) { + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, false, + defaultResources, startIndexForClusters, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(zone, services, failOnOutOfCapacity, false, false, 0, + return createModel(zone, services, failOnOutOfCapacity, false, false, NodeResources.unspecified(), 0, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, DeployState.Builder deployStateBuilder, String ... retiredHostNames) { - return createModel(zone, services, failOnOutOfCapacity, false, false, 0, + return createModel(zone, services, failOnOutOfCapacity, false, false, + NodeResources.unspecified(),0, Optional.empty(), deployStateBuilder, retiredHostNames); } public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, boolean useMaxResources, boolean alwaysReturnOneNode, + NodeResources defaultResources, int startIndexForClusters, Optional<VespaModel> previousModel, DeployState.Builder deployStatebuilder, String ... retiredHostNames) { return createModel(zone, services, null, failOnOutOfCapacity, useMaxResources, alwaysReturnOneNode, + defaultResources, startIndexForClusters, previousModel, deployStatebuilder, retiredHostNames); } /** @@ -189,6 +198,7 @@ public class VespaModelTester { */ public VespaModel createModel(Zone zone, String services, String hosts, boolean failOnOutOfCapacity, boolean useMaxResources, boolean alwaysReturnOneNode, + NodeResources defaultResources, int startIndexForClusters, Optional<VespaModel> previousModel, DeployState.Builder deployStatebuilder, String ... retiredHostNames) { VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(hosts, services, generateSchemas("type1")); @@ -200,7 +210,7 @@ public class VespaModelTester { useMaxResources, alwaysReturnOneNode, false, - NodeResources.unspecified(), + defaultResources, startIndexForClusters, retiredHostNames); provisioner.setEnvironment(zone.environment()); diff --git a/config-model/src/test/schema-test-files/services-hosted.xml b/config-model/src/test/schema-test-files/services-hosted.xml index 1246f06c58f..db4f9fa34ab 100644 --- a/config-model/src/test/schema-test-files/services-hosted.xml +++ b/config-model/src/test/schema-test-files/services-hosted.xml @@ -25,7 +25,7 @@ <content id="search" version="1.0"> <redundancy>2</redundancy> <nodes count="7" flavor="large" groups="12" no-vespamalloc="proton distributord"> - <resources vcpu="3.0" memory="32000.0Mb" disk="300 Gb"/> + <resources memory="32000.0Mb" disk="300 Gb"/> </nodes> </content> |