diff options
9 files changed, 145 insertions, 6 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Host.java b/config-model/src/main/java/com/yahoo/config/model/provision/Host.java index 360853d0f79..a030a8dde7a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/Host.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Host.java @@ -2,9 +2,11 @@ package com.yahoo.config.model.provision; import com.google.common.collect.ImmutableList; +import com.yahoo.config.provision.Flavor; import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * A hostname with zero or more aliases. This is immutable. @@ -15,15 +17,20 @@ public class Host { private final String hostname; private final ImmutableList<String> aliases; + private final Optional<Flavor> flavor; public Host(String hostname) { - this.hostname = hostname; - this.aliases = ImmutableList.of(); + this(hostname, ImmutableList.of(), Optional.empty()); } public Host(String hostname, List<String> hostAliases) { + this(hostname, hostAliases, Optional.empty()); + } + + public Host(String hostname, List<String> hostAliases, Optional<Flavor> flavor) { this.hostname = hostname; this.aliases = ImmutableList.copyOf(hostAliases); + this.flavor = flavor; } public String hostname() { return hostname; } @@ -31,9 +38,12 @@ public class Host { /** Returns an immutable list of the aliases of this node, which may be empty but never null */ public List<String> aliases() { return aliases; } + public Optional<Flavor> flavor() { return flavor; } + @Override public String toString() { - return hostname + (aliases.size() > 0 ? " (aliases: " + aliases + ")" : "" ); + return hostname + (aliases.size() > 0 ? " (aliases: " + aliases + ")" : "" ) + + (flavor.isPresent() ? " (flavor: " + flavor.get() + ")" : ""); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index c4ac4d91001..38bc3f39c62 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -88,7 +88,7 @@ public class InMemoryProvisioner implements HostProvisioner { List<Host> defaultHosts = freeNodes.get("default"); if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts of default flavor available"); Host newHost = freeNodes.removeValue("default", 0); - HostSpec hostSpec = new HostSpec(newHost.hostname(), newHost.aliases()); + HostSpec hostSpec = new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty()); legacyMapping.put(alias, hostSpec); return hostSpec; } @@ -142,7 +142,7 @@ public class InMemoryProvisioner implements HostProvisioner { if (freeNodes.get(flavor).isEmpty()) throw new IllegalArgumentException("Insufficient capacity of flavor '" + flavor + "'"); Host newHost = freeNodes.removeValue(flavor, 0); ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++); - allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(), membership)); + allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.of(membership))); } nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java new file mode 100644 index 00000000000..4ef42137c53 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java @@ -0,0 +1,27 @@ +package com.yahoo.vespa.model.search; + +import com.yahoo.config.provision.Flavor; +import com.yahoo.vespa.config.search.core.ProtonConfig; + +/** + * Tuning of proton config for a search node based on the node flavor of that node. + * + * @author geirst + */ +public class NodeFlavorTuning implements ProtonConfig.Producer { + + private final Flavor nodeFlavor; + + public NodeFlavorTuning(Flavor nodeFlavor) { + this.nodeFlavor = nodeFlavor; + } + + @Override + public void getConfig(ProtonConfig.Builder builder) { + ProtonConfig.Hwinfo.Disk.Builder diskInfo = new ProtonConfig.Hwinfo.Disk.Builder(); + if (!nodeFlavor.hasFastDisk()) { + diskInfo.writespeed(40); + } + builder.hwinfo(new ProtonConfig.Hwinfo.Builder().disk(diskInfo)); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 53ea11077e9..d5fc782a633 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -271,6 +271,9 @@ public class SearchNode extends AbstractService implements // to make sure the node failer has done its work builder.pruneremoveddocumentsage(4 * 24 * 3600 + 3600 + 60); } + if (getHostResource() != null && getHostResource().getFlavor().isPresent()) { + new NodeFlavorTuning(getHostResource().getFlavor().get()).getConfig(builder); + } } @Override diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 7d1d5c7c75b..ec10c84a335 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1,11 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.provision; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; import java.io.StringReader; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -14,7 +16,10 @@ import java.util.stream.Collectors; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.search.config.QrStartConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; @@ -22,7 +27,9 @@ import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Slobrok; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.search.Dispatch; +import com.yahoo.vespa.model.search.SearchNode; import com.yahoo.vespa.model.test.VespaModelTester; import org.junit.Ignore; import org.junit.Test; @@ -1413,5 +1420,41 @@ public class ModelProvisioningTest { cluster.getSearch().getConfig(b); return new QrStartConfig(b).jvm().heapSizeAsPercentageOfPhysicalMemory(); } + + @Test + public void require_that_proton_config_is_tuned_based_on_node_flavor() { + String services = joinLines("<?xml version='1.0' encoding='utf-8' ?>", + "<services>", + " <content version='1.0' id='test'>", + " <documents>", + " <document type='type1' mode='index'/>", + " </documents>", + " <nodes count='2' flavor='content-test-flavor'/>", + " </content>", + "</services>"); + + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(createFlavorFromDiskSetting("content-test-flavor", false), 2); + VespaModel model = tester.createModel(services, true, 0); + ContentSearchCluster cluster = model.getContentClusters().get("test").getSearch(); + assertEquals(2, cluster.getSearchNodes().size()); + assertEquals(40, getProtonConfig(cluster, 0).hwinfo().disk().writespeed(), 0.001); + assertEquals(40, getProtonConfig(cluster, 1).hwinfo().disk().writespeed(), 0.001); + } + + private static Flavor createFlavorFromDiskSetting(String name, boolean fastDisk) { + FlavorsConfig.Flavor.Builder builder = new FlavorsConfig.Flavor.Builder(); + builder.name(name); + builder.fastDisk(fastDisk); + return new Flavor(new FlavorsConfig.Flavor(builder)); + } + + private static ProtonConfig getProtonConfig(ContentSearchCluster cluster, int searchNodeIdx) { + ProtonConfig.Builder builder = new ProtonConfig.Builder(); + List<SearchNode> searchNodes = cluster.getSearchNodes(); + assertTrue(searchNodeIdx < searchNodes.size()); + searchNodes.get(searchNodeIdx).getConfig(builder); + return new ProtonConfig(builder); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java new file mode 100644 index 00000000000..40288a9b43c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java @@ -0,0 +1,39 @@ +package com.yahoo.vespa.model.search; + +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author geirst + */ +public class NodeFlavorTuningTest { + + @Test + public void require_that_fast_disk_is_reflected_in_proton_config() { + ProtonConfig cfg = configFromDiskSetting(true); + assertEquals(200, cfg.hwinfo().disk().writespeed(), 0.001); + assertEquals(100, cfg.hwinfo().disk().slowwritespeedlimit(), 0.001); + } + + @Test + public void require_that_slow_disk_is_reflected_in_proton_config() { + ProtonConfig cfg = configFromDiskSetting(false); + assertEquals(40, cfg.hwinfo().disk().writespeed(), 0.001); + assertEquals(100, cfg.hwinfo().disk().slowwritespeedlimit(), 0.001); + } + + private static ProtonConfig configFromDiskSetting(boolean fastDisk) { + FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder(); + flavorBuilder.fastDisk(fastDisk); + flavorBuilder.name("my_flavor"); + NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder))); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); + tuning.getConfig(protonBuilder); + return new ProtonConfig(protonBuilder); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index bedbe71e389..911bebe83ea 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.test; +import com.google.common.collect.ImmutableList; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.NullConfigModelRegistry; @@ -11,16 +12,18 @@ import com.yahoo.config.model.provision.Host; import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; +import com.yahoo.config.provision.Flavor; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; /** * Helper class which sets up a system with multiple hosts. @@ -60,6 +63,13 @@ public class VespaModelTester { this.hosts.put(flavor.isEmpty() ? "default" : flavor, hosts); return new Hosts(hosts); } + public void addHosts(Flavor flavor, int count) { + List<Host> hosts = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + hosts.add(new Host(flavor.name() + i, ImmutableList.of(), Optional.of(flavor))); + } + this.hosts.put(flavor.name(), hosts); + } /** Sets whether this sets up a model for a hosted system. Default: true */ public void setHosted(boolean hosted) { this.hosted = hosted; } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index fff1d53b50e..dbece80bf72 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -22,6 +22,7 @@ public class Flavor { private final double minCpuCores; private final double minMainMemoryAvailableGb; private final double minDiskAvailableGb; + private final boolean fastDisk; private final String description; private final boolean retired; private List<Flavor> replacesFlavors; @@ -39,6 +40,7 @@ public class Flavor { this.minCpuCores = flavorConfig.minCpuCores(); this.minMainMemoryAvailableGb = flavorConfig.minMainMemoryAvailableGb(); this.minDiskAvailableGb = flavorConfig.minDiskAvailableGb(); + this.fastDisk = flavorConfig.fastDisk(); this.description = flavorConfig.description(); this.retired = flavorConfig.retired(); } @@ -60,6 +62,8 @@ public class Flavor { public double getMinDiskAvailableGb() { return minDiskAvailableGb; } + public boolean hasFastDisk() { return fastDisk; } + public double getMinCpuCores() { return minCpuCores; } public String getDescription() { return description; } diff --git a/config-provisioning/src/main/resources/configdefinitions/flavors.def b/config-provisioning/src/main/resources/configdefinitions/flavors.def index edcd957c0b2..6c94194878a 100644 --- a/config-provisioning/src/main/resources/configdefinitions/flavors.def +++ b/config-provisioning/src/main/resources/configdefinitions/flavors.def @@ -37,6 +37,9 @@ flavor[].minMainMemoryAvailableGb double default=0.0 # The minimum amount of disk available. flavor[].minDiskAvailableGb double default=0.0 +# Whether the disk is fast (typically SSD) or slow (typically spinning HDD). +flavor[].fastDisk bool default=true + # Human readable free text for description of node. flavor[].description string default="" |