aboutsummaryrefslogtreecommitdiffstats
path: root/config-model
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@gmail.com>2020-05-25 10:27:35 +0200
committerJon Bratseth <bratseth@gmail.com>2020-05-25 10:27:35 +0200
commit5d0663a1f2eddb94b01d6ef09ea5cb8d149d4ab0 (patch)
treeb0a024b4129cb4d24877cc5b0ecd01d881983ea6 /config-model
parent3574f64e44a9d0ff9414cf1ff0566bb483a94260 (diff)
Revert "Merge pull request #13353 from vespa-engine/revert-13348-bratseth/real-resources-in-model"
This reverts commit 413ea623b5033593c2f2fe9951e1a685c8ff9dfc, reversing changes made to 5c0bfb06784075ac72c7891b7dce6454d85bdff8.
Diffstat (limited to 'config-model')
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java29
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java16
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/HostResource.java16
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java4
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java4
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java10
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java9
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java (renamed from config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java)23
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java (renamed from config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java)48
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java12
-rw-r--r--config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java10
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java7
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java14
-rwxr-xr-xconfig-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java (renamed from config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java)11
19 files changed, 123 insertions, 110 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
index e55686accca..ab2e0f632e4 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
@@ -7,6 +7,7 @@ import com.yahoo.vespa.model.container.Container;
import java.io.Reader;
import java.util.List;
+import java.util.Optional;
/**
* A host provisioner based on a hosts.xml file.
@@ -38,7 +39,7 @@ public class HostsXmlProvisioner implements HostProvisioner {
}
for (Host host : hosts.asCollection()) {
if (host.aliases().contains(alias)) {
- return new HostSpec(host.hostname(), host.aliases());
+ return new HostSpec(host.hostname(), host.aliases(), Optional.empty());
}
}
throw new IllegalArgumentException("Unable to find host for alias '" + alias + "'");
@@ -50,7 +51,7 @@ public class HostsXmlProvisioner implements HostProvisioner {
}
private HostSpec host2HostSpec(Host host) {
- return new HostSpec(host.hostname(), host.aliases());
+ return new HostSpec(host.hostname(), host.aliases(), Optional.empty());
}
private Host getFirstHost() {
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
index 1a1ed000478..009bc225037 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
@@ -9,7 +9,6 @@ import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.HostSpec;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.ProvisionLogger;
@@ -118,8 +117,7 @@ public class InMemoryProvisioner implements HostProvisioner {
List<Host> defaultHosts = freeNodes.get(defaultResources);
if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts with default resources available");
Host newHost = freeNodes.removeValue(defaultResources, 0);
- // Note: Always returns HostSpec with empty dockerImageRepo, which is OK since this method is never used when docker image repo is set
- return new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty(), newHost.version(), Optional.empty());
+ return new HostSpec(newHost.hostname(), List.of(alias), Optional.empty());
}
@Override
@@ -173,12 +171,12 @@ public class InMemoryProvisioner implements HostProvisioner {
private HostSpec retire(HostSpec host) {
return new HostSpec(host.hostname(),
- host.aliases(),
- host.flavor(),
- Optional.of(host.membership().get().retire()),
+ host.realResources(),
+ host.advertisedResources(),
+ host.requestedResources().orElse(NodeResources.unspecified()),
+ host.membership().get().retire(),
host.version(),
Optional.empty(),
- Optional.empty(),
host.dockerImageRepo());
}
@@ -189,11 +187,11 @@ public class InMemoryProvisioner implements HostProvisioner {
// Check if the current allocations are compatible with the new request
for (int i = allocation.size() - 1; i >= 0; i--) {
- Optional<NodeResources> currentResources = allocation.get(0).flavor().map(Flavor::resources);
- if (currentResources.isEmpty() || requestedResources == NodeResources.unspecified) continue;
- if (!currentResources.get().compatibleWith(requestedResources)) {
+ NodeResources currentResources = allocation.get(0).advertisedResources();
+ if (currentResources.isUnspecified() || requestedResources.isUnspecified()) continue;
+ if ( ! currentResources.compatibleWith(requestedResources)) {
HostSpec removed = allocation.remove(i);
- freeNodes.put(currentResources.get(), new Host(removed.hostname())); // Return the node back to free pool
+ freeNodes.put(currentResources, new Host(removed.hostname())); // Return the node back to free pool
}
}
@@ -202,7 +200,7 @@ public class InMemoryProvisioner implements HostProvisioner {
// Find the smallest host that can fit the requested requested
Optional<NodeResources> hostResources = freeNodes.keySet().stream()
.sorted(new MemoryDiskCpu())
- .filter(resources -> requestedResources == NodeResources.unspecified || resources.satisfies(requestedResources))
+ .filter(resources -> requestedResources.isUnspecified() || resources.satisfies(requestedResources))
.findFirst();
if (hostResources.isEmpty()) {
if (canFail)
@@ -214,10 +212,11 @@ public class InMemoryProvisioner implements HostProvisioner {
Host newHost = freeNodes.removeValue(hostResources.get(), 0);
if (freeNodes.get(hostResources.get()).isEmpty()) freeNodes.removeAll(hostResources.get());
ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++);
- allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(),
- hostResources.map(Flavor::new), Optional.of(membership),
+ allocation.add(new HostSpec(newHost.hostname(),
+ hostResources.get(), hostResources.get(), requestedResources,
+ membership,
newHost.version(), Optional.empty(),
- requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources)));
+ Optional.empty()));
}
nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex);
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
index 1f0e0755667..b4fd55c6d33 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
@@ -1,17 +1,21 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.config.model.provision;
+import com.yahoo.component.Version;
import com.yahoo.config.model.api.HostProvisioner;
import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.ProvisionLogger;
import com.yahoo.net.HostName;
+import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
/**
* A host provisioner used when there is no hosts.xml file (using localhost as the only host)
@@ -28,12 +32,15 @@ public class SingleNodeProvisioner implements HostProvisioner {
public SingleNodeProvisioner() {
host = new Host(HostName.getLocalhost());
- this.hostSpec = new HostSpec(host.hostname(), host.aliases());
+ this.hostSpec = new HostSpec(host.hostname(), host.aliases(), Optional.empty());
}
public SingleNodeProvisioner(Flavor flavor) {
host = new Host(HostName.getLocalhost());
- this.hostSpec = new HostSpec(host.hostname(), host.aliases(), flavor);
+ this.hostSpec = new HostSpec(host.hostname(),
+ flavor.resources(), flavor.resources(), flavor.resources(),
+ ClusterMembership.from(ClusterSpec.specification(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).group(ClusterSpec.Group.from(0)).vespaVersion("1").build(), 0),
+ Optional.empty(), Optional.empty(), Optional.empty());
}
@Override
@@ -45,7 +52,10 @@ public class SingleNodeProvisioner implements HostProvisioner {
public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
// TODO: This should fail if capacity requested is more than 1
List<HostSpec> hosts = new ArrayList<>();
- hosts.add(new HostSpec(host.hostname(), host.aliases(), ClusterMembership.from(cluster, counter++)));
+ hosts.add(new HostSpec(host.hostname(),
+ NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(),
+ ClusterMembership.from(cluster, counter++),
+ Optional.empty(), Optional.empty(), Optional.empty()));
return hosts;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
index 9dba6fde9d4..81931995485 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
@@ -1,23 +1,17 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model;
-import com.yahoo.component.Version;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.model.api.HostInfo;
-import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.HostSpec;
import com.yahoo.config.provision.NodeResources;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import java.util.stream.Collectors;
/**
@@ -44,7 +38,7 @@ public class HostResource implements Comparable<HostResource> {
* @param host {@link com.yahoo.vespa.model.Host} object to bind to.
*/
public HostResource(Host host) {
- this(host, new HostSpec(host.getHostname(), Optional.empty()));
+ this(host, new HostSpec(host.getHostname(), List.of(), Optional.empty()));
}
public HostResource(Host host, HostSpec spec) {
@@ -105,8 +99,16 @@ public class HostResource implements Comparable<HostResource> {
}
/** Returns the flavor of this resource. Empty for self-hosted Vespa. */
+ // TODO: Remove after June 2020
+ @Deprecated
public Optional<Flavor> getFlavor() { return spec.flavor(); }
+ /** The real resources available for Vespa processes on this node, after subtracting infrastructure overhead. */
+ public NodeResources realResources() { return spec.realResources(); }
+
+ /** The total advertised resources of this node, typically matching what's requested. */
+ public NodeResources advertisedResources() { return spec.advertisedResources(); }
+
@Override
public String toString() {
return "host '" + host.getHostname() + "'";
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
index 292d4761d8a..3d2918e0ee1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
@@ -120,7 +120,7 @@ public class HostSystem extends AbstractConfigProducer<Host> {
HostResource host = getExistingHost(spec).orElseGet(() -> addNewHost(spec));
retAllocatedHosts.put(host, spec.membership().orElse(null));
}
- retAllocatedHosts.keySet().forEach(host -> log.log(FINE, () -> "Allocated host " + host.getHostname() + " with flavor " + host.getFlavor()));
+ retAllocatedHosts.keySet().forEach(host -> log.log(FINE, () -> "Allocated host " + host.getHostname() + " with resources " + host.advertisedResources()));
return retAllocatedHosts;
}
@@ -131,7 +131,7 @@ public class HostSystem extends AbstractConfigProducer<Host> {
if (hosts.isEmpty()) {
return Optional.empty();
} else {
- log.log(FINE, () -> "Found existing host resource for " + key.hostname() + " with flavor " + hosts.get(0).getFlavor());
+ log.log(FINE, () -> "Found existing host resource for " + key.hostname() + " with resources" + hosts.get(0).advertisedResources());
return Optional.of(hosts.get(0));
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
index 5343a322382..24b7b0949f6 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
@@ -44,8 +44,8 @@ public class ResourcesReductionValidator implements ChangeValidator {
ClusterSpec.Id clusterId,
ValidationOverrides overrides,
Instant now) {
- if (current.minResources().nodeResources() == NodeResources.unspecified) return;
- if (next.minResources().nodeResources() == NodeResources.unspecified) return;
+ if (current.minResources().nodeResources().isUnspecified()) return;
+ if (next.minResources().nodeResources().isUnspecified()) return;
List<String> illegalChanges = Stream.of(
validateResource("vCPU",
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
index ea47e490b12..8f737f02dca 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
@@ -159,8 +159,8 @@ public class NodesSpecification {
* Returns a requirement from <code>count</code> non-dedicated nodes in one group
*/
public static NodesSpecification nonDedicated(int count, ConfigModelContext context) {
- return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified),
- new ClusterResources(count, 1, NodeResources.unspecified),
+ return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified()),
+ new ClusterResources(count, 1, NodeResources.unspecified()),
false,
context.getDeployState().getWantedNodeVespaVersion(),
false,
@@ -172,8 +172,8 @@ public class NodesSpecification {
/** Returns a requirement from <code>count</code> dedicated nodes in one group */
public static NodesSpecification dedicated(int count, ConfigModelContext context) {
- return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified),
- new ClusterResources(count, 1, NodeResources.unspecified),
+ return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified()),
+ new ClusterResources(count, 1, NodeResources.unspecified()),
true,
context.getDeployState().getWantedNodeVespaVersion(),
false,
@@ -224,7 +224,7 @@ public class NodesSpecification {
return new Pair<>(flavorResources, flavorResources);
}
else {
- return new Pair<>(NodeResources.unspecified, NodeResources.unspecified);
+ return new Pair<>(NodeResources.unspecified(), NodeResources.unspecified());
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
index 29bb578a67b..207e130eaaa 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container;
import com.yahoo.config.model.api.container.ContainerServiceType;
import com.yahoo.config.model.producer.AbstractConfigProducer;
-import com.yahoo.config.provision.Flavor;
import com.yahoo.container.bundle.BundleInstantiationSpecification;
import com.yahoo.container.handler.ThreadpoolConfig;
import com.yahoo.osgi.provider.model.ComponentModel;
@@ -45,8 +44,8 @@ public final class ApplicationContainer extends Container implements
@Override
public void getConfig(QrStartConfig.Builder builder) {
if (getHostResource() != null) {
- if (getHostResource().getFlavor().isPresent()) {
- NodeFlavorTuning flavorTuning = new NodeFlavorTuning(getHostResource().getFlavor().get());
+ if ( ! getHostResource().advertisedResources().isUnspecified()) {
+ NodeResourcesTuning flavorTuning = new NodeResourcesTuning(getHostResource().advertisedResources());
flavorTuning.getConfig(builder);
}
}
@@ -80,11 +79,11 @@ public final class ApplicationContainer extends Container implements
@Override
public void getConfig(ThreadpoolConfig.Builder builder) {
if (! (parent instanceof ContainerCluster)) return;
- if ((getHostResource() == null) || getHostResource().getFlavor().isEmpty()) return;
+ if ((getHostResource() == null) || getHostResource().advertisedResources().isUnspecified()) return;
ContainerCluster containerCluster = (ContainerCluster) parent;
if (containerCluster.getThreadPoolSizeFactor() <= 0.0) return;
- NodeFlavorTuning flavorTuning = new NodeFlavorTuning(getHostResource().getFlavor().get())
+ NodeResourcesTuning flavorTuning = new NodeResourcesTuning(getHostResource().advertisedResources())
.setThreadPoolSizeFactor(containerCluster.getThreadPoolSizeFactor())
.setQueueSizeFactor(containerCluster.getQueueSizeFactor());
flavorTuning.getConfig(builder);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java
index f9b50d0e641..7eb7a1fb518 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java
@@ -1,28 +1,25 @@
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.container;
-import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeResources;
import com.yahoo.container.handler.ThreadpoolConfig;
import com.yahoo.search.config.QrStartConfig;
/**
- * Tuning of qr-start config for a container service based on the node flavor of that node.
+ * Tuning of qr-start config for a container service based on node resources.
*
* @author balder
*/
-public class NodeFlavorTuning implements
- QrStartConfig.Producer,
- ThreadpoolConfig.Producer
-{
+public class NodeResourcesTuning implements QrStartConfig.Producer, ThreadpoolConfig.Producer {
- private final Flavor flavor;
+ private final NodeResources resources;
- public NodeFlavorTuning setThreadPoolSizeFactor(double threadPoolSizeFactor) {
+ public NodeResourcesTuning setThreadPoolSizeFactor(double threadPoolSizeFactor) {
this.threadPoolSizeFactor = threadPoolSizeFactor;
return this;
}
- public NodeFlavorTuning setQueueSizeFactor(double queueSizeFactor) {
+ public NodeResourcesTuning setQueueSizeFactor(double queueSizeFactor) {
this.queueSizeFactor = queueSizeFactor;
return this;
}
@@ -30,19 +27,19 @@ public class NodeFlavorTuning implements
private double threadPoolSizeFactor = 8.0;
private double queueSizeFactor = 8.0;
- NodeFlavorTuning(Flavor flavor) {
- this.flavor = flavor;
+ NodeResourcesTuning(NodeResources resources) {
+ this.resources = resources;
}
@Override
public void getConfig(QrStartConfig.Builder builder) {
- builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(flavor.getMinCpuCores())));
+ builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(resources.vcpu())));
}
@Override
public void getConfig(ThreadpoolConfig.Builder builder) {
// Controls max number of concurrent requests per container
- int workerThreads = Math.max(2, (int)Math.ceil(flavor.getMinCpuCores() * threadPoolSizeFactor));
+ int workerThreads = Math.max(2, (int)Math.ceil(resources.vcpu() * threadPoolSizeFactor));
builder.maxthreads(workerThreads);
// This controls your burst handling capability.
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 741e5ebffd1..559a4b8b668 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -664,7 +664,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
.dockerImageRepository(deployState.getWantedDockerImageRepo())
.build();
int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1;
- Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified),
+ Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified()),
false,
!deployState.getProperties().isBootstrap());
var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java
index c8220071373..028aee369c3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java
@@ -86,9 +86,8 @@ public class StorageNode extends ContentNode implements StorServerConfig.Produce
@Override
public void getConfig(StorFilestorConfig.Builder builder) {
- if (getHostResource() != null && getHostResource().getFlavor().isPresent()) {
- Flavor nodeFlavor = getHostResource().getFlavor().get();
- builder.num_threads(Math.max(4, (int)nodeFlavor.getMinCpuCores()));
+ if (getHostResource() != null && ! getHostResource().advertisedResources().isUnspecified()) {
+ builder.num_threads(Math.max(4, (int)getHostResource().advertisedResources().vcpu()));
}
cluster.getConfig(builder);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
index 0998d583d6c..ee854aaa9c3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
@@ -1,31 +1,31 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.search;
-import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeResources;
import com.yahoo.vespa.config.search.core.ProtonConfig;
import static java.lang.Long.min;
/**
- * Tuning of proton config for a search node based on the node flavor of that node.
+ * Tuning of proton config for a search node based on the resources on the node.
*
* @author geirst
*/
-public class NodeFlavorTuning implements ProtonConfig.Producer {
+public class NodeResourcesTuning implements ProtonConfig.Producer {
- static long MB = 1024 * 1024;
- static long GB = MB * 1024;
- private final Flavor nodeFlavor;
+ final static long MB = 1024 * 1024;
+ final static long GB = MB * 1024;
+ private final NodeResources resources;
private final int redundancy;
private final int searchableCopies;
private final int threadsPerSearch;
- public NodeFlavorTuning(Flavor nodeFlavor, int redundancy, int searchableCopies) {
- this(nodeFlavor, redundancy, searchableCopies, 1);
+ public NodeResourcesTuning(NodeResources resources, int redundancy, int searchableCopies) {
+ this(resources, redundancy, searchableCopies, 1);
}
- public NodeFlavorTuning(Flavor nodeFlavor, int redundancy, int searchableCopies, int threadsPerSearch) {
- this.nodeFlavor = nodeFlavor;
+ public NodeResourcesTuning(NodeResources resources, int redundancy, int searchableCopies, int threadsPerSearch) {
+ this.resources = resources;
this.redundancy = redundancy;
this.searchableCopies = searchableCopies;
this.threadsPerSearch = threadsPerSearch;
@@ -51,31 +51,31 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
private void getConfig(ProtonConfig.Documentdb.Builder builder) {
ProtonConfig.Documentdb dbCfg = builder.build();
if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) {
- long numDocs = (long)nodeFlavor.getMinMainMemoryAvailableGb()*GB/64L;
+ long numDocs = (long)resources.memoryGb() * GB / 64L;
builder.allocation.initialnumdocs(numDocs/Math.max(searchableCopies, redundancy));
}
}
private void tuneSummaryCache(ProtonConfig.Summary.Cache.Builder builder) {
- long memoryLimitBytes = (long) ((nodeFlavor.getMinMainMemoryAvailableGb() * 0.05) * GB);
+ long memoryLimitBytes = (long) ((resources.memoryGb() * 0.05) * GB);
builder.maxbytes(memoryLimitBytes);
}
private void setHwInfo(ProtonConfig.Builder builder) {
- builder.hwinfo.disk.shared(nodeFlavor.getType().equals(Flavor.Type.DOCKER_CONTAINER));
- builder.hwinfo.cpu.cores((int)nodeFlavor.getMinCpuCores());
- builder.hwinfo.memory.size((long)nodeFlavor.resources().memoryGb() * GB);
- builder.hwinfo.disk.size((long)nodeFlavor.resources().diskGb() * GB);
+ builder.hwinfo.disk.shared(true);
+ builder.hwinfo.cpu.cores((int)resources.vcpu());
+ builder.hwinfo.memory.size((long)resources.memoryGb() * GB);
+ builder.hwinfo.disk.size((long)resources.diskGb() * GB);
}
private void tuneDiskWriteSpeed(ProtonConfig.Builder builder) {
- if (!nodeFlavor.hasFastDisk()) {
+ if (resources.diskSpeed() != NodeResources.DiskSpeed.fast) {
builder.hwinfo.disk.writespeed(40);
}
}
private void tuneDocumentStoreMaxFileSize(ProtonConfig.Summary.Log.Builder builder) {
- double memoryGb = nodeFlavor.getMinMainMemoryAvailableGb();
+ double memoryGb = resources.memoryGb();
long fileSizeBytes = 4 * GB;
if (memoryGb <= 12.0) {
fileSizeBytes = 256 * MB;
@@ -88,31 +88,31 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
}
private void tuneFlushStrategyMemoryLimits(ProtonConfig.Flush.Memory.Builder builder) {
- long memoryLimitBytes = (long) ((nodeFlavor.getMinMainMemoryAvailableGb() / 8) * GB);
+ long memoryLimitBytes = (long) ((resources.memoryGb() / 8) * GB);
builder.maxmemory(memoryLimitBytes);
builder.each.maxmemory(memoryLimitBytes);
}
private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) {
- long tlsSizeBytes = (long) ((nodeFlavor.getMinDiskAvailableGb() * 0.07) * GB);
+ long tlsSizeBytes = (long) ((resources.diskGb() * 0.07) * GB);
tlsSizeBytes = min(tlsSizeBytes, 100 * GB);
builder.maxtlssize(tlsSizeBytes);
}
private void tuneSummaryReadIo(ProtonConfig.Summary.Read.Builder builder) {
- if (nodeFlavor.hasFastDisk()) {
+ if (resources.diskSpeed() == NodeResources.DiskSpeed.fast) {
builder.io(ProtonConfig.Summary.Read.Io.DIRECTIO);
}
}
private void tuneSearchReadIo(ProtonConfig.Search.Mmap.Builder builder) {
- if (nodeFlavor.hasFastDisk()) {
+ if (resources.diskSpeed() == NodeResources.DiskSpeed.fast) {
builder.advise(ProtonConfig.Search.Mmap.Advise.RANDOM);
}
}
private void tuneRequestThreads(ProtonConfig.Builder builder) {
- int numCores = (int)Math.ceil(nodeFlavor.getMinCpuCores());
+ int numCores = (int)Math.ceil(resources.vcpu());
builder.numsearcherthreads(numCores*threadsPerSearch);
builder.numsummarythreads(numCores);
builder.numthreadspersearch(threadsPerSearch);
@@ -122,7 +122,7 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
// "Reserve" 1GB of memory for other processes running on the content node (config-proxy, cluster-controller, metrics-proxy)
double reservedMemoryGb = 1;
double defaultMemoryLimit = new ProtonConfig.Writefilter(new ProtonConfig.Writefilter.Builder()).memorylimit();
- double scaledMemoryLimit = ((nodeFlavor.getMinMainMemoryAvailableGb() - reservedMemoryGb) * defaultMemoryLimit) / nodeFlavor.getMinMainMemoryAvailableGb();
+ double scaledMemoryLimit = ((resources.memoryGb() - reservedMemoryGb) * defaultMemoryLimit) / resources.memoryGb();
builder.memorylimit(scaledMemoryLimit);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
index 373c62c2eda..f908d091a72 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search;
import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.producer.AbstractConfigProducer;
-import com.yahoo.config.provision.Flavor;
import com.yahoo.metrics.MetricsmanagerConfig;
import com.yahoo.searchlib.TranslogserverConfig;
import com.yahoo.vespa.config.content.LoadTypeConfig;
@@ -270,12 +269,11 @@ public class SearchNode extends AbstractService implements
// to make sure the node failer has done its work
builder.pruneremoveddocumentsage(4 * 24 * 3600 + 3600 + 60);
}
- if (getHostResource() != null && getHostResource().getFlavor().isPresent()) {
- Flavor nodeFlavor = getHostResource().getFlavor().get();
- NodeFlavorTuning nodeFlavorTuning = tuning.isPresent()
- ? new NodeFlavorTuning(nodeFlavor, redundancy, searchableCopies, tuning.get().getNumThreadsPerSearch())
- : new NodeFlavorTuning(nodeFlavor, redundancy, searchableCopies);
- nodeFlavorTuning.getConfig(builder);
+ if (getHostResource() != null && ! getHostResource().advertisedResources().isUnspecified()) {
+ NodeResourcesTuning nodeResourcesTuning = tuning.isPresent()
+ ? new NodeResourcesTuning(getHostResource().advertisedResources(), redundancy, searchableCopies, tuning.get().getNumThreadsPerSearch())
+ : new NodeResourcesTuning(getHostResource().advertisedResources(), redundancy, searchableCopies);
+ nodeResourcesTuning.getConfig(builder);
if (tuning.isPresent()) {
tuning.get().getConfig(builder);
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java
index 51b039a7532..91f5fdc5f11 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java
@@ -6,6 +6,8 @@ import org.junit.Test;
import java.util.Arrays;
import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -17,10 +19,10 @@ public class HostSpecTest {
@Test
public void testEquals() {
- HostSpec h1 = new HostSpec("foo", Collections.<String>emptyList());
- HostSpec h2 = new HostSpec("foo", Collections.<String>emptyList());
- HostSpec h3 = new HostSpec("foo", Arrays.asList("my", "alias"));
- HostSpec h4 = new HostSpec("bar", Collections.<String>emptyList());
+ HostSpec h1 = new HostSpec("foo", List.of(), Optional.empty());
+ HostSpec h2 = new HostSpec("foo", List.of(), Optional.empty());
+ HostSpec h3 = new HostSpec("foo", List.of("my", "alias"), Optional.empty());
+ HostSpec h4 = new HostSpec("bar", List.of(), Optional.empty());
assertTrue(h1.equals(h1));
assertTrue(h1.equals(h2));
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java
index 6aea0593f8a..dff1d338ffe 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java
@@ -6,8 +6,10 @@ import com.yahoo.config.model.test.MockRoot;
import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.NodeResources;
import org.junit.Test;
+import java.util.List;
import java.util.Optional;
import static com.yahoo.config.provision.ClusterSpec.Type.container;
@@ -53,7 +55,10 @@ public class HostResourceTest {
private static HostResource hostResourceWithMemberships(ClusterMembership membership) {
return new HostResource(Host.createHost(null, "hostname"),
- new HostSpec("hostname", Optional.of(membership)));
+ new HostSpec("hostname",
+ NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(),
+ membership,
+ Optional.empty(), Optional.empty(), Optional.empty()));
}
private static int counter = 0;
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
index ac77b821d4a..a9bf8bdcc49 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
@@ -17,7 +17,9 @@ import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.ProvisionLogger;
+import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification;
import org.junit.Before;
import org.junit.Test;
@@ -103,17 +105,17 @@ public class VespaModelFactoryTest {
@Override
public HostSpec allocateHost(String alias) {
return new HostSpec(hostName,
- List.of(),
- ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.admin, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(),
- 0));
+ NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(),
+ ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.admin, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), 0),
+ Optional.empty(), Optional.empty(), Optional.empty());
}
@Override
public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
return List.of(new HostSpec(hostName,
- List.of(),
- ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(),
- 0)));
+ NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(),
+ ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), 0),
+ Optional.empty(), Optional.empty(), Optional.empty()));
}
};
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
index 0f94df80421..9a59907a230 100755
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
@@ -242,12 +242,12 @@ public class ContainerClusterTest {
@Test
public void requireThatPoolAndQueueCanBeControlledByPropertiesAndFlavor() {
FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder().name("my_flavor").minCpuCores(3);
- NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)))
+ NodeResourcesTuning nodeResourcesTuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources())
.setThreadPoolSizeFactor(13.3)
.setQueueSizeFactor(17.5);
ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder();
- nodeFlavorTuning.getConfig(tpBuilder);
+ nodeResourcesTuning.getConfig(tpBuilder);
ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder);
assertEquals(40, threadpoolConfig.maxthreads());
assertEquals(700, threadpoolConfig.queueSize());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
index ca3198637f3..fdd7ae57f0f 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
@@ -766,7 +766,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
assertEquals("KMP_SETTING=1 KMP_AFFINITY=granularity=fine,verbose,compact,1,0 ", qrStartConfig.qrs().env());
}
- private void verifyAvailableprocessors(boolean isHosted, Flavor flavor, int expectProcessors) throws IOException, SAXException {
+ private void verifyAvailableprocessors(boolean isHosted, Flavor flavor, int expectProcessors) {
DeployState deployState = new DeployState.Builder()
.modelHostProvisioner(flavor != null ? new SingleNodeProvisioner(flavor) : new SingleNodeProvisioner())
.properties(new TestProperties()
@@ -790,7 +790,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
}
@Test
- public void requireThatAvailableProcessorsFollowFlavor() throws IOException, SAXException {
+ public void requireThatAvailableProcessorsFollowFlavor() {
verifyAvailableprocessors(false, null,0);
verifyAvailableprocessors(true, null,0);
verifyAvailableprocessors(true, new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()), 9);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
index 023b7249939..fee0edacb90 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
@@ -11,13 +11,13 @@ import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.assertEquals;
-import static com.yahoo.vespa.model.search.NodeFlavorTuning.MB;
-import static com.yahoo.vespa.model.search.NodeFlavorTuning.GB;
+import static com.yahoo.vespa.model.search.NodeResourcesTuning.MB;
+import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB;
/**
* @author geirst
*/
-public class NodeFlavorTuningTest {
+public class NodeResourcesTuningTest {
private static double delta = 0.00001;
@@ -152,7 +152,6 @@ public class NodeFlavorTuningTest {
@Test
public void require_that_docker_node_is_tagged_with_shared_disk() {
assertSharedDisk(true, true);
- assertSharedDisk(false, false);
}
@Test
@@ -232,13 +231,13 @@ public class NodeFlavorTuningTest {
}
private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int redundancy, int searchableCopies) {
flavorBuilder.name("my_flavor");
- NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)), redundancy, searchableCopies);
+ NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), redundancy, searchableCopies);
tuning.getConfig(protonBuilder);
return new ProtonConfig(protonBuilder);
}
private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int redundancy, int searchableCopies, int numThreadsPerSearch) {
flavorBuilder.name("my_flavor");
- NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)), redundancy, searchableCopies, numThreadsPerSearch);
+ NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), redundancy, searchableCopies, numThreadsPerSearch);
tuning.getConfig(protonBuilder);
return new ProtonConfig(protonBuilder);
}