aboutsummaryrefslogtreecommitdiffstats
path: root/config-model
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2020-05-23 07:21:43 +0200
committerGitHub <noreply@github.com>2020-05-23 07:21:43 +0200
commit52e8214e1dfcc987b8227d19c288d05b54e711f9 (patch)
tree5e2355e5c08ba7698cf9c6e930496fb2a5ce9385 /config-model
parent5c0bfb06784075ac72c7891b7dce6454d85bdff8 (diff)
Revert "Bratseth/real resources in model"
Diffstat (limited to 'config-model')
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java29
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java16
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/HostResource.java16
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java4
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java4
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java10
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java9
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java (renamed from config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java)23
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java (renamed from config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java)48
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java12
-rw-r--r--config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java10
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java7
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java14
-rwxr-xr-xconfig-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java (renamed from config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java)11
19 files changed, 110 insertions, 123 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
index ab2e0f632e4..e55686accca 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
@@ -7,7 +7,6 @@ import com.yahoo.vespa.model.container.Container;
import java.io.Reader;
import java.util.List;
-import java.util.Optional;
/**
* A host provisioner based on a hosts.xml file.
@@ -39,7 +38,7 @@ public class HostsXmlProvisioner implements HostProvisioner {
}
for (Host host : hosts.asCollection()) {
if (host.aliases().contains(alias)) {
- return new HostSpec(host.hostname(), host.aliases(), Optional.empty());
+ return new HostSpec(host.hostname(), host.aliases());
}
}
throw new IllegalArgumentException("Unable to find host for alias '" + alias + "'");
@@ -51,7 +50,7 @@ public class HostsXmlProvisioner implements HostProvisioner {
}
private HostSpec host2HostSpec(Host host) {
- return new HostSpec(host.hostname(), host.aliases(), Optional.empty());
+ return new HostSpec(host.hostname(), host.aliases());
}
private Host getFirstHost() {
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
index 009bc225037..1a1ed000478 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.HostSpec;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.ProvisionLogger;
@@ -117,7 +118,8 @@ public class InMemoryProvisioner implements HostProvisioner {
List<Host> defaultHosts = freeNodes.get(defaultResources);
if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts with default resources available");
Host newHost = freeNodes.removeValue(defaultResources, 0);
- return new HostSpec(newHost.hostname(), List.of(alias), Optional.empty());
+ // Note: Always returns HostSpec with empty dockerImageRepo, which is OK since this method is never used when docker image repo is set
+ return new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty(), newHost.version(), Optional.empty());
}
@Override
@@ -171,12 +173,12 @@ public class InMemoryProvisioner implements HostProvisioner {
private HostSpec retire(HostSpec host) {
return new HostSpec(host.hostname(),
- host.realResources(),
- host.advertisedResources(),
- host.requestedResources().orElse(NodeResources.unspecified()),
- host.membership().get().retire(),
+ host.aliases(),
+ host.flavor(),
+ Optional.of(host.membership().get().retire()),
host.version(),
Optional.empty(),
+ Optional.empty(),
host.dockerImageRepo());
}
@@ -187,11 +189,11 @@ public class InMemoryProvisioner implements HostProvisioner {
// Check if the current allocations are compatible with the new request
for (int i = allocation.size() - 1; i >= 0; i--) {
- NodeResources currentResources = allocation.get(0).advertisedResources();
- if (currentResources.isUnspecified() || requestedResources.isUnspecified()) continue;
- if ( ! currentResources.compatibleWith(requestedResources)) {
+ Optional<NodeResources> currentResources = allocation.get(0).flavor().map(Flavor::resources);
+ if (currentResources.isEmpty() || requestedResources == NodeResources.unspecified) continue;
+ if (!currentResources.get().compatibleWith(requestedResources)) {
HostSpec removed = allocation.remove(i);
- freeNodes.put(currentResources, new Host(removed.hostname())); // Return the node back to free pool
+ freeNodes.put(currentResources.get(), new Host(removed.hostname())); // Return the node back to free pool
}
}
@@ -200,7 +202,7 @@ public class InMemoryProvisioner implements HostProvisioner {
// Find the smallest host that can fit the requested requested
Optional<NodeResources> hostResources = freeNodes.keySet().stream()
.sorted(new MemoryDiskCpu())
- .filter(resources -> requestedResources.isUnspecified() || resources.satisfies(requestedResources))
+ .filter(resources -> requestedResources == NodeResources.unspecified || resources.satisfies(requestedResources))
.findFirst();
if (hostResources.isEmpty()) {
if (canFail)
@@ -212,11 +214,10 @@ public class InMemoryProvisioner implements HostProvisioner {
Host newHost = freeNodes.removeValue(hostResources.get(), 0);
if (freeNodes.get(hostResources.get()).isEmpty()) freeNodes.removeAll(hostResources.get());
ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++);
- allocation.add(new HostSpec(newHost.hostname(),
- hostResources.get(), hostResources.get(), requestedResources,
- membership,
+ allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(),
+ hostResources.map(Flavor::new), Optional.of(membership),
newHost.version(), Optional.empty(),
- Optional.empty()));
+ requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources)));
}
nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex);
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
index b4fd55c6d33..1f0e0755667 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
@@ -1,21 +1,17 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.config.model.provision;
-import com.yahoo.component.Version;
import com.yahoo.config.model.api.HostProvisioner;
import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.HostSpec;
-import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.ProvisionLogger;
import com.yahoo.net.HostName;
-import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification;
import java.util.ArrayList;
import java.util.List;
-import java.util.Optional;
/**
* A host provisioner used when there is no hosts.xml file (using localhost as the only host)
@@ -32,15 +28,12 @@ public class SingleNodeProvisioner implements HostProvisioner {
public SingleNodeProvisioner() {
host = new Host(HostName.getLocalhost());
- this.hostSpec = new HostSpec(host.hostname(), host.aliases(), Optional.empty());
+ this.hostSpec = new HostSpec(host.hostname(), host.aliases());
}
public SingleNodeProvisioner(Flavor flavor) {
host = new Host(HostName.getLocalhost());
- this.hostSpec = new HostSpec(host.hostname(),
- flavor.resources(), flavor.resources(), flavor.resources(),
- ClusterMembership.from(ClusterSpec.specification(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).group(ClusterSpec.Group.from(0)).vespaVersion("1").build(), 0),
- Optional.empty(), Optional.empty(), Optional.empty());
+ this.hostSpec = new HostSpec(host.hostname(), host.aliases(), flavor);
}
@Override
@@ -52,10 +45,7 @@ public class SingleNodeProvisioner implements HostProvisioner {
public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
// TODO: This should fail if capacity requested is more than 1
List<HostSpec> hosts = new ArrayList<>();
- hosts.add(new HostSpec(host.hostname(),
- NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(),
- ClusterMembership.from(cluster, counter++),
- Optional.empty(), Optional.empty(), Optional.empty()));
+ hosts.add(new HostSpec(host.hostname(), host.aliases(), ClusterMembership.from(cluster, counter++)));
return hosts;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
index 81931995485..9dba6fde9d4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
@@ -1,17 +1,23 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model;
+import com.yahoo.component.Version;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.HostSpec;
import com.yahoo.config.provision.NodeResources;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.stream.Collectors;
/**
@@ -38,7 +44,7 @@ public class HostResource implements Comparable<HostResource> {
* @param host {@link com.yahoo.vespa.model.Host} object to bind to.
*/
public HostResource(Host host) {
- this(host, new HostSpec(host.getHostname(), List.of(), Optional.empty()));
+ this(host, new HostSpec(host.getHostname(), Optional.empty()));
}
public HostResource(Host host, HostSpec spec) {
@@ -99,16 +105,8 @@ public class HostResource implements Comparable<HostResource> {
}
/** Returns the flavor of this resource. Empty for self-hosted Vespa. */
- // TODO: Remove after June 2020
- @Deprecated
public Optional<Flavor> getFlavor() { return spec.flavor(); }
- /** The real resources available for Vespa processes on this node, after subtracting infrastructure overhead. */
- public NodeResources realResources() { return spec.realResources(); }
-
- /** The total advertised resources of this node, typically matching what's requested. */
- public NodeResources advertisedResources() { return spec.advertisedResources(); }
-
@Override
public String toString() {
return "host '" + host.getHostname() + "'";
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
index 3d2918e0ee1..292d4761d8a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
@@ -120,7 +120,7 @@ public class HostSystem extends AbstractConfigProducer<Host> {
HostResource host = getExistingHost(spec).orElseGet(() -> addNewHost(spec));
retAllocatedHosts.put(host, spec.membership().orElse(null));
}
- retAllocatedHosts.keySet().forEach(host -> log.log(FINE, () -> "Allocated host " + host.getHostname() + " with resources " + host.advertisedResources()));
+ retAllocatedHosts.keySet().forEach(host -> log.log(FINE, () -> "Allocated host " + host.getHostname() + " with flavor " + host.getFlavor()));
return retAllocatedHosts;
}
@@ -131,7 +131,7 @@ public class HostSystem extends AbstractConfigProducer<Host> {
if (hosts.isEmpty()) {
return Optional.empty();
} else {
- log.log(FINE, () -> "Found existing host resource for " + key.hostname() + " with resources" + hosts.get(0).advertisedResources());
+ log.log(FINE, () -> "Found existing host resource for " + key.hostname() + " with flavor " + hosts.get(0).getFlavor());
return Optional.of(hosts.get(0));
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
index 24b7b0949f6..5343a322382 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
@@ -44,8 +44,8 @@ public class ResourcesReductionValidator implements ChangeValidator {
ClusterSpec.Id clusterId,
ValidationOverrides overrides,
Instant now) {
- if (current.minResources().nodeResources().isUnspecified()) return;
- if (next.minResources().nodeResources().isUnspecified()) return;
+ if (current.minResources().nodeResources() == NodeResources.unspecified) return;
+ if (next.minResources().nodeResources() == NodeResources.unspecified) return;
List<String> illegalChanges = Stream.of(
validateResource("vCPU",
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
index 8f737f02dca..ea47e490b12 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
@@ -159,8 +159,8 @@ public class NodesSpecification {
* Returns a requirement from <code>count</code> non-dedicated nodes in one group
*/
public static NodesSpecification nonDedicated(int count, ConfigModelContext context) {
- return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified()),
- new ClusterResources(count, 1, NodeResources.unspecified()),
+ return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified),
+ new ClusterResources(count, 1, NodeResources.unspecified),
false,
context.getDeployState().getWantedNodeVespaVersion(),
false,
@@ -172,8 +172,8 @@ public class NodesSpecification {
/** Returns a requirement from <code>count</code> dedicated nodes in one group */
public static NodesSpecification dedicated(int count, ConfigModelContext context) {
- return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified()),
- new ClusterResources(count, 1, NodeResources.unspecified()),
+ return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified),
+ new ClusterResources(count, 1, NodeResources.unspecified),
true,
context.getDeployState().getWantedNodeVespaVersion(),
false,
@@ -224,7 +224,7 @@ public class NodesSpecification {
return new Pair<>(flavorResources, flavorResources);
}
else {
- return new Pair<>(NodeResources.unspecified(), NodeResources.unspecified());
+ return new Pair<>(NodeResources.unspecified, NodeResources.unspecified);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
index 207e130eaaa..29bb578a67b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.model.container;
import com.yahoo.config.model.api.container.ContainerServiceType;
import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.provision.Flavor;
import com.yahoo.container.bundle.BundleInstantiationSpecification;
import com.yahoo.container.handler.ThreadpoolConfig;
import com.yahoo.osgi.provider.model.ComponentModel;
@@ -44,8 +45,8 @@ public final class ApplicationContainer extends Container implements
@Override
public void getConfig(QrStartConfig.Builder builder) {
if (getHostResource() != null) {
- if ( ! getHostResource().advertisedResources().isUnspecified()) {
- NodeResourcesTuning flavorTuning = new NodeResourcesTuning(getHostResource().advertisedResources());
+ if (getHostResource().getFlavor().isPresent()) {
+ NodeFlavorTuning flavorTuning = new NodeFlavorTuning(getHostResource().getFlavor().get());
flavorTuning.getConfig(builder);
}
}
@@ -79,11 +80,11 @@ public final class ApplicationContainer extends Container implements
@Override
public void getConfig(ThreadpoolConfig.Builder builder) {
if (! (parent instanceof ContainerCluster)) return;
- if ((getHostResource() == null) || getHostResource().advertisedResources().isUnspecified()) return;
+ if ((getHostResource() == null) || getHostResource().getFlavor().isEmpty()) return;
ContainerCluster containerCluster = (ContainerCluster) parent;
if (containerCluster.getThreadPoolSizeFactor() <= 0.0) return;
- NodeResourcesTuning flavorTuning = new NodeResourcesTuning(getHostResource().advertisedResources())
+ NodeFlavorTuning flavorTuning = new NodeFlavorTuning(getHostResource().getFlavor().get())
.setThreadPoolSizeFactor(containerCluster.getThreadPoolSizeFactor())
.setQueueSizeFactor(containerCluster.getQueueSizeFactor());
flavorTuning.getConfig(builder);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java
index 7eb7a1fb518..f9b50d0e641 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java
@@ -1,25 +1,28 @@
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.container;
-import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.Flavor;
import com.yahoo.container.handler.ThreadpoolConfig;
import com.yahoo.search.config.QrStartConfig;
/**
- * Tuning of qr-start config for a container service based on node resources.
+ * Tuning of qr-start config for a container service based on the node flavor of that node.
*
* @author balder
*/
-public class NodeResourcesTuning implements QrStartConfig.Producer, ThreadpoolConfig.Producer {
+public class NodeFlavorTuning implements
+ QrStartConfig.Producer,
+ ThreadpoolConfig.Producer
+{
- private final NodeResources resources;
+ private final Flavor flavor;
- public NodeResourcesTuning setThreadPoolSizeFactor(double threadPoolSizeFactor) {
+ public NodeFlavorTuning setThreadPoolSizeFactor(double threadPoolSizeFactor) {
this.threadPoolSizeFactor = threadPoolSizeFactor;
return this;
}
- public NodeResourcesTuning setQueueSizeFactor(double queueSizeFactor) {
+ public NodeFlavorTuning setQueueSizeFactor(double queueSizeFactor) {
this.queueSizeFactor = queueSizeFactor;
return this;
}
@@ -27,19 +30,19 @@ public class NodeResourcesTuning implements QrStartConfig.Producer, ThreadpoolCo
private double threadPoolSizeFactor = 8.0;
private double queueSizeFactor = 8.0;
- NodeResourcesTuning(NodeResources resources) {
- this.resources = resources;
+ NodeFlavorTuning(Flavor flavor) {
+ this.flavor = flavor;
}
@Override
public void getConfig(QrStartConfig.Builder builder) {
- builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(resources.vcpu())));
+ builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(flavor.getMinCpuCores())));
}
@Override
public void getConfig(ThreadpoolConfig.Builder builder) {
// Controls max number of concurrent requests per container
- int workerThreads = Math.max(2, (int)Math.ceil(resources.vcpu() * threadPoolSizeFactor));
+ int workerThreads = Math.max(2, (int)Math.ceil(flavor.getMinCpuCores() * threadPoolSizeFactor));
builder.maxthreads(workerThreads);
// This controls your burst handling capability.
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 559a4b8b668..741e5ebffd1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -664,7 +664,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
.dockerImageRepository(deployState.getWantedDockerImageRepo())
.build();
int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1;
- Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified()),
+ Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified),
false,
!deployState.getProperties().isBootstrap());
var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java
index 028aee369c3..c8220071373 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java
@@ -86,8 +86,9 @@ public class StorageNode extends ContentNode implements StorServerConfig.Produce
@Override
public void getConfig(StorFilestorConfig.Builder builder) {
- if (getHostResource() != null && ! getHostResource().advertisedResources().isUnspecified()) {
- builder.num_threads(Math.max(4, (int)getHostResource().advertisedResources().vcpu()));
+ if (getHostResource() != null && getHostResource().getFlavor().isPresent()) {
+ Flavor nodeFlavor = getHostResource().getFlavor().get();
+ builder.num_threads(Math.max(4, (int)nodeFlavor.getMinCpuCores()));
}
cluster.getConfig(builder);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
index ee854aaa9c3..0998d583d6c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
@@ -1,31 +1,31 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.search;
-import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.Flavor;
import com.yahoo.vespa.config.search.core.ProtonConfig;
import static java.lang.Long.min;
/**
- * Tuning of proton config for a search node based on the resources on the node.
+ * Tuning of proton config for a search node based on the node flavor of that node.
*
* @author geirst
*/
-public class NodeResourcesTuning implements ProtonConfig.Producer {
+public class NodeFlavorTuning implements ProtonConfig.Producer {
- final static long MB = 1024 * 1024;
- final static long GB = MB * 1024;
- private final NodeResources resources;
+ static long MB = 1024 * 1024;
+ static long GB = MB * 1024;
+ private final Flavor nodeFlavor;
private final int redundancy;
private final int searchableCopies;
private final int threadsPerSearch;
- public NodeResourcesTuning(NodeResources resources, int redundancy, int searchableCopies) {
- this(resources, redundancy, searchableCopies, 1);
+ public NodeFlavorTuning(Flavor nodeFlavor, int redundancy, int searchableCopies) {
+ this(nodeFlavor, redundancy, searchableCopies, 1);
}
- public NodeResourcesTuning(NodeResources resources, int redundancy, int searchableCopies, int threadsPerSearch) {
- this.resources = resources;
+ public NodeFlavorTuning(Flavor nodeFlavor, int redundancy, int searchableCopies, int threadsPerSearch) {
+ this.nodeFlavor = nodeFlavor;
this.redundancy = redundancy;
this.searchableCopies = searchableCopies;
this.threadsPerSearch = threadsPerSearch;
@@ -51,31 +51,31 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
private void getConfig(ProtonConfig.Documentdb.Builder builder) {
ProtonConfig.Documentdb dbCfg = builder.build();
if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) {
- long numDocs = (long)resources.memoryGb() * GB / 64L;
+ long numDocs = (long)nodeFlavor.getMinMainMemoryAvailableGb()*GB/64L;
builder.allocation.initialnumdocs(numDocs/Math.max(searchableCopies, redundancy));
}
}
private void tuneSummaryCache(ProtonConfig.Summary.Cache.Builder builder) {
- long memoryLimitBytes = (long) ((resources.memoryGb() * 0.05) * GB);
+ long memoryLimitBytes = (long) ((nodeFlavor.getMinMainMemoryAvailableGb() * 0.05) * GB);
builder.maxbytes(memoryLimitBytes);
}
private void setHwInfo(ProtonConfig.Builder builder) {
- builder.hwinfo.disk.shared(true);
- builder.hwinfo.cpu.cores((int)resources.vcpu());
- builder.hwinfo.memory.size((long)resources.memoryGb() * GB);
- builder.hwinfo.disk.size((long)resources.diskGb() * GB);
+ builder.hwinfo.disk.shared(nodeFlavor.getType().equals(Flavor.Type.DOCKER_CONTAINER));
+ builder.hwinfo.cpu.cores((int)nodeFlavor.getMinCpuCores());
+ builder.hwinfo.memory.size((long)nodeFlavor.resources().memoryGb() * GB);
+ builder.hwinfo.disk.size((long)nodeFlavor.resources().diskGb() * GB);
}
private void tuneDiskWriteSpeed(ProtonConfig.Builder builder) {
- if (resources.diskSpeed() != NodeResources.DiskSpeed.fast) {
+ if (!nodeFlavor.hasFastDisk()) {
builder.hwinfo.disk.writespeed(40);
}
}
private void tuneDocumentStoreMaxFileSize(ProtonConfig.Summary.Log.Builder builder) {
- double memoryGb = resources.memoryGb();
+ double memoryGb = nodeFlavor.getMinMainMemoryAvailableGb();
long fileSizeBytes = 4 * GB;
if (memoryGb <= 12.0) {
fileSizeBytes = 256 * MB;
@@ -88,31 +88,31 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
}
private void tuneFlushStrategyMemoryLimits(ProtonConfig.Flush.Memory.Builder builder) {
- long memoryLimitBytes = (long) ((resources.memoryGb() / 8) * GB);
+ long memoryLimitBytes = (long) ((nodeFlavor.getMinMainMemoryAvailableGb() / 8) * GB);
builder.maxmemory(memoryLimitBytes);
builder.each.maxmemory(memoryLimitBytes);
}
private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) {
- long tlsSizeBytes = (long) ((resources.diskGb() * 0.07) * GB);
+ long tlsSizeBytes = (long) ((nodeFlavor.getMinDiskAvailableGb() * 0.07) * GB);
tlsSizeBytes = min(tlsSizeBytes, 100 * GB);
builder.maxtlssize(tlsSizeBytes);
}
private void tuneSummaryReadIo(ProtonConfig.Summary.Read.Builder builder) {
- if (resources.diskSpeed() == NodeResources.DiskSpeed.fast) {
+ if (nodeFlavor.hasFastDisk()) {
builder.io(ProtonConfig.Summary.Read.Io.DIRECTIO);
}
}
private void tuneSearchReadIo(ProtonConfig.Search.Mmap.Builder builder) {
- if (resources.diskSpeed() == NodeResources.DiskSpeed.fast) {
+ if (nodeFlavor.hasFastDisk()) {
builder.advise(ProtonConfig.Search.Mmap.Advise.RANDOM);
}
}
private void tuneRequestThreads(ProtonConfig.Builder builder) {
- int numCores = (int)Math.ceil(resources.vcpu());
+ int numCores = (int)Math.ceil(nodeFlavor.getMinCpuCores());
builder.numsearcherthreads(numCores*threadsPerSearch);
builder.numsummarythreads(numCores);
builder.numthreadspersearch(threadsPerSearch);
@@ -122,7 +122,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
// "Reserve" 1GB of memory for other processes running on the content node (config-proxy, cluster-controller, metrics-proxy)
double reservedMemoryGb = 1;
double defaultMemoryLimit = new ProtonConfig.Writefilter(new ProtonConfig.Writefilter.Builder()).memorylimit();
- double scaledMemoryLimit = ((resources.memoryGb() - reservedMemoryGb) * defaultMemoryLimit) / resources.memoryGb();
+ double scaledMemoryLimit = ((nodeFlavor.getMinMainMemoryAvailableGb() - reservedMemoryGb) * defaultMemoryLimit) / nodeFlavor.getMinMainMemoryAvailableGb();
builder.memorylimit(scaledMemoryLimit);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
index f908d091a72..373c62c2eda 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.search;
import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.provision.Flavor;
import com.yahoo.metrics.MetricsmanagerConfig;
import com.yahoo.searchlib.TranslogserverConfig;
import com.yahoo.vespa.config.content.LoadTypeConfig;
@@ -269,11 +270,12 @@ public class SearchNode extends AbstractService implements
// to make sure the node failer has done its work
builder.pruneremoveddocumentsage(4 * 24 * 3600 + 3600 + 60);
}
- if (getHostResource() != null && ! getHostResource().advertisedResources().isUnspecified()) {
- NodeResourcesTuning nodeResourcesTuning = tuning.isPresent()
- ? new NodeResourcesTuning(getHostResource().advertisedResources(), redundancy, searchableCopies, tuning.get().getNumThreadsPerSearch())
- : new NodeResourcesTuning(getHostResource().advertisedResources(), redundancy, searchableCopies);
- nodeResourcesTuning.getConfig(builder);
+ if (getHostResource() != null && getHostResource().getFlavor().isPresent()) {
+ Flavor nodeFlavor = getHostResource().getFlavor().get();
+ NodeFlavorTuning nodeFlavorTuning = tuning.isPresent()
+ ? new NodeFlavorTuning(nodeFlavor, redundancy, searchableCopies, tuning.get().getNumThreadsPerSearch())
+ : new NodeFlavorTuning(nodeFlavor, redundancy, searchableCopies);
+ nodeFlavorTuning.getConfig(builder);
if (tuning.isPresent()) {
tuning.get().getConfig(builder);
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java
index 91f5fdc5f11..51b039a7532 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java
@@ -6,8 +6,6 @@ import org.junit.Test;
import java.util.Arrays;
import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -19,10 +17,10 @@ public class HostSpecTest {
@Test
public void testEquals() {
- HostSpec h1 = new HostSpec("foo", List.of(), Optional.empty());
- HostSpec h2 = new HostSpec("foo", List.of(), Optional.empty());
- HostSpec h3 = new HostSpec("foo", List.of("my", "alias"), Optional.empty());
- HostSpec h4 = new HostSpec("bar", List.of(), Optional.empty());
+ HostSpec h1 = new HostSpec("foo", Collections.<String>emptyList());
+ HostSpec h2 = new HostSpec("foo", Collections.<String>emptyList());
+ HostSpec h3 = new HostSpec("foo", Arrays.asList("my", "alias"));
+ HostSpec h4 = new HostSpec("bar", Collections.<String>emptyList());
assertTrue(h1.equals(h1));
assertTrue(h1.equals(h2));
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java
index dff1d338ffe..6aea0593f8a 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java
@@ -6,10 +6,8 @@ import com.yahoo.config.model.test.MockRoot;
import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.HostSpec;
-import com.yahoo.config.provision.NodeResources;
import org.junit.Test;
-import java.util.List;
import java.util.Optional;
import static com.yahoo.config.provision.ClusterSpec.Type.container;
@@ -55,10 +53,7 @@ public class HostResourceTest {
private static HostResource hostResourceWithMemberships(ClusterMembership membership) {
return new HostResource(Host.createHost(null, "hostname"),
- new HostSpec("hostname",
- NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(),
- membership,
- Optional.empty(), Optional.empty(), Optional.empty()));
+ new HostSpec("hostname", Optional.of(membership)));
}
private static int counter = 0;
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
index a9bf8bdcc49..ac77b821d4a 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
@@ -17,9 +17,7 @@ import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.HostSpec;
-import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.ProvisionLogger;
-import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification;
import org.junit.Before;
import org.junit.Test;
@@ -105,17 +103,17 @@ public class VespaModelFactoryTest {
@Override
public HostSpec allocateHost(String alias) {
return new HostSpec(hostName,
- NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(),
- ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.admin, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), 0),
- Optional.empty(), Optional.empty(), Optional.empty());
+ List.of(),
+ ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.admin, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(),
+ 0));
}
@Override
public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
return List.of(new HostSpec(hostName,
- NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(),
- ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), 0),
- Optional.empty(), Optional.empty(), Optional.empty()));
+ List.of(),
+ ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(),
+ 0)));
}
};
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
index 9a59907a230..0f94df80421 100755
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
@@ -242,12 +242,12 @@ public class ContainerClusterTest {
@Test
public void requireThatPoolAndQueueCanBeControlledByPropertiesAndFlavor() {
FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder().name("my_flavor").minCpuCores(3);
- NodeResourcesTuning nodeResourcesTuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources())
+ NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)))
.setThreadPoolSizeFactor(13.3)
.setQueueSizeFactor(17.5);
ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder();
- nodeResourcesTuning.getConfig(tpBuilder);
+ nodeFlavorTuning.getConfig(tpBuilder);
ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder);
assertEquals(40, threadpoolConfig.maxthreads());
assertEquals(700, threadpoolConfig.queueSize());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
index fdd7ae57f0f..ca3198637f3 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
@@ -766,7 +766,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
assertEquals("KMP_SETTING=1 KMP_AFFINITY=granularity=fine,verbose,compact,1,0 ", qrStartConfig.qrs().env());
}
- private void verifyAvailableprocessors(boolean isHosted, Flavor flavor, int expectProcessors) {
+ private void verifyAvailableprocessors(boolean isHosted, Flavor flavor, int expectProcessors) throws IOException, SAXException {
DeployState deployState = new DeployState.Builder()
.modelHostProvisioner(flavor != null ? new SingleNodeProvisioner(flavor) : new SingleNodeProvisioner())
.properties(new TestProperties()
@@ -790,7 +790,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
}
@Test
- public void requireThatAvailableProcessorsFollowFlavor() {
+ public void requireThatAvailableProcessorsFollowFlavor() throws IOException, SAXException {
verifyAvailableprocessors(false, null,0);
verifyAvailableprocessors(true, null,0);
verifyAvailableprocessors(true, new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()), 9);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
index fee0edacb90..023b7249939 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
@@ -11,13 +11,13 @@ import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.assertEquals;
-import static com.yahoo.vespa.model.search.NodeResourcesTuning.MB;
-import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB;
+import static com.yahoo.vespa.model.search.NodeFlavorTuning.MB;
+import static com.yahoo.vespa.model.search.NodeFlavorTuning.GB;
/**
* @author geirst
*/
-public class NodeResourcesTuningTest {
+public class NodeFlavorTuningTest {
private static double delta = 0.00001;
@@ -152,6 +152,7 @@ public class NodeResourcesTuningTest {
@Test
public void require_that_docker_node_is_tagged_with_shared_disk() {
assertSharedDisk(true, true);
+ assertSharedDisk(false, false);
}
@Test
@@ -231,13 +232,13 @@ public class NodeResourcesTuningTest {
}
private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int redundancy, int searchableCopies) {
flavorBuilder.name("my_flavor");
- NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), redundancy, searchableCopies);
+ NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)), redundancy, searchableCopies);
tuning.getConfig(protonBuilder);
return new ProtonConfig(protonBuilder);
}
private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int redundancy, int searchableCopies, int numThreadsPerSearch) {
flavorBuilder.name("my_flavor");
- NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), redundancy, searchableCopies, numThreadsPerSearch);
+ NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)), redundancy, searchableCopies, numThreadsPerSearch);
tuning.getConfig(protonBuilder);
return new ProtonConfig(protonBuilder);
}