diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2020-05-23 09:00:56 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-05-23 09:00:56 +0200 |
commit | 413ea623b5033593c2f2fe9951e1a685c8ff9dfc (patch) | |
tree | 5e2355e5c08ba7698cf9c6e930496fb2a5ce9385 | |
parent | 5c0bfb06784075ac72c7891b7dce6454d85bdff8 (diff) | |
parent | 52e8214e1dfcc987b8227d19c288d05b54e711f9 (diff) |
Merge pull request #13353 from vespa-engine/revert-13348-bratseth/real-resources-in-model
Revert "Bratseth/real resources in model"
40 files changed, 217 insertions, 395 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java index ab2e0f632e4..e55686accca 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.model.container.Container; import java.io.Reader; import java.util.List; -import java.util.Optional; /** * A host provisioner based on a hosts.xml file. @@ -39,7 +38,7 @@ public class HostsXmlProvisioner implements HostProvisioner { } for (Host host : hosts.asCollection()) { if (host.aliases().contains(alias)) { - return new HostSpec(host.hostname(), host.aliases(), Optional.empty()); + return new HostSpec(host.hostname(), host.aliases()); } } throw new IllegalArgumentException("Unable to find host for alias '" + alias + "'"); @@ -51,7 +50,7 @@ public class HostsXmlProvisioner implements HostProvisioner { } private HostSpec host2HostSpec(Host host) { - return new HostSpec(host.hostname(), host.aliases(), Optional.empty()); + return new HostSpec(host.hostname(), host.aliases()); } private Host getFirstHost() { diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 009bc225037..1a1ed000478 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ProvisionLogger; @@ -117,7 +118,8 @@ public class InMemoryProvisioner implements HostProvisioner { List<Host> defaultHosts = freeNodes.get(defaultResources); if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts with default resources available"); Host newHost = freeNodes.removeValue(defaultResources, 0); - return new HostSpec(newHost.hostname(), List.of(alias), Optional.empty()); + // Note: Always returns HostSpec with empty dockerImageRepo, which is OK since this method is never used when docker image repo is set + return new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty(), newHost.version(), Optional.empty()); } @Override @@ -171,12 +173,12 @@ public class InMemoryProvisioner implements HostProvisioner { private HostSpec retire(HostSpec host) { return new HostSpec(host.hostname(), - host.realResources(), - host.advertisedResources(), - host.requestedResources().orElse(NodeResources.unspecified()), - host.membership().get().retire(), + host.aliases(), + host.flavor(), + Optional.of(host.membership().get().retire()), host.version(), Optional.empty(), + Optional.empty(), host.dockerImageRepo()); } @@ -187,11 +189,11 @@ public class InMemoryProvisioner implements HostProvisioner { // Check if the current allocations are compatible with the new request for (int i = allocation.size() - 1; i >= 0; i--) { - NodeResources currentResources = allocation.get(0).advertisedResources(); - if (currentResources.isUnspecified() || requestedResources.isUnspecified()) continue; - if ( ! currentResources.compatibleWith(requestedResources)) { + Optional<NodeResources> currentResources = allocation.get(0).flavor().map(Flavor::resources); + if (currentResources.isEmpty() || requestedResources == NodeResources.unspecified) continue; + if (!currentResources.get().compatibleWith(requestedResources)) { HostSpec removed = allocation.remove(i); - freeNodes.put(currentResources, new Host(removed.hostname())); // Return the node back to free pool + freeNodes.put(currentResources.get(), new Host(removed.hostname())); // Return the node back to free pool } } @@ -200,7 +202,7 @@ public class InMemoryProvisioner implements HostProvisioner { // Find the smallest host that can fit the requested requested Optional<NodeResources> hostResources = freeNodes.keySet().stream() .sorted(new MemoryDiskCpu()) - .filter(resources -> requestedResources.isUnspecified() || resources.satisfies(requestedResources)) + .filter(resources -> requestedResources == NodeResources.unspecified || resources.satisfies(requestedResources)) .findFirst(); if (hostResources.isEmpty()) { if (canFail) @@ -212,11 +214,10 @@ public class InMemoryProvisioner implements HostProvisioner { Host newHost = freeNodes.removeValue(hostResources.get(), 0); if (freeNodes.get(hostResources.get()).isEmpty()) freeNodes.removeAll(hostResources.get()); ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++); - allocation.add(new HostSpec(newHost.hostname(), - hostResources.get(), hostResources.get(), requestedResources, - membership, + allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(), + hostResources.map(Flavor::new), Optional.of(membership), newHost.version(), Optional.empty(), - Optional.empty())); + requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources))); } nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex); diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java index b4fd55c6d33..1f0e0755667 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -1,21 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.provision; -import com.yahoo.component.Version; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.net.HostName; -import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification; import java.util.ArrayList; import java.util.List; -import java.util.Optional; /** * A host provisioner used when there is no hosts.xml file (using localhost as the only host) @@ -32,15 +28,12 @@ public class SingleNodeProvisioner implements HostProvisioner { public SingleNodeProvisioner() { host = new Host(HostName.getLocalhost()); - this.hostSpec = new HostSpec(host.hostname(), host.aliases(), Optional.empty()); + this.hostSpec = new HostSpec(host.hostname(), host.aliases()); } public SingleNodeProvisioner(Flavor flavor) { host = new Host(HostName.getLocalhost()); - this.hostSpec = new HostSpec(host.hostname(), - flavor.resources(), flavor.resources(), flavor.resources(), - ClusterMembership.from(ClusterSpec.specification(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).group(ClusterSpec.Group.from(0)).vespaVersion("1").build(), 0), - Optional.empty(), Optional.empty(), Optional.empty()); + this.hostSpec = new HostSpec(host.hostname(), host.aliases(), flavor); } @Override @@ -52,10 +45,7 @@ public class SingleNodeProvisioner implements HostProvisioner { public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1 List<HostSpec> hosts = new ArrayList<>(); - hosts.add(new HostSpec(host.hostname(), - NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), - ClusterMembership.from(cluster, counter++), - Optional.empty(), Optional.empty(), Optional.empty())); + hosts.add(new HostSpec(host.hostname(), host.aliases(), ClusterMembership.from(cluster, counter++))); return hosts; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 81931995485..9dba6fde9d4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -1,17 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; /** @@ -38,7 +44,7 @@ public class HostResource implements Comparable<HostResource> { * @param host {@link com.yahoo.vespa.model.Host} object to bind to. */ public HostResource(Host host) { - this(host, new HostSpec(host.getHostname(), List.of(), Optional.empty())); + this(host, new HostSpec(host.getHostname(), Optional.empty())); } public HostResource(Host host, HostSpec spec) { @@ -99,16 +105,8 @@ public class HostResource implements Comparable<HostResource> { } /** Returns the flavor of this resource. Empty for self-hosted Vespa. */ - // TODO: Remove after June 2020 - @Deprecated public Optional<Flavor> getFlavor() { return spec.flavor(); } - /** The real resources available for Vespa processes on this node, after subtracting infrastructure overhead. */ - public NodeResources realResources() { return spec.realResources(); } - - /** The total advertised resources of this node, typically matching what's requested. */ - public NodeResources advertisedResources() { return spec.advertisedResources(); } - @Override public String toString() { return "host '" + host.getHostname() + "'"; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 3d2918e0ee1..292d4761d8a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -120,7 +120,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { HostResource host = getExistingHost(spec).orElseGet(() -> addNewHost(spec)); retAllocatedHosts.put(host, spec.membership().orElse(null)); } - retAllocatedHosts.keySet().forEach(host -> log.log(FINE, () -> "Allocated host " + host.getHostname() + " with resources " + host.advertisedResources())); + retAllocatedHosts.keySet().forEach(host -> log.log(FINE, () -> "Allocated host " + host.getHostname() + " with flavor " + host.getFlavor())); return retAllocatedHosts; } @@ -131,7 +131,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { if (hosts.isEmpty()) { return Optional.empty(); } else { - log.log(FINE, () -> "Found existing host resource for " + key.hostname() + " with resources" + hosts.get(0).advertisedResources()); + log.log(FINE, () -> "Found existing host resource for " + key.hostname() + " with flavor " + hosts.get(0).getFlavor()); return Optional.of(hosts.get(0)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index 24b7b0949f6..5343a322382 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -44,8 +44,8 @@ public class ResourcesReductionValidator implements ChangeValidator { ClusterSpec.Id clusterId, ValidationOverrides overrides, Instant now) { - if (current.minResources().nodeResources().isUnspecified()) return; - if (next.minResources().nodeResources().isUnspecified()) return; + if (current.minResources().nodeResources() == NodeResources.unspecified) return; + if (next.minResources().nodeResources() == NodeResources.unspecified) return; List<String> illegalChanges = Stream.of( validateResource("vCPU", diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index 8f737f02dca..ea47e490b12 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -159,8 +159,8 @@ public class NodesSpecification { * Returns a requirement from <code>count</code> non-dedicated nodes in one group */ public static NodesSpecification nonDedicated(int count, ConfigModelContext context) { - return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified()), - new ClusterResources(count, 1, NodeResources.unspecified()), + return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified), + new ClusterResources(count, 1, NodeResources.unspecified), false, context.getDeployState().getWantedNodeVespaVersion(), false, @@ -172,8 +172,8 @@ public class NodesSpecification { /** Returns a requirement from <code>count</code> dedicated nodes in one group */ public static NodesSpecification dedicated(int count, ConfigModelContext context) { - return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified()), - new ClusterResources(count, 1, NodeResources.unspecified()), + return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified), + new ClusterResources(count, 1, NodeResources.unspecified), true, context.getDeployState().getWantedNodeVespaVersion(), false, @@ -224,7 +224,7 @@ public class NodesSpecification { return new Pair<>(flavorResources, flavorResources); } else { - return new Pair<>(NodeResources.unspecified(), NodeResources.unspecified()); + return new Pair<>(NodeResources.unspecified, NodeResources.unspecified); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 207e130eaaa..29bb578a67b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.container; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.Flavor; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; @@ -44,8 +45,8 @@ public final class ApplicationContainer extends Container implements @Override public void getConfig(QrStartConfig.Builder builder) { if (getHostResource() != null) { - if ( ! getHostResource().advertisedResources().isUnspecified()) { - NodeResourcesTuning flavorTuning = new NodeResourcesTuning(getHostResource().advertisedResources()); + if (getHostResource().getFlavor().isPresent()) { + NodeFlavorTuning flavorTuning = new NodeFlavorTuning(getHostResource().getFlavor().get()); flavorTuning.getConfig(builder); } } @@ -79,11 +80,11 @@ public final class ApplicationContainer extends Container implements @Override public void getConfig(ThreadpoolConfig.Builder builder) { if (! (parent instanceof ContainerCluster)) return; - if ((getHostResource() == null) || getHostResource().advertisedResources().isUnspecified()) return; + if ((getHostResource() == null) || getHostResource().getFlavor().isEmpty()) return; ContainerCluster containerCluster = (ContainerCluster) parent; if (containerCluster.getThreadPoolSizeFactor() <= 0.0) return; - NodeResourcesTuning flavorTuning = new NodeResourcesTuning(getHostResource().advertisedResources()) + NodeFlavorTuning flavorTuning = new NodeFlavorTuning(getHostResource().getFlavor().get()) .setThreadPoolSizeFactor(containerCluster.getThreadPoolSizeFactor()) .setQueueSizeFactor(containerCluster.getQueueSizeFactor()); flavorTuning.getConfig(builder); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java index 7eb7a1fb518..f9b50d0e641 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java @@ -1,25 +1,28 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; -import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.Flavor; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; /** - * Tuning of qr-start config for a container service based on node resources. + * Tuning of qr-start config for a container service based on the node flavor of that node. * * @author balder */ -public class NodeResourcesTuning implements QrStartConfig.Producer, ThreadpoolConfig.Producer { +public class NodeFlavorTuning implements + QrStartConfig.Producer, + ThreadpoolConfig.Producer +{ - private final NodeResources resources; + private final Flavor flavor; - public NodeResourcesTuning setThreadPoolSizeFactor(double threadPoolSizeFactor) { + public NodeFlavorTuning setThreadPoolSizeFactor(double threadPoolSizeFactor) { this.threadPoolSizeFactor = threadPoolSizeFactor; return this; } - public NodeResourcesTuning setQueueSizeFactor(double queueSizeFactor) { + public NodeFlavorTuning setQueueSizeFactor(double queueSizeFactor) { this.queueSizeFactor = queueSizeFactor; return this; } @@ -27,19 +30,19 @@ public class NodeResourcesTuning implements QrStartConfig.Producer, ThreadpoolCo private double threadPoolSizeFactor = 8.0; private double queueSizeFactor = 8.0; - NodeResourcesTuning(NodeResources resources) { - this.resources = resources; + NodeFlavorTuning(Flavor flavor) { + this.flavor = flavor; } @Override public void getConfig(QrStartConfig.Builder builder) { - builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(resources.vcpu()))); + builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(flavor.getMinCpuCores()))); } @Override public void getConfig(ThreadpoolConfig.Builder builder) { // Controls max number of concurrent requests per container - int workerThreads = Math.max(2, (int)Math.ceil(resources.vcpu() * threadPoolSizeFactor)); + int workerThreads = Math.max(2, (int)Math.ceil(flavor.getMinCpuCores() * threadPoolSizeFactor)); builder.maxthreads(workerThreads); // This controls your burst handling capability. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 559a4b8b668..741e5ebffd1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -664,7 +664,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .dockerImageRepository(deployState.getWantedDockerImageRepo()) .build(); int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1; - Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified()), + Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified), false, !deployState.getProperties().isBootstrap()); var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java index 028aee369c3..c8220071373 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java @@ -86,8 +86,9 @@ public class StorageNode extends ContentNode implements StorServerConfig.Produce @Override public void getConfig(StorFilestorConfig.Builder builder) { - if (getHostResource() != null && ! getHostResource().advertisedResources().isUnspecified()) { - builder.num_threads(Math.max(4, (int)getHostResource().advertisedResources().vcpu())); + if (getHostResource() != null && getHostResource().getFlavor().isPresent()) { + Flavor nodeFlavor = getHostResource().getFlavor().get(); + builder.num_threads(Math.max(4, (int)nodeFlavor.getMinCpuCores())); } cluster.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java index ee854aaa9c3..0998d583d6c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java @@ -1,31 +1,31 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; -import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.Flavor; import com.yahoo.vespa.config.search.core.ProtonConfig; import static java.lang.Long.min; /** - * Tuning of proton config for a search node based on the resources on the node. + * Tuning of proton config for a search node based on the node flavor of that node. * * @author geirst */ -public class NodeResourcesTuning implements ProtonConfig.Producer { +public class NodeFlavorTuning implements ProtonConfig.Producer { - final static long MB = 1024 * 1024; - final static long GB = MB * 1024; - private final NodeResources resources; + static long MB = 1024 * 1024; + static long GB = MB * 1024; + private final Flavor nodeFlavor; private final int redundancy; private final int searchableCopies; private final int threadsPerSearch; - public NodeResourcesTuning(NodeResources resources, int redundancy, int searchableCopies) { - this(resources, redundancy, searchableCopies, 1); + public NodeFlavorTuning(Flavor nodeFlavor, int redundancy, int searchableCopies) { + this(nodeFlavor, redundancy, searchableCopies, 1); } - public NodeResourcesTuning(NodeResources resources, int redundancy, int searchableCopies, int threadsPerSearch) { - this.resources = resources; + public NodeFlavorTuning(Flavor nodeFlavor, int redundancy, int searchableCopies, int threadsPerSearch) { + this.nodeFlavor = nodeFlavor; this.redundancy = redundancy; this.searchableCopies = searchableCopies; this.threadsPerSearch = threadsPerSearch; @@ -51,31 +51,31 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private void getConfig(ProtonConfig.Documentdb.Builder builder) { ProtonConfig.Documentdb dbCfg = builder.build(); if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) { - long numDocs = (long)resources.memoryGb() * GB / 64L; + long numDocs = (long)nodeFlavor.getMinMainMemoryAvailableGb()*GB/64L; builder.allocation.initialnumdocs(numDocs/Math.max(searchableCopies, redundancy)); } } private void tuneSummaryCache(ProtonConfig.Summary.Cache.Builder builder) { - long memoryLimitBytes = (long) ((resources.memoryGb() * 0.05) * GB); + long memoryLimitBytes = (long) ((nodeFlavor.getMinMainMemoryAvailableGb() * 0.05) * GB); builder.maxbytes(memoryLimitBytes); } private void setHwInfo(ProtonConfig.Builder builder) { - builder.hwinfo.disk.shared(true); - builder.hwinfo.cpu.cores((int)resources.vcpu()); - builder.hwinfo.memory.size((long)resources.memoryGb() * GB); - builder.hwinfo.disk.size((long)resources.diskGb() * GB); + builder.hwinfo.disk.shared(nodeFlavor.getType().equals(Flavor.Type.DOCKER_CONTAINER)); + builder.hwinfo.cpu.cores((int)nodeFlavor.getMinCpuCores()); + builder.hwinfo.memory.size((long)nodeFlavor.resources().memoryGb() * GB); + builder.hwinfo.disk.size((long)nodeFlavor.resources().diskGb() * GB); } private void tuneDiskWriteSpeed(ProtonConfig.Builder builder) { - if (resources.diskSpeed() != NodeResources.DiskSpeed.fast) { + if (!nodeFlavor.hasFastDisk()) { builder.hwinfo.disk.writespeed(40); } } private void tuneDocumentStoreMaxFileSize(ProtonConfig.Summary.Log.Builder builder) { - double memoryGb = resources.memoryGb(); + double memoryGb = nodeFlavor.getMinMainMemoryAvailableGb(); long fileSizeBytes = 4 * GB; if (memoryGb <= 12.0) { fileSizeBytes = 256 * MB; @@ -88,31 +88,31 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneFlushStrategyMemoryLimits(ProtonConfig.Flush.Memory.Builder builder) { - long memoryLimitBytes = (long) ((resources.memoryGb() / 8) * GB); + long memoryLimitBytes = (long) ((nodeFlavor.getMinMainMemoryAvailableGb() / 8) * GB); builder.maxmemory(memoryLimitBytes); builder.each.maxmemory(memoryLimitBytes); } private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { - long tlsSizeBytes = (long) ((resources.diskGb() * 0.07) * GB); + long tlsSizeBytes = (long) ((nodeFlavor.getMinDiskAvailableGb() * 0.07) * GB); tlsSizeBytes = min(tlsSizeBytes, 100 * GB); builder.maxtlssize(tlsSizeBytes); } private void tuneSummaryReadIo(ProtonConfig.Summary.Read.Builder builder) { - if (resources.diskSpeed() == NodeResources.DiskSpeed.fast) { + if (nodeFlavor.hasFastDisk()) { builder.io(ProtonConfig.Summary.Read.Io.DIRECTIO); } } private void tuneSearchReadIo(ProtonConfig.Search.Mmap.Builder builder) { - if (resources.diskSpeed() == NodeResources.DiskSpeed.fast) { + if (nodeFlavor.hasFastDisk()) { builder.advise(ProtonConfig.Search.Mmap.Advise.RANDOM); } } private void tuneRequestThreads(ProtonConfig.Builder builder) { - int numCores = (int)Math.ceil(resources.vcpu()); + int numCores = (int)Math.ceil(nodeFlavor.getMinCpuCores()); builder.numsearcherthreads(numCores*threadsPerSearch); builder.numsummarythreads(numCores); builder.numthreadspersearch(threadsPerSearch); @@ -122,7 +122,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { // "Reserve" 1GB of memory for other processes running on the content node (config-proxy, cluster-controller, metrics-proxy) double reservedMemoryGb = 1; double defaultMemoryLimit = new ProtonConfig.Writefilter(new ProtonConfig.Writefilter.Builder()).memorylimit(); - double scaledMemoryLimit = ((resources.memoryGb() - reservedMemoryGb) * defaultMemoryLimit) / resources.memoryGb(); + double scaledMemoryLimit = ((nodeFlavor.getMinMainMemoryAvailableGb() - reservedMemoryGb) * defaultMemoryLimit) / nodeFlavor.getMinMainMemoryAvailableGb(); builder.memorylimit(scaledMemoryLimit); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index f908d091a72..373c62c2eda 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.search; import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.Flavor; import com.yahoo.metrics.MetricsmanagerConfig; import com.yahoo.searchlib.TranslogserverConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; @@ -269,11 +270,12 @@ public class SearchNode extends AbstractService implements // to make sure the node failer has done its work builder.pruneremoveddocumentsage(4 * 24 * 3600 + 3600 + 60); } - if (getHostResource() != null && ! getHostResource().advertisedResources().isUnspecified()) { - NodeResourcesTuning nodeResourcesTuning = tuning.isPresent() - ? new NodeResourcesTuning(getHostResource().advertisedResources(), redundancy, searchableCopies, tuning.get().getNumThreadsPerSearch()) - : new NodeResourcesTuning(getHostResource().advertisedResources(), redundancy, searchableCopies); - nodeResourcesTuning.getConfig(builder); + if (getHostResource() != null && getHostResource().getFlavor().isPresent()) { + Flavor nodeFlavor = getHostResource().getFlavor().get(); + NodeFlavorTuning nodeFlavorTuning = tuning.isPresent() + ? new NodeFlavorTuning(nodeFlavor, redundancy, searchableCopies, tuning.get().getNumThreadsPerSearch()) + : new NodeFlavorTuning(nodeFlavor, redundancy, searchableCopies); + nodeFlavorTuning.getConfig(builder); if (tuning.isPresent()) { tuning.get().getConfig(builder); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java index 91f5fdc5f11..51b039a7532 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java @@ -6,8 +6,6 @@ import org.junit.Test; import java.util.Arrays; import java.util.Collections; -import java.util.List; -import java.util.Optional; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -19,10 +17,10 @@ public class HostSpecTest { @Test public void testEquals() { - HostSpec h1 = new HostSpec("foo", List.of(), Optional.empty()); - HostSpec h2 = new HostSpec("foo", List.of(), Optional.empty()); - HostSpec h3 = new HostSpec("foo", List.of("my", "alias"), Optional.empty()); - HostSpec h4 = new HostSpec("bar", List.of(), Optional.empty()); + HostSpec h1 = new HostSpec("foo", Collections.<String>emptyList()); + HostSpec h2 = new HostSpec("foo", Collections.<String>emptyList()); + HostSpec h3 = new HostSpec("foo", Arrays.asList("my", "alias")); + HostSpec h4 = new HostSpec("bar", Collections.<String>emptyList()); assertTrue(h1.equals(h1)); assertTrue(h1.equals(h2)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java index dff1d338ffe..6aea0593f8a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -6,10 +6,8 @@ import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.NodeResources; import org.junit.Test; -import java.util.List; import java.util.Optional; import static com.yahoo.config.provision.ClusterSpec.Type.container; @@ -55,10 +53,7 @@ public class HostResourceTest { private static HostResource hostResourceWithMemberships(ClusterMembership membership) { return new HostResource(Host.createHost(null, "hostname"), - new HostSpec("hostname", - NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), - membership, - Optional.empty(), Optional.empty(), Optional.empty())); + new HostSpec("hostname", Optional.of(membership))); } private static int counter = 0; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java index a9bf8bdcc49..ac77b821d4a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java @@ -17,9 +17,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ProvisionLogger; -import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification; import org.junit.Before; import org.junit.Test; @@ -105,17 +103,17 @@ public class VespaModelFactoryTest { @Override public HostSpec allocateHost(String alias) { return new HostSpec(hostName, - NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), - ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.admin, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), 0), - Optional.empty(), Optional.empty(), Optional.empty()); + List.of(), + ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.admin, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), + 0)); } @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { return List.of(new HostSpec(hostName, - NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), - ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), 0), - Optional.empty(), Optional.empty(), Optional.empty())); + List.of(), + ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), + 0))); } }; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 9a59907a230..0f94df80421 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -242,12 +242,12 @@ public class ContainerClusterTest { @Test public void requireThatPoolAndQueueCanBeControlledByPropertiesAndFlavor() { FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder().name("my_flavor").minCpuCores(3); - NodeResourcesTuning nodeResourcesTuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources()) + NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder))) .setThreadPoolSizeFactor(13.3) .setQueueSizeFactor(17.5); ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); - nodeResourcesTuning.getConfig(tpBuilder); + nodeFlavorTuning.getConfig(tpBuilder); ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(40, threadpoolConfig.maxthreads()); assertEquals(700, threadpoolConfig.queueSize()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index fdd7ae57f0f..ca3198637f3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -766,7 +766,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertEquals("KMP_SETTING=1 KMP_AFFINITY=granularity=fine,verbose,compact,1,0 ", qrStartConfig.qrs().env()); } - private void verifyAvailableprocessors(boolean isHosted, Flavor flavor, int expectProcessors) { + private void verifyAvailableprocessors(boolean isHosted, Flavor flavor, int expectProcessors) throws IOException, SAXException { DeployState deployState = new DeployState.Builder() .modelHostProvisioner(flavor != null ? new SingleNodeProvisioner(flavor) : new SingleNodeProvisioner()) .properties(new TestProperties() @@ -790,7 +790,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void requireThatAvailableProcessorsFollowFlavor() { + public void requireThatAvailableProcessorsFollowFlavor() throws IOException, SAXException { verifyAvailableprocessors(false, null,0); verifyAvailableprocessors(true, null,0); verifyAvailableprocessors(true, new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()), 9); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java index fee0edacb90..023b7249939 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java @@ -11,13 +11,13 @@ import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; -import static com.yahoo.vespa.model.search.NodeResourcesTuning.MB; -import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; +import static com.yahoo.vespa.model.search.NodeFlavorTuning.MB; +import static com.yahoo.vespa.model.search.NodeFlavorTuning.GB; /** * @author geirst */ -public class NodeResourcesTuningTest { +public class NodeFlavorTuningTest { private static double delta = 0.00001; @@ -152,6 +152,7 @@ public class NodeResourcesTuningTest { @Test public void require_that_docker_node_is_tagged_with_shared_disk() { assertSharedDisk(true, true); + assertSharedDisk(false, false); } @Test @@ -231,13 +232,13 @@ public class NodeResourcesTuningTest { } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int redundancy, int searchableCopies) { flavorBuilder.name("my_flavor"); - NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), redundancy, searchableCopies); + NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)), redundancy, searchableCopies); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int redundancy, int searchableCopies, int numThreadsPerSearch) { flavorBuilder.name("my_flavor"); - NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), redundancy, searchableCopies, numThreadsPerSearch); + NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)), redundancy, searchableCopies, numThreadsPerSearch); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index 1a1cd243f89..f723575c342 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -90,7 +90,7 @@ public final class Capacity { /** Creates this from a node type */ public static Capacity fromRequiredNodeType(NodeType type) { - return from(new ClusterResources(0, 0, NodeResources.unspecified()), true, false, type); + return from(new ClusterResources(0, 0, NodeResources.unspecified), true, false, type); } private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index c8ec45b0ec8..8d767e9f4ad 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -109,25 +109,15 @@ public class Flavor { public Optional<FlavorOverrides> flavorOverrides() { return flavorOverrides; } - // TODO: Remove after June 2020 - @Deprecated public double getMinMainMemoryAvailableGb() { return resources.memoryGb(); } - // TODO: Remove after June 2020 - @Deprecated public double getMinDiskAvailableGb() { return resources.diskGb(); } - // TODO: Remove after June 2020 - @Deprecated public boolean hasFastDisk() { return resources.diskSpeed() == NodeResources.DiskSpeed.fast; } - // TODO: Remove after June 2020 - @Deprecated public double getBandwidthGbps() { return resources.bandwidthGbps(); } /** Returns the number of cores available in this flavor, not scaled for speed. */ - // TODO: Remove after June 2020 - @Deprecated public double getMinCpuCores() { return minCpuCores; } public Type getType() { return type; } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index 52a2008b658..2a5d27a0fe7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -22,167 +22,73 @@ public class HostSpec implements Comparable<HostSpec> { /** Aliases of this host */ private final List<String> aliases; - private final NodeResources realResources; - private final NodeResources advertisedResources; - private final NodeResources requestedResources; - /** The current membership role of this host in the cluster it belongs to */ private final Optional<ClusterMembership> membership; + private final Optional<Flavor> flavor; + private final Optional<Version> version; private final Optional<DockerImage> dockerImageRepo; private final Optional<NetworkPorts> networkPorts; - /** Create a host in a non-cloud system, where hosts are specified in config */ - public HostSpec(String hostname, List<String> aliases, Optional<NetworkPorts> networkPorts) { - this(hostname, aliases, - NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), - Optional.empty(), Optional.empty(), networkPorts, Optional.empty()); - } - - /** Create a host in a hosted system */ - public HostSpec(String hostname, - NodeResources realResources, - NodeResources advertisedResurces, - NodeResources requestedResources, - ClusterMembership membership, - Optional<Version> version, - Optional<NetworkPorts> networkPorts, - Optional<DockerImage> dockerImageRepo) { - this(hostname, List.of(), - realResources, - advertisedResurces, - requestedResources, - Optional.of(membership), - version, - networkPorts, - dockerImageRepo); - } - - private HostSpec(String hostname, - List<String> aliases, - NodeResources realResources, - NodeResources advertisedResurces, - NodeResources requestedResources, - Optional<ClusterMembership> membership, - Optional<Version> version, - Optional<NetworkPorts> networkPorts, - Optional<DockerImage> dockerImageRepo) { - if (hostname == null || hostname.isEmpty()) throw new IllegalArgumentException("Hostname must be specified"); - this.hostname = hostname; - this.aliases = List.copyOf(aliases); - this.realResources = Objects.requireNonNull(realResources); - this.advertisedResources = Objects.requireNonNull(advertisedResurces); - this.requestedResources = Objects.requireNonNull(requestedResources, "RequestedResources cannot be null"); - this.membership = Objects.requireNonNull(membership); - this.version = Objects.requireNonNull(version, "Version cannot be null but can be empty"); - this.networkPorts = Objects.requireNonNull(networkPorts, "Network ports cannot be null but can be empty"); - this.dockerImageRepo = Objects.requireNonNull(dockerImageRepo, "Docker image repo cannot be null but can be empty"); - } + private final Optional<NodeResources> requestedResources; - // TODO: Remove after June 2020 - @Deprecated public HostSpec(String hostname, Optional<ClusterMembership> membership) { - this(hostname, new ArrayList<>(), - NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), - membership, - Optional.empty(), Optional.empty(), Optional.empty()); + this(hostname, new ArrayList<>(), Optional.empty(), membership); } - // TODO: Remove after June 2020 - @Deprecated public HostSpec(String hostname, ClusterMembership membership, Flavor flavor, Optional<Version> version) { - this(hostname, new ArrayList<>(), - flavor.resources(), flavor.resources(), NodeResources.unspecified(), - Optional.of(membership), version, Optional.empty(), Optional.empty()); + this(hostname, new ArrayList<>(), Optional.of(flavor), Optional.of(membership), version); } - // TODO: Remove after June 2020 - @Deprecated public HostSpec(String hostname, List<String> aliases) { - this(hostname, aliases, - NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), - Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + this(hostname, aliases, Optional.empty(), Optional.empty()); } - // TODO: Remove after June 2020 - @Deprecated public HostSpec(String hostname, List<String> aliases, Flavor flavor) { - this(hostname, aliases, - flavor.resources(), flavor.resources(), NodeResources.unspecified(), - Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + this(hostname, aliases, Optional.of(flavor), Optional.empty()); } - // TODO: Remove after June 2020 - @Deprecated public HostSpec(String hostname, List<String> aliases, ClusterMembership membership) { - this(hostname, aliases, - NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), - Optional.of(membership), - Optional.empty(), Optional.empty(), Optional.empty()); + this(hostname, aliases, Optional.empty(), Optional.of(membership)); } - // TODO: Remove after June 2020 - @Deprecated public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership) { - this(hostname, aliases, - flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), - flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), - NodeResources.unspecified(), - membership, Optional.empty(), Optional.empty(), Optional.empty()); + this(hostname, aliases, flavor, membership, Optional.empty()); } - // TODO: Remove after June 2020 - @Deprecated public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<Version> version) { - this(hostname, aliases, - flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), - flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), - NodeResources.unspecified(), - membership, version, - Optional.empty(), Optional.empty()); + this(hostname, aliases, flavor, membership, version, Optional.empty()); } - // TODO: Remove after June 2020 - @Deprecated public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<Version> version, Optional<NetworkPorts> networkPorts) { - this(hostname, aliases, - flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), - flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), - NodeResources.unspecified(), - membership, version, networkPorts, - Optional.empty()); + this(hostname, aliases, flavor, membership, version, networkPorts, Optional.empty()); } - // TODO: Remove after June 2020 - @Deprecated - public HostSpec(String hostname, List<String> aliases, - Optional<Flavor> flavor, + public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<Version> version, Optional<NetworkPorts> networkPorts, Optional<NodeResources> requestedResources) { - this(hostname, aliases, - flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), - flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), - requestedResources.orElse(NodeResources.unspecified()), - membership, version, networkPorts, Optional.empty()); + this(hostname, aliases, flavor, membership, version, networkPorts, requestedResources, Optional.empty()); } - // TODO: Remove after June 2020 - @Deprecated public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<Version> version, Optional<NetworkPorts> networkPorts, Optional<NodeResources> requestedResources, Optional<DockerImage> dockerImageRepo) { - this(hostname, aliases, - flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), - flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), - requestedResources.orElse(NodeResources.unspecified()), - membership, version, networkPorts, dockerImageRepo); + if (hostname == null || hostname.isEmpty()) throw new IllegalArgumentException("Hostname must be specified"); + this.hostname = hostname; + this.aliases = List.copyOf(aliases); + this.flavor = flavor; + this.membership = membership; + this.version = Objects.requireNonNull(version, "Version cannot be null but can be empty"); + this.networkPorts = Objects.requireNonNull(networkPorts, "Network ports cannot be null but can be empty"); + this.requestedResources = Objects.requireNonNull(requestedResources, "RequestedResources cannot be null"); + this.dockerImageRepo = Objects.requireNonNull(dockerImageRepo, "Docker image repo cannot be null but can be empty"); } /** Returns the name identifying this host */ @@ -191,17 +97,7 @@ public class HostSpec implements Comparable<HostSpec> { /** Returns the aliases of this host as an immutable list. This may be empty but never null. */ public List<String> aliases() { return aliases; } - /** The real resources available for Vespa processes on this node, after subtracting infrastructure overhead. */ - public NodeResources realResources() { return realResources; } - - /** The total advertised resources of this node, typically matching what's requested. */ - public NodeResources advertisedResources() { return advertisedResources; } - - /** A flavor contained the advertised resources of this host */ - // TODO: Remove after June 2020 - public Optional<Flavor> flavor() { - return advertisedResources.asOptional().map(resources -> new Flavor(resources)); - } + public Optional<Flavor> flavor() { return flavor; } /** Returns the current version of Vespa running on this node, or empty if not known */ public Optional<com.yahoo.component.Version> version() { return version; } @@ -212,13 +108,13 @@ public class HostSpec implements Comparable<HostSpec> { /** Returns the network port allocations on this host, or empty if not present */ public Optional<NetworkPorts> networkPorts() { return networkPorts; } - /** Returns the requested resources leading to this host being provisioned, or empty if unspecified */ - public Optional<NodeResources> requestedResources() { return requestedResources.asOptional(); } + /** Returns the requested resources leading to this host being provisioned, or empty if not known */ + public Optional<NodeResources> requestedResources() { return requestedResources; } public Optional<DockerImage> dockerImageRepo() { return dockerImageRepo; } public HostSpec withPorts(Optional<NetworkPorts> ports) { - return new HostSpec(hostname, aliases, realResources, advertisedResources, requestedResources, membership, version, ports, dockerImageRepo); + return new HostSpec(hostname, aliases, flavor, membership, version, ports, requestedResources, dockerImageRepo); } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index e02aa7a5f22..91604cd667d 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -3,7 +3,6 @@ package com.yahoo.config.provision; import java.util.Locale; import java.util.Objects; -import java.util.Optional; /** * The node resources required by an application cluster @@ -17,7 +16,6 @@ public class NodeResources { private static final double memoryUnitCost = 0.012; private static final double diskUnitCost = 0.0004; - // TODO: Make private after June 2020 public static final NodeResources unspecified = new NodeResources(0, 0, 0, 0); public enum DiskSpeed { @@ -254,14 +252,7 @@ public class NodeResources { return true; } - public static NodeResources unspecified() { return unspecified; } - - public boolean isUnspecified() { return this.equals(unspecified); } - - /** Returns this.isUnspecified() ? Optional.empty() : Optional.of(this) */ - public Optional<NodeResources> asOptional() { - return this.isUnspecified() ? Optional.empty() : Optional.of(this); - } + public boolean isUnspecified() { return this == unspecified; } private boolean equal(double a, double b) { return Math.abs(a - b) < 0.00000001; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index 96b189cefe6..9ba26be072c 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -45,9 +45,6 @@ public class AllocatedHostsSerializer { private static final String aliasesKey = "aliases"; private static final String hostSpecMembershipKey = "membership"; - private static final String realResourcesKey = "realResources"; - private static final String advertisedResourcesKey = "advertisedResources"; - // Flavor can be removed when all allocated nodes are docker nodes private static final String flavorKey = "flavor"; @@ -93,9 +90,7 @@ public class AllocatedHostsSerializer { object.setString(hostSpecDockerImageRepoKey, repo.repository()); }); }); - host.flavor().ifPresent(flavor -> toSlime(flavor, object)); // TODO: Remove this line after June 2020 - toSlime(host.realResources(), object.setObject(realResourcesKey)); - toSlime(host.advertisedResources(), object.setObject(advertisedResourcesKey)); + host.flavor().ifPresent(flavor -> toSlime(flavor, object)); host.requestedResources().ifPresent(resources -> toSlime(resources, object.setObject(requestedResourcesKey))); host.version().ifPresent(version -> object.setString(hostSpecCurrentVespaVersionKey, version.toFullString())); host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray(hostSpecNetworkPortsKey))); @@ -138,25 +133,18 @@ public class AllocatedHostsSerializer { } private static HostSpec hostFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { - if (object.field(hostSpecMembershipKey).valid()) { // Hosted - return new HostSpec(object.field(hostSpecHostNameKey).asString(), - nodeResourcesFromSlime(object.field(realResourcesKey), object, nodeFlavors), - nodeResourcesFromSlime(object.field(advertisedResourcesKey), object, nodeFlavors), - optionalNodeResourcesFromSlime(object.field(requestedResourcesKey)), // TODO: Make non-optional after June 2020 - membershipFromSlime(object), - optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new), - NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)), - optionalDockerImage(object.field(hostSpecDockerImageRepoKey))); - } - else { - return new HostSpec(object.field(hostSpecHostNameKey).asString(), - aliasesFromSlime(object), - NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey))); - } + return new HostSpec(object.field(hostSpecHostNameKey).asString(), + aliasesFromSlime(object), + flavorFromSlime(object, nodeFlavors), + object.field(hostSpecMembershipKey).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(), + optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new), + NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)), + nodeResourcesFromSlime(object.field(requestedResourcesKey)), + optionalDockerImage(object.field(hostSpecDockerImageRepoKey))); } private static List<String> aliasesFromSlime(Inspector object) { - if ( ! object.field(aliasesKey).valid()) return List.of(); + if ( ! object.field(aliasesKey).valid()) return Collections.emptyList(); List<String> aliases = new ArrayList<>(); object.field(aliasesKey).traverse((ArrayTraverser)(index, alias) -> aliases.add(alias.asString())); return aliases; @@ -166,28 +154,17 @@ public class AllocatedHostsSerializer { if (object.field(flavorKey).valid() && nodeFlavors.isPresent() && nodeFlavors.get().exists(object.field(flavorKey).asString())) return nodeFlavors.get().getFlavor(object.field(flavorKey).asString()); else - return Optional.empty(); - } - - private static NodeResources nodeResourcesFromSlime(Inspector resources) { - return new NodeResources(resources.field(vcpuKey).asDouble(), - resources.field(memoryKey).asDouble(), - resources.field(diskKey).asDouble(), - resources.field(bandwidthKey).asDouble(), - diskSpeedFromSlime(resources.field(diskSpeedKey)), - storageTypeFromSlime(resources.field(storageTypeKey))); - } - - private static NodeResources optionalNodeResourcesFromSlime(Inspector resources) { - if ( ! resources.valid()) return NodeResources.unspecified(); - return nodeResourcesFromSlime(resources); + return nodeResourcesFromSlime(object.field(resourcesKey)).map(resources -> new Flavor(resources)); } - private static NodeResources nodeResourcesFromSlime(Inspector resources, Inspector parent, - Optional<NodeFlavors> nodeFlavors) { - if ( ! resources.valid()) // TODO: Remove the fallback using nodeFlavors after June 2020 - return flavorFromSlime(parent, nodeFlavors).map(f -> f.resources()).orElse(NodeResources.unspecified); - return nodeResourcesFromSlime(resources); + private static Optional<NodeResources> nodeResourcesFromSlime(Inspector resources) { + if ( ! resources.valid()) return Optional.empty(); + return Optional.of(new NodeResources(resources.field(vcpuKey).asDouble(), + resources.field(memoryKey).asDouble(), + resources.field(diskKey).asDouble(), + resources.field(bandwidthKey).asDouble(), + diskSpeedFromSlime(resources.field(diskSpeedKey)), + storageTypeFromSlime(resources.field(storageTypeKey)))); } private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java index e184e85c399..31eec77de6b 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java @@ -34,7 +34,7 @@ public class NodeFlavorsTest { NodeFlavors nodeFlavors = new NodeFlavors(config); Flavor banana = nodeFlavors.getFlavor("banana").get(); assertEquals(3, banana.cost()); - assertEquals(13, banana.resources().vcpu(), delta); + assertEquals(10, banana.getMinCpuCores(), delta); assertEquals("10 * 1.3", 13, banana.resources().vcpu(), delta); } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java index 30810b79104..33157a01c07 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java @@ -14,6 +14,7 @@ import com.yahoo.config.provisioning.FlavorsConfig; import org.junit.Test; import java.io.IOException; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; @@ -32,44 +33,34 @@ public class AllocatedHostsSerializerTest { public void testAllocatedHostsSerialization() throws IOException { NodeFlavors configuredFlavors = configuredFlavorsFrom("C/12/45/100", 12, 45, 100, 50, Flavor.Type.BARE_METAL); Set<HostSpec> hosts = new LinkedHashSet<>(); - hosts.add(new HostSpec("empty", List.of(), Optional.empty())); - hosts.add(new HostSpec("with-aliases", List.of("alias1", "alias2"), Optional.empty())); + hosts.add(new HostSpec("empty", + Optional.empty())); + hosts.add(new HostSpec("with-aliases", + List.of("alias1", "alias2"))); hosts.add(new HostSpec("allocated", - NodeResources.unspecified(), - NodeResources.unspecified(), - NodeResources.unspecified(), - ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), - Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))), - Optional.empty(), + List.of(), Optional.empty(), + Optional.of(ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), + Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar")))), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar")))); + hosts.add(new HostSpec("flavor-from-resources-1", + Collections.emptyList(), new Flavor(new NodeResources(0.5, 3.1, 4, 1)))); hosts.add(new HostSpec("flavor-from-resources-2", - new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.slow), - new NodeResources(1.0, 6.2, 8, 2, NodeResources.DiskSpeed.slow), - new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.any), - ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), - Optional.empty()), + Collections.emptyList(), + Optional.of(new Flavor(new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.slow))), Optional.empty(), Optional.empty(), - Optional.empty())); + Optional.empty(), + Optional.of(new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.any)))); + hosts.add(new HostSpec("configured-flavor", + Collections.emptyList(), configuredFlavors.getFlavorOrThrow("C/12/45/100"))); hosts.add(new HostSpec("with-version", - NodeResources.unspecified(), - NodeResources.unspecified(), - NodeResources.unspecified(), - ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), - Optional.empty()), - Optional.of(Version.fromString("3.4.5")), - Optional.empty(), Optional.empty())); + Collections.emptyList(), Optional.empty(), Optional.empty(), Optional.of(Version.fromString("3.4.5")))); hosts.add(new HostSpec("with-ports", - NodeResources.unspecified(), - NodeResources.unspecified(), - NodeResources.unspecified(), - ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), - Optional.empty()), - Optional.empty(), + Collections.emptyList(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(new NetworkPorts(List.of(new NetworkPorts.Allocation(1234, "service1", "configId1", "suffix1"), - new NetworkPorts.Allocation(4567, "service2", "configId2", "suffix2")))), - Optional.empty())); + new NetworkPorts.Allocation(4567, "service2", "configId2", "suffix2")))))); assertAllocatedHosts(AllocatedHosts.withHosts(hosts), configuredFlavors); } @@ -82,12 +73,11 @@ public class AllocatedHostsSerializerTest { HostSpec deserializedHost = requireHost(expectedHost.hostname(), deserializedHosts); assertEquals(expectedHost.hostname(), deserializedHost.hostname()); assertEquals(expectedHost.membership(), deserializedHost.membership()); - assertEquals(expectedHost.realResources(), deserializedHost.realResources()); - assertEquals(expectedHost.advertisedResources(), deserializedHost.advertisedResources()); - assertEquals(expectedHost.requestedResources(), deserializedHost.requestedResources()); + assertEquals(expectedHost.flavor(), deserializedHost.flavor()); assertEquals(expectedHost.version(), deserializedHost.version()); assertEquals(expectedHost.networkPorts(), deserializedHost.networkPorts()); assertEquals(expectedHost.aliases(), deserializedHost.aliases()); + assertEquals(expectedHost.requestedResources(), deserializedHost.requestedResources()); assertEquals(expectedHost.dockerImageRepo(), deserializedHost.dockerImageRepo()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 4d0253cd3f8..8ee76843e94 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -422,9 +422,7 @@ public class ApplicationRepositoryTest { assertNotNull(info); assertThat(info.getHosts().size(), is(1)); System.out.println(info.getHosts()); - assertTrue(info.getHosts().contains(new HostSpec("mytesthost", - Collections.emptyList(), - Optional.empty()))); + assertTrue(info.getHosts().contains(new HostSpec("mytesthost", Collections.emptyList()))); Optional<NetworkPorts> portsCopy = info.getHosts().iterator().next().networkPorts(); assertTrue(portsCopy.isPresent()); assertThat(portsCopy.get().allocations(), is(list)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index 8394611737e..14fa0cb2dbe 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -193,8 +193,8 @@ public class ZooKeeperClientTest { Path app = Path.fromString("/1"); ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, true, app); zooKeeperClient.setupZooKeeper(); - HostSpec host1 = new HostSpec("host1.yahoo.com", Collections.emptyList(), Optional.empty()); - HostSpec host2 = new HostSpec("host2.yahoo.com", Collections.emptyList(), Optional.empty()); + HostSpec host1 = new HostSpec("host1.yahoo.com", Collections.emptyList()); + HostSpec host2 = new HostSpec("host2.yahoo.com", Collections.emptyList()); ImmutableSet<HostSpec> hosts = ImmutableSet.of(host1, host2); zooKeeperClient.write(AllocatedHosts.withHosts(hosts)); Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 7a76b70dc27..6ba3c33d37b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -9,7 +9,9 @@ import com.yahoo.config.model.application.provider.DeployData; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -56,7 +58,6 @@ import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.Optional; -import java.util.Set; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.CONFLICT; @@ -308,7 +309,7 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { ActivateRequest invoke() throws Exception { SessionZooKeeperClient zkClient = new MockSessionZKClient(curator, tenantName, sessionId, - Optional.of(AllocatedHosts.withHosts(Set.of(new HostSpec("bar", Collections.emptyList(), Optional.empty()))))); + Optional.of(AllocatedHosts.withHosts(Collections.singleton(new HostSpec("bar", Collections.emptyList()))))); session = createRemoteSession(sessionId, initialStatus, zkClient); addLocalSession(sessionId, deployData, zkClient); tenantRepository.getTenant(tenantName).getApplicationRepo().createApplication(deployData.getApplicationId()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 4397e087fb7..9c7da7134e6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -5,12 +5,10 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -41,14 +39,9 @@ public class ZKApplicationPackageTest { private static final String TEST_FLAVOR_NAME = "test-flavor"; private static final Optional<Flavor> TEST_FLAVOR = new MockNodeFlavors().getFlavor(TEST_FLAVOR_NAME); private static final AllocatedHosts ALLOCATED_HOSTS = AllocatedHosts.withHosts( - Collections.singleton(new HostSpec("foo.yahoo.com", - TEST_FLAVOR.get().resources(), - TEST_FLAVOR.get().resources(), - NodeResources.unspecified(), - ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), - Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))), + Collections.singleton(new HostSpec("foo.yahoo.com", Collections.emptyList(), TEST_FLAVOR, Optional.empty(), Optional.of(Version.fromString("6.0.1")), Optional.empty(), - Optional.of(DockerImage.fromString("docker repo"))))); + Optional.empty(), Optional.of(DockerImage.fromString("docker repo"))))); private ConfigCurator configCurator; @@ -86,8 +79,8 @@ public class ZKApplicationPackageTest { assertFalse(zkApp.getFileRegistries().containsKey(new Version(0, 0, 0))); assertThat(zkApp.getFileRegistries().get(goodVersion).fileSourceHost(), is("dummyfiles")); AllocatedHosts readInfo = zkApp.getAllocatedHosts().get(); - assertEquals(Utf8.toString(toJson(ALLOCATED_HOSTS)), Utf8.toString(toJson(readInfo))); - assertEquals(TEST_FLAVOR.get().resources(), readInfo.getHosts().iterator().next().advertisedResources()); + assertThat(Utf8.toString(toJson(readInfo)), is(Utf8.toString(toJson(ALLOCATED_HOSTS)))); + assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR)); assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); // TODO: Enable when dockerImageRepo is written to zk //assertEquals("docker repo", readInfo.getHosts().iterator().next().dockerImageRepo().get()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 475f2feaebd..f0f53161d9b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -40,8 +40,8 @@ public class AllocationOptimizer { AllocatableClusterResources current, Limits limits) { if (limits.isEmpty()) - limits = Limits.of(new ClusterResources(minimumNodes, 1, NodeResources.unspecified()), - new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified())); + limits = Limits.of(new ClusterResources(minimumNodes, 1, NodeResources.unspecified), + new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified)); Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); for (int groups = limits.min().groups(); groups <= limits.max().groups(); groups++) { for (int nodes = limits.min().nodes(); nodes <= limits.max().nodes(); nodes++) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index ebe9327967e..36034b62cfb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.transaction.Mutex; @@ -184,8 +183,8 @@ class Activator { for (Node node : nodes) { HostSpec hostSpec = getHost(node.hostname(), hosts); node = hostSpec.membership().get().retired() ? node.retire(nodeRepository.clock().instant()) : node.unretire(); - if (! hostSpec.advertisedResources().equals(node.flavor().resources())) // A resized node - node = node.with(new Flavor(hostSpec.advertisedResources())); + if (hostSpec.flavor().isPresent()) // Docker nodes may change flavor + node = node.with(hostSpec.flavor().get()); Allocation allocation = node.allocation().get() .with(hostSpec.membership().get()) .withRequestedResources(hostSpec.requestedResources() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 78ccf258675..bd92357ea79 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; @@ -197,12 +196,12 @@ public class NodeRepositoryProvisioner implements Provisioner { log.log(Level.FINE, () -> "Prepared node " + node.hostname() + " - " + node.flavor()); Allocation nodeAllocation = node.allocation().orElseThrow(IllegalStateException::new); hosts.add(new HostSpec(node.hostname(), - nodeRepository.resourcesCalculator().realResourcesOf(node, nodeRepository), - node.flavor().resources(), - requestedResources, - nodeAllocation.membership(), + List.of(), + Optional.of(node.flavor()), + Optional.of(nodeAllocation.membership()), node.status().vespaVersion(), nodeAllocation.networkPorts(), + requestedResources.isUnspecified() ? Optional.empty() : Optional.of(requestedResources), node.status().dockerImage())); if (nodeAllocation.networkPorts().isPresent()) { log.log(Level.FINE, () -> "Prepared node " + node.hostname() + " has port allocations"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index e3edf394da6..9d02d2907cc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -145,7 +145,7 @@ class NodesResponse extends HttpResponse { object.setString("flavor", node.flavor().name()); node.reservedTo().ifPresent(reservedTo -> object.setString("reservedTo", reservedTo.value())); if (node.flavor().isConfigured()) - object.setDouble("cpuCores", node.flavor().resources().vcpu()); + object.setDouble("cpuCores", node.flavor().getMinCpuCores()); NodeResourcesSerializer.toSlime(node.flavor().resources(), object.setObject("resources")); if (node.flavor().cost() > 0) object.setLong("cost", node.flavor().cost()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 914008af227..6b03def7e57 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -229,6 +229,7 @@ public class OsVersionsTest { // All hosts are deprovisioning assertEquals(hostCount, hostNodes.get().deprovisioning().size()); + // Nodes complete their upgrade by being reprovisioned completeUpgradeOf(hostNodes.get().deprovisioning().asList(), NodeType.confighost); assertEquals(hostCount, hostNodes.get().onOsVersion(version1).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index f3fe1fc4915..134b3bc1cd7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -275,12 +275,12 @@ public class NodeSerializerTest { @Test public void flavor_overrides_serialization() { Node node = createNode(); - assertEquals(20, node.flavor().resources().diskGb(), 0); + assertEquals(20, node.flavor().getMinDiskAvailableGb(), 0); node = node.with(node.flavor().with(FlavorOverrides.ofDisk(1234))); - assertEquals(1234, node.flavor().resources().diskGb(), 0); + assertEquals(1234, node.flavor().getMinDiskAvailableGb(), 0); Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); - assertEquals(1234, copy.flavor().resources().diskGb(), 0); + assertEquals(1234, copy.flavor().getMinDiskAvailableGb(), 0); assertEquals(node, copy); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java index ea4386f2fd5..fd2374c55b7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java @@ -102,13 +102,13 @@ public class AllocationVisualizer extends JPanel { if (isHost) { g.setColor(Color.GRAY); - for (int i = 0; i < node.flavor().resources().memoryGb(); i++) { + for (int i = 0; i < node.flavor().getMinMainMemoryAvailableGb(); i++) { g.fillRect(x, y - nodeHeight, nodeWidth, nodeHeight); y = y - (nodeHeight + 2); } } else { g.setColor(Color.YELLOW); - int multi = (int) node.flavor().resources().memoryGb(); + int multi = (int) node.flavor().getMinMainMemoryAvailableGb(); int height = multi * nodeHeight + ((multi - 1) * 2); g.fillRect(x, y - height, nodeWidth, height); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 7350df40718..24c1b0b5d2f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -346,8 +346,8 @@ public class DynamicDockerAllocationTest { List<HostSpec> hosts = tester.prepare(application, cluster, 2, 1, resources); assertEquals(2, hosts.size()); - assertEquals(NodeResources.DiskSpeed.slow, hosts.get(0).advertisedResources().diskSpeed()); - assertEquals(NodeResources.DiskSpeed.slow, hosts.get(1).advertisedResources().diskSpeed()); + assertEquals(NodeResources.DiskSpeed.slow, hosts.get(0).flavor().get().resources().diskSpeed()); + assertEquals(NodeResources.DiskSpeed.slow, hosts.get(1).flavor().get().resources().diskSpeed()); tester.activate(application, hosts); } @@ -386,9 +386,9 @@ public class DynamicDockerAllocationTest { List<HostSpec> hosts = tester.prepare(application, cluster, 4, 1, resources); assertEquals(1, hosts.size()); tester.activate(application, hosts); - assertEquals(0.1, hosts.get(0).advertisedResources().vcpu(), 0.000001); + assertEquals(0.1, hosts.get(0).flavor().get().resources().vcpu(), 0.000001); assertEquals("Slow nodes are allowed in dev and preferred because they are cheaper", - NodeResources.DiskSpeed.slow, hosts.get(0).advertisedResources().diskSpeed()); + NodeResources.DiskSpeed.slow, hosts.get(0).flavor().get().resources().diskSpeed()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 7ef780b8b7f..e1a66d2ae21 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -270,8 +270,8 @@ public class DynamicDockerProvisionTest { app1, cluster1); // Stop specifying node resources - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(6, 3, NodeResources.unspecified()), - new ClusterResources(9, 3, NodeResources.unspecified()))); + tester.activate(app1, cluster1, Capacity.from(new ClusterResources(6, 3, NodeResources.unspecified), + new ClusterResources(9, 3, NodeResources.unspecified))); tester.assertNodes("Existing allocation is preserved", 9, 3, 2, 20, 15, app1, cluster1); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 9e211fd497c..03c07515cd5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -418,8 +418,8 @@ public class ProvisioningTest { ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, NodeResources.unspecified()), - new ClusterResources(4, 1, NodeResources.unspecified()))); + tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, NodeResources.unspecified), + new ClusterResources(4, 1, NodeResources.unspecified))); tester.assertNodes("Initial allocation at min with default resources", 2, 1, 1.5, 8, 50, 0.3, app1, cluster1); @@ -479,8 +479,8 @@ public class ProvisioningTest { app1, cluster1); // Stop specifying node resources - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(6, 3, NodeResources.unspecified()), - new ClusterResources(9, 3, NodeResources.unspecified()))); + tester.activate(app1, cluster1, Capacity.from(new ClusterResources(6, 3, NodeResources.unspecified), + new ClusterResources(9, 3, NodeResources.unspecified))); tester.assertNodes("No change", 9, 3, 8, 35, 15, app1, cluster1); @@ -622,7 +622,7 @@ public class ProvisioningTest { tester.makeReadyHosts(4, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); - tester.prepare(application, cluster, Capacity.from(new ClusterResources(5, 1, NodeResources.unspecified()), false, false)); + tester.prepare(application, cluster, Capacity.from(new ClusterResources(5, 1, NodeResources.unspecified), false, false)); // No exception; Success } @@ -881,7 +881,7 @@ public class ProvisioningTest { allHosts.addAll(content0); allHosts.addAll(content1); - Function<Integer, Capacity> capacity = count -> Capacity.from(new ClusterResources(count, 1, NodeResources.unspecified()), required, true); + Function<Integer, Capacity> capacity = count -> Capacity.from(new ClusterResources(count, 1, NodeResources.unspecified), required, true); int expectedContainer0Size = tester.capacityPolicies().decideSize(container0Size, capacity.apply(container0Size), containerCluster0, application); int expectedContainer1Size = tester.capacityPolicies().decideSize(container1Size, capacity.apply(container1Size), containerCluster1, application); int expectedContent0Size = tester.capacityPolicies().decideSize(content0Size, capacity.apply(content0Size), contentCluster0, application); |