diff options
author | Jon Bratseth <bratseth@oath.com> | 2021-05-26 18:03:59 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-05-26 18:03:59 +0200 |
commit | ee058429eb26d9c3a4d794bc251a0dd70546a97a (patch) | |
tree | 30d03227b060f0d851a8853d3a750d03e8573601 | |
parent | b68b47e1df3820f0d7e87c92745ae5a2b6e16483 (diff) | |
parent | 291391e306c04aeeb52d7b0ef58f64dcd8f29d4f (diff) |
Merge pull request #17994 from vespa-engine/bratseth/nearest-flavor-info
Show info on nearest flavor also with autoscaling
9 files changed, 129 insertions, 33 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index b9ee973664e..b5cae857ce0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -25,7 +25,6 @@ import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.admin.Configserver; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerComponent; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerConfigurer; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java index eb462c86f4f..c3b2bd9f69a 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java @@ -7,6 +7,7 @@ import com.yahoo.config.provisioning.FlavorsConfig; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 0548bc7520f..4454517d5ec 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -280,6 +280,17 @@ public class NodeResources { public boolean isUnspecified() { return this.equals(unspecified); } + // Returns squared euclidean distance of the relevant numerical values of two node resources + public double distanceTo(NodeResources other) { + if ( ! this.diskSpeed().compatibleWith(other.diskSpeed())) return Double.MAX_VALUE; + if ( ! this.storageType().compatibleWith(other.storageType())) return Double.MAX_VALUE; + + double distance = Math.pow(this.vcpu() - other.vcpu(), 2) + Math.pow(this.memoryGb() - other.memoryGb(), 2); + if (this.storageType() == StorageType.local || other.storageType() == StorageType.local) + distance += Math.pow(this.diskGb() - other.diskGb(), 2); + return distance; + } + /** Returns this.isUnspecified() ? Optional.empty() : Optional.of(this) */ public Optional<NodeResources> asOptional() { return this.isUnspecified() ? Optional.empty() : Optional.of(this); diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterResourcesTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterResourcesTest.java new file mode 100644 index 00000000000..2ae5347da12 --- /dev/null +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterResourcesTest.java @@ -0,0 +1,20 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class ClusterResourcesTest { + + @Test + public void testCost() { + ClusterResources r1 = new ClusterResources(3, 1, new NodeResources(2, 8, 50, 1)); + ClusterResources r2 = new ClusterResources(3, 1, new NodeResources(2, 16, 50, 1)); + assertEquals(1.818, r1.cost() + r2.cost(), 0.01); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 79b09348d21..0a1c6c5df6b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -49,7 +49,6 @@ public class AllocationOptimizer { new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified())); else limits = atLeast(minimumNodes, limits); - Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); NodeList hosts = nodeRepository.nodes().list().hosts(); for (int groups = limits.min().groups(); groups <= limits.max().groups(); groups++) { @@ -67,8 +66,8 @@ public class AllocationOptimizer { nodeResourcesWith(nodesAdjustedForRedundancy, groupsAdjustedForRedundancy, limits, target, current, clusterModel)); - var allocatableResources = AllocatableClusterResources.from(next, current.clusterSpec(), limits, hosts, nodeRepository); + if (allocatableResources.isEmpty()) continue; if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) bestAllocation = allocatableResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 81a56e4d47e..def992a264b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -62,8 +62,8 @@ public class GroupPreparer { List<Node> surplusActiveNodes, NodeIndices indices, int wantedGroups) { String allocateOsRequirement = allocateOsRequirementFlag - .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) - .value(); + .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) + .value(); // Try preparing in memory without global unallocated lock. Most of the time there should be no changes and we // can return nodes previously allocated. @@ -89,24 +89,16 @@ public class GroupPreparer { allocateOsRequirement); NodeType hostType = allocation.nodeType().hostType(); if (canProvisionDynamically(hostType)) { - final Version osVersion; - if (allocateOsRequirement.equals("rhel8")) { - osVersion = new Version(8, Integer.MAX_VALUE /* always use latest 8 version */, 0); - } else if (allocateOsRequirement.equals("rhel7")) { - osVersion = new Version(7, Integer.MAX_VALUE /* always use latest 7 version */, 0); - } else { - osVersion = nodeRepository.osVersions().targetFor(hostType).orElse(Version.emptyVersion); - } HostSharing sharing = hostSharing(requestedNodes, hostType); List<ProvisionedHost> provisionedHosts = allocation.hostDeficit() - .map(deficit -> { - return hostProvisioner.get().provisionHosts(allocation.provisionIndices(deficit.count()), - hostType, - deficit.resources(), - application, - osVersion, - sharing); - }) + .map(deficit -> + hostProvisioner.get().provisionHosts(allocation.provisionIndices(deficit.count()), + hostType, + deficit.resources(), + application, + decideOsVersion(allocateOsRequirement, hostType), + sharing) + ) .orElseGet(List::of); // At this point we have started provisioning of the hosts, the first priority is to make sure that @@ -141,12 +133,17 @@ public class GroupPreparer { List<Node> surplusActiveNodes, Supplier<Integer> nextIndex, int wantedGroups, Mutex allocationLock, String allocateOsRequirement) { LockedNodeList allNodes = nodeRepository.nodes().list(allocationLock); - NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requestedNodes, - nextIndex, nodeRepository); - NodePrioritizer prioritizer = new NodePrioritizer( - allNodes, application, cluster, requestedNodes, wantedGroups, - nodeRepository.zone().getCloud().dynamicProvisioning(), nodeRepository.nameResolver(), - nodeRepository.resourcesCalculator(), nodeRepository.spareCount(), allocateOsRequirement); + NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requestedNodes, nextIndex, nodeRepository); + NodePrioritizer prioritizer = new NodePrioritizer(allNodes, + application, + cluster, + requestedNodes, + wantedGroups, + nodeRepository.zone().getCloud().dynamicProvisioning(), + nodeRepository.nameResolver(), + nodeRepository.resourcesCalculator(), + nodeRepository.spareCount(), + allocateOsRequirement); allocation.offer(prioritizer.collect(surplusActiveNodes)); return allocation; } @@ -164,4 +161,13 @@ public class GroupPreparer { return sharing; } + private Version decideOsVersion(String allocateOsRequirement, NodeType hostType) { + if (allocateOsRequirement.equals("rhel8")) + return new Version(8, Integer.MAX_VALUE /* always use latest 8 version */, 0); + else if (allocateOsRequirement.equals("rhel7")) + return new Version(7, Integer.MAX_VALUE /* always use latest 7 version */, 0); + else + return nodeRepository.osVersions().targetFor(hostType).orElse(Version.emptyVersion); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index ab881a68ebe..ee8ce23a5c0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; @@ -27,7 +28,6 @@ import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.AllocationOptimizer; import com.yahoo.vespa.hosted.provision.autoscale.ClusterModel; import com.yahoo.vespa.hosted.provision.autoscale.Limits; -import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.autoscale.ResourceTarget; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; @@ -184,7 +184,7 @@ public class NodeRepositoryProvisioner implements Provisioner { current, clusterModel, limits) - .orElseThrow(() -> new IllegalArgumentException("No allocation possible within " + limits)) + .orElseThrow(() -> newNoAllocationPossible(current.clusterSpec(), limits)) .advertisedResources(); } @@ -224,4 +224,37 @@ public class NodeRepositoryProvisioner implements Provisioner { } } + private IllegalArgumentException newNoAllocationPossible(ClusterSpec spec, Limits limits) { + StringBuilder message = new StringBuilder("No allocation possible within ").append(limits); + + boolean exclusiveHosts = spec.isExclusive() || nodeRepository.zone().getCloud().dynamicProvisioning(); + if (exclusiveHosts) + message.append(". Nearest allowed node resources: ").append(findNearestNodeResources(limits)); + + return new IllegalArgumentException(message.toString()); + } + + private NodeResources findNearestNodeResources(Limits limits) { + NodeResources nearestMin = nearestFlavorResources(limits.min().nodeResources()); + NodeResources nearestMax = nearestFlavorResources(limits.max().nodeResources()); + if (limits.min().nodeResources().distanceTo(nearestMin) < limits.max().nodeResources().distanceTo(nearestMax)) + return nearestMin; + else + return nearestMax; + } + + /** Returns the advertised flavor resources which are nearest to the given resources */ + private NodeResources nearestFlavorResources(NodeResources requestedResources) { + NodeResources nearestHostResources = nodeRepository.flavors().getFlavors().stream() + .map(flavor -> nodeRepository.resourcesCalculator().advertisedResourcesOf(flavor)) + .filter(resources -> resources.diskSpeed().compatibleWith(requestedResources.diskSpeed())) + .filter(resources -> resources.storageType().compatibleWith(requestedResources.storageType())) + .min(Comparator.comparingDouble(resources -> resources.distanceTo(requestedResources))) + .orElseThrow() + .withBandwidthGbps(requestedResources.bandwidthGbps()); + if ( nearestHostResources.storageType() == NodeResources.StorageType.remote) + nearestHostResources = nearestHostResources.withDiskGb(requestedResources.diskGb()); + return nearestHostResources; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index d2b701e5312..97f935d273b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -45,8 +45,7 @@ class Preparer { catch (OutOfCapacityException e) { throw new OutOfCapacityException("Could not satisfy " + requestedNodes + ( wantedGroups > 1 ? " (in " + wantedGroups + " groups)" : "") + - " in " + application + " " + cluster + - ": " + e.getMessage()); + " in " + application + " " + cluster + ": " + e.getMessage()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index 3ad9041cdbb..f2ca993f4d7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -338,7 +338,8 @@ public class DockerProvisioningTest { tester.makeReadyHosts(2, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.applicationId("app1"); - ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, + new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); // 5 Gb requested memory becomes 5-3=2 Gb real memory, which is an illegally small amount var resources = new NodeResources(1, 5, 10, 1); @@ -346,7 +347,34 @@ public class DockerProvisioningTest { new ClusterResources(4, 1, resources))); } catch (IllegalArgumentException e) { - assertEquals("No allocation possible within limits: from 2 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps] to 4 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps]", + assertEquals("No allocation possible within limits: " + + "from 2 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps] " + + "to 4 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps]", + e.getMessage()); + } + } + + @Test + public void exclusive_resources_not_matching_host_causes_failure() { + try { + Flavor hostFlavor1 = new Flavor(new NodeResources(20, 40, 100, 4)); + Flavor hostFlavor2 = new Flavor(new NodeResources(30, 40, 100, 4)); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) + .flavors(List.of(hostFlavor1, hostFlavor2)) + .build(); + ApplicationId app1 = ProvisioningTester.applicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, + new ClusterSpec.Id("cluster1")).exclusive(true).vespaVersion("7").build(); + + var resources = new NodeResources(20, 37, 100, 1); + tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, resources), + new ClusterResources(4, 1, resources))); + } + catch (IllegalArgumentException e) { + assertEquals("No allocation possible within limits: " + + "from 2 nodes with [vcpu: 20.0, memory: 37.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps] " + + "to 4 nodes with [vcpu: 20.0, memory: 37.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps]. " + + "Nearest allowed node resources: [vcpu: 20.0, memory: 40.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote]", e.getMessage()); } } |