From c8ae53bbc4499fe8fe0e8b480440fbbbb2737779 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Tue, 26 May 2020 22:16:47 +0200 Subject: Make room for overhead when allocating on shared nodes --- .../provision/autoscale/AllocatableClusterResources.java | 4 ++-- .../hosted/provision/autoscale/AllocationOptimizer.java | 6 +----- .../provisioning/EmptyProvisionServiceProvider.java | 2 +- .../hosted/provision/autoscale/AutoscalingTest.java | 16 +++++++--------- .../hosted/provision/autoscale/AutoscalingTester.java | 5 ++++- .../provisioning/DynamicDockerProvisionTest.java | 3 ++- 6 files changed, 17 insertions(+), 19 deletions(-) (limited to 'node-repository/src') diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index 17a163eed2a..65673c12209 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -141,9 +141,9 @@ public class AllocatableClusterResources { NodeResources advertisedResources = cappedNodeResources.add(nodeRepository.resourcesCalculator().overheadAllocating(cappedNodeResources, exclusive)); NodeResources realResources = cappedNodeResources; for (Flavor flavor : nodeRepository.flavors().getFlavors()) { - if (flavor.resources().satisfies(cappedNodeResources)) // TODO: advertisedResources + if (flavor.resources().satisfies(advertisedResources)) return Optional.of(new AllocatableClusterResources(resources.with(realResources), - cappedNodeResources, // TODO: advertisedResources + advertisedResources, resources.nodeResources(), clusterType)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index de53fd8e5b6..d2589b15421 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -44,7 +44,6 @@ public class AllocationOptimizer { limits = Limits.of(new ClusterResources(minimumNodes, 1, NodeResources.unspecified()), new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified())); Optional bestAllocation = Optional.empty(); - System.out.println("Optimizing for " + target); for (int groups = limits.min().groups(); groups <= limits.max().groups(); groups++) { for (int nodes = limits.min().nodes(); nodes <= limits.max().nodes(); nodes++) { if (nodes % groups != 0) continue; @@ -61,11 +60,8 @@ public class AllocationOptimizer { var allocatableResources = AllocatableClusterResources.from(next, exclusive, current.clusterType(), limits, nodeRepository); if (allocatableResources.isEmpty()) continue; - System.out.println("Considering " + allocatableResources.get()); - if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) { - System.out.println(" ... better"); + if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) bestAllocation = allocatableResources; - } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index e9634906fb1..a6caa7edf8b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -45,7 +45,7 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { @Override public NodeResources overheadAllocating(NodeResources resources, boolean exclusive) { - return resources.withVcpu(0).withMemoryGb(0).withDiskGb(0); + return resources.withVcpu(0).withMemoryGb(0).withDiskGb(0).withBandwidthGbps(0); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 0f13ebd8f89..969a5788fca 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -300,34 +300,32 @@ public class AutoscalingTest { ClusterResources max = new ClusterResources(4, 1, new NodeResources(60, 100, 1000, 1)); { // No memory tax - System.out.println("------- No memory tax"); AutoscalingTester tester = new AutoscalingTester(hostResources, new OnlySubtractingWhenForecastingCalculator(0)); ApplicationId application1 = tester.applicationId("app1"); ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1"); tester.deploy(application1, cluster1, min); - tester.addMeasurements(Resource.memory, 1.0f, 1000, application1); // Need more memory, which can be achieved in two way - tester.addMeasurements(Resource.cpu, 1.0f, 1000, application1); // Pushes towards more nodes due to fixed cpu cost per node + tester.addMeasurements(Resource.cpu, 1.0f, 1000, application1); + tester.addMeasurements(Resource.memory, 1.0f, 1000, application1); tester.addMeasurements(Resource.disk, 0.7f, 1000, application1); tester.assertResources("Scaling up", - 4, 1, 7.0, 20.0, 200.0, + 4, 1, 7.0, 20, 200, tester.autoscale(application1, cluster1.id(), min, max)); } { // 15 Gb memory tax - System.out.println("------- With 15 Gb memory tax"); AutoscalingTester tester = new AutoscalingTester(hostResources, new OnlySubtractingWhenForecastingCalculator(15)); ApplicationId application1 = tester.applicationId("app1"); ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1"); tester.deploy(application1, cluster1, min); - tester.addMeasurements(Resource.memory, 1.0f, 1000, application1); // Need more memory, which can be achieved in two way - tester.addMeasurements(Resource.cpu, 1.0f, 1000, application1); // Pushes towards more nodes due to fixed cpu cost per node + tester.addMeasurements(Resource.cpu, 1.0f, 1000, application1); + tester.addMeasurements(Resource.memory, 1.0f, 1000, application1); tester.addMeasurements(Resource.disk, 0.7f, 1000, application1); tester.assertResources("Scaling up", - 4, 1, 7.0, 20.0, 200.0, + 4, 1, 7.0, 35, 200, tester.autoscale(application1, cluster1.id(), min, max)); } } @@ -394,7 +392,7 @@ public class AutoscalingTest { @Override public NodeResources overheadAllocating(NodeResources resources, boolean exclusive) { - return resources.withVcpu(0).withMemoryGb(memoryTaxGb).withDiskGb(0); + return resources.withVcpu(0).withMemoryGb(memoryTaxGb).withDiskGb(0).withBandwidthGbps(0); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index bcae145f1e6..940499854e8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -221,7 +221,10 @@ class AutoscalingTester { @Override public NodeResources overheadAllocating(NodeResources resources, boolean exclusive) { - return resources.withVcpu(0).withMemoryGb(zone.getCloud().dynamicProvisioning() ? 3 : 0).withDiskGb(0); + return resources.withVcpu(0) + .withMemoryGb(zone.getCloud().dynamicProvisioning() ? 3 : 0) + .withDiskGb(0) + .withBandwidthGbps(0); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 1757efea51c..ea7651b0c9b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -394,7 +394,8 @@ public class DynamicDockerProvisionTest { public NodeResources overheadAllocating(NodeResources resources, boolean exclusive) { return resources.withVcpu(0) .withMemoryGb(memoryTaxGb) - .withDiskGb(resources.storageType() == local ? localDiskTax : 0); + .withDiskGb(resources.storageType() == local ? localDiskTax : 0) + .withBandwidthGbps(0); } } -- cgit v1.2.3