diff options
13 files changed, 232 insertions, 71 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index 2711406c216..d11d7137226 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -49,6 +49,11 @@ public class Flavor { this(resources.toString(), resources, Optional.empty(), Type.DOCKER_CONTAINER, false, 0, resources.vcpu()); } + /** Creates a *host* flavor for testing */ + public Flavor(String name, NodeResources resources) { + this(name, resources, Optional.empty(), Flavor.Type.VIRTUAL_MACHINE, true, 0, resources.vcpu()); + } + private Flavor(String name, NodeResources resources, Optional<FlavorOverrides> flavorOverrides, diff --git a/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java b/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java index 6fb6e4f0860..0565b1cff09 100644 --- a/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java +++ b/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java @@ -39,6 +39,7 @@ public class Defaults { vespaPortConfigServerHttp = vespaPortConfigServerRpc + 1; vespaPortConfigProxyRpc = findConfigProxyPort(vespaPortBase + 90); } + static private String findVespaHome(String defHome) { Optional<String> vespaHomeEnv = Optional.ofNullable(System.getenv("VESPA_HOME")); if ( ! vespaHomeEnv.isPresent() || vespaHomeEnv.get().trim().isEmpty()) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 321c5632302..d5dd2138a4e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -83,7 +83,7 @@ public final class Node { this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null"); if (state == State.active) - requireNonEmpty(ipConfig.primary(), "An active node must have at least one valid IP address"); + requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); if (parentHostname.isPresent()) { if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 750b22484c9..f1be485d368 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import java.time.Duration; import java.util.List; @@ -43,10 +44,14 @@ public class Autoscaler { private static final double memoryUnitCost = 1.2; private static final double diskUnitCost = 0.045; + private final HostResourcesCalculator hostResourcesCalculator; private final NodeMetricsDb metricsDb; private final NodeRepository nodeRepository; - public Autoscaler(NodeMetricsDb metricsDb, NodeRepository nodeRepository) { + public Autoscaler(HostResourcesCalculator hostResourcesCalculator, + NodeMetricsDb metricsDb, + NodeRepository nodeRepository) { + this.hostResourcesCalculator = hostResourcesCalculator; this.metricsDb = metricsDb; this.nodeRepository = nodeRepository; } @@ -56,7 +61,6 @@ public class Autoscaler { node.allocation().get().membership().retired() || node.allocation().get().isRemovable())) return Optional.empty(); // Don't autoscale clusters that are in flux - ClusterResources currentAllocation = new ClusterResources(clusterNodes); Optional<Double> totalCpuSpent = averageUseOf(Resource.cpu, applicationId, cluster, clusterNodes); Optional<Double> totalMemorySpent = averageUseOf(Resource.memory, applicationId, cluster, clusterNodes); @@ -74,24 +78,23 @@ public class Autoscaler { private Optional<ClusterResources> findBestAllocation(double totalCpu, double totalMemory, double totalDisk, ClusterResources currentAllocation) { - Optional<ClusterResources> bestAllocation = Optional.empty(); + Optional<ClusterResourcesWithCost> bestAllocation = Optional.empty(); for (ResourceIterator i = new ResourceIterator(totalCpu, totalMemory, totalDisk, currentAllocation); i.hasNext(); ) { ClusterResources allocation = i.next(); - Optional<NodeResources> allocatableResources = toAllocatableResources(allocation.resources()); + Optional<ClusterResourcesWithCost> allocatableResources = toAllocatableResources(allocation); if (allocatableResources.isEmpty()) continue; - ClusterResources effectiveAllocation = allocation.with(allocatableResources.get()); - if (bestAllocation.isEmpty() || effectiveAllocation.cost() < bestAllocation.get().cost()) - bestAllocation = Optional.of(effectiveAllocation); + if (bestAllocation.isEmpty() || allocatableResources.get().cost() < bestAllocation.get().cost()) + bestAllocation = allocatableResources; } - return bestAllocation; + return bestAllocation.map(a -> a.clusterResources()); } private boolean isSimilar(ClusterResources a1, ClusterResources a2) { if (a1.nodes() != a2.nodes()) return false; // A full node is always a significant difference - return isSimilar(a1.resources().vcpu(), a2.resources().vcpu()) && - isSimilar(a1.resources().memoryGb(), a2.resources().memoryGb()) && - isSimilar(a1.resources().diskGb(), a2.resources().diskGb()); + return isSimilar(a1.nodeResources().vcpu(), a2.nodeResources().vcpu()) && + isSimilar(a1.nodeResources().memoryGb(), a2.nodeResources().memoryGb()) && + isSimilar(a1.nodeResources().diskGb(), a2.nodeResources().diskGb()); } private boolean isSimilar(double r1, double r2) { @@ -102,11 +105,13 @@ public class Autoscaler { * Returns the smallest allocatable node resources larger than the given node resources, * or empty if none available. */ - private Optional<NodeResources> toAllocatableResources(NodeResources nodeResources) { + private Optional<ClusterResourcesWithCost> toAllocatableResources(ClusterResources resources) { if (allowsHostSharing(nodeRepository.zone().cloud())) { // Return the requested resources, or empty if they cannot fit on existing hosts for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) - if (flavor.resources().satisfies(nodeResources)) return Optional.of(nodeResources); + if (flavor.resources().satisfies(resources.nodeResources())) + return Optional.of(new ClusterResourcesWithCost(resources, + costOf(resources.nodeResources()) * resources.nodes())); return Optional.empty(); } else { @@ -114,14 +119,17 @@ public class Autoscaler { double bestCost = Double.MAX_VALUE; Optional<Flavor> bestFlavor = Optional.empty(); for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { - // TODO: Use effective not advertised flavor resources - if ( ! flavor.resources().satisfies(nodeResources)) continue; + if ( ! flavor.resources().satisfies(resources.nodeResources())) continue; if (bestFlavor.isEmpty() || bestCost > costOf(flavor.resources())) { bestFlavor = Optional.of(flavor); - bestCost = costOf(flavor.resources()); + bestCost = costOf(flavor); } } - return bestFlavor.map(flavor -> flavor.resources()); + if (bestFlavor.isEmpty()) + return Optional.empty(); + else + return Optional.of(new ClusterResourcesWithCost(resources.with(bestFlavor.get().resources()), + bestCost * resources.nodes())); } } @@ -154,6 +162,11 @@ public class Autoscaler { return true; } + private double costOf(Flavor flavor) { + NodeResources chargedResources = hostResourcesCalculator.availableCapacityOf(flavor.name(), flavor.resources()); + return costOf(chargedResources); + } + static double costOf(NodeResources resources) { return resources.vcpu() * cpuUnitCost + resources.memoryGb() * memoryUnitCost + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java index 3fdf3c87601..4f66c8bb8f5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java @@ -17,7 +17,7 @@ public class ClusterResources { private final int groups; /** The resources of each node in the cluster */ - private final NodeResources resources; + private final NodeResources nodeResources; public ClusterResources(List<Node> nodes) { this(nodes.size(), @@ -25,25 +25,21 @@ public class ClusterResources { nodes.get(0).flavor().resources()); } - public ClusterResources(int nodes, int groups, NodeResources resources) { + public ClusterResources(int nodes, int groups, NodeResources nodeResources) { this.nodes = nodes; this.groups = groups; - this.resources = resources; + this.nodeResources = nodeResources; } /** Returns the total number of allocated nodes (over all groups) */ public int nodes() { return nodes; } public int groups() { return groups; } - public NodeResources resources() { return resources; } + public NodeResources nodeResources() { return nodeResources; } public ClusterResources with(NodeResources resources) { return new ClusterResources(nodes, groups, resources); } - public double cost() { - return Autoscaler.costOf(resources) * nodes; - } - @Override public boolean equals(Object o) { if (o == this) return true; @@ -52,18 +48,18 @@ public class ClusterResources { ClusterResources other = (ClusterResources)o; if (other.nodes != this.nodes) return false; if (other.groups != this.groups) return false; - if (other.resources != this.resources) return false; + if (other.nodeResources != this.nodeResources) return false; return true; } @Override public int hashCode() { - return Objects.hash(nodes, groups, resources); + return Objects.hash(nodes, groups, nodeResources); } @Override public String toString() { - return "cluster resources: " + nodes + " * " + resources + (groups > 1 ? " in " + groups + " groups" : ""); + return "cluster resources: " + nodes + " * " + nodeResources + (groups > 1 ? " in " + groups + " groups" : ""); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java new file mode 100644 index 00000000000..c581d2304fb --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java @@ -0,0 +1,21 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +/** + * @author bratseth + */ +public class ClusterResourcesWithCost { + + private final ClusterResources resources; + private final double cost; + + public ClusterResourcesWithCost(ClusterResources resources, double cost) { + this.resources = resources; + this.cost = cost; + } + + public ClusterResources clusterResources() { return resources;} + + public double cost() { return cost; } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java index 2d3d7c83d4f..2dd1ae4b5a0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java @@ -30,7 +30,7 @@ public class ResourceIterator { // ceil: If the division does not produce a whole number we assume some node is missing groupSize = (int)Math.ceil((double)currentAllocation.nodes() / currentAllocation.groups()); - resourcesPrototype = currentAllocation.resources(); + resourcesPrototype = currentAllocation.nodeResources(); // What number of nodes is it effective to add or remove at the time from this cluster? // This is the group size, since we (for now) assume the group size is decided by someone wiser than us diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 2279d7a9eeb..30c25b3771e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; import com.yahoo.vespa.hosted.provision.autoscale.ClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import java.time.Duration; import java.util.List; @@ -25,9 +26,12 @@ public class AutoscalingMaintainer extends Maintainer { private final Autoscaler autoscaler; - public AutoscalingMaintainer(NodeRepository nodeRepository, NodeMetricsDb metricsDb, Duration interval) { + public AutoscalingMaintainer(NodeRepository nodeRepository, + HostResourcesCalculator hostResourcesCalculator, + NodeMetricsDb metricsDb, + Duration interval) { super(nodeRepository, interval); - this.autoscaler = new Autoscaler(metricsDb, nodeRepository); + this.autoscaler = new Autoscaler(hostResourcesCalculator, metricsDb, nodeRepository); } @Override @@ -42,7 +46,7 @@ public class AutoscalingMaintainer extends Maintainer { Optional<ClusterResources> target = autoscaler.autoscale(applicationId, clusterSpec, clusterNodes); target.ifPresent(t -> log.info("Autoscale: Application " + applicationId + " cluster " + clusterSpec + " from " + applicationNodes.size() + " * " + applicationNodes.get(0).flavor().resources() + - " to " + t.nodes() + " * " + t.resources())); + " to " + t.nodes() + " * " + t.nodeResources())); }); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index a49049f8b04..9b06a3161e9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -91,7 +91,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval); rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval); - autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, nodeMetricsDb, defaults.autoscalingInterval); + autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, defaults.autoscalingInterval); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintain(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 5753bbb3c5a..af6fa8edf64 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -64,7 +64,6 @@ public class GroupPreparer { .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) .value(); boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty(); - try (Mutex lock = nodeRepository.lock(application)) { // Lock ready pool to ensure that the same nodes are not simultaneously allocated by others diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 5dce8815a16..f478e58420c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -2,10 +2,19 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; import org.junit.Test; +import java.util.ArrayList; +import java.util.List; + import static org.junit.Assert.assertTrue; /** @@ -83,4 +92,36 @@ public class AutoscalingTest { tester.autoscale(application1, cluster1)); } + @Test + public void testAutoscalingAws() { + List<Flavor> flavors = new ArrayList<>(); + flavors.add(new Flavor("aws-xlarge", new NodeResources(6, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); + flavors.add(new Flavor("aws-large", new NodeResources(3, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); + flavors.add(new Flavor("aws-medium", new NodeResources(2, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); + flavors.add(new Flavor("aws-small", new NodeResources(1, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); + AutoscalingTester tester = new AutoscalingTester(new Zone(CloudName.from("aws"), SystemName.main, + Environment.prod, RegionName.from("us-east")), + flavors); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 1, new NodeResources(3, 100, 100, 1)); + + tester.addMeasurements( 0.25f, 120, application1); + ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", + 7, 1, 3, 100, 100, + tester.autoscale(application1, cluster1)); + + tester.deploy(application1, cluster1, scaledResources); + tester.deactivateRetired(application1, cluster1, scaledResources); + + tester.addMeasurements( 0.05f, 1000, application1); + System.out.println("Assuming downscaling"); + tester.assertResources("Scaling down since resource usage has gone down significantly", + 8, 1, 1, 100, 100, + tester.autoscale(application1, cluster1)); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index d9c41d20b5d..0a3eaa0a95b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; @@ -14,13 +15,24 @@ import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.test.ManualClock; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.IP; +import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; +import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -30,16 +42,32 @@ class AutoscalingTester { private final ProvisioningTester provisioningTester; private final Autoscaler autoscaler; private final NodeMetricsDb db; + private final MockHostResourcesCalculator hostResourcesCalculator; + /** Creates an autoscaling tester with a single host type ready */ public AutoscalingTester(NodeResources hostResources) { - provisioningTester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) - .flavorsConfig(asConfig(hostResources)) - .build(); - provisioningTester.makeReadyNodes(20, "flavor0", NodeType.host, 8); + this(new Zone(Environment.prod, RegionName.from("us-east")), null, null, asConfig(hostResources)); + provisioningTester.makeReadyNodes(20, "hostFlavor", NodeType.host, 8); // "hostFlavor" generated by asConfig provisioningTester.deployZoneApp(); + } + public AutoscalingTester(Zone zone, List<Flavor> flavors) { + this(zone, + new MockHostProvisioner(flavors), + new InMemoryFlagSource().withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true), + asConfig(flavors)); + } + + private AutoscalingTester(Zone zone, MockHostProvisioner hostProvisioner, FlagSource flagSource, FlavorsConfig flavorsConfig) { + provisioningTester = new ProvisioningTester.Builder().zone(zone) + .flavorsConfig(flavorsConfig) + .hostProvisioner(hostProvisioner) + .flagSource(flagSource) + .build(); + + hostResourcesCalculator = new MockHostResourcesCalculator(); db = new NodeMetricsDb(); - autoscaler = new Autoscaler(db, nodeRepository()); + autoscaler = new Autoscaler(hostResourcesCalculator, db, nodeRepository()); } public ApplicationId applicationId(String applicationName) { @@ -54,13 +82,24 @@ class AutoscalingTester { } public void deploy(ApplicationId application, ClusterSpec cluster, ClusterResources resources) { - deploy(application, cluster, resources.nodes(), resources.groups(), resources.resources()); + deploy(application, cluster, resources.nodes(), resources.groups(), resources.nodeResources()); } public void deploy(ApplicationId application, ClusterSpec cluster, int nodes, int groups, NodeResources resources) { List<HostSpec> hosts = provisioningTester.prepare(application, cluster, Capacity.fromCount(nodes, resources), groups); + for (HostSpec host : hosts) + makeReady(host.hostname()); + provisioningTester.deployZoneApp(); provisioningTester.activate(application, hosts); + } + public void makeReady(String hostname) { + Node node = nodeRepository().getNode(hostname).get(); + nodeRepository().write(node.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of())), nodeRepository().lock(node)); + Node host = nodeRepository().getNode(node.parentHostname().get()).get(); + host = host.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); + if (host.state() == Node.State.provisioned) + nodeRepository().setReady(List.of(host), Agent.system, getClass().getSimpleName()); } public void deactivateRetired(ApplicationId application, ClusterSpec cluster, ClusterResources resources) { @@ -104,9 +143,9 @@ class AutoscalingTester { assertTrue(message, actualResources.isPresent()); assertEquals("Node count " + message, nodeCount, actualResources.get().nodes()); assertEquals("Group count " + message, groupCount, actualResources.get().groups()); - assertEquals("Cpu: " + message, approxCpu, Math.round(actualResources.get().resources().vcpu() * 10) / 10.0, delta); - assertEquals("Memory: " + message, approxMemory, Math.round(actualResources.get().resources().memoryGb() * 10) / 10.0, delta); - assertEquals("Disk: " + message, approxDisk, Math.round(actualResources.get().resources().diskGb() * 10) / 10.0, delta); + assertEquals("Cpu: " + message, approxCpu, Math.round(actualResources.get().nodeResources().vcpu() * 10) / 10.0, delta); + assertEquals("Memory: " + message, approxMemory, Math.round(actualResources.get().nodeResources().memoryGb() * 10) / 10.0, delta); + assertEquals("Disk: " + message, approxDisk, Math.round(actualResources.get().nodeResources().diskGb() * 10) / 10.0, delta); return actualResources.get(); } @@ -118,19 +157,74 @@ class AutoscalingTester { return provisioningTester.nodeRepository(); } - private FlavorsConfig asConfig(NodeResources ... resources) { + private static FlavorsConfig asConfig(NodeResources hostResources) { FlavorsConfig.Builder b = new FlavorsConfig.Builder(); - int i = 0; - for (NodeResources nodeResources : resources) { - FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); - flavor.name("flavor" + (i++)); - flavor.minCpuCores(nodeResources.vcpu()); - flavor.minMainMemoryAvailableGb(nodeResources.memoryGb()); - flavor.minDiskAvailableGb(nodeResources.diskGb()); - flavor.bandwidth(nodeResources.bandwidthGbps() * 1000); - b.flavor(flavor); - } + b.flavor(asFlavorConfig("hostFlavor", hostResources)); return b.build(); } + private static FlavorsConfig asConfig(List<Flavor> flavors) { + FlavorsConfig.Builder b = new FlavorsConfig.Builder(); + for (Flavor flavor : flavors) + b.flavor(asFlavorConfig(flavor.name(), flavor.resources())); + return b.build(); + } + + private static FlavorsConfig.Flavor.Builder asFlavorConfig(String flavorName, NodeResources resources) { + FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); + flavor.name(flavorName); + flavor.minCpuCores(resources.vcpu()); + flavor.minMainMemoryAvailableGb(resources.memoryGb()); + flavor.minDiskAvailableGb(resources.diskGb()); + flavor.bandwidth(resources.bandwidthGbps() * 1000); + return flavor; + } + + private static class MockHostResourcesCalculator implements HostResourcesCalculator { + + @Override + public NodeResources availableCapacityOf(String flavorName, NodeResources hostResources) { + return hostResources; + } + + } + + private static class MockHostProvisioner implements HostProvisioner { + + private final List<Flavor> hostFlavors; + + public MockHostProvisioner(List<Flavor> hostFlavors) { + this.hostFlavors = hostFlavors; + } + + @Override + public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, ApplicationId applicationId) { + for (Flavor f : hostFlavors) + System.out.println(f + ": " + f.resources()); + Flavor hostFlavor = hostFlavors.stream().filter(f -> f.resources().justNumbers().equals(resources.justNumbers())).findAny() + .orElseThrow(() -> new RuntimeException("No flavor matching " + resources + ". Flavors: " + hostFlavors)); + + List<ProvisionedHost> hosts = new ArrayList<>(); + for (int index : provisionIndexes) { + hosts.add(new ProvisionedHost("host" + index, + "hostname" + index, + hostFlavor, + "nodename" + index, + resources)); + } + return hosts; + } + + @Override + public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException { + throw new RuntimeException("Not implemented"); + } + + @Override + public void deprovision(Node host) { + throw new RuntimeException("Not implemented"); + } + + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 2b770625060..ad2e75570ea 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -62,7 +62,7 @@ public class DynamicDockerProvisionTest { @Test public void does_not_allocate_to_available_empty_hosts() { tester.makeReadyNodes(3, "small", NodeType.host, 10); - deployZoneApp(tester); + tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); NodeResources flavor = new NodeResources(1, 4, 10, 1); @@ -82,7 +82,7 @@ public class DynamicDockerProvisionTest { tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor); verify(hostProvisioner).provisionHosts(expectedProvisionIndexes, flavor, application); - // Ready the provisioned hosts, add an IP addreses to pool and activate them + // Ready the provisioned hosts, add an IP addresses to pool and activate them for (Integer i : expectedProvisionIndexes) { String hostname = "host-" + i; var ipConfig = new IP.Config(Set.of("::" + i + ":0"), Set.of("::" + i + ":2")); @@ -90,7 +90,7 @@ public class DynamicDockerProvisionTest { tester.nodeRepository().setReady(List.of(host), Agent.system, getClass().getSimpleName()); nameResolver.addRecord(hostname + "-2", "::" + i + ":2"); } - deployZoneApp(tester); + tester.deployZoneApp(); mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("small")); tester.prepare(application, clusterSpec("another-id"), 2, 1, flavor); @@ -103,19 +103,6 @@ public class DynamicDockerProvisionTest { assertEquals(4, tester.nodeRepository().getNodes(NodeType.tenant, Node.State.reserved).size()); } - private static void deployZoneApp(ProvisioningTester tester) { - ApplicationId applicationId = tester.makeApplicationId(); - List<HostSpec> list = tester.prepare(applicationId, - ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from("node-admin"), - Version.fromString("6.42"), - false), - Capacity.fromRequiredNodeType(NodeType.host), - 1); - tester.activate(applicationId, ImmutableSet.copyOf(list)); - } - - private static ClusterSpec clusterSpec(String clusterId) { return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false); } |