diff options
author | Jon Bratseth <bratseth@oath.com> | 2019-06-14 14:35:39 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-06-14 14:35:39 +0200 |
commit | e0964e009b2a2d2686b2479eaa194e16271808b0 (patch) | |
tree | c666e70f71f445d8eb45ab862d2570be8fcac3ae | |
parent | 940196ffa5880adf18a35303adc117a097ec65ce (diff) | |
parent | 0b88f88ae82228e7c5ae88449f6b16151828e166 (diff) |
Merge pull request #9797 from vespa-engine/freva/reserve-host-resources
Reserve host resources
13 files changed, 156 insertions, 229 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 5de621634ea..a79ac908b9e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -3,25 +3,22 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; -import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.History; -import com.yahoo.vespa.hosted.provision.provisioning.DockerHostCapacity; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.service.monitor.ServiceMonitor; import java.time.Duration; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -30,6 +27,8 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import static com.yahoo.config.provision.NodeResources.DiskSpeed.any; + /** * @author oyving */ @@ -56,8 +55,7 @@ public class MetricsReporter extends Maintainer { @Override public void maintain() { - LockedNodeList nodes = nodeRepository().list(() -> {}); // Ignore locking for the purposes of reporting metrics - + NodeList nodes = nodeRepository().list(); Map<HostName, List<ServiceInstance>> servicesByHost = serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName(); @@ -122,12 +120,8 @@ public class MetricsReporter extends Maintainer { metric.set("wantToRetire", node.status().wantToRetire() ? 1 : 0, context); metric.set("wantToDeprovision", node.status().wantToDeprovision() ? 1 : 0, context); - metric.set("hardwareFailure", - node.status().hardwareFailureDescription().isPresent() ? 1 : 0, - context); - metric.set("hardwareDivergence", - node.status().hardwareDivergence().isPresent() ? 1 : 0, - context); + metric.set("hardwareFailure", node.status().hardwareFailureDescription().isPresent() ? 1 : 0, context); + metric.set("hardwareDivergence", node.status().hardwareDivergence().isPresent() ? 1 : 0, context); orchestrator.apply(new HostName(node.hostname())) .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN ? 1 : 0) @@ -183,63 +177,57 @@ public class MetricsReporter extends Maintainer { } private Metric.Context getContextAt(String... point) { - if (point.length % 2 != 0) { + if (point.length % 2 != 0) throw new IllegalArgumentException("Dimension specification comes in pairs"); - } Map<String, String> dimensions = new HashMap<>(); for (int i = 0; i < point.length; i += 2) { dimensions.put(point[i], point[i + 1]); } - Metric.Context context = contextMap.get(dimensions); - if (context != null) { - return context; - } - - context = metric.createContext(dimensions); - contextMap.put(dimensions, context); - return context; + return contextMap.computeIfAbsent(dimensions, metric::createContext); } - private void updateStateMetrics(LockedNodeList nodes) { - Map<Node.State, List<Node>> nodesByState = nodes.asList().stream() + private void updateStateMetrics(NodeList nodes) { + Map<Node.State, List<Node>> nodesByState = nodes.nodeType(NodeType.tenant).asList().stream() .collect(Collectors.groupingBy(Node::state)); // Metrics pr state for (Node.State state : Node.State.values()) { - List<Node> nodesInState = nodesByState.getOrDefault(state, new ArrayList<>()); - long size = nodesInState.stream().filter(node -> node.type() == NodeType.tenant).count(); - metric.set("hostedVespa." + state.name() + "Hosts", size, null); + List<Node> nodesInState = nodesByState.getOrDefault(state, List.of()); + metric.set("hostedVespa." + state.name() + "Hosts", nodesInState.size(), null); } } - private void updateDockerMetrics(LockedNodeList nodes) { - // Capacity flavors for docker - DockerHostCapacity capacity = new DockerHostCapacity(nodes); - metric.set("hostedVespa.docker.totalCapacityCpu", - capacity.getCapacityTotal(NodeResources.DiskSpeed.any).vcpu(), null); - metric.set("hostedVespa.docker.totalCapacityMem", - capacity.getCapacityTotal(NodeResources.DiskSpeed.any).memoryGb(), null); - metric.set("hostedVespa.docker.totalCapacityDisk", - capacity.getCapacityTotal(NodeResources.DiskSpeed.any).diskGb(), null); - metric.set("hostedVespa.docker.freeCapacityCpu", - capacity.getFreeCapacityTotal(NodeResources.DiskSpeed.any).vcpu(), null); - metric.set("hostedVespa.docker.freeCapacityMem", - capacity.getFreeCapacityTotal(NodeResources.DiskSpeed.any).memoryGb(), null); - metric.set("hostedVespa.docker.freeCapacityDisk", - capacity.getFreeCapacityTotal(NodeResources.DiskSpeed.any).diskGb(), null); - - List<Flavor> dockerFlavors = nodeRepository().getAvailableFlavors().getFlavors().stream() - .filter(f -> f.getType().equals(Flavor.Type.DOCKER_CONTAINER)) - .collect(Collectors.toList()); - for (Flavor flavor : dockerFlavors) { - Metric.Context context = getContextAt("flavor", flavor.name()); - metric.set("hostedVespa.docker.freeCapacityFlavor", - capacity.freeCapacityInFlavorEquivalence(flavor), context); - metric.set("hostedVespa.docker.hostsAvailableFlavor", - capacity.getNofHostsAvailableFor(flavor), context); - } + private void updateDockerMetrics(NodeList nodes) { + NodeResources totalCapacity = getCapacityTotal(nodes); + metric.set("hostedVespa.docker.totalCapacityCpu", totalCapacity.vcpu(), null); + metric.set("hostedVespa.docker.totalCapacityMem", totalCapacity.memoryGb(), null); + metric.set("hostedVespa.docker.totalCapacityDisk", totalCapacity.diskGb(), null); + + NodeResources totalFreeCapacity = getFreeCapacityTotal(nodes); + metric.set("hostedVespa.docker.freeCapacityCpu", totalFreeCapacity.vcpu(), null); + metric.set("hostedVespa.docker.freeCapacityMem", totalFreeCapacity.memoryGb(), null); + metric.set("hostedVespa.docker.freeCapacityDisk", totalFreeCapacity.diskGb(), null); + } + + private static NodeResources getCapacityTotal(NodeList nodes) { + return nodes.nodeType(NodeType.host).asList().stream() + .map(host -> host.flavor().resources()) + .map(resources -> resources.withDiskSpeed(any)) + .reduce(new NodeResources(0, 0, 0, any), NodeResources::add); } + private static NodeResources getFreeCapacityTotal(NodeList nodes) { + return nodes.nodeType(NodeType.host).asList().stream() + .map(n -> freeCapacityOf(nodes, n)) + .map(resources -> resources.withDiskSpeed(any)) + .reduce(new NodeResources(0, 0, 0, any), NodeResources::add); + } + + private static NodeResources freeCapacityOf(NodeList nodes, Node dockerHost) { + return nodes.childrenOf(dockerHost).asList().stream() + .map(node -> node.flavor().resources().withDiskSpeed(any)) + .reduce(dockerHost.flavor().resources().withDiskSpeed(any), NodeResources::subtract); + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java index feccfb430e6..a5969dc69cb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; @@ -20,25 +19,15 @@ import java.util.Objects; public class DockerHostCapacity { private final LockedNodeList allNodes; + private final HostResourcesCalculator hostResourcesCalculator; - public DockerHostCapacity(LockedNodeList allNodes) { + DockerHostCapacity(LockedNodeList allNodes, HostResourcesCalculator hostResourcesCalculator) { this.allNodes = Objects.requireNonNull(allNodes, "allNodes must be non-null"); - } - - /** - * Compare hosts on free capacity. - * Used in prioritizing hosts for allocation in <b>descending</b> order. - */ - int compare(Node hostA, Node hostB) { - int result = compare(freeCapacityOf(hostB, false), freeCapacityOf(hostA, false)); - if (result != 0) return result; - - // If resources are equal we want to assign to the one with the most IPaddresses free - return freeIPs(hostB) - freeIPs(hostA); + this.hostResourcesCalculator = Objects.requireNonNull(hostResourcesCalculator, "hostResourcesCalculator must be non-null"); } int compareWithoutInactive(Node hostA, Node hostB) { - int result = compare(freeCapacityOf(hostB, true), freeCapacityOf(hostA, true)); + int result = compare(freeCapacityOf(hostB, true), freeCapacityOf(hostA, true)); if (result != 0) return result; // If resources are equal we want to assign to the one with the most IPaddresses free @@ -64,79 +53,29 @@ public class DockerHostCapacity { return dockerHost.ipAddressPool().findUnused(allNodes).size(); } - /** Return total free capacity for a given disk speed (or for any disk speed) */ - public NodeResources getFreeCapacityTotal(NodeResources.DiskSpeed speed) { - return allNodes.asList().stream() - .filter(n -> n.type().equals(NodeType.host)) - .filter(n -> speed == NodeResources.DiskSpeed.any || n.flavor().resources().diskSpeed() == speed) - .map(n -> freeCapacityOf(n, false)) - .map(resources -> resources.withDiskSpeed(speed)) // Set speed to 'any' if necessary - .reduce(new NodeResources(0, 0, 0, speed), NodeResources::add); - } - - /** Return total capacity for a given disk speed (or for any disk speed) */ - public NodeResources getCapacityTotal(NodeResources.DiskSpeed speed) { - return allNodes.asList().stream() - .filter(n -> n.type().equals(NodeType.host)) - .filter(n -> speed == NodeResources.DiskSpeed.any || n.flavor().resources().diskSpeed() == speed) - .map(host -> host.flavor().resources()) - .map(resources -> resources.withDiskSpeed(speed)) // Set speed to 'any' if necessary - .reduce(new NodeResources(0, 0, 0, speed), NodeResources::add); - } - - public int freeCapacityInFlavorEquivalence(Flavor flavor) { - return allNodes.asList().stream() - .filter(n -> n.type().equals(NodeType.host)) - .map(n -> canFitNumberOf(n, flavor)) - .reduce(0, (a, b) -> a + b); - } - - public long getNofHostsAvailableFor(Flavor flavor) { - return allNodes.asList().stream() - .filter(n -> n.type().equals(NodeType.host)) - .filter(n -> hasCapacity(n, flavor.resources())) - .count(); - } - - private int canFitNumberOf(Node node, Flavor flavor) { - NodeResources freeCapacity = freeCapacityOf(node, false); - int capacityFactor = freeCapacityInFlavorEquivalence(freeCapacity, flavor); - int ips = freeIPs(node); - return Math.min(capacityFactor, ips); - } - - int freeCapacityInFlavorEquivalence(NodeResources freeCapacity, Flavor flavor) { - if ( ! freeCapacity.satisfies(flavor.resources())) return 0; - - double cpuFactor = Math.floor(freeCapacity.vcpu() / flavor.getMinCpuCores()); - double memoryFactor = Math.floor(freeCapacity.memoryGb() / flavor.getMinMainMemoryAvailableGb()); - double diskFactor = Math.floor(freeCapacity.diskGb() / flavor.getMinDiskAvailableGb()); - - return (int) Math.min(Math.min(memoryFactor, cpuFactor), diskFactor); - } - /** * Calculate the remaining capacity for the dockerHost. * * @param dockerHost The host to find free capacity of. * @return A default (empty) capacity if not a docker host, otherwise the free/unallocated/rest capacity */ - public NodeResources freeCapacityOf(Node dockerHost, boolean includeInactive) { + NodeResources freeCapacityOf(Node dockerHost, boolean excludeInactive) { // Only hosts have free capacity - if ( ! dockerHost.type().equals(NodeType.host)) return new NodeResources(0, 0, 0); + if (dockerHost.type() != NodeType.host) return new NodeResources(0, 0, 0); + NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(dockerHost.flavor().resources()); // Subtract used resources without taking disk speed into account since existing allocations grandfathered in // may not use reflect the actual disk speed (as of May 2019). This (the 3 diskSpeed assignments below) // can be removed when all node allocations accurately reflect the true host disk speed return allNodes.childrenOf(dockerHost).asList().stream() - .filter(node -> !(includeInactive && isInactiveOrRetired(node))) + .filter(node -> !(excludeInactive && isInactiveOrRetired(node))) .map(node -> node.flavor().resources().withDiskSpeed(NodeResources.DiskSpeed.any)) - .reduce(dockerHost.flavor().resources().withDiskSpeed(NodeResources.DiskSpeed.any), NodeResources::subtract) + .reduce(hostResources.withDiskSpeed(NodeResources.DiskSpeed.any), NodeResources::subtract) .withDiskSpeed(dockerHost.flavor().resources().diskSpeed()); } - private boolean isInactiveOrRetired(Node node) { - if (node.state().equals(Node.State.inactive)) return true; + private static boolean isInactiveOrRetired(Node node) { + if (node.state() == Node.State.inactive) return true; if (node.allocation().isPresent() && node.allocation().get().membership().retired()) return true; return false; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index eb72e1a2081..05915b82bae 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import java.util.Optional; @@ -9,6 +10,8 @@ import java.util.Optional; * @author freva */ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { + private final HostResourcesCalculator hostResourcesCalculator = new NoopHostResourcesCalculator(); + @Override public Optional<LoadBalancerService> getLoadBalancerService() { return Optional.empty(); @@ -18,4 +21,17 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { public Optional<HostProvisioner> getHostProvisioner() { return Optional.empty(); } + + @Override + public HostResourcesCalculator getHostResourcesCalculator() { + return hostResourcesCalculator; + } + + public static class NoopHostResourcesCalculator implements HostResourcesCalculator { + + @Override + public NodeResources availableCapacityOf(NodeResources hostResources) { + return hostResources; + } + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index ff412c6c99b..30d39a4717f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -25,12 +25,14 @@ public class GroupPreparer { private final NodeRepository nodeRepository; private final Optional<HostProvisioner> hostProvisioner; + private final HostResourcesCalculator hostResourcesCalculator; private final BooleanFlag dynamicProvisioningEnabledFlag; public GroupPreparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, - BooleanFlag dynamicProvisioningEnabledFlag) { + HostResourcesCalculator hostResourcesCalculator, BooleanFlag dynamicProvisioningEnabledFlag) { this.nodeRepository = nodeRepository; this.hostProvisioner = hostProvisioner; + this.hostResourcesCalculator = hostResourcesCalculator; this.dynamicProvisioningEnabledFlag = dynamicProvisioningEnabledFlag; } @@ -65,7 +67,8 @@ public class GroupPreparer { LockedNodeList nodeList = nodeRepository.list(allocationLock); NodePrioritizer prioritizer = new NodePrioritizer(nodeList, application, cluster, requestedNodes, spareCount, nodeRepository.nameResolver(), - nodeRepository.getAvailableFlavors()); + nodeRepository.getAvailableFlavors(), + hostResourcesCalculator); prioritizer.addApplicationNodes(); prioritizer.addSurplusNodes(surplusActiveNodes); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java new file mode 100644 index 00000000000..50aeb464e9f --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java @@ -0,0 +1,13 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.NodeResources; + +/** + * @author freva + */ +public interface HostResourcesCalculator { + + /** Calculates the resources that are reserved for host level processes and returns the remainder. */ + NodeResources availableCapacityOf(NodeResources hostResources); +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 294ba12497a..6b27662448c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -51,9 +51,9 @@ class NodePrioritizer { private final Set<Node> spareHosts; NodePrioritizer(LockedNodeList allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec, - int spares, NameResolver nameResolver, NodeFlavors flavors) { + int spares, NameResolver nameResolver, NodeFlavors flavors, HostResourcesCalculator hostResourcesCalculator) { this.allNodes = allNodes; - this.capacity = new DockerHostCapacity(allNodes); + this.capacity = new DockerHostCapacity(allNodes, hostResourcesCalculator); this.requestedNodes = nodeSpec; this.clusterSpec = clusterSpec; this.appId = appId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index c3281253b6b..21bfc1b6886 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -6,10 +6,10 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -64,8 +63,9 @@ public class NodeRepositoryProvisioner implements Provisioner { this.capacityPolicies = new CapacityPolicies(zone, flavors); this.zone = zone; this.preparer = new Preparer(nodeRepository, - zone.environment().equals(Environment.prod) ? SPARE_CAPACITY_PROD : SPARE_CAPACITY_NONPROD, + zone.environment() == Environment.prod ? SPARE_CAPACITY_PROD : SPARE_CAPACITY_NONPROD, provisionServiceProvider.getHostProvisioner(), + provisionServiceProvider.getHostResourcesCalculator(), Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource)); this.activator = new Activator(nodeRepository); this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService().map(lbService -> @@ -140,7 +140,7 @@ public class NodeRepositoryProvisioner implements Provisioner { log.log(LogLevel.DEBUG, () -> "Prepared node " + node.hostname() + " - " + node.flavor()); Allocation nodeAllocation = node.allocation().orElseThrow(IllegalStateException::new); hosts.add(new HostSpec(node.hostname(), - Collections.emptyList(), + List.of(), Optional.of(node.flavor()), Optional.of(nodeAllocation.membership()), node.status().vespaVersion(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 75960e4416a..ca958f15c69 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -27,10 +27,10 @@ class Preparer { private final int spareCount; public Preparer(NodeRepository nodeRepository, int spareCount, Optional<HostProvisioner> hostProvisioner, - BooleanFlag dynamicProvisioningEnabled) { + HostResourcesCalculator hostResourcesCalculator, BooleanFlag dynamicProvisioningEnabled) { this.nodeRepository = nodeRepository; this.spareCount = spareCount; - this.groupPreparer = new GroupPreparer(nodeRepository, hostProvisioner, dynamicProvisioningEnabled); + this.groupPreparer = new GroupPreparer(nodeRepository, hostProvisioner, hostResourcesCalculator, dynamicProvisioningEnabled); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java index ca9e629e1ea..a86bd581516 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java @@ -15,4 +15,6 @@ public interface ProvisionServiceProvider { Optional<LoadBalancerService> getLoadBalancerService(); Optional<HostProvisioner> getHostProvisioner(); + + HostResourcesCalculator getHostResourcesCalculator(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java index ea48d8a6b47..0d5950fe33a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java @@ -4,7 +4,9 @@ package com.yahoo.vespa.hosted.provision.testutils; import com.google.inject.Inject; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock; +import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; import java.util.Optional; @@ -16,6 +18,7 @@ public class MockProvisionServiceProvider implements ProvisionServiceProvider { private final Optional<LoadBalancerService> loadBalancerService; private final Optional<HostProvisioner> hostProvisioner; + private final HostResourcesCalculator hostResourcesCalculator; @Inject public MockProvisionServiceProvider() { @@ -23,8 +26,14 @@ public class MockProvisionServiceProvider implements ProvisionServiceProvider { } public MockProvisionServiceProvider(LoadBalancerService loadBalancerService, HostProvisioner hostProvisioner) { + this(loadBalancerService, hostProvisioner, new EmptyProvisionServiceProvider.NoopHostResourcesCalculator()); + } + + public MockProvisionServiceProvider(LoadBalancerService loadBalancerService, HostProvisioner hostProvisioner, + HostResourcesCalculator hostResourcesCalculator) { this.loadBalancerService = Optional.ofNullable(loadBalancerService); this.hostProvisioner = Optional.ofNullable(hostProvisioner); + this.hostResourcesCalculator = hostResourcesCalculator; } @Override @@ -36,4 +45,9 @@ public class MockProvisionServiceProvider implements ProvisionServiceProvider { public Optional<HostProvisioner> getHostProvisioner() { return hostProvisioner; } + + @Override + public HostResourcesCalculator getHostResourcesCalculator() { + return hostResourcesCalculator; + } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index e1a0f478b86..4eba95cb817 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -30,7 +30,6 @@ import org.junit.Test; import java.time.Clock; import java.time.Duration; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -56,19 +55,19 @@ public class MetricsReporterTest { DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); - nodeRepository.addNodes(Collections.singletonList(node)); + nodeRepository.addNodes(List.of(node)); Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy); - nodeRepository.addNodes(Collections.singletonList(hostNode)); + nodeRepository.addNodes(List.of(hostNode)); Map<String, Number> expectedMetrics = new HashMap<>(); - expectedMetrics.put("hostedVespa.provisionedHosts", 1L); - expectedMetrics.put("hostedVespa.parkedHosts", 0L); - expectedMetrics.put("hostedVespa.readyHosts", 0L); - expectedMetrics.put("hostedVespa.reservedHosts", 0L); - expectedMetrics.put("hostedVespa.activeHosts", 0L); - expectedMetrics.put("hostedVespa.inactiveHosts", 0L); - expectedMetrics.put("hostedVespa.dirtyHosts", 0L); - expectedMetrics.put("hostedVespa.failedHosts", 0L); + expectedMetrics.put("hostedVespa.provisionedHosts", 1); + expectedMetrics.put("hostedVespa.parkedHosts", 0); + expectedMetrics.put("hostedVespa.readyHosts", 0); + expectedMetrics.put("hostedVespa.reservedHosts", 0); + expectedMetrics.put("hostedVespa.activeHosts", 0); + expectedMetrics.put("hostedVespa.inactiveHosts", 0); + expectedMetrics.put("hostedVespa.dirtyHosts", 0); + expectedMetrics.put("hostedVespa.failedHosts", 0); expectedMetrics.put("hostedVespa.pendingRedeployments", 42); expectedMetrics.put("hostedVespa.docker.totalCapacityDisk", 0.0); expectedMetrics.put("hostedVespa.docker.totalCapacityMem", 0.0); @@ -92,7 +91,7 @@ public class MetricsReporterTest { when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS)); ServiceModel serviceModel = mock(ServiceModel.class); when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); - when(serviceModel.getServiceInstancesByHostName()).thenReturn(Collections.emptyMap()); + when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); TestMetric metric = new TestMetric(); MetricsReporter metricsReporter = new MetricsReporter( @@ -120,18 +119,18 @@ public class MetricsReporterTest { // Allow 4 containers Set<String> ipAddressPool = ImmutableSet.of("::2", "::3", "::4", "::5"); - Node dockerHost = Node.create("openStackId1", Collections.singleton("::1"), ipAddressPool, "dockerHost", + Node dockerHost = Node.create("openStackId1", Set.of("::1"), ipAddressPool, "dockerHost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); - nodeRepository.addNodes(Collections.singletonList(dockerHost)); + nodeRepository.addNodes(List.of(dockerHost)); nodeRepository.dirtyRecursively("dockerHost", Agent.system, getClass().getSimpleName()); nodeRepository.setReady("dockerHost", Agent.system, getClass().getSimpleName()); - Node container1 = Node.createDockerNode(Collections.singleton("::2"), Collections.emptySet(), "container1", + Node container1 = Node.createDockerNode(Set.of("::2"), Set.of(), "container1", Optional.of("dockerHost"), new NodeResources(1, 3, 2), NodeType.tenant); container1 = container1.with(allocation(Optional.of("app1")).get()); nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockAllocation())); - Node container2 = Node.createDockerNode(Collections.singleton("::3"), Collections.emptySet(), "container2", + Node container2 = Node.createDockerNode(Set.of("::3"), Set.of(), "container2", Optional.of("dockerHost"), new NodeResources(2, 4, 4), NodeType.tenant); container2 = container2.with(allocation(Optional.of("app2")).get()); nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockAllocation())); @@ -141,7 +140,7 @@ public class MetricsReporterTest { when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS)); ServiceModel serviceModel = mock(ServiceModel.class); when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); - when(serviceModel.getServiceInstancesByHostName()).thenReturn(Collections.emptyMap()); + when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); TestMetric metric = new TestMetric(); MetricsReporter metricsReporter = new MetricsReporter( @@ -154,8 +153,8 @@ public class MetricsReporterTest { ); metricsReporter.maintain(); - assertEquals(0L, metric.values.get("hostedVespa.readyHosts")); /** Only tenants counts **/ - assertEquals(2L, metric.values.get("hostedVespa.reservedHosts")); + assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts + assertEquals(2, metric.values.get("hostedVespa.reservedHosts")); assertEquals(12.0, metric.values.get("hostedVespa.docker.totalCapacityDisk")); assertEquals(10.0, metric.values.get("hostedVespa.docker.totalCapacityMem")); @@ -164,27 +163,6 @@ public class MetricsReporterTest { assertEquals(6.0, metric.values.get("hostedVespa.docker.freeCapacityDisk")); assertEquals(3.0, metric.values.get("hostedVespa.docker.freeCapacityMem")); assertEquals(4.0, metric.values.get("hostedVespa.docker.freeCapacityCpu")); - - assertContext(metric, "hostedVespa.docker.freeCapacityFlavor", 1, 0); - assertContext(metric, "hostedVespa.docker.hostsAvailableFlavor", 1l, 0l); - } - - private void assertContext(TestMetric metric, String key, Number dockerValue, Number docker2Value) { - List<Metric.Context> freeCapacityFlavor = metric.context.get(key); - assertEquals(freeCapacityFlavor.size(), 2); - - // Get the value for the two flavors - TestMetric.TestContext contextFlavorDocker = (TestMetric.TestContext)freeCapacityFlavor.get(0); - TestMetric.TestContext contextFlavorDocker2 = (TestMetric.TestContext)freeCapacityFlavor.get(1); - if (!contextFlavorDocker.properties.containsValue("docker")) { - TestMetric.TestContext temp = contextFlavorDocker; - contextFlavorDocker = contextFlavorDocker2; - contextFlavorDocker2 = temp; - } - - assertEquals(dockerValue, contextFlavorDocker.value); - assertEquals(docker2Value, contextFlavorDocker2.value); - } private ApplicationId app(String tenant) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java index d5d4bcc954d..60a31f1c804 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java @@ -11,7 +11,6 @@ import org.junit.Before; import org.junit.Test; import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; @@ -20,12 +19,16 @@ import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; /** * @author smorgrav */ public class DockerHostCapacityTest { + private final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class); private DockerHostCapacity capacity; private List<Node> nodes; private Node host1, host2, host3; @@ -33,41 +36,32 @@ public class DockerHostCapacityTest { @Before public void setup() { + doAnswer(invocation -> invocation.getArguments()[0]).when(hostResourcesCalculator).availableCapacityOf(any()); + // Create flavors NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); flavorDocker = nodeFlavors.getFlavorOrThrow("docker"); flavorDocker2 = nodeFlavors.getFlavorOrThrow("docker2"); // Create three docker hosts - host1 = Node.create("host1", Collections.singleton("::1"), generateIPs(2, 4), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); - host2 = Node.create("host2", Collections.singleton("::11"), generateIPs(12, 3), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); - host3 = Node.create("host3", Collections.singleton("::21"), generateIPs(22, 1), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + host1 = Node.create("host1", Set.of("::1"), generateIPs(2, 4), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + host2 = Node.create("host2", Set.of("::11"), generateIPs(12, 3), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + host3 = Node.create("host3", Set.of("::21"), generateIPs(22, 1), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); // Add two containers to host1 - var nodeA = Node.create("nodeA", Collections.singleton("::2"), Collections.emptySet(), "nodeA", Optional.of("host1"), Optional.empty(), flavorDocker, NodeType.tenant); - var nodeB = Node.create("nodeB", Collections.singleton("::3"), Collections.emptySet(), "nodeB", Optional.of("host1"), Optional.empty(), flavorDocker, NodeType.tenant); + var nodeA = Node.create("nodeA", Set.of("::2"), Set.of(), "nodeA", Optional.of("host1"), Optional.empty(), flavorDocker, NodeType.tenant); + var nodeB = Node.create("nodeB", Set.of("::3"), Set.of(), "nodeB", Optional.of("host1"), Optional.empty(), flavorDocker, NodeType.tenant); // Add two containers to host 2 (same as host 1) - var nodeC = Node.create("nodeC", Collections.singleton("::12"), Collections.emptySet(), "nodeC", Optional.of("host2"), Optional.empty(), flavorDocker, NodeType.tenant); - var nodeD = Node.create("nodeD", Collections.singleton("::13"), Collections.emptySet(), "nodeD", Optional.of("host2"), Optional.empty(), flavorDocker, NodeType.tenant); + var nodeC = Node.create("nodeC", Set.of("::12"), Set.of(), "nodeC", Optional.of("host2"), Optional.empty(), flavorDocker, NodeType.tenant); + var nodeD = Node.create("nodeD", Set.of("::13"), Set.of(), "nodeD", Optional.of("host2"), Optional.empty(), flavorDocker, NodeType.tenant); // Add a larger container to host3 - var nodeE = Node.create("nodeE", Collections.singleton("::22"), Collections.emptySet(), "nodeE", Optional.of("host3"), Optional.empty(), flavorDocker2, NodeType.tenant); + var nodeE = Node.create("nodeE", Set.of("::22"), Set.of(), "nodeE", Optional.of("host3"), Optional.empty(), flavorDocker2, NodeType.tenant); // init docker host capacity - nodes = new ArrayList<>(); - Collections.addAll(nodes, host1, host2, host3, nodeA, nodeB, nodeC, nodeD, nodeE); - capacity = new DockerHostCapacity(new LockedNodeList(nodes, () -> {})); - } - - @Test - public void compare_used_to_sort_in_decending_order() { - assertEquals(host1, nodes.get(0)); // Make sure it is unsorted here - - Collections.sort(nodes, capacity::compare); - assertEquals(host3, nodes.get(0)); - assertEquals(host1, nodes.get(1)); - assertEquals(host2, nodes.get(2)); + nodes = new ArrayList<>(List.of(host1, host2, host3, nodeA, nodeB, nodeC, nodeD, nodeE)); + capacity = new DockerHostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); } @Test @@ -80,10 +74,10 @@ public class DockerHostCapacityTest { assertFalse(capacity.hasCapacity(host3, flavorDocker2.resources())); // No ip available // Add a new node to host1 to deplete the memory resource - Node nodeF = Node.create("nodeF", Collections.singleton("::6"), Collections.emptySet(), + Node nodeF = Node.create("nodeF", Set.of("::6"), Set.of(), "nodeF", Optional.of("host1"), Optional.empty(), flavorDocker, NodeType.tenant); nodes.add(nodeF); - capacity = new DockerHostCapacity(new LockedNodeList(nodes, () -> {})); + capacity = new DockerHostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); assertFalse(capacity.hasCapacity(host1, flavorDocker.resources())); assertFalse(capacity.hasCapacity(host1, flavorDocker2.resources())); } @@ -96,31 +90,17 @@ public class DockerHostCapacityTest { } @Test - public void getCapacityTotal() { - NodeResources total = capacity.getCapacityTotal(NodeResources.DiskSpeed.any); - assertEquals(21.0, total.vcpu(), 0.1); - assertEquals(30.0, total.memoryGb(), 0.1); - assertEquals(36.0, total.diskGb(), 0.1); - } - - @Test - public void getFreeCapacityTotal() { - NodeResources totalFree = capacity.getFreeCapacityTotal(NodeResources.DiskSpeed.any); - assertEquals(15.0, totalFree.vcpu(), 0.1); - assertEquals(14.0, totalFree.memoryGb(), 0.1); - assertEquals(24.0, totalFree.diskGb(), 0.1); - } + public void freeCapacityOf() { + assertEquals(new NodeResources(5, 4, 8), capacity.freeCapacityOf(host1, false)); + assertEquals(new NodeResources(5, 6, 8), capacity.freeCapacityOf(host3, false)); - @Test - public void freeCapacityInFlavorEquivalence() { - assertEquals(2, capacity.freeCapacityInFlavorEquivalence(flavorDocker)); - assertEquals(2, capacity.freeCapacityInFlavorEquivalence(flavorDocker2)); - } + doAnswer(invocation -> { + NodeResources totalHostResources = (NodeResources) invocation.getArguments()[0]; + return totalHostResources.subtract(new NodeResources(1, 2, 3, NodeResources.DiskSpeed.any)); + }).when(hostResourcesCalculator).availableCapacityOf(any()); - @Test - public void getNofHostsAvailableFor() { - assertEquals(2, capacity.getNofHostsAvailableFor(flavorDocker)); - assertEquals(2, capacity.getNofHostsAvailableFor(flavorDocker2)); + assertEquals(new NodeResources(4, 2, 5), capacity.freeCapacityOf(host1, false)); + assertEquals(new NodeResources(4, 4, 5), capacity.freeCapacityOf(host3, false)); } private Set<String> generateIPs(int start, int count) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 664430a2de8..50e19e15da5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -19,7 +19,6 @@ import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.transaction.CuratorTransaction; -import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -37,7 +36,6 @@ import java.util.Set; import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.Matchers.greaterThan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; @@ -221,17 +219,13 @@ public class DynamicDockerAllocationTest { List<HostSpec> hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor); tester.activate(application1, ImmutableSet.copyOf(hosts)); - DockerHostCapacity capacity = new DockerHostCapacity(new LockedNodeList(tester.nodeRepository().getNodes(Node.State.values()), () -> {})); - assertThat(capacity.freeCapacityInFlavorEquivalence(new Flavor(flavor)), greaterThan(0)); - List<Node> initialSpareCapacity = findSpareCapacity(tester); assertThat(initialSpareCapacity.size(), is(2)); try { hosts = tester.prepare(application1, clusterSpec, 4, 1, flavor); fail("Was able to deploy with 4 nodes, should not be able to use spare capacity"); - } catch (OutOfCapacityException e) { - } + } catch (OutOfCapacityException ignored) { } tester.fail(hosts.get(0)); hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor); |