diff options
author | Harald Musum <musum@verizonmedia.com> | 2020-02-26 15:23:52 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-02-26 15:23:52 +0100 |
commit | 001fe069ffa2a1766fb431dc76fefcd20773e8f4 (patch) | |
tree | 01af8518c759f975b23caa3c563147a284e23509 /node-repository/src/main | |
parent | 76221d0b9b7bda577ac61ce8a79c7b6ee3a8dbb4 (diff) |
Revert "Bratseth/node metrics"
Diffstat (limited to 'node-repository/src/main')
23 files changed, 16 insertions, 1032 deletions
diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml index 186f052a274..274be6d572a 100644 --- a/node-repository/src/main/config/node-repository.xml +++ b/node-repository/src/main/config/node-repository.xml @@ -1,8 +1,6 @@ <!-- services.xml snippet for the node repository. Included in config server services.xml if the package is installed--> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <component id="com.yahoo.vespa.hosted.provision.provisioning.InfraDeployerImpl" bundle="node-repository"/> -<component id="com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsFetcher" bundle="node-repository"/> -<component id="com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb" bundle="node-repository"/> <component id="com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner" bundle="node-repository" /> <component id="NodeRepository" class="com.yahoo.vespa.hosted.provision.NodeRepository" bundle="node-repository"/> <component id="com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance" bundle="node-repository"/> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index f881f888752..efb2a71264a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -83,7 +83,7 @@ public final class Node { this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null"); if (state == State.active) - requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); + requireNonEmpty(ipConfig.primary(), "An active node must have at least one valid IP address"); if (parentHostname.isPresent()) { if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); @@ -375,6 +375,8 @@ public final class Node { .deviation(); } + + @Override public boolean equals(Object o) { if (this == o) return true; @@ -434,7 +436,6 @@ public final class Node { public static Set<State> allocatedStates() { return Set.of(reserved, active, inactive, failed, parked); } - } /** The mean and mean deviation (squared difference) of a bunch of numbers */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java deleted file mode 100644 index 71f7dc3701e..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; -import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits; - -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * The autoscaler makes decisions about the flavor and node count that should be allocated to a cluster - * based on observed behavior. - * - * @author bratseth - */ -public class Autoscaler { - - /* - TODO: - - Scale group size - - Have a better idea about whether we have sufficient information to make decisions - - Consider taking spikes/variance into account - - Measure observed regulation lag (startup+redistribution) into account when deciding regulation observation window - - Test AutoscalingMaintainer - - Scale by performance not just load+cost - */ - - private static final int minimumMeasurements = 500; // TODO: Per node instead? Also say something about interval? - - /** What cost difference factor warrants reallocation? */ - private static final double costDifferenceRatioWorthReallocation = 0.1; - /** What difference factor from ideal (for any resource) warrants a change? */ - private static final double idealDivergenceWorthReallocation = 0.1; - - // We only depend on the ratios between these values - private static final double cpuUnitCost = 12.0; - private static final double memoryUnitCost = 1.2; - private static final double diskUnitCost = 0.045; - - private final HostResourcesCalculator hostResourcesCalculator; - private final NodeMetricsDb metricsDb; - private final NodeRepository nodeRepository; - private final NodeResourceLimits nodeResourceLimits; - - public Autoscaler(HostResourcesCalculator hostResourcesCalculator, - NodeMetricsDb metricsDb, - NodeRepository nodeRepository) { - this.hostResourcesCalculator = hostResourcesCalculator; - this.metricsDb = metricsDb; - this.nodeRepository = nodeRepository; - this.nodeResourceLimits = new NodeResourceLimits(nodeRepository.zone()); - } - - public Optional<ClusterResources> autoscale(ApplicationId applicationId, ClusterSpec cluster, List<Node> clusterNodes) { - if (clusterNodes.stream().anyMatch(node -> node.status().wantToRetire() || - node.allocation().get().membership().retired() || - node.allocation().get().isRemovable())) - return Optional.empty(); // Don't autoscale clusters that are in flux - ClusterResources currentAllocation = new ClusterResources(clusterNodes); - Optional<Double> cpuLoad = averageLoad(Resource.cpu, cluster, clusterNodes); - Optional<Double> memoryLoad = averageLoad(Resource.memory, cluster, clusterNodes); - Optional<Double> diskLoad = averageLoad(Resource.disk, cluster, clusterNodes); - if (cpuLoad.isEmpty() || memoryLoad.isEmpty() || diskLoad.isEmpty()) return Optional.empty(); - - Optional<ClusterResourcesWithCost> bestAllocation = findBestAllocation(cpuLoad.get(), - memoryLoad.get(), - diskLoad.get(), - currentAllocation, - cluster); - if (bestAllocation.isEmpty()) return Optional.empty(); - - if (closeToIdeal(Resource.cpu, cpuLoad.get()) && - closeToIdeal(Resource.memory, memoryLoad.get()) && - closeToIdeal(Resource.disk, diskLoad.get()) && - similarCost(bestAllocation.get().cost(), currentAllocation.nodes() * costOf(currentAllocation.nodeResources()))) - return Optional.empty(); // Avoid small, unnecessary changes - return bestAllocation.map(a -> a.clusterResources()); - } - - private Optional<ClusterResourcesWithCost> findBestAllocation(double cpuLoad, double memoryLoad, double diskLoad, - ClusterResources currentAllocation, ClusterSpec cluster) { - Optional<ClusterResourcesWithCost> bestAllocation = Optional.empty(); - for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation); i.hasNext(); ) { - ClusterResources allocation = i.next(); - Optional<ClusterResourcesWithCost> allocatableResources = toAllocatableResources(allocation, cluster); - if (allocatableResources.isEmpty()) continue; - if (bestAllocation.isEmpty() || allocatableResources.get().cost() < bestAllocation.get().cost()) - bestAllocation = allocatableResources; - } - return bestAllocation; - } - - private boolean similarCost(double cost1, double cost2) { - return similar(cost1, cost2, costDifferenceRatioWorthReallocation); - } - - private boolean closeToIdeal(Resource resource, double value) { - return similar(resource.idealAverageLoad(), value, idealDivergenceWorthReallocation); - } - - private boolean similar(double r1, double r2, double threshold) { - return Math.abs(r1 - r2) / r1 < threshold; - } - - /** - * Returns the smallest allocatable node resources larger than the given node resources, - * or empty if none available. - */ - private Optional<ClusterResourcesWithCost> toAllocatableResources(ClusterResources resources, ClusterSpec cluster) { - if (allowsHostSharing(nodeRepository.zone().cloud())) { - // Return the requested resources, adjusted to be legal or empty if they cannot fit on existing hosts - NodeResources nodeResources = nodeResourceLimits.enlargeToLegal(resources.nodeResources(), cluster.type()); - for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) - if (flavor.resources().satisfies(nodeResources)) - return Optional.of(new ClusterResourcesWithCost(resources.with(nodeResources), - costOf(nodeResources) * resources.nodes())); - return Optional.empty(); - } - else { - // return the cheapest flavor satisfying the target resources, if any - double bestCost = Double.MAX_VALUE; - Optional<Flavor> bestFlavor = Optional.empty(); - for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { - if ( ! flavor.resources().satisfies(resources.nodeResources())) continue; - if (bestFlavor.isEmpty() || bestCost > costOf(flavor.resources())) { - bestFlavor = Optional.of(flavor); - bestCost = costOf(flavor); - } - } - if (bestFlavor.isEmpty()) - return Optional.empty(); - else - return Optional.of(new ClusterResourcesWithCost(resources.with(bestFlavor.get().resources()), - bestCost * resources.nodes())); - } - } - - /** - * Returns the average load of this resource in the measurement window, - * or empty if we are not in a position to make decisions from these measurements at this time. - */ - private Optional<Double> averageLoad(Resource resource, ClusterSpec cluster, List<Node> clusterNodes) { - NodeMetricsDb.Window window = metricsDb.getWindow(nodeRepository.clock().instant().minus(scalingWindow(cluster.type())), - resource, - clusterNodes.stream().map(Node::hostname).collect(Collectors.toList())); - - if (window.measurementCount() < minimumMeasurements) return Optional.empty(); - if (window.hostnames() != clusterNodes.size()) return Optional.empty(); // Regulate only when all nodes are measured - - return Optional.of(window.average()); - } - - /** The duration of the window we need to consider to make a scaling decision */ - private Duration scalingWindow(ClusterSpec.Type clusterType) { - if (clusterType.isContent()) return Duration.ofHours(12); // Ideally we should use observed redistribution time - return Duration.ofHours(12); // TODO: Measure much more often to get this down to minutes. And, ideally we should take node startup time into account - } - - // TODO: Put this in zone config instead? - private boolean allowsHostSharing(CloudName cloudName) { - if (cloudName.value().equals("aws")) return false; - return true; - } - - private double costOf(Flavor flavor) { - NodeResources chargedResources = hostResourcesCalculator.availableCapacityOf(flavor.name(), flavor.resources()); - return costOf(chargedResources); - } - - private double costOf(NodeResources resources) { - return resources.vcpu() * cpuUnitCost + - resources.memoryGb() * memoryUnitCost + - resources.diskGb() * diskUnitCost; - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java deleted file mode 100644 index e068b4404d8..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import com.yahoo.config.provision.NodeResources; -import com.yahoo.vespa.hosted.provision.Node; - -import java.util.List; -import java.util.Objects; - -/** A description of the resources of a cluster */ -public class ClusterResources { - - /** The node count in the cluster */ - private final int nodes; - - /** The number of node groups in the cluster */ - private final int groups; - - /** The resources of each node in the cluster */ - private final NodeResources nodeResources; - - public ClusterResources(List<Node> nodes) { - this(nodes.size(), - (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(), - nodes.get(0).flavor().resources()); - } - - public ClusterResources(int nodes, int groups, NodeResources nodeResources) { - this.nodes = nodes; - this.groups = groups; - this.nodeResources = nodeResources; - } - - /** Returns the total number of allocated nodes (over all groups) */ - public int nodes() { return nodes; } - public int groups() { return groups; } - public NodeResources nodeResources() { return nodeResources; } - - public ClusterResources with(NodeResources resources) { - return new ClusterResources(nodes, groups, resources); - } - - @Override - public boolean equals(Object o) { - if (o == this) return true; - if ( ! (o instanceof ClusterResources)) return false; - - ClusterResources other = (ClusterResources)o; - if (other.nodes != this.nodes) return false; - if (other.groups != this.groups) return false; - if (other.nodeResources != this.nodeResources) return false; - return true; - } - - @Override - public int hashCode() { - return Objects.hash(nodes, groups, nodeResources); - } - - @Override - public String toString() { - return "cluster resources: " + nodes + " * " + nodeResources + (groups > 1 ? " in " + groups + " groups" : ""); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java deleted file mode 100644 index 55b28ef3ce1..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -/** - * @author bratseth - */ -public class ClusterResourcesWithCost { - - private final ClusterResources resources; - private final double cost; - - public ClusterResourcesWithCost(ClusterResources resources, double cost) { - this.resources = resources; - this.cost = cost; - } - - public ClusterResources clusterResources() { return resources;} - - public double cost() { return cost; } - - @Override - public String toString() { - return "$" + cost + ": " + clusterResources(); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java deleted file mode 100644 index a599606c314..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Consumes a response from the metrics/v2 API and populates the fields of this with the resulting values - * - * @author bratseth - */ -public class MetricsResponse { - - private final List<NodeMetrics.MetricValue> metricValues = new ArrayList<>(); - - public MetricsResponse(byte[] response) { - this(SlimeUtils.jsonToSlime(response)); - } - - public MetricsResponse(String response) { - this(SlimeUtils.jsonToSlime(response)); - } - - public List<NodeMetrics.MetricValue> metrics() { return metricValues; } - - private MetricsResponse(Slime response) { - Inspector root = response.get(); - Inspector nodes = root.field("nodes"); - nodes.traverse((ArrayTraverser)(__, node) -> consumeNode(node)); - } - - private void consumeNode(Inspector node) { - String hostname = node.field("hostname").asString(); - consumeNodeMetrics(hostname, node.field("node")); - consumeServiceMetrics(hostname, node.field("services")); - } - - private void consumeNodeMetrics(String hostname, Inspector node) { - long timestamp = node.field("timestamp").asLong(); - Map<String, Double> values = consumeMetrics(node.field("metrics")); - for (Resource resource : Resource.values()) - addMetricIfPresent(hostname, resource.metricName(), timestamp, values); - } - - private void addMetricIfPresent(String hostname, String metricName, long timestamp, Map<String, Double> values) { - if (values.containsKey(metricName)) - metricValues.add(new NodeMetrics.MetricValue(hostname, metricName, timestamp, values.get(metricName).floatValue())); - } - - private void consumeServiceMetrics(String hostname, Inspector node) { - String name = node.field("name").asString(); - long timestamp = node.field("timestamp").asLong(); - Map<String, Double> values = consumeMetrics(node.field("metrics")); - } - - private Map<String, Double> consumeMetrics(Inspector metrics) { - Map<String, Double> values = new HashMap<>(); - metrics.traverse((ArrayTraverser) (__, item) -> consumeMetricsItem(item, values)); - return values; - } - - private void consumeMetricsItem(Inspector item, Map<String, Double> values) { - item.field("values").traverse((ObjectTraverser)(name, value) -> values.put(name, value.asDouble())); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java deleted file mode 100644 index 97ac1e72be9..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import com.yahoo.config.provision.ApplicationId; - -import java.util.Collection; - -/** - * Interface to retrieve metrics on (tenant) nodes. - * - * @author bratseth - */ -public interface NodeMetrics { - - /** - * Fetches metrics for an application. This call may be expensive. - * - * @param application the application to fetch metrics from - */ - Collection<MetricValue> fetchMetrics(ApplicationId application); - - final class MetricValue { - - private final String hostname; - private final String name; - private long timestamp; - private final float value; - - public MetricValue(String hostname, String name, long timestamp, float value) { - this.hostname = hostname; - this.name = name; - this.timestamp = timestamp; - this.value = value; - } - - public String hostname() { return hostname; } - public String name() { return name; } - public long timestamp() { return timestamp; } - public float value() { return value; } - - @Override - public String toString() { - return "metric value " + name + ": " + value + " at " + timestamp + " for " + hostname; - } - - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java deleted file mode 100644 index 14a35e3efbc..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * An in-memory time-series "database" of node metrics. - * Thread model: One writer, many readers. - * - * @author bratseth - */ -public class NodeMetricsDb { - - private static final Duration dbWindow = Duration.ofHours(24); - - /** Measurements by key. Each list of measurements is sorted by increasing timestamp */ - private Map<MeasurementKey, List<Measurement>> db = new HashMap<>(); - - /** Lock all access for now since we modify lists inside a map */ - private final Object lock = new Object(); - - /** Add a measurement to this */ - public void add(Collection<NodeMetrics.MetricValue> metricValues) { - synchronized (lock) { - for (var value : metricValues) { - List<Measurement> measurements = db.computeIfAbsent(new MeasurementKey(value.hostname(), - Resource.fromMetric(value.name())), - (__) -> new ArrayList<>()); - measurements.add(new Measurement(value.timestamp(), value.value())); - } - } - } - - /** Must be called intermittently (as long as add is called) to gc old measurements */ - public void gc(Clock clock) { - synchronized (lock) { - // TODO: We may need to do something more complicated to avoid spending too much memory to - // lower the measurement interval (see NodeRepositoryMaintenance) - // Each measurement is Object + long + float = 16 + 8 + 4 = 28 bytes - // 24 hours with 1k nodes and 3 resources and 1 measurement/sec is about 10Gb - - long oldestTimestamp = clock.instant().minus(dbWindow).toEpochMilli(); - for (Iterator<List<Measurement>> i = db.values().iterator(); i.hasNext(); ) { - List<Measurement> measurements = i.next(); - - while (!measurements.isEmpty() && measurements.get(0).timestamp < oldestTimestamp) - measurements.remove(0); - - if (measurements.isEmpty()) - i.remove(); - } - } - } - - /** Returns a window within which we can ask for specific information from this db */ - public Window getWindow(Instant startTime, Resource resource, List<String> hostnames) { - return new Window(startTime, resource, hostnames); - } - - public class Window { - - private final long startTime; - private List<MeasurementKey> keys; - - private Window(Instant startTime, Resource resource, List<String> hostnames) { - this.startTime = startTime.toEpochMilli(); - keys = hostnames.stream().map(hostname -> new MeasurementKey(hostname, resource)).collect(Collectors.toList()); - } - - public int measurementCount() { - synchronized (lock) { - return (int) keys.stream() - .flatMap(key -> db.getOrDefault(key, List.of()).stream()) - .filter(measurement -> measurement.timestamp >= startTime) - .count(); - } - } - - /** Returns the count of hostnames which have measurements in this window */ - public int hostnames() { - synchronized (lock) { - int count = 0; - for (MeasurementKey key : keys) { - List<Measurement> measurements = db.get(key); - if (measurements == null || measurements.isEmpty()) continue; - - if (measurements.get(measurements.size() - 1).timestamp >= startTime) - count++; - } - return count; - } - } - - public double average() { - synchronized (lock) { - double sum = 0; - int count = 0; - for (MeasurementKey key : keys) { - List<Measurement> measurements = db.get(key); - if (measurements == null) continue; - - int index = measurements.size() - 1; - while (index >= 0 && measurements.get(index).timestamp >= startTime) { - sum += measurements.get(index).value; - count++; - - index--; - } - } - return sum / count; - } - } - - } - - private static class MeasurementKey { - - private final String hostname; - private final Resource resource; - - public MeasurementKey(String hostname, Resource resource) { - this.hostname = hostname; - this.resource = resource; - } - - @Override - public int hashCode() { - return Objects.hash(hostname, resource); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if ( ! (o instanceof MeasurementKey)) return false; - MeasurementKey other = (MeasurementKey)o; - if ( ! this.hostname.equals(other.hostname)) return false; - if ( ! this.resource.equals(other.resource)) return false; - return true; - } - - } - - private static class Measurement { - - /** The time of this measurement in epoch millis */ - private final long timestamp; - - /** The measured value */ - private final float value; - - public Measurement(long timestamp, float value) { - this.timestamp = timestamp; - this.value = value; - } - - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java deleted file mode 100644 index 54d8eac238f..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import ai.vespa.util.http.VespaHttpClientBuilder; -import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.orchestrator.HostNameNotFoundException; -import com.yahoo.vespa.orchestrator.Orchestrator; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.BasicResponseHandler; -import org.apache.http.impl.client.CloseableHttpClient; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Collection; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Fetches node metrics over the metrics/v2 API - * - * @author bratseth - */ -public class NodeMetricsFetcher extends AbstractComponent implements NodeMetrics { - - private static final Logger log = Logger.getLogger(NodeMetricsFetcher.class.getName()); - - private static final String apiPath = "/metrics/v2/values"; - - private final NodeRepository nodeRepository; - private final Orchestrator orchestrator; - private final HttpClient httpClient; - - @Inject - public NodeMetricsFetcher(NodeRepository nodeRepository, Orchestrator orchestrator) { - this(nodeRepository, orchestrator, new ApacheHttpClient()); - } - - NodeMetricsFetcher(NodeRepository nodeRepository, Orchestrator orchestrator, HttpClient httpClient) { - this.nodeRepository = nodeRepository; - this.orchestrator = orchestrator; - this.httpClient = httpClient; - } - - @Override - public Collection<MetricValue> fetchMetrics(ApplicationId application) { - Node metricsV2Container = nodeRepository.list() - .owner(application) - .state(Node.State.active) - .container() - .filter(node -> expectedUp(node)) - .asList().get(0); - String url = "http://" + metricsV2Container.hostname() + ":" + 4080 + apiPath + "?consumer=vespa-consumer-metrics"; - String response = httpClient.get(url); - return new MetricsResponse(response).metrics(); - } - - @Override - public void deconstruct() { - httpClient.close(); - } - - private boolean expectedUp(Node node) { - try { - return ! orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended(); - } - catch (HostNameNotFoundException e) { - return false; - } - } - - /** The simplest possible http client interface */ - public interface HttpClient { - - String get(String url); - void close(); - - } - - /** Implements the HttpClient interface by delegating to an Apache HTTP client */ - public static class ApacheHttpClient implements HttpClient { - - private final CloseableHttpClient httpClient = VespaHttpClientBuilder.createWithBasicConnectionManager().build(); - - @Override - public String get(String url) { - try { - return httpClient.execute(new HttpGet(url), new BasicResponseHandler()); - } - catch (IOException e) { - throw new UncheckedIOException("Could not get " + url, e); - } - } - - @Override - public void close() { - try { - httpClient.close(); - } - catch (IOException e) { - log.log(Level.WARNING, "Exception deconstructing", e); - } - } - - - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java deleted file mode 100644 index 9c85ca870d5..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import com.yahoo.config.provision.NodeResources; - -/** - * A resource subject to autoscaling - * - * @author bratseth - */ -public enum Resource { - - cpu { - String metricName() { return "cpu.util"; } - double idealAverageLoad() { return 0.2; } - double valueFrom(NodeResources resources) { return resources.vcpu(); } - }, - - memory { - String metricName() { return "memory.util"; } - double idealAverageLoad() { return 0.7; } - double valueFrom(NodeResources resources) { return resources.memoryGb(); } - }, - - disk { - String metricName() { return "disk.util"; } - double idealAverageLoad() { return 0.7; } - double valueFrom(NodeResources resources) { return resources.diskGb(); } - }; - - abstract String metricName(); - - /** The load we should have of this resource on average, when one node in the cluster is down */ - abstract double idealAverageLoad(); - - abstract double valueFrom(NodeResources resources); - - public static Resource fromMetric(String metricName) { - for (Resource resource : values()) - if (resource.metricName().equals(metricName)) return resource; - throw new IllegalArgumentException("Metric '" + metricName + "' does not map to a resource"); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java deleted file mode 100644 index 464fe570b95..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import com.yahoo.config.provision.NodeResources; - -/** - * Provides iteration over possible cluster resource allocations given a target total load - * and current groups/nodes allocation. - */ -public class ResourceIterator { - - // Configured min and max nodes TODO: These should come from the application package - private static final int minimumNodesPerCluster = 3; // Since this is with redundancy it cannot be lower than 2 - private static final int maximumNodesPerCluster = 150; - - // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component - // proportional to document count. We must account for this when comparing configurations with more or fewer nodes. - // TODO: Measure this, and only take it into account with queries - private static final double fixedCpuCostFraction = 0.1; - - // Describes the observed state - private final ClusterResources allocation; - private final double cpuLoad; - private final double memoryLoad; - private final double diskLoad; - private final int groupSize; - - // Derived from the observed state - private final int nodeIncrement; - private final boolean singleGroupMode; - - // Iterator state - private int currentNodes; - - public ResourceIterator(double cpuLoad, double memoryLoad, double diskLoad, ClusterResources currentAllocation) { - this.cpuLoad = cpuLoad; - this.memoryLoad = memoryLoad; - this.diskLoad = diskLoad; - - // ceil: If the division does not produce a whole number we assume some node is missing - groupSize = (int)Math.ceil((double)currentAllocation.nodes() / currentAllocation.groups()); - allocation = currentAllocation; - - // What number of nodes is it effective to add or remove at the time from this cluster? - // This is the group size, since we (for now) assume the group size is decided by someone wiser than us - // and we decide the number of groups. - // The exception is when we only have one group, where we can add and remove single nodes in it. - singleGroupMode = currentAllocation.groups() == 1; - nodeIncrement = singleGroupMode ? 1 : groupSize; - - currentNodes = currentAllocation.nodes(); - while (currentNodes - nodeIncrement >= minimumNodesPerCluster - && (singleGroupMode || currentNodes - nodeIncrement > groupSize)) // group level redundancy - currentNodes -= nodeIncrement; - } - - public ClusterResources next() { - int nodesWithRedundancy = currentNodes - (singleGroupMode ? 1 : groupSize); - ClusterResources next = new ClusterResources(currentNodes, - singleGroupMode ? 1 : currentNodes / groupSize, - resourcesFor(nodesWithRedundancy)); - currentNodes += nodeIncrement; - return next; - } - - public boolean hasNext() { - return currentNodes <= maximumNodesPerCluster; - } - - /** - * For the observed load this instance is initialized with, returns the resources needed per node to be at - * ideal load given a target node count - */ - private NodeResources resourcesFor(int nodeCount) { - // Cpu: Scales with cluster size (TODO: Only reads, writes scales with group size) - // Memory and disk: Scales with group size - - double cpu, memory, disk; - if (singleGroupMode) { - // The fixed cost portion of cpu does not scale with changes to the node count - // TODO: Only for the portion of cpu consumed by queries - double totalCpu = totalUsage(Resource.cpu, cpuLoad); - cpu = fixedCpuCostFraction * totalCpu / groupSize / Resource.cpu.idealAverageLoad() + - (1 - fixedCpuCostFraction) * totalCpu / nodeCount / Resource.cpu.idealAverageLoad(); - memory = totalGroupUsage(Resource.memory, memoryLoad) / nodeCount / Resource.memory.idealAverageLoad(); - disk = totalGroupUsage(Resource.disk, diskLoad) / nodeCount / Resource.disk.idealAverageLoad(); - } - else { - cpu = totalUsage(Resource.cpu, cpuLoad) / nodeCount / Resource.cpu.idealAverageLoad(); - memory = totalGroupUsage(Resource.memory, memoryLoad) / groupSize / Resource.memory.idealAverageLoad(); - disk = totalGroupUsage(Resource.disk, diskLoad) / groupSize / Resource.disk.idealAverageLoad(); - } - return allocation.nodeResources().withVcpu(cpu).withMemoryGb(memory).withDiskGb(disk); - } - - private double totalUsage(Resource resource, double load) { - return load * resource.valueFrom(allocation.nodeResources()) * allocation.nodes(); - } - - private double totalGroupUsage(Resource resource, double load) { - return load * resource.valueFrom(allocation.nodeResources()) * groupSize; - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java deleted file mode 100644 index f3c1d8603b3..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Deployer; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; -import com.yahoo.vespa.hosted.provision.autoscale.ClusterResources; -import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; -import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; - -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * Maintainer making automatic scaling decisions - * - * @author bratseth - */ -public class AutoscalingMaintainer extends Maintainer { - - private final Autoscaler autoscaler; - private final Deployer deployer; - - public AutoscalingMaintainer(NodeRepository nodeRepository, - HostResourcesCalculator hostResourcesCalculator, - NodeMetricsDb metricsDb, - Deployer deployer, - Duration interval) { - super(nodeRepository, interval); - this.autoscaler = new Autoscaler(hostResourcesCalculator, metricsDb, nodeRepository); - this.deployer = deployer; - } - - @Override - protected void maintain() { - if ( ! nodeRepository().zone().environment().isProduction()) return; - - activeNodesByApplication().forEach((applicationId, nodes) -> autoscale(applicationId, nodes)); - } - - private void autoscale(ApplicationId application, List<Node> applicationNodes) { - MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, nodeRepository()); - if ( ! deployment.isValid()) return; // Another config server will consider this application - nodesByCluster(applicationNodes).forEach((clusterSpec, clusterNodes) -> { - Optional<ClusterResources> target = autoscaler.autoscale(application, clusterSpec, clusterNodes); - target.ifPresent(t -> log.info("Autoscale: Application " + application + " cluster " + clusterSpec + - " from " + applicationNodes.size() + " * " + applicationNodes.get(0).flavor().resources() + - " to " + t.nodes() + " * " + t.nodeResources())); - }); - } - - private Map<ClusterSpec, List<Node>> nodesByCluster(List<Node> applicationNodes) { - return applicationNodes.stream().collect(Collectors.groupingBy(n -> n.allocation().get().membership().cluster())); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java index 27fba9e8f8e..0d5a8587902 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java @@ -2,22 +2,17 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.AbstractComponent; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; /** * A maintainer is some job which runs at a fixed rate to perform some maintenance task on the node repo. @@ -80,12 +75,6 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { private String name() { return this.getClass().getSimpleName(); } - /** A utility to group active tenant applications by application */ - protected Map<ApplicationId, List<Node>> activeNodesByApplication() { - return nodeRepository().list().nodeType(NodeType.tenant).state(Node.State.active).asList() - .stream().collect(Collectors.groupingBy(n -> n.allocation().get().owner())); - } - static long staggeredDelay(List<HostName> cluster, HostName host, Instant now, Duration interval) { if ( ! cluster.contains(host)) return interval.toMillis(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java deleted file mode 100644 index 178e8385008..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; - -import java.time.Duration; -import java.util.logging.Level; - -/** - * Maintainer which keeps the node metric db up to date by periodically fetching metrics from all - * active nodes. - * - * @author bratseth - */ -public class NodeMetricsDbMaintainer extends Maintainer { - - private static final int maxWarningsPerInvocation = 2; - - private final NodeMetrics nodeMetrics; - private final NodeMetricsDb nodeMetricsDb; - - public NodeMetricsDbMaintainer(NodeRepository nodeRepository, - NodeMetrics nodeMetrics, - NodeMetricsDb nodeMetricsDb, - Duration interval) { - super(nodeRepository, interval); - this.nodeMetrics = nodeMetrics; - this.nodeMetricsDb = nodeMetricsDb; - } - - @Override - protected void maintain() { - int warnings = 0; - for (ApplicationId application : activeNodesByApplication().keySet()) { - try { - nodeMetricsDb.add(nodeMetrics.fetchMetrics(application)); - } - catch (Exception e) { - if (warnings++ < maxWarningsPerInvocation) - log.log(Level.WARNING, "Could not update metrics for " + application, e); - } - } - nodeMetricsDb.gc(nodeRepository().clock()); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index ecc550527fc..37620e17a95 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -8,11 +8,9 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.monitor.ServiceMonitor; @@ -50,25 +48,22 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final CapacityReportMaintainer capacityReportMaintainer; private final OsUpgradeActivator osUpgradeActivator; private final Rebalancer rebalancer; - private final NodeMetricsDbMaintainer nodeMetricsDbMaintainer; - private final AutoscalingMaintainer autoscalingMaintainer; @SuppressWarnings("unused") @Inject public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, InfraDeployer infraDeployer, HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Orchestrator orchestrator, Metric metric, - ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource, - NodeMetrics nodeMetrics, NodeMetricsDb nodeMetricsDb) { + ProvisionServiceProvider provisionServiceProvider, + FlagSource flagSource) { this(nodeRepository, deployer, infraDeployer, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(), - orchestrator, metric, provisionServiceProvider, flagSource, nodeMetrics, nodeMetricsDb); + orchestrator, metric, provisionServiceProvider, flagSource); } public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, InfraDeployer infraDeployer, HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Clock clock, Orchestrator orchestrator, Metric metric, - ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource, - NodeMetrics nodeMetrics, NodeMetricsDb nodeMetricsDb) { + ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource) { DefaultTimes defaults = new DefaultTimes(zone); nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, defaults.failGrace, clock, orchestrator, throttlePolicyFromEnv().orElse(defaults.throttlePolicy), metric); @@ -90,8 +85,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent { capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval); osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval); rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); - nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval); - autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, deployer, defaults.autoscalingInterval); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintainButThrowOnException(); @@ -116,8 +109,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent { dynamicProvisioningMaintainer.ifPresent(Maintainer::deconstruct); osUpgradeActivator.deconstruct(); rebalancer.deconstruct(); - nodeMetricsDbMaintainer.deconstruct(); - autoscalingMaintainer.deconstruct(); } private static Optional<NodeFailer.ThrottlePolicy> throttlePolicyFromEnv() { @@ -158,8 +149,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration dynamicProvisionerInterval; private final Duration osUpgradeActivatorInterval; private final Duration rebalancerInterval; - private final Duration nodeMetricsCollectionInterval; - private final Duration autoscalingInterval; private final NodeFailer.ThrottlePolicy throttlePolicy; @@ -180,8 +169,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent { dynamicProvisionerInterval = Duration.ofMinutes(5); osUpgradeActivatorInterval = zone.system().isCd() ? Duration.ofSeconds(30) : Duration.ofMinutes(5); rebalancerInterval = Duration.ofMinutes(40); - nodeMetricsCollectionInterval = Duration.ofMinutes(1); - autoscalingInterval = Duration.ofMinutes(5); if (zone.environment().equals(Environment.prod) && ! zone.system().isCd()) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 179d7f2703c..7c5ff35878b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -19,15 +19,11 @@ import java.util.Locale; public class CapacityPolicies { private final Zone zone; - - private final NodeResourceLimits nodeResourceLimits; - /* Deployments must match 1-to-1 the advertised resources of a physical host */ private final boolean isUsingAdvertisedResources; public CapacityPolicies(Zone zone) { this.zone = zone; - this.nodeResourceLimits = new NodeResourceLimits(zone); this.isUsingAdvertisedResources = zone.cloud().value().equals("aws"); } @@ -68,7 +64,7 @@ public class CapacityPolicies { } private void ensureSufficientResources(NodeResources resources, ClusterSpec cluster) { - double minMemoryGb = nodeResourceLimits.minMemoryGb(cluster.type()); + double minMemoryGb = minMemoryGb(cluster.type()); if (resources.memoryGb() >= minMemoryGb) return; throw new IllegalArgumentException(String.format(Locale.ENGLISH, @@ -76,6 +72,12 @@ public class CapacityPolicies { minMemoryGb, cluster.type().name(), cluster.id().value(), resources.memoryGb())); } + private int minMemoryGb(ClusterSpec.Type clusterType) { + if (zone.system() == SystemName.dev) return 1; // Allow small containers in dev system + if (clusterType == ClusterSpec.Type.admin) return 2; + return 4; + } + private NodeResources defaultNodeResources(ClusterSpec.Type clusterType) { if (clusterType == ClusterSpec.Type.admin) { if (zone.system() == SystemName.dev) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index af6fa8edf64..5753bbb3c5a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -64,6 +64,7 @@ public class GroupPreparer { .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) .value(); boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty(); + try (Mutex lock = nodeRepository.lock(application)) { // Lock ready pool to ensure that the same nodes are not simultaneously allocated by others diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 0423f762f2b..394549e4141 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -18,7 +18,7 @@ public interface HostProvisioner { /** * Schedule provisioning of a given number of hosts. * - * @param provisionIndexes list of unique provision indexes which will be used to generate the node hostnames + * @param provisionIndexes List of unique provision indexes which will be used to generate the node hostnames * on the form of <code>[prefix][index].[domain]</code> * @param resources the resources needed per node * @param applicationId id of the application that will own the provisioned host diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index c92f7889496..ebd6a01e61f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -390,5 +390,4 @@ class NodeAllocation { return count; } } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java deleted file mode 100644 index ca04bf66ce3..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; - -/** - * Defines the resource limits for nodes in various zones - * - * @author bratseth - */ -public class NodeResourceLimits { - - private final Zone zone; - - public NodeResourceLimits(Zone zone) { - this.zone = zone; - } - - public int minMemoryGb(ClusterSpec.Type clusterType) { - if (zone.system() == SystemName.dev) return 1; // Allow small containers in dev system - if (clusterType == ClusterSpec.Type.admin) return 2; - return 4; - } - - public NodeResources enlargeToLegal(NodeResources resources, ClusterSpec.Type clusterType) { - return resources.withMemoryGb(Math.max(minMemoryGb(clusterType), resources.memoryGb())); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index d26accd7a84..49d0ba5cf70 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -25,8 +25,6 @@ public class ContainerConfig { " <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockDuperModel'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>\n" + - " <component id='com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb'/>\n" + - " <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeMetrics'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance'/>\n" + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java index e7ebf049e51..915ef0d9125 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java @@ -18,7 +18,6 @@ import java.util.concurrent.ConcurrentHashMap; * @author hakonhall */ public class MockDuperModel implements DuperModelInfraApi { - private final Map<ApplicationId, InfraApplicationApi> supportedInfraApps = new HashMap<>(); private final ConcurrentHashMap<ApplicationId, List<HostName>> activeApps = new ConcurrentHashMap<>(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java deleted file mode 100644 index d5397aa421c..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.testutils; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; - -import java.util.ArrayList; -import java.util.Collection; - -/** - * @author bratseth - */ -public class MockNodeMetrics implements NodeMetrics { - - @Override - public Collection<MetricValue> fetchMetrics(ApplicationId application) { - return new ArrayList<>(); - } - -} |