summaryrefslogtreecommitdiffstats
path: root/node-repository/src/main
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2020-02-26 15:23:52 +0100
committerGitHub <noreply@github.com>2020-02-26 15:23:52 +0100
commit001fe069ffa2a1766fb431dc76fefcd20773e8f4 (patch)
tree01af8518c759f975b23caa3c563147a284e23509 /node-repository/src/main
parent76221d0b9b7bda577ac61ce8a79c7b6ee3a8dbb4 (diff)
Revert "Bratseth/node metrics"
Diffstat (limited to 'node-repository/src/main')
-rw-r--r--node-repository/src/main/config/node-repository.xml2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java5
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java185
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java65
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java26
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java74
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java48
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java169
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java112
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java44
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java104
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java62
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java11
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java49
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java21
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java12
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java1
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java1
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java32
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java1
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java20
23 files changed, 16 insertions, 1032 deletions
diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml
index 186f052a274..274be6d572a 100644
--- a/node-repository/src/main/config/node-repository.xml
+++ b/node-repository/src/main/config/node-repository.xml
@@ -1,8 +1,6 @@
<!-- services.xml snippet for the node repository. Included in config server services.xml if the package is installed-->
<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
<component id="com.yahoo.vespa.hosted.provision.provisioning.InfraDeployerImpl" bundle="node-repository"/>
-<component id="com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsFetcher" bundle="node-repository"/>
-<component id="com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb" bundle="node-repository"/>
<component id="com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner" bundle="node-repository" />
<component id="NodeRepository" class="com.yahoo.vespa.hosted.provision.NodeRepository" bundle="node-repository"/>
<component id="com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance" bundle="node-repository"/>
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
index f881f888752..efb2a71264a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
@@ -83,7 +83,7 @@ public final class Node {
this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null");
if (state == State.active)
- requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address");
+ requireNonEmpty(ipConfig.primary(), "An active node must have at least one valid IP address");
if (parentHostname.isPresent()) {
if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool");
@@ -375,6 +375,8 @@ public final class Node {
.deviation();
}
+
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
@@ -434,7 +436,6 @@ public final class Node {
public static Set<State> allocatedStates() {
return Set.of(reserved, active, inactive, failed, parked);
}
-
}
/** The mean and mean deviation (squared difference) of a bunch of numbers */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
deleted file mode 100644
index 71f7dc3701e..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
+++ /dev/null
@@ -1,185 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.autoscale;
-
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.CloudName;
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.Flavor;
-import com.yahoo.config.provision.NodeResources;
-import com.yahoo.vespa.hosted.provision.Node;
-import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
-import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits;
-
-import java.time.Duration;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-/**
- * The autoscaler makes decisions about the flavor and node count that should be allocated to a cluster
- * based on observed behavior.
- *
- * @author bratseth
- */
-public class Autoscaler {
-
- /*
- TODO:
- - Scale group size
- - Have a better idea about whether we have sufficient information to make decisions
- - Consider taking spikes/variance into account
- - Measure observed regulation lag (startup+redistribution) into account when deciding regulation observation window
- - Test AutoscalingMaintainer
- - Scale by performance not just load+cost
- */
-
- private static final int minimumMeasurements = 500; // TODO: Per node instead? Also say something about interval?
-
- /** What cost difference factor warrants reallocation? */
- private static final double costDifferenceRatioWorthReallocation = 0.1;
- /** What difference factor from ideal (for any resource) warrants a change? */
- private static final double idealDivergenceWorthReallocation = 0.1;
-
- // We only depend on the ratios between these values
- private static final double cpuUnitCost = 12.0;
- private static final double memoryUnitCost = 1.2;
- private static final double diskUnitCost = 0.045;
-
- private final HostResourcesCalculator hostResourcesCalculator;
- private final NodeMetricsDb metricsDb;
- private final NodeRepository nodeRepository;
- private final NodeResourceLimits nodeResourceLimits;
-
- public Autoscaler(HostResourcesCalculator hostResourcesCalculator,
- NodeMetricsDb metricsDb,
- NodeRepository nodeRepository) {
- this.hostResourcesCalculator = hostResourcesCalculator;
- this.metricsDb = metricsDb;
- this.nodeRepository = nodeRepository;
- this.nodeResourceLimits = new NodeResourceLimits(nodeRepository.zone());
- }
-
- public Optional<ClusterResources> autoscale(ApplicationId applicationId, ClusterSpec cluster, List<Node> clusterNodes) {
- if (clusterNodes.stream().anyMatch(node -> node.status().wantToRetire() ||
- node.allocation().get().membership().retired() ||
- node.allocation().get().isRemovable()))
- return Optional.empty(); // Don't autoscale clusters that are in flux
- ClusterResources currentAllocation = new ClusterResources(clusterNodes);
- Optional<Double> cpuLoad = averageLoad(Resource.cpu, cluster, clusterNodes);
- Optional<Double> memoryLoad = averageLoad(Resource.memory, cluster, clusterNodes);
- Optional<Double> diskLoad = averageLoad(Resource.disk, cluster, clusterNodes);
- if (cpuLoad.isEmpty() || memoryLoad.isEmpty() || diskLoad.isEmpty()) return Optional.empty();
-
- Optional<ClusterResourcesWithCost> bestAllocation = findBestAllocation(cpuLoad.get(),
- memoryLoad.get(),
- diskLoad.get(),
- currentAllocation,
- cluster);
- if (bestAllocation.isEmpty()) return Optional.empty();
-
- if (closeToIdeal(Resource.cpu, cpuLoad.get()) &&
- closeToIdeal(Resource.memory, memoryLoad.get()) &&
- closeToIdeal(Resource.disk, diskLoad.get()) &&
- similarCost(bestAllocation.get().cost(), currentAllocation.nodes() * costOf(currentAllocation.nodeResources())))
- return Optional.empty(); // Avoid small, unnecessary changes
- return bestAllocation.map(a -> a.clusterResources());
- }
-
- private Optional<ClusterResourcesWithCost> findBestAllocation(double cpuLoad, double memoryLoad, double diskLoad,
- ClusterResources currentAllocation, ClusterSpec cluster) {
- Optional<ClusterResourcesWithCost> bestAllocation = Optional.empty();
- for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation); i.hasNext(); ) {
- ClusterResources allocation = i.next();
- Optional<ClusterResourcesWithCost> allocatableResources = toAllocatableResources(allocation, cluster);
- if (allocatableResources.isEmpty()) continue;
- if (bestAllocation.isEmpty() || allocatableResources.get().cost() < bestAllocation.get().cost())
- bestAllocation = allocatableResources;
- }
- return bestAllocation;
- }
-
- private boolean similarCost(double cost1, double cost2) {
- return similar(cost1, cost2, costDifferenceRatioWorthReallocation);
- }
-
- private boolean closeToIdeal(Resource resource, double value) {
- return similar(resource.idealAverageLoad(), value, idealDivergenceWorthReallocation);
- }
-
- private boolean similar(double r1, double r2, double threshold) {
- return Math.abs(r1 - r2) / r1 < threshold;
- }
-
- /**
- * Returns the smallest allocatable node resources larger than the given node resources,
- * or empty if none available.
- */
- private Optional<ClusterResourcesWithCost> toAllocatableResources(ClusterResources resources, ClusterSpec cluster) {
- if (allowsHostSharing(nodeRepository.zone().cloud())) {
- // Return the requested resources, adjusted to be legal or empty if they cannot fit on existing hosts
- NodeResources nodeResources = nodeResourceLimits.enlargeToLegal(resources.nodeResources(), cluster.type());
- for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors())
- if (flavor.resources().satisfies(nodeResources))
- return Optional.of(new ClusterResourcesWithCost(resources.with(nodeResources),
- costOf(nodeResources) * resources.nodes()));
- return Optional.empty();
- }
- else {
- // return the cheapest flavor satisfying the target resources, if any
- double bestCost = Double.MAX_VALUE;
- Optional<Flavor> bestFlavor = Optional.empty();
- for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) {
- if ( ! flavor.resources().satisfies(resources.nodeResources())) continue;
- if (bestFlavor.isEmpty() || bestCost > costOf(flavor.resources())) {
- bestFlavor = Optional.of(flavor);
- bestCost = costOf(flavor);
- }
- }
- if (bestFlavor.isEmpty())
- return Optional.empty();
- else
- return Optional.of(new ClusterResourcesWithCost(resources.with(bestFlavor.get().resources()),
- bestCost * resources.nodes()));
- }
- }
-
- /**
- * Returns the average load of this resource in the measurement window,
- * or empty if we are not in a position to make decisions from these measurements at this time.
- */
- private Optional<Double> averageLoad(Resource resource, ClusterSpec cluster, List<Node> clusterNodes) {
- NodeMetricsDb.Window window = metricsDb.getWindow(nodeRepository.clock().instant().minus(scalingWindow(cluster.type())),
- resource,
- clusterNodes.stream().map(Node::hostname).collect(Collectors.toList()));
-
- if (window.measurementCount() < minimumMeasurements) return Optional.empty();
- if (window.hostnames() != clusterNodes.size()) return Optional.empty(); // Regulate only when all nodes are measured
-
- return Optional.of(window.average());
- }
-
- /** The duration of the window we need to consider to make a scaling decision */
- private Duration scalingWindow(ClusterSpec.Type clusterType) {
- if (clusterType.isContent()) return Duration.ofHours(12); // Ideally we should use observed redistribution time
- return Duration.ofHours(12); // TODO: Measure much more often to get this down to minutes. And, ideally we should take node startup time into account
- }
-
- // TODO: Put this in zone config instead?
- private boolean allowsHostSharing(CloudName cloudName) {
- if (cloudName.value().equals("aws")) return false;
- return true;
- }
-
- private double costOf(Flavor flavor) {
- NodeResources chargedResources = hostResourcesCalculator.availableCapacityOf(flavor.name(), flavor.resources());
- return costOf(chargedResources);
- }
-
- private double costOf(NodeResources resources) {
- return resources.vcpu() * cpuUnitCost +
- resources.memoryGb() * memoryUnitCost +
- resources.diskGb() * diskUnitCost;
- }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java
deleted file mode 100644
index e068b4404d8..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.autoscale;
-
-import com.yahoo.config.provision.NodeResources;
-import com.yahoo.vespa.hosted.provision.Node;
-
-import java.util.List;
-import java.util.Objects;
-
-/** A description of the resources of a cluster */
-public class ClusterResources {
-
- /** The node count in the cluster */
- private final int nodes;
-
- /** The number of node groups in the cluster */
- private final int groups;
-
- /** The resources of each node in the cluster */
- private final NodeResources nodeResources;
-
- public ClusterResources(List<Node> nodes) {
- this(nodes.size(),
- (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(),
- nodes.get(0).flavor().resources());
- }
-
- public ClusterResources(int nodes, int groups, NodeResources nodeResources) {
- this.nodes = nodes;
- this.groups = groups;
- this.nodeResources = nodeResources;
- }
-
- /** Returns the total number of allocated nodes (over all groups) */
- public int nodes() { return nodes; }
- public int groups() { return groups; }
- public NodeResources nodeResources() { return nodeResources; }
-
- public ClusterResources with(NodeResources resources) {
- return new ClusterResources(nodes, groups, resources);
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == this) return true;
- if ( ! (o instanceof ClusterResources)) return false;
-
- ClusterResources other = (ClusterResources)o;
- if (other.nodes != this.nodes) return false;
- if (other.groups != this.groups) return false;
- if (other.nodeResources != this.nodeResources) return false;
- return true;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(nodes, groups, nodeResources);
- }
-
- @Override
- public String toString() {
- return "cluster resources: " + nodes + " * " + nodeResources + (groups > 1 ? " in " + groups + " groups" : "");
- }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java
deleted file mode 100644
index 55b28ef3ce1..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java
+++ /dev/null
@@ -1,26 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.autoscale;
-
-/**
- * @author bratseth
- */
-public class ClusterResourcesWithCost {
-
- private final ClusterResources resources;
- private final double cost;
-
- public ClusterResourcesWithCost(ClusterResources resources, double cost) {
- this.resources = resources;
- this.cost = cost;
- }
-
- public ClusterResources clusterResources() { return resources;}
-
- public double cost() { return cost; }
-
- @Override
- public String toString() {
- return "$" + cost + ": " + clusterResources();
- }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
deleted file mode 100644
index a599606c314..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
+++ /dev/null
@@ -1,74 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.autoscale;
-
-import com.yahoo.slime.ArrayTraverser;
-import com.yahoo.slime.Inspector;
-import com.yahoo.slime.ObjectTraverser;
-import com.yahoo.slime.Slime;
-import com.yahoo.slime.SlimeUtils;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Consumes a response from the metrics/v2 API and populates the fields of this with the resulting values
- *
- * @author bratseth
- */
-public class MetricsResponse {
-
- private final List<NodeMetrics.MetricValue> metricValues = new ArrayList<>();
-
- public MetricsResponse(byte[] response) {
- this(SlimeUtils.jsonToSlime(response));
- }
-
- public MetricsResponse(String response) {
- this(SlimeUtils.jsonToSlime(response));
- }
-
- public List<NodeMetrics.MetricValue> metrics() { return metricValues; }
-
- private MetricsResponse(Slime response) {
- Inspector root = response.get();
- Inspector nodes = root.field("nodes");
- nodes.traverse((ArrayTraverser)(__, node) -> consumeNode(node));
- }
-
- private void consumeNode(Inspector node) {
- String hostname = node.field("hostname").asString();
- consumeNodeMetrics(hostname, node.field("node"));
- consumeServiceMetrics(hostname, node.field("services"));
- }
-
- private void consumeNodeMetrics(String hostname, Inspector node) {
- long timestamp = node.field("timestamp").asLong();
- Map<String, Double> values = consumeMetrics(node.field("metrics"));
- for (Resource resource : Resource.values())
- addMetricIfPresent(hostname, resource.metricName(), timestamp, values);
- }
-
- private void addMetricIfPresent(String hostname, String metricName, long timestamp, Map<String, Double> values) {
- if (values.containsKey(metricName))
- metricValues.add(new NodeMetrics.MetricValue(hostname, metricName, timestamp, values.get(metricName).floatValue()));
- }
-
- private void consumeServiceMetrics(String hostname, Inspector node) {
- String name = node.field("name").asString();
- long timestamp = node.field("timestamp").asLong();
- Map<String, Double> values = consumeMetrics(node.field("metrics"));
- }
-
- private Map<String, Double> consumeMetrics(Inspector metrics) {
- Map<String, Double> values = new HashMap<>();
- metrics.traverse((ArrayTraverser) (__, item) -> consumeMetricsItem(item, values));
- return values;
- }
-
- private void consumeMetricsItem(Inspector item, Map<String, Double> values) {
- item.field("values").traverse((ObjectTraverser)(name, value) -> values.put(name, value.asDouble()));
- }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java
deleted file mode 100644
index 97ac1e72be9..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java
+++ /dev/null
@@ -1,48 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.autoscale;
-
-import com.yahoo.config.provision.ApplicationId;
-
-import java.util.Collection;
-
-/**
- * Interface to retrieve metrics on (tenant) nodes.
- *
- * @author bratseth
- */
-public interface NodeMetrics {
-
- /**
- * Fetches metrics for an application. This call may be expensive.
- *
- * @param application the application to fetch metrics from
- */
- Collection<MetricValue> fetchMetrics(ApplicationId application);
-
- final class MetricValue {
-
- private final String hostname;
- private final String name;
- private long timestamp;
- private final float value;
-
- public MetricValue(String hostname, String name, long timestamp, float value) {
- this.hostname = hostname;
- this.name = name;
- this.timestamp = timestamp;
- this.value = value;
- }
-
- public String hostname() { return hostname; }
- public String name() { return name; }
- public long timestamp() { return timestamp; }
- public float value() { return value; }
-
- @Override
- public String toString() {
- return "metric value " + name + ": " + value + " at " + timestamp + " for " + hostname;
- }
-
- }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java
deleted file mode 100644
index 14a35e3efbc..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java
+++ /dev/null
@@ -1,169 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.autoscale;
-
-import java.time.Clock;
-import java.time.Duration;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * An in-memory time-series "database" of node metrics.
- * Thread model: One writer, many readers.
- *
- * @author bratseth
- */
-public class NodeMetricsDb {
-
- private static final Duration dbWindow = Duration.ofHours(24);
-
- /** Measurements by key. Each list of measurements is sorted by increasing timestamp */
- private Map<MeasurementKey, List<Measurement>> db = new HashMap<>();
-
- /** Lock all access for now since we modify lists inside a map */
- private final Object lock = new Object();
-
- /** Add a measurement to this */
- public void add(Collection<NodeMetrics.MetricValue> metricValues) {
- synchronized (lock) {
- for (var value : metricValues) {
- List<Measurement> measurements = db.computeIfAbsent(new MeasurementKey(value.hostname(),
- Resource.fromMetric(value.name())),
- (__) -> new ArrayList<>());
- measurements.add(new Measurement(value.timestamp(), value.value()));
- }
- }
- }
-
- /** Must be called intermittently (as long as add is called) to gc old measurements */
- public void gc(Clock clock) {
- synchronized (lock) {
- // TODO: We may need to do something more complicated to avoid spending too much memory to
- // lower the measurement interval (see NodeRepositoryMaintenance)
- // Each measurement is Object + long + float = 16 + 8 + 4 = 28 bytes
- // 24 hours with 1k nodes and 3 resources and 1 measurement/sec is about 10Gb
-
- long oldestTimestamp = clock.instant().minus(dbWindow).toEpochMilli();
- for (Iterator<List<Measurement>> i = db.values().iterator(); i.hasNext(); ) {
- List<Measurement> measurements = i.next();
-
- while (!measurements.isEmpty() && measurements.get(0).timestamp < oldestTimestamp)
- measurements.remove(0);
-
- if (measurements.isEmpty())
- i.remove();
- }
- }
- }
-
- /** Returns a window within which we can ask for specific information from this db */
- public Window getWindow(Instant startTime, Resource resource, List<String> hostnames) {
- return new Window(startTime, resource, hostnames);
- }
-
- public class Window {
-
- private final long startTime;
- private List<MeasurementKey> keys;
-
- private Window(Instant startTime, Resource resource, List<String> hostnames) {
- this.startTime = startTime.toEpochMilli();
- keys = hostnames.stream().map(hostname -> new MeasurementKey(hostname, resource)).collect(Collectors.toList());
- }
-
- public int measurementCount() {
- synchronized (lock) {
- return (int) keys.stream()
- .flatMap(key -> db.getOrDefault(key, List.of()).stream())
- .filter(measurement -> measurement.timestamp >= startTime)
- .count();
- }
- }
-
- /** Returns the count of hostnames which have measurements in this window */
- public int hostnames() {
- synchronized (lock) {
- int count = 0;
- for (MeasurementKey key : keys) {
- List<Measurement> measurements = db.get(key);
- if (measurements == null || measurements.isEmpty()) continue;
-
- if (measurements.get(measurements.size() - 1).timestamp >= startTime)
- count++;
- }
- return count;
- }
- }
-
- public double average() {
- synchronized (lock) {
- double sum = 0;
- int count = 0;
- for (MeasurementKey key : keys) {
- List<Measurement> measurements = db.get(key);
- if (measurements == null) continue;
-
- int index = measurements.size() - 1;
- while (index >= 0 && measurements.get(index).timestamp >= startTime) {
- sum += measurements.get(index).value;
- count++;
-
- index--;
- }
- }
- return sum / count;
- }
- }
-
- }
-
- private static class MeasurementKey {
-
- private final String hostname;
- private final Resource resource;
-
- public MeasurementKey(String hostname, Resource resource) {
- this.hostname = hostname;
- this.resource = resource;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(hostname, resource);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if ( ! (o instanceof MeasurementKey)) return false;
- MeasurementKey other = (MeasurementKey)o;
- if ( ! this.hostname.equals(other.hostname)) return false;
- if ( ! this.resource.equals(other.resource)) return false;
- return true;
- }
-
- }
-
- private static class Measurement {
-
- /** The time of this measurement in epoch millis */
- private final long timestamp;
-
- /** The measured value */
- private final float value;
-
- public Measurement(long timestamp, float value) {
- this.timestamp = timestamp;
- this.value = value;
- }
-
- }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java
deleted file mode 100644
index 54d8eac238f..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java
+++ /dev/null
@@ -1,112 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.autoscale;
-
-import ai.vespa.util.http.VespaHttpClientBuilder;
-import com.google.inject.Inject;
-import com.yahoo.component.AbstractComponent;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.vespa.applicationmodel.HostName;
-import com.yahoo.vespa.hosted.provision.Node;
-import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.orchestrator.HostNameNotFoundException;
-import com.yahoo.vespa.orchestrator.Orchestrator;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.BasicResponseHandler;
-import org.apache.http.impl.client.CloseableHttpClient;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.Collection;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Fetches node metrics over the metrics/v2 API
- *
- * @author bratseth
- */
-public class NodeMetricsFetcher extends AbstractComponent implements NodeMetrics {
-
- private static final Logger log = Logger.getLogger(NodeMetricsFetcher.class.getName());
-
- private static final String apiPath = "/metrics/v2/values";
-
- private final NodeRepository nodeRepository;
- private final Orchestrator orchestrator;
- private final HttpClient httpClient;
-
- @Inject
- public NodeMetricsFetcher(NodeRepository nodeRepository, Orchestrator orchestrator) {
- this(nodeRepository, orchestrator, new ApacheHttpClient());
- }
-
- NodeMetricsFetcher(NodeRepository nodeRepository, Orchestrator orchestrator, HttpClient httpClient) {
- this.nodeRepository = nodeRepository;
- this.orchestrator = orchestrator;
- this.httpClient = httpClient;
- }
-
- @Override
- public Collection<MetricValue> fetchMetrics(ApplicationId application) {
- Node metricsV2Container = nodeRepository.list()
- .owner(application)
- .state(Node.State.active)
- .container()
- .filter(node -> expectedUp(node))
- .asList().get(0);
- String url = "http://" + metricsV2Container.hostname() + ":" + 4080 + apiPath + "?consumer=vespa-consumer-metrics";
- String response = httpClient.get(url);
- return new MetricsResponse(response).metrics();
- }
-
- @Override
- public void deconstruct() {
- httpClient.close();
- }
-
- private boolean expectedUp(Node node) {
- try {
- return ! orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended();
- }
- catch (HostNameNotFoundException e) {
- return false;
- }
- }
-
- /** The simplest possible http client interface */
- public interface HttpClient {
-
- String get(String url);
- void close();
-
- }
-
- /** Implements the HttpClient interface by delegating to an Apache HTTP client */
- public static class ApacheHttpClient implements HttpClient {
-
- private final CloseableHttpClient httpClient = VespaHttpClientBuilder.createWithBasicConnectionManager().build();
-
- @Override
- public String get(String url) {
- try {
- return httpClient.execute(new HttpGet(url), new BasicResponseHandler());
- }
- catch (IOException e) {
- throw new UncheckedIOException("Could not get " + url, e);
- }
- }
-
- @Override
- public void close() {
- try {
- httpClient.close();
- }
- catch (IOException e) {
- log.log(Level.WARNING, "Exception deconstructing", e);
- }
- }
-
-
- }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java
deleted file mode 100644
index 9c85ca870d5..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.autoscale;
-
-import com.yahoo.config.provision.NodeResources;
-
-/**
- * A resource subject to autoscaling
- *
- * @author bratseth
- */
-public enum Resource {
-
- cpu {
- String metricName() { return "cpu.util"; }
- double idealAverageLoad() { return 0.2; }
- double valueFrom(NodeResources resources) { return resources.vcpu(); }
- },
-
- memory {
- String metricName() { return "memory.util"; }
- double idealAverageLoad() { return 0.7; }
- double valueFrom(NodeResources resources) { return resources.memoryGb(); }
- },
-
- disk {
- String metricName() { return "disk.util"; }
- double idealAverageLoad() { return 0.7; }
- double valueFrom(NodeResources resources) { return resources.diskGb(); }
- };
-
- abstract String metricName();
-
- /** The load we should have of this resource on average, when one node in the cluster is down */
- abstract double idealAverageLoad();
-
- abstract double valueFrom(NodeResources resources);
-
- public static Resource fromMetric(String metricName) {
- for (Resource resource : values())
- if (resource.metricName().equals(metricName)) return resource;
- throw new IllegalArgumentException("Metric '" + metricName + "' does not map to a resource");
- }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java
deleted file mode 100644
index 464fe570b95..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java
+++ /dev/null
@@ -1,104 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.autoscale;
-
-import com.yahoo.config.provision.NodeResources;
-
-/**
- * Provides iteration over possible cluster resource allocations given a target total load
- * and current groups/nodes allocation.
- */
-public class ResourceIterator {
-
- // Configured min and max nodes TODO: These should come from the application package
- private static final int minimumNodesPerCluster = 3; // Since this is with redundancy it cannot be lower than 2
- private static final int maximumNodesPerCluster = 150;
-
- // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component
- // proportional to document count. We must account for this when comparing configurations with more or fewer nodes.
- // TODO: Measure this, and only take it into account with queries
- private static final double fixedCpuCostFraction = 0.1;
-
- // Describes the observed state
- private final ClusterResources allocation;
- private final double cpuLoad;
- private final double memoryLoad;
- private final double diskLoad;
- private final int groupSize;
-
- // Derived from the observed state
- private final int nodeIncrement;
- private final boolean singleGroupMode;
-
- // Iterator state
- private int currentNodes;
-
- public ResourceIterator(double cpuLoad, double memoryLoad, double diskLoad, ClusterResources currentAllocation) {
- this.cpuLoad = cpuLoad;
- this.memoryLoad = memoryLoad;
- this.diskLoad = diskLoad;
-
- // ceil: If the division does not produce a whole number we assume some node is missing
- groupSize = (int)Math.ceil((double)currentAllocation.nodes() / currentAllocation.groups());
- allocation = currentAllocation;
-
- // What number of nodes is it effective to add or remove at the time from this cluster?
- // This is the group size, since we (for now) assume the group size is decided by someone wiser than us
- // and we decide the number of groups.
- // The exception is when we only have one group, where we can add and remove single nodes in it.
- singleGroupMode = currentAllocation.groups() == 1;
- nodeIncrement = singleGroupMode ? 1 : groupSize;
-
- currentNodes = currentAllocation.nodes();
- while (currentNodes - nodeIncrement >= minimumNodesPerCluster
- && (singleGroupMode || currentNodes - nodeIncrement > groupSize)) // group level redundancy
- currentNodes -= nodeIncrement;
- }
-
- public ClusterResources next() {
- int nodesWithRedundancy = currentNodes - (singleGroupMode ? 1 : groupSize);
- ClusterResources next = new ClusterResources(currentNodes,
- singleGroupMode ? 1 : currentNodes / groupSize,
- resourcesFor(nodesWithRedundancy));
- currentNodes += nodeIncrement;
- return next;
- }
-
- public boolean hasNext() {
- return currentNodes <= maximumNodesPerCluster;
- }
-
- /**
- * For the observed load this instance is initialized with, returns the resources needed per node to be at
- * ideal load given a target node count
- */
- private NodeResources resourcesFor(int nodeCount) {
- // Cpu: Scales with cluster size (TODO: Only reads, writes scales with group size)
- // Memory and disk: Scales with group size
-
- double cpu, memory, disk;
- if (singleGroupMode) {
- // The fixed cost portion of cpu does not scale with changes to the node count
- // TODO: Only for the portion of cpu consumed by queries
- double totalCpu = totalUsage(Resource.cpu, cpuLoad);
- cpu = fixedCpuCostFraction * totalCpu / groupSize / Resource.cpu.idealAverageLoad() +
- (1 - fixedCpuCostFraction) * totalCpu / nodeCount / Resource.cpu.idealAverageLoad();
- memory = totalGroupUsage(Resource.memory, memoryLoad) / nodeCount / Resource.memory.idealAverageLoad();
- disk = totalGroupUsage(Resource.disk, diskLoad) / nodeCount / Resource.disk.idealAverageLoad();
- }
- else {
- cpu = totalUsage(Resource.cpu, cpuLoad) / nodeCount / Resource.cpu.idealAverageLoad();
- memory = totalGroupUsage(Resource.memory, memoryLoad) / groupSize / Resource.memory.idealAverageLoad();
- disk = totalGroupUsage(Resource.disk, diskLoad) / groupSize / Resource.disk.idealAverageLoad();
- }
- return allocation.nodeResources().withVcpu(cpu).withMemoryGb(memory).withDiskGb(disk);
- }
-
- private double totalUsage(Resource resource, double load) {
- return load * resource.valueFrom(allocation.nodeResources()) * allocation.nodes();
- }
-
- private double totalGroupUsage(Resource resource, double load) {
- return load * resource.valueFrom(allocation.nodeResources()) * groupSize;
- }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
deleted file mode 100644
index f3c1d8603b3..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.maintenance;
-
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.Deployer;
-import com.yahoo.vespa.hosted.provision.Node;
-import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler;
-import com.yahoo.vespa.hosted.provision.autoscale.ClusterResources;
-import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb;
-import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
-
-import java.time.Duration;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-/**
- * Maintainer making automatic scaling decisions
- *
- * @author bratseth
- */
-public class AutoscalingMaintainer extends Maintainer {
-
- private final Autoscaler autoscaler;
- private final Deployer deployer;
-
- public AutoscalingMaintainer(NodeRepository nodeRepository,
- HostResourcesCalculator hostResourcesCalculator,
- NodeMetricsDb metricsDb,
- Deployer deployer,
- Duration interval) {
- super(nodeRepository, interval);
- this.autoscaler = new Autoscaler(hostResourcesCalculator, metricsDb, nodeRepository);
- this.deployer = deployer;
- }
-
- @Override
- protected void maintain() {
- if ( ! nodeRepository().zone().environment().isProduction()) return;
-
- activeNodesByApplication().forEach((applicationId, nodes) -> autoscale(applicationId, nodes));
- }
-
- private void autoscale(ApplicationId application, List<Node> applicationNodes) {
- MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, nodeRepository());
- if ( ! deployment.isValid()) return; // Another config server will consider this application
- nodesByCluster(applicationNodes).forEach((clusterSpec, clusterNodes) -> {
- Optional<ClusterResources> target = autoscaler.autoscale(application, clusterSpec, clusterNodes);
- target.ifPresent(t -> log.info("Autoscale: Application " + application + " cluster " + clusterSpec +
- " from " + applicationNodes.size() + " * " + applicationNodes.get(0).flavor().resources() +
- " to " + t.nodes() + " * " + t.nodeResources()));
- });
- }
-
- private Map<ClusterSpec, List<Node>> nodesByCluster(List<Node> applicationNodes) {
- return applicationNodes.stream().collect(Collectors.groupingBy(n -> n.allocation().get().membership().cluster()));
- }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
index 27fba9e8f8e..0d5a8587902 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
@@ -2,22 +2,17 @@
package com.yahoo.vespa.hosted.provision.maintenance;
import com.yahoo.component.AbstractComponent;
-import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
-import java.util.stream.Collectors;
/**
* A maintainer is some job which runs at a fixed rate to perform some maintenance task on the node repo.
@@ -80,12 +75,6 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
private String name() { return this.getClass().getSimpleName(); }
- /** A utility to group active tenant applications by application */
- protected Map<ApplicationId, List<Node>> activeNodesByApplication() {
- return nodeRepository().list().nodeType(NodeType.tenant).state(Node.State.active).asList()
- .stream().collect(Collectors.groupingBy(n -> n.allocation().get().owner()));
- }
-
static long staggeredDelay(List<HostName> cluster, HostName host, Instant now, Duration interval) {
if ( ! cluster.contains(host))
return interval.toMillis();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
deleted file mode 100644
index 178e8385008..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.maintenance;
-
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics;
-import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb;
-
-import java.time.Duration;
-import java.util.logging.Level;
-
-/**
- * Maintainer which keeps the node metric db up to date by periodically fetching metrics from all
- * active nodes.
- *
- * @author bratseth
- */
-public class NodeMetricsDbMaintainer extends Maintainer {
-
- private static final int maxWarningsPerInvocation = 2;
-
- private final NodeMetrics nodeMetrics;
- private final NodeMetricsDb nodeMetricsDb;
-
- public NodeMetricsDbMaintainer(NodeRepository nodeRepository,
- NodeMetrics nodeMetrics,
- NodeMetricsDb nodeMetricsDb,
- Duration interval) {
- super(nodeRepository, interval);
- this.nodeMetrics = nodeMetrics;
- this.nodeMetricsDb = nodeMetricsDb;
- }
-
- @Override
- protected void maintain() {
- int warnings = 0;
- for (ApplicationId application : activeNodesByApplication().keySet()) {
- try {
- nodeMetricsDb.add(nodeMetrics.fetchMetrics(application));
- }
- catch (Exception e) {
- if (warnings++ < maxWarningsPerInvocation)
- log.log(Level.WARNING, "Could not update metrics for " + application, e);
- }
- }
- nodeMetricsDb.gc(nodeRepository().clock());
- }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index ecc550527fc..37620e17a95 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -8,11 +8,9 @@ import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.HostLivenessTracker;
import com.yahoo.config.provision.InfraDeployer;
import com.yahoo.config.provision.Zone;
-import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics;
import com.yahoo.jdisc.Metric;
import com.yahoo.vespa.flags.FlagSource;
import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb;
import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider;
import com.yahoo.vespa.orchestrator.Orchestrator;
import com.yahoo.vespa.service.monitor.ServiceMonitor;
@@ -50,25 +48,22 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
private final CapacityReportMaintainer capacityReportMaintainer;
private final OsUpgradeActivator osUpgradeActivator;
private final Rebalancer rebalancer;
- private final NodeMetricsDbMaintainer nodeMetricsDbMaintainer;
- private final AutoscalingMaintainer autoscalingMaintainer;
@SuppressWarnings("unused")
@Inject
public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, InfraDeployer infraDeployer,
HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
Zone zone, Orchestrator orchestrator, Metric metric,
- ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource,
- NodeMetrics nodeMetrics, NodeMetricsDb nodeMetricsDb) {
+ ProvisionServiceProvider provisionServiceProvider,
+ FlagSource flagSource) {
this(nodeRepository, deployer, infraDeployer, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(),
- orchestrator, metric, provisionServiceProvider, flagSource, nodeMetrics, nodeMetricsDb);
+ orchestrator, metric, provisionServiceProvider, flagSource);
}
public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, InfraDeployer infraDeployer,
HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
Zone zone, Clock clock, Orchestrator orchestrator, Metric metric,
- ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource,
- NodeMetrics nodeMetrics, NodeMetricsDb nodeMetricsDb) {
+ ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource) {
DefaultTimes defaults = new DefaultTimes(zone);
nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, defaults.failGrace, clock, orchestrator, throttlePolicyFromEnv().orElse(defaults.throttlePolicy), metric);
@@ -90,8 +85,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval);
osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval);
rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval);
- nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval);
- autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, deployer, defaults.autoscalingInterval);
// The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now
infrastructureProvisioner.maintainButThrowOnException();
@@ -116,8 +109,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
dynamicProvisioningMaintainer.ifPresent(Maintainer::deconstruct);
osUpgradeActivator.deconstruct();
rebalancer.deconstruct();
- nodeMetricsDbMaintainer.deconstruct();
- autoscalingMaintainer.deconstruct();
}
private static Optional<NodeFailer.ThrottlePolicy> throttlePolicyFromEnv() {
@@ -158,8 +149,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
private final Duration dynamicProvisionerInterval;
private final Duration osUpgradeActivatorInterval;
private final Duration rebalancerInterval;
- private final Duration nodeMetricsCollectionInterval;
- private final Duration autoscalingInterval;
private final NodeFailer.ThrottlePolicy throttlePolicy;
@@ -180,8 +169,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
dynamicProvisionerInterval = Duration.ofMinutes(5);
osUpgradeActivatorInterval = zone.system().isCd() ? Duration.ofSeconds(30) : Duration.ofMinutes(5);
rebalancerInterval = Duration.ofMinutes(40);
- nodeMetricsCollectionInterval = Duration.ofMinutes(1);
- autoscalingInterval = Duration.ofMinutes(5);
if (zone.environment().equals(Environment.prod) && ! zone.system().isCd()) {
inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
index 179d7f2703c..7c5ff35878b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
@@ -19,15 +19,11 @@ import java.util.Locale;
public class CapacityPolicies {
private final Zone zone;
-
- private final NodeResourceLimits nodeResourceLimits;
-
/* Deployments must match 1-to-1 the advertised resources of a physical host */
private final boolean isUsingAdvertisedResources;
public CapacityPolicies(Zone zone) {
this.zone = zone;
- this.nodeResourceLimits = new NodeResourceLimits(zone);
this.isUsingAdvertisedResources = zone.cloud().value().equals("aws");
}
@@ -68,7 +64,7 @@ public class CapacityPolicies {
}
private void ensureSufficientResources(NodeResources resources, ClusterSpec cluster) {
- double minMemoryGb = nodeResourceLimits.minMemoryGb(cluster.type());
+ double minMemoryGb = minMemoryGb(cluster.type());
if (resources.memoryGb() >= minMemoryGb) return;
throw new IllegalArgumentException(String.format(Locale.ENGLISH,
@@ -76,6 +72,12 @@ public class CapacityPolicies {
minMemoryGb, cluster.type().name(), cluster.id().value(), resources.memoryGb()));
}
+ private int minMemoryGb(ClusterSpec.Type clusterType) {
+ if (zone.system() == SystemName.dev) return 1; // Allow small containers in dev system
+ if (clusterType == ClusterSpec.Type.admin) return 2;
+ return 4;
+ }
+
private NodeResources defaultNodeResources(ClusterSpec.Type clusterType) {
if (clusterType == ClusterSpec.Type.admin) {
if (zone.system() == SystemName.dev) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index af6fa8edf64..5753bbb3c5a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -64,6 +64,7 @@ public class GroupPreparer {
.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm())
.value();
boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty();
+
try (Mutex lock = nodeRepository.lock(application)) {
// Lock ready pool to ensure that the same nodes are not simultaneously allocated by others
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
index 0423f762f2b..394549e4141 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
@@ -18,7 +18,7 @@ public interface HostProvisioner {
/**
* Schedule provisioning of a given number of hosts.
*
- * @param provisionIndexes list of unique provision indexes which will be used to generate the node hostnames
+ * @param provisionIndexes List of unique provision indexes which will be used to generate the node hostnames
* on the form of <code>[prefix][index].[domain]</code>
* @param resources the resources needed per node
* @param applicationId id of the application that will own the provisioned host
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
index c92f7889496..ebd6a01e61f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
@@ -390,5 +390,4 @@ class NodeAllocation {
return count;
}
}
-
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
deleted file mode 100644
index ca04bf66ce3..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
+++ /dev/null
@@ -1,32 +0,0 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.provisioning;
-
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.NodeResources;
-import com.yahoo.config.provision.SystemName;
-import com.yahoo.config.provision.Zone;
-
-/**
- * Defines the resource limits for nodes in various zones
- *
- * @author bratseth
- */
-public class NodeResourceLimits {
-
- private final Zone zone;
-
- public NodeResourceLimits(Zone zone) {
- this.zone = zone;
- }
-
- public int minMemoryGb(ClusterSpec.Type clusterType) {
- if (zone.system() == SystemName.dev) return 1; // Allow small containers in dev system
- if (clusterType == ClusterSpec.Type.admin) return 2;
- return 4;
- }
-
- public NodeResources enlargeToLegal(NodeResources resources, ClusterSpec.Type clusterType) {
- return resources.withMemoryGb(Math.max(minMemoryGb(clusterType), resources.memoryGb()));
- }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
index d26accd7a84..49d0ba5cf70 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
@@ -25,8 +25,6 @@ public class ContainerConfig {
" <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>\n" +
" <component id='com.yahoo.vespa.hosted.provision.testutils.MockDuperModel'/>\n" +
" <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>\n" +
- " <component id='com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb'/>\n" +
- " <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeMetrics'/>\n" +
" <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/>\n" +
" <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider'/>\n" +
" <component id='com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance'/>\n" +
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
index e7ebf049e51..915ef0d9125 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
@@ -18,7 +18,6 @@ import java.util.concurrent.ConcurrentHashMap;
* @author hakonhall
*/
public class MockDuperModel implements DuperModelInfraApi {
-
private final Map<ApplicationId, InfraApplicationApi> supportedInfraApps = new HashMap<>();
private final ConcurrentHashMap<ApplicationId, List<HostName>> activeApps = new ConcurrentHashMap<>();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java
deleted file mode 100644
index d5397aa421c..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.testutils;
-
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics;
-
-import java.util.ArrayList;
-import java.util.Collection;
-
-/**
- * @author bratseth
- */
-public class MockNodeMetrics implements NodeMetrics {
-
- @Override
- public Collection<MetricValue> fetchMetrics(ApplicationId application) {
- return new ArrayList<>();
- }
-
-}