summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@gmail.com>2020-10-23 19:33:05 +0200
committerJon Bratseth <bratseth@gmail.com>2020-10-23 19:33:05 +0200
commitbe08b4378e8e2c54847961434047dbfb91c06136 (patch)
tree4110d460ecd45afb40ac9c6b59011c941f48c03c
parent836f93ca25c729795f5401272f087cf11793c0e0 (diff)
Don't autoscale if a deployment is ongoing
This avoids trying to do another rescaling (which will fail on activate) if a deployment is already ongoing, for example if another config server has already decided to autoscale.
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java18
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java2
3 files changed, 14 insertions, 8 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
index c73a19bd9e2..eace7457615 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
@@ -61,7 +61,7 @@ public class Autoscaler {
private Optional<AllocatableClusterResources> autoscale(Cluster cluster,
List<Node> clusterNodes, Limits limits, boolean exclusive) {
- if (unstable(clusterNodes)) return Optional.empty();
+ if (unstable(clusterNodes, nodeRepository)) return Optional.empty();
AllocatableClusterResources currentAllocation = new AllocatableClusterResources(clusterNodes, nodeRepository);
@@ -111,10 +111,18 @@ public class Autoscaler {
return 20;
}
- public static boolean unstable(List<Node> nodes) {
- return nodes.stream().anyMatch(node -> node.status().wantToRetire() ||
- node.allocation().get().membership().retired() ||
- node.allocation().get().isRemovable());
+ public static boolean unstable(List<Node> nodes, NodeRepository nodeRepository) {
+ // The cluster is processing recent changes
+ if (nodes.stream().anyMatch(node -> node.status().wantToRetire() ||
+ node.allocation().get().membership().retired() ||
+ node.allocation().get().isRemovable()))
+ return true;
+
+ // A deployment is ongoing
+ if (nodeRepository.getNodes(nodes.get(0).allocation().get().owner(), Node.State.reserved).size() > 0)
+ return true;
+
+ return false;
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java
index b4a63175548..4597fc04e17 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java
@@ -56,7 +56,7 @@ public class MetricsV2MetricsFetcher extends AbstractComponent implements Metric
NodeList applicationNodes = nodeRepository.list(application).state(Node.State.active);
// Do not try to draw conclusions from utilization while unstable
- if (Autoscaler.unstable(applicationNodes.asList())) return Collections.emptyList();
+ if (Autoscaler.unstable(applicationNodes.asList(), nodeRepository)) return Collections.emptyList();
Optional<Node> metricsV2Container = applicationNodes.container()
.matching(node -> expectedUp(node))
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
index 3b01f678982..c0fd7df9b2e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
@@ -28,7 +28,6 @@ import java.util.stream.Collectors;
*/
public class AutoscalingMaintainer extends NodeRepositoryMaintainer {
- private final MetricsDb metricsDb;
private final Autoscaler autoscaler;
private final Deployer deployer;
private final Metric metric;
@@ -40,7 +39,6 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer {
Duration interval) {
super(nodeRepository, interval, metric);
this.autoscaler = new Autoscaler(metricsDb, nodeRepository);
- this.metricsDb = metricsDb;
this.metric = metric;
this.deployer = deployer;
}