From dd6fde6bfd38f24b6fff64f0eb81bb834f109adb Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Tue, 20 Oct 2020 11:20:40 +0200 Subject: Store scaling events in ZooKeeper --- .../com/yahoo/vespa/hosted/provision/NodeList.java | 8 ++ .../vespa/hosted/provision/NodeRepository.java | 11 +- .../hosted/provision/applications/Application.java | 3 +- .../provision/applications/Applications.java | 12 +- .../hosted/provision/applications/Cluster.java | 20 +++- .../provision/applications/ScalingEvent.java | 59 ++++++++++ .../hosted/provision/autoscale/Autoscaler.java | 9 +- .../hosted/provision/autoscale/MetricSnapshot.java | 22 ++-- .../hosted/provision/autoscale/NodeMetricsDb.java | 39 ------- .../hosted/provision/autoscale/QuestMetricsDb.java | 122 ++++++++++++++++++++- .../maintenance/AutoscalingMaintainer.java | 8 +- .../hosted/provision/maintenance/NodeFailer.java | 2 +- .../com/yahoo/vespa/hosted/provision/node/IP.java | 2 +- .../persistence/ApplicationSerializer.java | 34 +++++- .../persistence/CuratorDatabaseClient.java | 9 +- .../hosted/provision/provisioning/Activator.java | 74 +++++++++---- .../provision/provisioning/InfraDeployerImpl.java | 4 +- .../provisioning/LoadBalancerProvisioner.java | 15 +-- .../provisioning/NodeRepositoryProvisioner.java | 27 ++++- .../hosted/provision/testutils/MockDeployer.java | 10 +- .../provision/testutils/MockNodeRepository.java | 4 +- .../provision/testutils/MockProvisioner.java | 24 ++-- .../provision/applications/ApplicationsTest.java | 11 +- .../maintenance/AutoscalingMaintainerTest.java | 27 +++-- .../provision/maintenance/FailedExpirerTest.java | 4 +- .../provision/maintenance/NodeFailTester.java | 4 +- .../provision/maintenance/RebalancerTest.java | 4 +- .../provision/maintenance/RetiredExpirerTest.java | 2 + .../persistence/ApplicationSerializerTest.java | 15 ++- .../provisioning/InfraDeployerImplTest.java | 8 +- .../provision/provisioning/ProvisioningTester.java | 7 +- 31 files changed, 440 insertions(+), 160 deletions(-) create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/ScalingEvent.java (limited to 'node-repository') diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index 5827d7a0f7d..4aedb5dde07 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision; import com.yahoo.collections.AbstractFilteringList; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; @@ -179,6 +180,13 @@ public class NodeList extends AbstractFilteringList { .findFirst()); } + public ClusterResources toResources() { + if (isEmpty()) return new ClusterResources(0, 0, NodeResources.unspecified()); + return new ClusterResources(size(), + (int)stream().distinct().count(), + first().get().resources()); + } + /** Returns the nodes of this as a stream */ public Stream stream() { return asList().stream(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index fd7dbc9716b..3fdade27410 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -7,6 +7,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; @@ -504,17 +505,17 @@ public class NodeRepository extends AbstractComponent { } /** Deactivate nodes owned by application guarded by given lock */ - public void deactivate(NestedTransaction transaction, ProvisionLock lock) { - deactivate(db.readNodes(lock.application(), State.reserved, State.active), transaction, lock); - applications.remove(lock.application(), transaction, lock); + public void deactivate(ApplicationTransaction transaction) { + deactivate(db.readNodes(transaction.application(), State.reserved, State.active), transaction); + applications.remove(transaction); } /** * Deactivates these nodes in a transaction and returns the nodes in the new state which will hold if the * transaction commits. */ - public List deactivate(List nodes, NestedTransaction transaction, @SuppressWarnings("unused") ProvisionLock lock) { - return db.writeTo(State.inactive, nodes, Agent.application, Optional.empty(), transaction); + public List deactivate(List nodes, ApplicationTransaction transaction) { + return db.writeTo(State.inactive, nodes, Agent.application, Optional.empty(), transaction.nested()); } /** Move nodes to the dirty state */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java index e9e91910281..fd92b5b0ca0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.HashMap; import java.util.Optional; @@ -56,7 +57,7 @@ public class Application { public Application withCluster(ClusterSpec.Id id, boolean exclusive, ClusterResources min, ClusterResources max) { Cluster cluster = clusters.get(id); if (cluster == null) - cluster = new Cluster(id, exclusive, min, max, Optional.empty(), Optional.empty()); + cluster = new Cluster(id, exclusive, min, max, Optional.empty(), Optional.empty(), List.of()); else cluster = cluster.withConfiguration(exclusive, min, max); return with(cluster); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java index 9f45839f1c3..9db28652d0e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ProvisionLock; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; @@ -42,17 +43,16 @@ public class Applications { // TODO: Require ProvisionLock instead of Mutex public void put(Application application, Mutex applicationLock) { NestedTransaction transaction = new NestedTransaction(); - put(application, transaction, applicationLock); + db.writeApplication(application, transaction); transaction.commit(); } - // TODO: Require ProvisionLock instead of Mutex - public void put(Application application, NestedTransaction transaction, Mutex applicationLock) { - db.writeApplication(application, transaction); + public void put(Application application, ApplicationTransaction transaction) { + db.writeApplication(application, transaction.nested()); } - public void remove(ApplicationId application, NestedTransaction transaction, @SuppressWarnings("unused") ProvisionLock lock) { - db.deleteApplication(application, transaction); + public void remove(ApplicationTransaction transaction) { + db.deleteApplication(transaction); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 3aae47a9088..a17ee081447 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -23,13 +24,15 @@ public class Cluster { private final ClusterResources min, max; private final Optional suggested; private final Optional target; + private final List scalingEvents; public Cluster(ClusterSpec.Id id, boolean exclusive, ClusterResources minResources, ClusterResources maxResources, Optional suggestedResources, - Optional targetResources) { + Optional targetResources, + List scalingEvents) { this.id = Objects.requireNonNull(id); this.exclusive = exclusive; this.min = Objects.requireNonNull(minResources); @@ -40,6 +43,7 @@ public class Cluster { this.target = Optional.empty(); else this.target = targetResources; + this.scalingEvents = scalingEvents; } public ClusterSpec.Id id() { return id; } @@ -66,16 +70,24 @@ public class Cluster { */ public Optional suggestedResources() { return suggested; } + /** Returns the recent scaling events in this cluster */ + public List scalingEvents() { return scalingEvents; } + public Cluster withConfiguration(boolean exclusive, ClusterResources min, ClusterResources max) { - return new Cluster(id, exclusive, min, max, suggested, target); + return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents); } public Cluster withSuggested(Optional suggested) { - return new Cluster(id, exclusive, min, max, suggested, target); + return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents); } public Cluster withTarget(Optional target) { - return new Cluster(id, exclusive, min, max, suggested, target); + return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents); + } + + public Cluster with(ScalingEvent scalingEvent) { + // NOTE: We're just storing the latest scaling event so far + return new Cluster(id, exclusive, min, max, suggested, target, List.of(scalingEvent)); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/ScalingEvent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/ScalingEvent.java new file mode 100644 index 00000000000..68e65d10d69 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/ScalingEvent.java @@ -0,0 +1,59 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.applications; + +import com.yahoo.config.provision.ClusterResources; + +import java.time.Instant; +import java.util.Objects; + +/** + * A recording of a change in resources for an application cluster + * + * @author bratseth + */ +public class ScalingEvent { + + private final ClusterResources from, to; + private final long generation; + private final Instant at; + + public ScalingEvent(ClusterResources from, ClusterResources to, long generation, Instant at) { + this.from = from; + this.to = to; + this.generation = generation; + this.at = at; + } + + /** Returns the resources we changed from */ + public ClusterResources from() { return from; } + + /** Returns the resources we changed to */ + public ClusterResources to() { return to; } + + /** Returns the application config generation resulting from this deployment */ + public long generation() { return generation; } + + /** Returns the time of this deployment */ + public Instant at() { return at; } + + @Override + public int hashCode() { return Objects.hash(from, to, generation, at); } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof ScalingEvent)) return true; + ScalingEvent other = (ScalingEvent)o; + if ( other.generation != this.generation) return false; + if ( ! other.at.equals(this.at)) return false; + if ( ! other.from.equals(this.from)) return false; + if ( ! other.to.equals(this.to)) return false; + return true; + } + + @Override + public String toString() { + return "scaling event from " + from + " to " + to + ", generation " + generation + " at " + at; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index a267d59f1dc..58175a459d9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -42,7 +42,7 @@ public class Autoscaler { * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time */ public Optional suggest(Cluster cluster, List clusterNodes) { - return autoscale(clusterNodes, Limits.empty(), cluster.exclusive()) + return autoscale(cluster, clusterNodes, Limits.empty(), cluster.exclusive()) .map(AllocatableClusterResources::toAdvertisedClusterResources); } @@ -55,16 +55,17 @@ public class Autoscaler { */ public Optional autoscale(Cluster cluster, List clusterNodes) { if (cluster.minResources().equals(cluster.maxResources())) return Optional.empty(); // Shortcut - return autoscale(clusterNodes, Limits.of(cluster), cluster.exclusive()) + return autoscale(cluster, clusterNodes, Limits.of(cluster), cluster.exclusive()) .map(AllocatableClusterResources::toAdvertisedClusterResources); } - private Optional autoscale(List clusterNodes, Limits limits, boolean exclusive) { + private Optional autoscale(Cluster cluster, + List clusterNodes, Limits limits, boolean exclusive) { if (unstable(clusterNodes)) return Optional.empty(); AllocatableClusterResources currentAllocation = new AllocatableClusterResources(clusterNodes, nodeRepository); - MetricSnapshot metricSnapshot = new MetricSnapshot(clusterNodes, metricsDb, nodeRepository); + MetricSnapshot metricSnapshot = new MetricSnapshot(cluster, clusterNodes, metricsDb, nodeRepository); Optional cpuLoad = metricSnapshot.averageLoad(Resource.cpu); Optional memoryLoad = metricSnapshot.averageLoad(Resource.memory); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java index 46ba4351082..43636c6573a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java @@ -1,10 +1,10 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Cluster; import java.time.Instant; import java.util.HashMap; @@ -25,29 +25,29 @@ public class MetricSnapshot { private final NodeRepository nodeRepository; private final Map startTimePerHost; - public MetricSnapshot(List clusterNodes, NodeMetricsDb db, NodeRepository nodeRepository) { + public MetricSnapshot(Cluster cluster, List clusterNodes, NodeMetricsDb db, NodeRepository nodeRepository) { this.clusterNodes = clusterNodes; this.db = db; this.nodeRepository = nodeRepository; - this.startTimePerHost = metricStartTimes(clusterNodes, db, nodeRepository); + this.startTimePerHost = metricStartTimes(cluster, clusterNodes, db, nodeRepository); + startTimePerHost.forEach((a,b) -> System.out.println(a + " = " + b)); } /** * Returns the instant of the oldest metric to consider for each node, or an empty map if metrics from the * entire (max) window should be considered. */ - private static Map metricStartTimes(List clusterNodes, + private static Map metricStartTimes(Cluster cluster, + List clusterNodes, NodeMetricsDb db, NodeRepository nodeRepository) { - ApplicationId application = clusterNodes.get(0).allocation().get().owner(); - List deployments = db.getEvents(application); Map startTimePerHost = new HashMap<>(); - if (!deployments.isEmpty()) { - var deployment = deployments.get(deployments.size() - 1); + if ( ! cluster.scalingEvents().isEmpty()) { + var deployment = cluster.scalingEvents().get(cluster.scalingEvents().size() - 1); List generationMeasurements = - db.getMeasurements(deployment.time(), - Metric.generation, - clusterNodes.stream().map(Node::hostname).collect(Collectors.toList())); + db.getMeasurements(deployment.at(), + Metric.generation, + clusterNodes.stream().map(Node::hostname).collect(Collectors.toList())); for (Node node : clusterNodes) { startTimePerHost.put(node.hostname(), nodeRepository.clock().instant()); // Discard all unless we can prove otherwise var nodeGenerationMeasurements = diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java index 635f3ffc081..1c643b0ddb5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java @@ -32,9 +32,6 @@ public class NodeMetricsDb { /** Measurements by key. Each list of measurements is sorted by increasing timestamp */ private final Map db = new HashMap<>(); - /** Events */ - private final List events = new ArrayList<>(); - /** Lock all access for now since we modify lists inside a map */ private final Object lock = new Object(); @@ -65,13 +62,6 @@ public class NodeMetricsDb { } } - /** Adds an event to this */ - public void add(AutoscalingEvent event) { - synchronized (lock) { - events.add(event); - } - } - /** Must be called intermittently (as long as any add methods are called) to gc old data */ public void gc(Clock clock) { synchronized (lock) { @@ -106,12 +96,6 @@ public class NodeMetricsDb { } } - public List getEvents(ApplicationId application) { - synchronized (lock) { - return events.stream().filter(event -> event.application().equals(application)).collect(Collectors.toList()); - } - } - private static class NodeMeasurementsKey { private final String hostname; @@ -210,27 +194,4 @@ public class NodeMetricsDb { } - public static class AutoscalingEvent { - - private final ApplicationId application; - private final long generation; - private final long timestamp; - - public AutoscalingEvent(ApplicationId application, long generation, Instant times) { - this.application = application; - this.generation = generation; - this.timestamp = times.toEpochMilli(); - } - - /** Returns the deployed application */ - public ApplicationId application() { return application; } - - /** Returns the application config generation resulting from this deployment */ - public long generation() { return generation; } - - /** Returns the time of this deployment */ - public Instant time() { return Instant.ofEpochMilli(timestamp); } - - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index 20720d4c35b..244aee3c117 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -1,2 +1,122 @@ -package com.yahoo.vespa.hosted.provision.autoscale;public class QuestMetricsDb { +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.io.IOUtils; +import io.questdb.cairo.CairoConfiguration; +import io.questdb.cairo.CairoEngine; +import io.questdb.cairo.DefaultCairoConfiguration; +import io.questdb.cairo.TableWriter; +import io.questdb.cairo.sql.Record; +import io.questdb.cairo.sql.RecordCursor; +import io.questdb.cairo.sql.RecordCursorFactory; +import io.questdb.griffin.SqlCompiler; +import io.questdb.griffin.SqlException; +import io.questdb.griffin.SqlExecutionContext; +import io.questdb.griffin.SqlExecutionContextImpl; +import io.questdb.std.Os; +import io.questdb.std.str.Path; + +import java.io.File; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class QuestMetricsDb implements AutoCloseable { + + private static final String tableName = "metrics"; + + private final String dataDir; + private final CairoEngine engine; + + public QuestMetricsDb() { + System.setProperty("questdbLog", "etc/quest-log.conf"); // silence Questdb's custom logging system + dataDir = "data"; + IOUtils.createDirectory(dataDir + "/" + tableName); + CairoConfiguration configuration = new DefaultCairoConfiguration(dataDir); + engine = new CairoEngine(configuration); + ensureExists(tableName); + } + + @Override + public void close() { + if (engine != null) + engine.close(); + } + + public void addMetrics() { + try (TableWriter writer = engine.getWriter(newContext().getCairoSecurityContext(), tableName)) { + for (int i = 0; i < 10; i++) { + TableWriter.Row row = writer.newRow(Os.currentTimeMicros()); + row.putStr(0, "host" + i); + row.putTimestamp(1, Instant.now().toEpochMilli()); + row.putFloat(2, i * 1.1F); + row.putFloat(3, i * 2.2F); + row.putFloat(4, i * 3.3F); + row.putFloat(5, i); // really a long, but keep this uniform? + row.append(); + } + writer.commit(); + } + } + + private void ensureExists(String tableName) { + SqlExecutionContext context = newContext(); + if (0 == engine.getStatus(context.getCairoSecurityContext(), new Path(), tableName)) return; + + try (SqlCompiler compiler = new SqlCompiler(engine)) { + compiler.compile("create table " + tableName + + " (host string, at timestamp, cpu_util float, mem_total_util float, disk_util float, application_generation float)" + + " timestamp(at)" + + "PARTITION BY DAY;", + context); + } + catch (SqlException e) { + throw new IllegalStateException("Could not create Quest db table '" + tableName + "'", e); + } + } + + private void readData(String tableName, CairoEngine engine, SqlExecutionContextImpl context) throws SqlException { + try (SqlCompiler compiler = new SqlCompiler(engine)) { + try (RecordCursorFactory factory = compiler.compile(tableName, context).getRecordCursorFactory()) { + try (RecordCursor cursor = factory.getCursor(context)) { + Record record = cursor.getRecord(); + double cpuUtilSum = 0; + int rowCount = 0; + while (cursor.hasNext()) { + cpuUtilSum += record.getFloat(2); + rowCount++; + } + } + } + } + } + + private void gc() throws SqlException { + int maxAgeDays = 3; + SqlExecutionContext context = newContext(); + try (SqlCompiler compiler = new SqlCompiler(engine)) { + File tableRoot = new File(dataDir, tableName); + List removeList = new ArrayList<>(); + for (String dirEntry : tableRoot.list()) { + File partitionDir = new File(tableRoot, dirEntry); + if ( ! partitionDir.isDirectory()) continue; + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of("UTC")); + Instant partitionDay = Instant.from(formatter.parse(dirEntry + "T00:00:00")); + if (partitionDay.isBefore(Instant.now().minus(Duration.ofDays(maxAgeDays)))) + removeList.add(dirEntry); + } + compiler.compile("alter table " + tableName + " drop partition " + + removeList.stream().map(dir -> "'" + dir + "'").collect(Collectors.joining(",")), + context); + } + } + + private SqlExecutionContext newContext() { + return new SqlExecutionContextImpl(engine, 1); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index bdc16cadab6..2a38dec3230 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; @@ -73,12 +74,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { applications().put(application.with(cluster.get().withTarget(target)), deployment.applicationLock().get()); if (target.isPresent()) { logAutoscaling(target.get(), applicationId, clusterId, clusterNodes); - Optional resultingGeneration = deployment.activate(); - if (resultingGeneration.isEmpty()) return; // Failed to activate - - metricsDb.add(new NodeMetricsDb.AutoscalingEvent(applicationId, - resultingGeneration.get(), - nodeRepository().clock().instant())); + deployment.activate(); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 38511db6c4d..a651c4e52c2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -357,7 +357,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { private boolean failActive(Node node, String reason) { Optional deployment = deployer.deployFromLocalActive(node.allocation().get().owner(), Duration.ofMinutes(30)); - if (deployment.isEmpty()) return false; // this will be done at another config server + if (deployment.isEmpty()) return false; try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) { // If the active node that we are trying to fail is of type host, we need to successfully fail all diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index 6b2e1da0432..955936931be 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -245,7 +245,7 @@ public class IP { /** * Finds all unused addresses in this pool * - * @param nodes Locked list of all nodes in the repository + * @param nodes a list of all nodes in the repository */ public Set findUnused(NodeList nodes) { var unusedAddresses = new LinkedHashSet<>(asSet()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index 3464e9dd881..2ddbd6def6f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -11,13 +11,16 @@ import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * Application JSON serializer @@ -43,6 +46,11 @@ public class ApplicationSerializer { private static final String nodesKey = "nodes"; private static final String groupsKey = "groups"; private static final String nodeResourcesKey = "resources"; + private static final String scalingEventsKey = "scalingEvents"; + private static final String fromKey = "from"; + private static final String toKey = "to"; + private static final String generationKey = "generation"; + private static final String atKey = "at"; public static byte[] toJson(Application application) { Slime slime = new Slime(); @@ -86,6 +94,7 @@ public class ApplicationSerializer { toSlime(cluster.maxResources(), clusterObject.setObject(maxResourcesKey)); cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject(suggestedResourcesKey))); cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject(targetResourcesKey))); + scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray(scalingEventsKey)); } private static Cluster clusterFromSlime(String id, Inspector clusterObject) { @@ -94,7 +103,8 @@ public class ApplicationSerializer { clusterResourcesFromSlime(clusterObject.field(minResourcesKey)), clusterResourcesFromSlime(clusterObject.field(maxResourcesKey)), optionalClusterResourcesFromSlime(clusterObject.field(suggestedResourcesKey)), - optionalClusterResourcesFromSlime(clusterObject.field(targetResourcesKey))); + optionalClusterResourcesFromSlime(clusterObject.field(targetResourcesKey)), + scalingEventsFromSlime(clusterObject.field(scalingEventsKey))); } private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) { @@ -114,4 +124,26 @@ public class ApplicationSerializer { : Optional.empty(); } + private static void scalingEventsToSlime(List scalingEvents, Cursor eventArray) { + scalingEvents.forEach(event -> toSlime(event, eventArray.addObject())); + } + + private static List scalingEventsFromSlime(Inspector eventArray) { + return SlimeUtils.entriesStream(eventArray).map(item -> scalingEventFromSlime(item)).collect(Collectors.toList()); + } + + private static void toSlime(ScalingEvent event, Cursor object) { + toSlime(event.from(), object.setObject(fromKey)); + toSlime(event.to(), object.setObject(toKey)); + object.setLong(generationKey, event.generation()); + object.setLong(atKey, event.at().toEpochMilli()); + } + + private static ScalingEvent scalingEventFromSlime(Inspector inspector) { + return new ScalingEvent(clusterResourcesFromSlime(inspector.field(fromKey)), + clusterResourcesFromSlime(inspector.field(toKey)), + inspector.field(generationKey).asLong(), + Instant.ofEpochMilli(inspector.field(atKey).asLong())); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index e6564b52216..9e83960335d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -5,6 +5,7 @@ import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; @@ -380,10 +381,10 @@ public class CuratorDatabaseClient { ApplicationSerializer.toJson(application))); } - public void deleteApplication(ApplicationId application, NestedTransaction transaction) { - if (db.exists(applicationPath(application))) - db.newCuratorTransactionIn(transaction) - .add(CuratorOperations.delete(applicationPath(application).getAbsolute())); + public void deleteApplication(ApplicationTransaction transaction) { + if (db.exists(applicationPath(transaction.application()))) + db.newCuratorTransactionIn(transaction.nested()) + .add(CuratorOperations.delete(applicationPath(transaction.application()).getAbsolute())); } private Path applicationPath(ApplicationId id) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index b85862446a8..ffeed74d944 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -2,7 +2,9 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; @@ -13,6 +15,8 @@ import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.node.Allocation; import java.util.ArrayList; @@ -39,9 +43,9 @@ class Activator { } /** Activate required resources for application guarded by given lock */ - public void activate(Collection hosts, NestedTransaction transaction, ProvisionLock lock) { - activateNodes(hosts, transaction, lock); - activateLoadBalancers(hosts, transaction, lock); + public void activate(Collection hosts, long generation, ApplicationTransaction transaction) { + activateNodes(hosts, generation, transaction); + activateLoadBalancers(hosts, transaction); } /** @@ -53,41 +57,69 @@ class Activator { * Post condition: Nodes in reserved which are present in hosts are moved to active. * Nodes in active which are not present in hosts are moved to inactive. * - * @param transaction Transaction with operations to commit together with any operations done within the repository. * @param hosts the hosts to make the set of active nodes of this - * @param lock provision lock that must be held when calling this + * @param generation the application config generation that is activated + * @param transaction transaction with operations to commit together with any operations done within the repository, + * while holding the node repository lock on this application */ - private void activateNodes(Collection hosts, NestedTransaction transaction, ProvisionLock lock) { - ApplicationId application = lock.application(); + private void activateNodes(Collection hosts, long generation, ApplicationTransaction transaction) { + ApplicationId application = transaction.application(); Set hostnames = hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); NodeList allNodes = nodeRepository.list(); NodeList applicationNodes = allNodes.owner(application); List reserved = applicationNodes.state(Node.State.reserved).asList(); - List reservedToActivate = retainHostsInList(hostnames, reserved); - List active = applicationNodes.state(Node.State.active).asList(); - List continuedActive = retainHostsInList(hostnames, active); - List allActive = new ArrayList<>(continuedActive); - allActive.addAll(reservedToActivate); - if (!containsAll(hostnames, allActive)) + List reservedToActivate = updatePortsFrom(hosts, retainHostsInList(hostnames, reserved)); + List oldActive = applicationNodes.state(Node.State.active).asList(); // All nodes active now + List continuedActive = retainHostsInList(hostnames, oldActive); + List newActive = updateFrom(hosts, continuedActive); // All nodes that will be active when this is committed + newActive.addAll(reservedToActivate); + if ( ! containsAll(hostnames, newActive)) throw new IllegalArgumentException("Activation of " + application + " failed. " + "Could not find all requested hosts." + "\nRequested: " + hosts + "\nReserved: " + toHostNames(reserved) + - "\nActive: " + toHostNames(active) + + "\nActive: " + toHostNames(oldActive) + "\nThis might happen if the time from reserving host to activation takes " + "longer time than reservation expiry (the hosts will then no longer be reserved)"); validateParentHosts(application, allNodes, reservedToActivate); - List activeToRemove = removeHostsFromList(hostnames, active); - activeToRemove = activeToRemove.stream().map(Node::unretire).collect(Collectors.toList()); // only active nodes can be retired - nodeRepository.deactivate(activeToRemove, transaction, lock); - nodeRepository.activate(updateFrom(hosts, continuedActive), transaction); // update active with any changes - nodeRepository.activate(updatePortsFrom(hosts, reservedToActivate), transaction); + List activeToRemove = removeHostsFromList(hostnames, oldActive); + activeToRemove = activeToRemove.stream().map(Node::unretire).collect(Collectors.toList()); // only active nodes can be retired. TODO: Move this line to deactivate + nodeRepository.deactivate(activeToRemove, transaction); + nodeRepository.activate(newActive, transaction.nested()); // activate also continued active to update node state + + rememberResourceChange(transaction, generation, + NodeList.copyOf(oldActive).not().retired(), + NodeList.copyOf(newActive).not().retired()); unreserveParentsOf(reservedToActivate); } + private void rememberResourceChange(ApplicationTransaction transaction, long generation, + NodeList oldNodes, NodeList newNodes) { + if (nodeRepository.applications().get(transaction.application()).isEmpty()) return; // infrastructure app, hopefully + Application application = nodeRepository.applications().get(transaction.application()).get(); + + var currentNodesByCluster = newNodes.stream() + .collect(Collectors.groupingBy(node -> node.allocation().get().membership().cluster().id())); + Application modified = application; + for (var clusterEntry : currentNodesByCluster.entrySet()) { + var previousResources = oldNodes.cluster(clusterEntry.getKey()).toResources(); + var currentResources = NodeList.copyOf(clusterEntry.getValue()).toResources(); + if ( ! previousResources.equals(currentResources)) { + modified = modified.with(application.cluster(clusterEntry.getKey()).get() + .with(new ScalingEvent(previousResources, + currentResources, + generation, + nodeRepository.clock().instant()))); + } + } + + if (modified != application) + nodeRepository.applications().put(modified, transaction); + } + /** When a tenant node is activated on a host, we can open up that host for use by others */ private void unreserveParentsOf(List nodes) { for (Node node : nodes) { @@ -104,8 +136,8 @@ class Activator { } /** Activate load balancers */ - private void activateLoadBalancers(Collection hosts, NestedTransaction transaction, ProvisionLock lock) { - loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, allClustersOf(hosts), lock)); + private void activateLoadBalancers(Collection hosts, ApplicationTransaction transaction) { + loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(allClustersOf(hosts), transaction)); } private static Set allClustersOf(Collection hosts) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index b3506a0c102..34ad8ef6b00 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.google.inject.Inject; import com.yahoo.component.Version; +import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostName; @@ -105,7 +107,7 @@ public class InfraDeployerImpl implements InfraDeployer { removeApplication(application.getApplicationId()); } else { NestedTransaction nestedTransaction = new NestedTransaction(); - provisioner.activate(nestedTransaction, hostSpecs, lock); + provisioner.activate(hostSpecs, new ActivationContext(0), new ApplicationTransaction(lock, nestedTransaction)); nestedTransaction.commit(); duperModel.infraApplicationActivated( diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 634726f9d71..bb5498a2459 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; @@ -96,24 +97,24 @@ public class LoadBalancerProvisioner { * * Calling this when no load balancer has been prepared for given cluster is a no-op. */ - public void activate(NestedTransaction transaction, Set clusters, ProvisionLock lock) { - for (var cluster : loadBalancedClustersOf(lock.application()).entrySet()) { + public void activate(Set clusters, ApplicationTransaction transaction) { + for (var cluster : loadBalancedClustersOf(transaction.application()).entrySet()) { // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(lock.application(), cluster.getKey(), cluster.getValue(), true, lock); + provision(transaction.application(), cluster.getKey(), cluster.getValue(), true, transaction.lock()); } // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed - var surplusLoadBalancers = surplusLoadBalancersOf(lock.application(), clusters.stream() + var surplusLoadBalancers = surplusLoadBalancersOf(transaction.application(), clusters.stream() .map(LoadBalancerProvisioner::effectiveId) .collect(Collectors.toSet())); - deactivate(surplusLoadBalancers, transaction); + deactivate(surplusLoadBalancers, transaction.nested()); } /** * Deactivate all load balancers assigned to given application. This is a no-op if an application does not have any * load balancer(s). */ - public void deactivate(NestedTransaction transaction, ProvisionLock lock) { - deactivate(nodeRepository.loadBalancers(lock.application()).asList(), transaction); + public void deactivate(ApplicationTransaction transaction) { + deactivate(nodeRepository.loadBalancers(transaction.application()).asList(), transaction.nested()); } /** Returns load balancers of given application that are no longer referenced by given clusters */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index b79b87ae86c..d8e102f02c2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -2,7 +2,9 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.google.inject.Inject; +import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -122,14 +124,21 @@ public class NodeRepositoryProvisioner implements Provisioner { // TODO(mpolden): Remove public void activate(NestedTransaction transaction, ApplicationId application, Collection hosts) { try (var lock = lock(application)) { - activate(transaction, hosts, lock); + activate(hosts, new ActivationContext(0), new ApplicationTransaction(lock, transaction)); } } @Override + // TODO: Remove after November 2020 public void activate(NestedTransaction transaction, Collection hosts, ProvisionLock lock) { validate(hosts); - activator.activate(hosts, transaction, lock); + activator.activate(hosts, 0, new ApplicationTransaction(lock, transaction)); + } + + @Override + public void activate(Collection hosts, ActivationContext context, ApplicationTransaction transaction) { + validate(hosts); + activator.activate(hosts, context.generation(), transaction); } @Override @@ -137,18 +146,24 @@ public class NodeRepositoryProvisioner implements Provisioner { nodeRepository.restart(ApplicationFilter.from(application, NodeHostFilter.from(filter))); } - @Override // TODO(mpolden): Remove + @Override public void remove(NestedTransaction transaction, ApplicationId application) { try (var lock = lock(application)) { - remove(transaction, lock); + remove(new ApplicationTransaction(lock, transaction)); } } + // TODO: Remove after November 2020 @Override public void remove(NestedTransaction transaction, ProvisionLock lock) { - nodeRepository.deactivate(transaction, lock); - loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.deactivate(transaction, lock)); + remove(new ApplicationTransaction(lock, transaction)); + } + + @Override + public void remove(ApplicationTransaction transaction) { + nodeRepository.deactivate(transaction); + loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.deactivate(transaction)); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java index a5522a93a6e..8099b08ae89 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java @@ -2,7 +2,9 @@ package com.yahoo.vespa.hosted.provision.testutils; import com.google.inject.Inject; +import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; @@ -36,7 +38,7 @@ public class MockDeployer implements Deployer { // For mock deploy anything, changing wantToRetire to retired only private final NodeRepository nodeRepository; - /** The number of redeployments done to this */ + /** The number of redeployments done to this, which is also the config generation */ public int redeployments = 0; private final Map lastDeployTimes = new HashMap<>(); @@ -159,14 +161,16 @@ public class MockDeployer implements Deployer { prepare(); if (failActivate) throw new IllegalStateException("failActivate is true"); + + redeployments++; try (var lock = provisioner.lock(application.id)) { try (NestedTransaction t = new NestedTransaction()) { - provisioner.activate(t, preparedHosts, lock); + provisioner.activate(preparedHosts, new ActivationContext(redeployments), new ApplicationTransaction(lock, t)); t.commit(); lastDeployTimes.put(application.id, clock.instant()); } } - return redeployments++; + return redeployments; } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 0509ccc81c1..fed0ab6a8ee 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -2,8 +2,10 @@ package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.component.Version; +import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -198,7 +200,7 @@ public class MockNodeRepository extends NodeRepository { private void activate(List hosts, ApplicationId application, NodeRepositoryProvisioner provisioner) { try (var lock = provisioner.lock(application)) { NestedTransaction transaction = new NestedTransaction(); - provisioner.activate(transaction, hosts, lock); + provisioner.activate(hosts, new ActivationContext(0), new ApplicationTransaction(lock, transaction)); transaction.commit(); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java index 3dc96e0011b..d5d3bb6c950 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java @@ -1,7 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; +import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; @@ -25,29 +27,25 @@ public class MockProvisioner implements Provisioner { } @Override - public void activate(NestedTransaction transaction, ApplicationId application, Collection hosts) { - - } + public void activate(NestedTransaction transaction, ApplicationId application, Collection hosts) { } @Override - public void activate(NestedTransaction transaction, Collection hosts, ProvisionLock lock) { - - } + public void activate(NestedTransaction transaction, Collection hosts, ProvisionLock lock) { } @Override - public void remove(NestedTransaction transaction, ApplicationId application) { - - } + public void activate(Collection hosts, ActivationContext context, ApplicationTransaction transaction) { } @Override - public void remove(NestedTransaction transaction, ProvisionLock lock) { + public void remove(NestedTransaction transaction, ApplicationId application) { } - } + @Override + public void remove(NestedTransaction transaction, ProvisionLock lock) { } @Override - public void restart(ApplicationId application, HostFilter filter) { + public void remove(ApplicationTransaction transaction) { } - } + @Override + public void restart(ApplicationId application, HostFilter filter) { } @Override public ProvisionLock lock(ApplicationId application) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java index 2f331c53f74..cc988b2ec1e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ProvisionLock; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; @@ -30,7 +31,7 @@ public class ApplicationsTest { assertEquals(app1, applications.get(app1).get().id()); assertEquals(List.of(app1), applications.ids()); NestedTransaction t = new NestedTransaction(); - applications.remove(app1, t, provisionLock(app1)); + applications.remove(new ApplicationTransaction(provisionLock(app1), t)); t.commit(); assertTrue(applications.get(app1).isEmpty()); assertEquals(List.of(), applications.ids()); @@ -38,14 +39,14 @@ public class ApplicationsTest { applications.put(new Application(app1), () -> {}); applications.put(new Application(app2), () -> {}); t = new NestedTransaction(); - applications.put(new Application(app3), t, () -> {}); + applications.put(new Application(app3), new ApplicationTransaction(provisionLock(app1), t)); assertEquals(List.of(app1, app2), applications.ids()); t.commit(); assertEquals(List.of(app1, app2, app3), applications.ids()); t = new NestedTransaction(); - applications.remove(app1, t, provisionLock(app1)); - applications.remove(app2, t, provisionLock(app2)); - applications.remove(app3, t, provisionLock(app3)); + applications.remove(new ApplicationTransaction(provisionLock(app1), t)); + applications.remove(new ApplicationTransaction(provisionLock(app2), t)); + applications.remove(new ApplicationTransaction(provisionLock(app3), t)); assertEquals(List.of(app1, app2, app3), applications.ids()); t.commit(); assertTrue(applications.get(app1).isEmpty()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 2635258956a..5c2bbf1742f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -6,13 +6,18 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.autoscale.Metric; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; import org.junit.Test; import java.time.Duration; import java.time.Instant; +import java.util.List; + +import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; +import static com.yahoo.config.provision.NodeResources.StorageType.remote; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -78,24 +83,32 @@ public class AutoscalingMaintainerTest { var tester = new AutoscalingMaintainerTester(new MockDeployer.ApplicationContext(app1, cluster1, app1Capacity)); // Initial deployment at time 0 + System.out.println("Initial deploy"); tester.deploy(app1, cluster1, app1Capacity); // Measure overload tester.clock().advance(Duration.ofSeconds(1)); + System.out.println("Advance by 1 second to " + tester.clock().instant()); + System.out.println("Emit metrics"); + tester.addMeasurements(Metric.generation, 0, 1, app1); tester.addMeasurements(Metric.cpu, 0.9f, 500, app1); tester.addMeasurements(Metric.memory, 0.9f, 500, app1); tester.addMeasurements(Metric.disk, 0.9f, 500, app1); // Causes autoscaling tester.clock().advance(Duration.ofSeconds(1)); + System.out.println("Advance by 1 second to " + tester.clock().instant()); Instant firstMaintenanceTime = tester.clock().instant(); + System.out.println("Run maintenance"); tester.maintainer().maintain(); assertTrue(tester.deployer().lastDeployTime(app1).isPresent()); assertEquals(firstMaintenanceTime.toEpochMilli(), tester.deployer().lastDeployTime(app1).get().toEpochMilli()); - assertEquals(1, tester.nodeMetricsDb().getEvents(app1).size()); - assertEquals(app1, tester.nodeMetricsDb().getEvents(app1).get(0).application()); - assertEquals(0, tester.nodeMetricsDb().getEvents(app1).get(0).generation()); - assertEquals(firstMaintenanceTime.toEpochMilli(), tester.nodeMetricsDb().getEvents(app1).get(0).time().toEpochMilli()); + List events = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().scalingEvents(); + assertEquals(1, events.size()); + assertEquals(2, events.get(0).from().nodes()); + assertEquals(4, events.get(0).to().nodes()); + assertEquals(1, events.get(0).generation()); + assertEquals(firstMaintenanceTime.toEpochMilli(), events.get(0).at().toEpochMilli()); // Measure overload still, since change is not applied, but metrics are discarded tester.clock().advance(Duration.ofSeconds(1)); @@ -117,7 +130,7 @@ public class AutoscalingMaintainerTest { // Add measurement of the expected generation, leading to rescaling tester.clock().advance(Duration.ofSeconds(1)); - tester.addMeasurements(Metric.generation, 0, 1, app1); + tester.addMeasurements(Metric.generation, 1, 1, app1); tester.addMeasurements(Metric.cpu, 0.1f, 500, app1); tester.addMeasurements(Metric.memory, 0.1f, 500, app1); tester.addMeasurements(Metric.disk, 0.1f, 500, app1); @@ -125,8 +138,8 @@ public class AutoscalingMaintainerTest { Instant lastMaintenanceTime = tester.clock().instant(); tester.maintainer().maintain(); assertEquals(lastMaintenanceTime.toEpochMilli(), tester.deployer().lastDeployTime(app1).get().toEpochMilli()); - assertEquals(2, tester.nodeMetricsDb().getEvents(app1).size()); - assertEquals(1, tester.nodeMetricsDb().getEvents(app1).get(1).generation()); + events = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().scalingEvents(); + assertEquals(2, events.get(0).generation()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 58a8edb4631..ceedb41ab31 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -1,8 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -330,7 +332,7 @@ public class FailedExpirerTest { (level, message) -> System.out.println(level + ": " + message) ); try (var lock = provisioner.lock(applicationId)) { NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); - provisioner.activate(transaction, Set.copyOf(preparedNodes), lock); + provisioner.activate(Set.copyOf(preparedNodes), new ActivationContext(0), new ApplicationTransaction(lock, transaction)); transaction.commit(); } return this; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 5117a7b7397..824ebe40ea8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -1,7 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -250,7 +252,7 @@ public class NodeFailTester { List hosts = provisioner.prepare(applicationId, cluster, capacity, null); try (var lock = provisioner.lock(applicationId)) { NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); - provisioner.activate(transaction, hosts, lock); + provisioner.activate(hosts, new ActivationContext(0), new ApplicationTransaction(lock, transaction)); transaction.commit(); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index 35c8a9a9251..21003324696 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -82,7 +83,8 @@ public class RebalancerTest { // --- Making the system stable enables rebalancing NestedTransaction tx = new NestedTransaction(); - tester.nodeRepository().deactivate(List.of(cpuSkewedNode), tx, new ProvisionLock(cpuApp, () -> {})); + tester.nodeRepository().deactivate(List.of(cpuSkewedNode), + new ApplicationTransaction(new ProvisionLock(cpuApp, () -> {}), tx)); tx.commit(); // ... if activation fails when trying, we clean up the state diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index aba5810784b..bdae9d28fe2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -1,8 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index 598831d1eeb..72f9e9597de 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -7,8 +7,10 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import org.junit.Test; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -30,13 +32,19 @@ public class ApplicationSerializerTest { new ClusterResources( 8, 4, new NodeResources(1, 2, 3, 4)), new ClusterResources(12, 6, new NodeResources(3, 6, 21, 24)), Optional.empty(), - Optional.empty())); + Optional.empty(), + List.of())); + var minResources = new NodeResources(1, 2, 3, 4); clusters.add(new Cluster(ClusterSpec.Id.from("c2"), true, - new ClusterResources( 8, 4, new NodeResources(1, 2, 3, 4)), + new ClusterResources( 8, 4, minResources), new ClusterResources(14, 7, new NodeResources(3, 6, 21, 24)), Optional.of(new ClusterResources(20, 10, new NodeResources(0.5, 4, 14, 16))), - Optional.of(new ClusterResources(10, 5, new NodeResources(2, 4, 14, 16))))); + Optional.of(new ClusterResources(10, 5, new NodeResources(2, 4, 14, 16))), + List.of(new ScalingEvent(new ClusterResources(10, 5, minResources), + new ClusterResources(12, 6, minResources), + 7L, + Instant.ofEpochMilli(12345L))))); Application original = new Application(ApplicationId.from("myTenant", "myApplication", "myInstance"), clusters); @@ -56,6 +64,7 @@ public class ApplicationSerializerTest { assertEquals(originalCluster.maxResources(), serializedCluster.maxResources()); assertEquals(originalCluster.suggestedResources(), serializedCluster.suggestedResources()); assertEquals(originalCluster.targetResources(), serializedCluster.targetResources()); + assertEquals(originalCluster.scalingEvents(), serializedCluster.scalingEvents()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java index 4fae7cf0ab9..fb732c641b8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; @@ -50,7 +51,6 @@ import static org.mockito.Mockito.when; @RunWith(Parameterized.class) public class InfraDeployerImplTest { - @Parameterized.Parameters(name = "application={0}") public static Iterable parameters() { return List.of( @@ -136,15 +136,15 @@ public class InfraDeployerImplTest { private void verifyActivated(String... hostnames) { verify(duperModelInfraApi).infraApplicationActivated( eq(application.getApplicationId()), eq(Stream.of(hostnames).map(HostName::from).collect(Collectors.toList()))); - ArgumentMatcher lockMatcher = lock -> { - assertEquals(application.getApplicationId(), lock.application()); + ArgumentMatcher transactionMatcher = t -> { + assertEquals(application.getApplicationId(), t.application()); return true; }; ArgumentMatcher> hostsMatcher = hostSpecs -> { assertEquals(Set.of(hostnames), hostSpecs.stream().map(HostSpec::hostname).collect(Collectors.toSet())); return true; }; - verify(provisioner).activate(any(), argThat(hostsMatcher), argThat(lockMatcher)); + verify(provisioner).activate(argThat(hostsMatcher), any(), argThat(transactionMatcher)); } private Node addNode(int id, Node.State state, Optional wantedVespaVersion) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 2a43b9f44f6..5acfc6f69d4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -2,8 +2,10 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; +import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -209,7 +211,7 @@ public class ProvisioningTester { try (var lock = provisioner.lock(application)) { NestedTransaction transaction = new NestedTransaction(); transaction.add(new CuratorTransaction(curator)); - provisioner.activate(transaction, hosts, lock); + provisioner.activate(hosts, new ActivationContext(0), new ApplicationTransaction(lock, transaction)); transaction.commit(); } assertEquals(toHostNames(hosts), toHostNames(nodeRepository.getNodes(application, Node.State.active))); @@ -239,7 +241,8 @@ public class ProvisioningTester { public void deactivate(ApplicationId applicationId) { try (var lock = nodeRepository.lock(applicationId)) { NestedTransaction deactivateTransaction = new NestedTransaction(); - nodeRepository.deactivate(deactivateTransaction, new ProvisionLock(applicationId, lock)); + nodeRepository.deactivate(new ApplicationTransaction(new ProvisionLock(applicationId, lock), + deactivateTransaction)); deactivateTransaction.commit(); } } -- cgit v1.2.3