Maintain application min, max and target resources

7 files changed, 193 insertions, 48 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 3cf2442f6f7..4ba480b73b1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -19,6 +19,7 @@ import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.hosted.provision.applications.Applications;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance;
@@ -98,6 +99,7 @@ public class NodeRepository extends AbstractComponent {
     private final FirmwareChecks firmwareChecks;
     private final DockerImages dockerImages;
     private final JobControl jobControl;
+    private final Applications applications;
 
     /**
      * Creates a node repository from a zookeeper provider.
@@ -124,6 +126,7 @@ public class NodeRepository extends AbstractComponent {
         this.firmwareChecks = new FirmwareChecks(db, clock);
         this.dockerImages = new DockerImages(db, dockerImage);
         this.jobControl = new JobControl(db);
+        this.applications = new Applications();
 
         // read and write all nodes to make sure they are stored in the latest version of the serialized format
         for (State state : State.values())
@@ -154,6 +157,9 @@ public class NodeRepository extends AbstractComponent {
     /** Returns the status of maintenance jobs managed by this. */
     public JobControl jobControl() { return jobControl; }
 
+    /** Returns this node repo's view of the applications deployed to it */
+    public Applications applications() { return applications; }
+
     // ---------------- Query API ----------------------------------------------------------------
 
     /**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java
new file mode 100644
index 00000000000..7dd2dc7be17
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java
@@ -0,0 +1,37 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.applications;
+
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.transaction.Mutex;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Optional;
+
+/**
+ * The node repository's view of an application deployment.
+ *
+ * This is immutable, and must be locked with the application lock on read-modify-write.
+ *
+ * @author bratseth
+ */
+public class Application {
+
+    private Map<ClusterSpec.Id, Cluster> clusters = new HashMap<>();
+
+    /** Returns the cluster with the given id or null if none */
+    public Cluster cluster(ClusterSpec.Id id) { return clusters.get(id); }
+
+    /**
+     * Sets the min and max resource limits of the given cluster.
+     * This will create the cluster with these limits if it does not exist.
+     * If the cluster has a target which is not inside the new limits, the target is removed.
+     */
+    public void setClusterLimits(ClusterSpec.Id id, ClusterResources min, ClusterResources max, Mutex applicationLock) {
+        Cluster cluster = clusters.computeIfAbsent(id, clusterId -> new Cluster(min, max, Optional.empty()));
+        if (cluster.targetResources().isPresent() && ! cluster.targetResources().get().isWithin(min, max))
+            clusters.put(id, cluster.withoutTarget());
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java
new file mode 100644
index 00000000000..1409857df1a
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java
@@ -0,0 +1,24 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.applications;
+
+import com.yahoo.config.provision.ApplicationId;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * An (in-memory, for now) repository of the node repo's view of applications.
+ *
+ * This is multithread safe.
+ *
+ * @author bratseth
+ */
+public class Applications {
+
+    private final ConcurrentHashMap<ApplicationId, Application> applications = new ConcurrentHashMap<>();
+
+    /** Returns the application with the given id, or null if it does not exist and should not be created */
+    public Application get(ApplicationId applicationId, boolean create) {
+        return applications.computeIfAbsent(applicationId, id -> create ? new Application() : null);
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
new file mode 100644
index 00000000000..d4f7f10abfd
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
@@ -0,0 +1,46 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.applications;
+
+import com.yahoo.config.provision.ClusterResources;
+
+import java.util.Optional;
+
+/**
+ * The node repo's view of a cluster in an application deployment.
+ *
+ * This is immutable, and must be locked with the application lock on read-modify-write.
+ *
+ * @author bratseth
+ */
+public class Cluster {
+
+    private final ClusterResources minResources, maxResources;
+    private final Optional<ClusterResources> targetResources;
+
+    Cluster(ClusterResources minResources, ClusterResources maxResources, Optional<ClusterResources> targetResources) {
+        this.minResources = minResources;
+        this.maxResources = maxResources;
+        this.targetResources = targetResources;
+    }
+
+    /** Returns the configured minimal resources in this cluster */
+    public ClusterResources minResources() { return minResources; }
+
+    /** Returns the configured maximal resources in this cluster */
+    public ClusterResources maxResources() { return maxResources; }
+
+    /**
+     * Returns the computed resources (between min and max, inclusive) this cluster should
+     * have allocated at the moment, or empty if the system currently have no opinion on this.
+     */
+    public Optional<ClusterResources> targetResources() { return targetResources; }
+
+    public Cluster withTarget(ClusterResources target) {
+        return new Cluster(minResources, maxResources, Optional.of(target));
+    }
+
+    public Cluster withoutTarget() {
+        return new Cluster(minResources, maxResources, Optional.empty());
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
index a23c1a932d4..d1cd4752c93 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.NodeResources;
@@ -31,42 +32,37 @@ public class CapacityPolicies {
         this.isUsingAdvertisedResources = zone.cloud().value().equals("aws");
     }
 
-    public int decideSize(Capacity capacity, ClusterSpec cluster, ApplicationId application) {
-        int requestedNodes = capacity.minResources().nodes();
-
+    public int decideSize(int requested, Capacity capacity, ClusterSpec cluster, ApplicationId application) {
         if (application.instance().isTester()) return 1;
 
-        ensureRedundancy(requestedNodes, cluster, capacity.canFail());
-
-        if (capacity.isRequired()) return requestedNodes;
-
+        ensureRedundancy(requested, cluster, capacity.canFail());
+        if (capacity.isRequired()) return requested;
         switch(zone.environment()) {
             case dev : case test : return 1;
-            case perf : return Math.min(capacity.minResources().nodes(), 3);
-            case staging: return requestedNodes <= 1 ? requestedNodes : Math.max(2, requestedNodes / 10);
-            case prod : return requestedNodes;
+            case perf : return Math.min(requested, 3);
+            case staging: return requested <= 1 ? requested : Math.max(2, requested / 10);
+            case prod : return requested;
             default : throw new IllegalArgumentException("Unsupported environment " + zone.environment());
         }
     }
 
-    public NodeResources decideNodeResources(Capacity capacity, ClusterSpec cluster) {
-        NodeResources resources = capacity.minResources().nodeResources();
-        if (resources == NodeResources.unspecified)
-            resources = defaultNodeResources(cluster.type());
-        ensureSufficientResources(resources, cluster);
+    public NodeResources decideNodeResources(NodeResources requested, Capacity capacity, ClusterSpec cluster) {
+        if (requested == NodeResources.unspecified)
+            requested = defaultNodeResources(cluster.type());
+        ensureSufficientResources(requested, cluster);
 
-        if (capacity.isRequired()) return resources;
+        if (capacity.isRequired()) return requested;
 
         // Allow slow storage in zones which are not performance sensitive
         if (zone.system().isCd() || zone.environment() == Environment.dev || zone.environment() == Environment.test)
-            resources = resources.with(NodeResources.DiskSpeed.any).with(NodeResources.StorageType.any);
+            requested = requested.with(NodeResources.DiskSpeed.any).with(NodeResources.StorageType.any);
 
         // Dev does not cap the cpu of containers since usage is spotty: Allocate just a small amount exclusively
         // Do not cap in AWS as hosts are allocated on demand and 1-to-1, so the node can use the entire host
         if (zone.environment() == Environment.dev && !zone.region().value().contains("aws-"))
-            resources = resources.withVcpu(0.1);
+            requested = requested.withVcpu(0.1);
 
-        return resources;
+        return requested;
     }
 
     private void ensureSufficientResources(NodeResources resources, ClusterSpec cluster) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
index 2c2c927034b..083f8db5aa5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 import com.google.inject.Inject;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
@@ -15,10 +16,14 @@ import com.yahoo.config.provision.ProvisionLogger;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.log.LogLevel;
+import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.applications.Application;
+import com.yahoo.vespa.hosted.provision.applications.Cluster;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter;
@@ -87,38 +92,36 @@ public class NodeRepositoryProvisioner implements Provisioner {
      * The nodes are ordered by increasing index number.
      */
     @Override
-    public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity requestedCapacity, 
+    public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity requested,
                                   ProvisionLogger logger) {
-        if (cluster.group().isPresent()) throw new IllegalArgumentException("Node requests cannot specify a group");
-
         log.log(zone.system().isCd() ? Level.INFO : LogLevel.DEBUG,
-                () -> "Received deploy prepare request for " + requestedCapacity +
+                () -> "Received deploy prepare request for " + requested +
                       " for application " + application + ", cluster " + cluster);
 
-        int effectiveGroups;
-        NodeSpec requestedNodes;
-        NodeResources resources = requestedCapacity.minResources().nodeResources();
-        if ( requestedCapacity.type() == NodeType.tenant) {
-            int nodeCount = capacityPolicies.decideSize(requestedCapacity, cluster, application);
-            if (zone.environment().isManuallyDeployed() && nodeCount < requestedCapacity.minResources().nodes())
-                logger.log(Level.INFO, "Requested " + requestedCapacity.minResources().nodes() + " nodes for " + cluster +
-                                       ", downscaling to " + nodeCount + " nodes in " + zone.environment());
-            resources = capacityPolicies.decideNodeResources(requestedCapacity, cluster);
-            boolean exclusive = capacityPolicies.decideExclusivity(cluster.isExclusive());
-            effectiveGroups = Math.min(requestedCapacity.minResources().groups(), nodeCount); // cannot have more groups than nodes
-            requestedNodes = NodeSpec.from(nodeCount, resources, exclusive, requestedCapacity.canFail());
+        if (cluster.group().isPresent()) throw new IllegalArgumentException("Node requests cannot specify a group");
 
-            if ( ! hasQuota(application, nodeCount))
-                throw new IllegalArgumentException(requestedCapacity + " requested for " + cluster +
-                                                   (requestedCapacity.minResources().nodes() != nodeCount ? " resolved to " + nodeCount + " nodes" : "") +
-                                                   " exceeds your quota. Resolve this at https://cloud.vespa.ai/quota");
+        if ( ! hasQuota(application, requested.maxResources().nodes()))
+            throw new IllegalArgumentException(requested + " requested for " + cluster +
+                                               ". Max value exceeds your quota. Resolve this at https://cloud.vespa.ai/quota");
+
+        int groups;
+        NodeResources resources;
+        NodeSpec nodeSpec;
+        if ( requested.type() == NodeType.tenant) {
+            ClusterResources target = decideTargetResources(application, cluster.id(), requested);
+            int nodeCount = capacityPolicies.decideSize(target.nodes(), requested, cluster, application);
+            resources = capacityPolicies.decideNodeResources(target.nodeResources(), requested, cluster);
+            boolean exclusive = capacityPolicies.decideExclusivity(cluster.isExclusive());
+            groups = Math.min(target.groups(), nodeCount); // cannot have more groups than nodes
+            nodeSpec = NodeSpec.from(nodeCount, resources, exclusive, requested.canFail());
+            logIfDownscaled(target.nodes(), nodeCount, cluster, logger);
         }
         else {
-            requestedNodes = NodeSpec.from(requestedCapacity.type());
-            effectiveGroups = 1; // type request with multiple groups is not supported
+            groups = 1; // type request with multiple groups is not supported
+            resources = requested.minResources().nodeResources();
+            nodeSpec = NodeSpec.from(requested.type());
         }
-
-        return asSortedHosts(preparer.prepare(application, cluster, requestedNodes, effectiveGroups), resources);
+        return asSortedHosts(preparer.prepare(application, cluster, nodeSpec, groups), resources);
     }
 
     @Override
@@ -138,6 +141,39 @@ public class NodeRepositoryProvisioner implements Provisioner {
         loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.deactivate(application, transaction));
     }
 
+    /**
+     * Returns the target cluster resources, a value between the min and max in the requested capacity,
+     * and updates the application store with the received min and max,
+     */
+    private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec.Id clusterId, Capacity requested) {
+        try (Mutex lock = nodeRepository.lock(applicationId)) {
+            Application application = nodeRepository.applications().get(applicationId, true);
+            application.setClusterLimits(clusterId, requested.minResources(), requested.maxResources(), lock);
+            return application.cluster(clusterId).targetResources()
+                    .orElse(currentResources(applicationId, clusterId, requested)
+                    .orElse(requested.minResources()));
+        }
+    }
+
+    /** Returns the current resources of this cluster, if it'1s already depoyed and inside the requested limits */
+    private Optional<ClusterResources> currentResources(ApplicationId applicationId,
+                                                        ClusterSpec.Id clusterId,
+                                                        Capacity requested) {
+        List<Node> nodes = NodeList.copyOf(nodeRepository.getNodes(applicationId, Node.State.active))
+                                   .cluster(clusterId).not().retired().asList();
+        if (nodes.size() < 1) return Optional.empty();
+        long groups = nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count();
+        var resources = new ClusterResources(nodes.size(), (int)groups, nodes.get(0).flavor().resources());
+        if ( ! resources.isWithin(requested.minResources(), requested.maxResources())) return Optional.empty();
+        return Optional.of(resources);
+    }
+
+    private void logIfDownscaled(int targetNodes, int actualNodes, ClusterSpec cluster, ProvisionLogger logger) {
+        if (zone.environment().isManuallyDeployed() && actualNodes < targetNodes)
+            logger.log(Level.INFO, "Requested " + targetNodes + " nodes for " + cluster +
+                                   ", downscaling to " + actualNodes + " nodes in " + zone.environment());
+    }
+
     private boolean hasQuota(ApplicationId application, int requestedNodes) {
         if ( ! this.zone.system().isPublic()) return true; // no quota management
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
index f88cb839946..76258e86de9 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
@@ -493,7 +493,7 @@ public class ProvisioningTest {
             fail("Expected exception");
         }
         catch (IllegalArgumentException e) {
-            assertEquals("6 nodes with [vcpu: 1.0, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 4.0 Gbps] requested for content cluster 'content0' 6.42 exceeds your quota. Resolve this at https://cloud.vespa.ai/quota",
+            assertEquals("6 nodes with [vcpu: 1.0, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 4.0 Gbps] requested for content cluster 'content0' 6.42. Max value exceeds your quota. Resolve this at https://cloud.vespa.ai/quota",
                          e.getMessage());
         }
     }
@@ -772,10 +772,10 @@ public class ProvisioningTest {
         allHosts.addAll(content1);
 
         Function<Integer, Capacity> capacity = count -> Capacity.from(new ClusterResources(count, 1, NodeResources.unspecified), required, true);
-        int expectedContainer0Size = tester.capacityPolicies().decideSize(capacity.apply(container0Size), containerCluster0, application);
-        int expectedContainer1Size = tester.capacityPolicies().decideSize(capacity.apply(container1Size), containerCluster1, application);
-        int expectedContent0Size = tester.capacityPolicies().decideSize(capacity.apply(content0Size), contentCluster0, application);
-        int expectedContent1Size = tester.capacityPolicies().decideSize(capacity.apply(content1Size), contentCluster1, application);
+        int expectedContainer0Size = tester.capacityPolicies().decideSize(container0Size, capacity.apply(container0Size), containerCluster0, application);
+        int expectedContainer1Size = tester.capacityPolicies().decideSize(container1Size, capacity.apply(container1Size), containerCluster1, application);
+        int expectedContent0Size = tester.capacityPolicies().decideSize(content0Size, capacity.apply(content0Size), contentCluster0, application);
+        int expectedContent1Size = tester.capacityPolicies().decideSize(content1Size, capacity.apply(content1Size), contentCluster1, application);
 
         assertEquals("Hosts in each group cluster is disjunct and the total number of unretired nodes is correct",
                      expectedContainer0Size + expectedContainer1Size + expectedContent0Size + expectedContent1Size,