Improve documentation and get application lock when maintainging info and utils

8 files changed, 48 insertions, 19 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java
index 1958c5bd0ff..a96f7637738 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java
@@ -66,4 +66,6 @@ public interface ConfigServerClient {
      * @throws IOException If trouble contacting the server
      */
     EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint) throws IOException;
+
+    //JsonNode getClusterInfo(DeploymentId deployment, String endpoint) throws IOException;
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java
index 468d00eaf80..03d0cd28ca1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java
@@ -2,7 +2,21 @@
 package com.yahoo.vespa.hosted.controller.application;
 
 /**
- * Calculate tco and waste for one cluster within one deployment.
+ * Calculate utilization relative to the target utilization,
+ * tco and waste for one cluster of one deployment.
+ *
+ * The target utilization is defined the following assumptions:
+ * 1. CPU contention starts to cause problems on 0.8
+ * 2. Memory management starts to casue problems on 0.7
+ * 3. Load is evenly divided between two deployments - each deployments can handle the other.
+ * 4. Memory and disk are agnostic to query load.
+ * 5. Peak utilization (daily variations) are twice the size of the average.
+ *
+ * With this in mind we get:
+ * CPU: 0.8/2/2 = 0.2
+ * Mem: 0.7
+ * Disk: 0.7
+ * Disk busy: 0.3
  *
  * @author smorgrav
  */
@@ -30,10 +44,12 @@ public class ClusterCost {
         this.waste  = clusterInfo.getCost() - tco;
     }
 
+    /** @return TCO in dollars */
     public double getTco() {
         return tco;
     }
 
+    /** @return Waste in dollars */
     public double getWaste() {
         return waste;
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java
index 259caa9616d..cb39177c811 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java
@@ -1,4 +1,5 @@
-package com.yahoo.vespa.hosted.controller.application;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
 
 import com.yahoo.config.provision.ClusterSpec;
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilization.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilization.java
index 3f6e807a581..ff92ce36d1b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilization.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilization.java
@@ -1,4 +1,5 @@
-package com.yahoo.vespa.hosted.controller.application;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
 
 /**
  * System resources as _ratios_ of available resources.
@@ -17,7 +18,7 @@ public class ClusterUtilization {
 
     /**
      * Resource utilization as ratios. The ratio is normally between 0 and 1 where
-     * one is fully utilized but can be higher as it consumes more than it are guaranteed.
+     * one is fully utilized - but can be higher as it consumes more than it are guaranteed.
      *
      * @param memory Memory utilization
      * @param cpu  CPU utilization
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
index 9eb2fb87de2..01219e940a3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
@@ -33,8 +33,8 @@ public class Deployment {
         Objects.requireNonNull(revision, "revision cannot be null");
         Objects.requireNonNull(version, "version cannot be null");
         Objects.requireNonNull(deployTime, "deployTime cannot be null");
-        Objects.requireNonNull(clusterUtils, "deployTime cannot be null");
-        Objects.requireNonNull(clusterInfo, "deployTime cannot be null");
+        Objects.requireNonNull(clusterUtils, "clusterUtils cannot be null");
+        Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null");
         this.zone = zone;
         this.revision = revision;
         this.version = version;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java
index b0bbbf7ca8c..fce825bd99e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java
@@ -5,7 +5,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 /**
- * Calculates cost for for an application instance.
+ * Calculates cost for for an application deployment.
  *
  * @author smorgrav
  */
@@ -17,11 +17,7 @@ public class DeploymentCost {
 
     private final Map<String, ClusterCost> clusters;
 
-    public DeploymentCost() {
-        this(new HashMap<>());
-    }
-
-    public DeploymentCost(Map<String, ClusterCost> clusterCosts) {
+    DeploymentCost(Map<String, ClusterCost> clusterCosts) {
         clusters = new HashMap<>(clusterCosts);
 
         double tco = 0;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java
index 5fba945844b..5aef1512d63 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java
@@ -1,4 +1,5 @@
-package com.yahoo.vespa.hosted.controller.maintenance;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -6,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
@@ -20,7 +22,9 @@ import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
- * Fetch info about hardware, hostnames and cluster specifications and update applications.
+ * Maintain info about hardware, hostnames and cluster specifications.
+ *
+ * This is used to calculate cost metrics for the application api.
  *
  * @author smorgrav
  */
@@ -28,14 +32,13 @@ public class ClusterInfoMaintainer extends Maintainer {
 
     private final Controller controller;
 
-    public ClusterInfoMaintainer(Controller controller, Duration duration, JobControl jobControl) {
+    ClusterInfoMaintainer(Controller controller, Duration duration, JobControl jobControl) {
         super(controller, duration, jobControl);
         this.controller = controller;
     }
 
     private static String clusterid(NodeRepositoryJsonModel.Node node) {
         return node.membership.clusterId;
-
     }
 
     private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(NodeRepositoryJsonModel nodes) {
@@ -46,6 +49,7 @@ public class ClusterInfoMaintainer extends Maintainer {
                 .filter(node -> node.membership != null)
                 .collect(Collectors.groupingBy(ClusterInfoMaintainer::clusterid));
 
+        // For each cluster - get info
         for (String id : clusters.keySet()) {
             List<NodeRepositoryJsonModel.Node> clusterNodes = clusters.get(id);
 
@@ -61,6 +65,7 @@ public class ClusterInfoMaintainer extends Maintainer {
         return infoMap;
     }
 
+    // TODO use appId in url
     private NodeRepositoryJsonModel getApplicationNodes(ApplicationId appId, Zone zone) {
         NodeRepositoryJsonModel nodesResponse = null;
         ObjectMapper mapper = new ObjectMapper();
@@ -77,11 +82,14 @@ public class ClusterInfoMaintainer extends Maintainer {
 
     @Override
     protected void maintain() {
+
         for (Application application : controller().applications().asList()) {
+            Lock lock = controller().applications().lock(application.id());
             for (Deployment deployment : application.deployments().values()) {
                 NodeRepositoryJsonModel appNodes = getApplicationNodes(application.id(), deployment.zone());
                 Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(appNodes);
-                application.with(deployment.withClusterInfo(clusterInfo));
+                Application app = application.with(deployment.withClusterInfo(clusterInfo));
+                controller.applications().store(app, lock);
             }
         }
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java
index 0986b0abf7c..71557ab2536 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java
@@ -1,8 +1,10 @@
-package com.yahoo.vespa.hosted.controller.maintenance;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
@@ -43,10 +45,13 @@ public class ClusterUtilizationMaintainer extends Maintainer {
 
     @Override
     protected void maintain() {
+
         for (Application application : controller().applications().asList()) {
+            Lock lock = controller().applications().lock(application.id());
             for (Deployment deployment : application.deployments().values()) {
                 Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getUpdatedClusterUtilizations(application.id(), deployment.zone());
-                application.with(deployment.withClusterUtils(clusterUtilization));
+                Application app = application.with(deployment.withClusterUtils(clusterUtilization));
+                controller.applications().store(app, lock);
             }
         }
     }