Add support for version ACK-dependent tasks in cluster controller

Used to enable synchronous operation for set-node-state calls, which
ensure that side-effects of the call are visible when the response
returns.

If controller leadership is lost before state is published, tasks will
be failed back to the client.

4 files changed, 46 insertions, 8 deletions

diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/UnknownMasterException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/UnknownMasterException.java
index 7108a941277..eade2e807c9 100644
--- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/UnknownMasterException.java
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/UnknownMasterException.java
@@ -3,6 +3,10 @@ package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors;
 
 public class UnknownMasterException extends NotMasterException {
 
+    public UnknownMasterException(String message) {
+        super(message);
+    }
+
     public UnknownMasterException() {
         super("No known master cluster controller currently exists.");
     }
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/SetUnitStateRequest.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/SetUnitStateRequest.java
index db2b33c68e8..5a9b85e734b 100644
--- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/SetUnitStateRequest.java
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/SetUnitStateRequest.java
@@ -16,7 +16,7 @@ public interface SetUnitStateRequest extends UnitRequest {
 
         public final int value;
 
-        private Condition(int value) {
+        Condition(int value) {
             this.value = value;
         }
 
@@ -24,9 +24,34 @@ public interface SetUnitStateRequest extends UnitRequest {
             try {
                 return Condition.valueOf(value.toUpperCase());
             } catch (IllegalArgumentException e) {
-                throw new InvalidContentException("Invalid value for my enum Condition: " + value);
+                throw new InvalidContentException(String.format("Invalid value for condition: '%s', expected one of 'force', 'safe'", value));
             }
         }
     }
     Condition getCondition();
+
+    enum ResponseWait {
+        WAIT_UNTIL_CLUSTER_ACKED("wait-until-cluster-acked"), // Wait for state change to be ACKed by cluster
+        NO_WAIT("no-wait"); // Return without waiting for state change to be ACKed by cluster
+
+        private final String name;
+
+        ResponseWait(String name) { this.name = name; }
+
+        public String getName() { return this.name; }
+
+        @Override
+        public String toString() { return name; }
+
+        public static ResponseWait fromString(String value) throws InvalidContentException {
+            if (value.equalsIgnoreCase(WAIT_UNTIL_CLUSTER_ACKED.name)) {
+                return WAIT_UNTIL_CLUSTER_ACKED;
+            } else if (value.equalsIgnoreCase(NO_WAIT.name)) {
+                return NO_WAIT;
+            }
+            throw new InvalidContentException(String.format("Invalid value for response-wait: '%s', expected one of '%s', '%s'",
+                    value, WAIT_UNTIL_CLUSTER_ACKED.name, NO_WAIT.name));
+        }
+    }
+    ResponseWait getResponseWait();
 }
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonReader.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonReader.java
index f5ab406179b..04dcb582389 100644
--- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonReader.java
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/JsonReader.java
@@ -35,9 +35,12 @@ public class JsonReader {
     static class SetRequestData {
         final Map<String, UnitState> stateMap;
         final SetUnitStateRequest.Condition condition;
-        public SetRequestData(Map<String, UnitState> stateMap, SetUnitStateRequest.Condition condition) {
+        final SetUnitStateRequest.ResponseWait responseWait;
+        public SetRequestData(Map<String, UnitState> stateMap, SetUnitStateRequest.Condition condition,
+                              SetUnitStateRequest.ResponseWait responseWait) {
             this.stateMap = stateMap;
             this.condition = condition;
+            this.responseWait = responseWait;
         }
     }
 
@@ -47,11 +50,15 @@ public class JsonReader {
         final SetUnitStateRequest.Condition condition;
 
         if (json.has("condition")) {
-            condition = SetUnitStateRequest.Condition.valueOf(json.getString("condition"));
+            condition = SetUnitStateRequest.Condition.fromString(json.getString("condition"));
         } else {
             condition = SetUnitStateRequest.Condition.FORCE;
         }
 
+        final SetUnitStateRequest.ResponseWait responseWait = json.has("response-wait")
+                ? SetUnitStateRequest.ResponseWait.fromString(json.getString("response-wait"))
+                : SetUnitStateRequest.ResponseWait.WAIT_UNTIL_CLUSTER_ACKED;
+
         Map<String, UnitState> stateMap = new HashMap<>();
         if (!json.has("state")) {
             throw new InvalidContentException("Set state requests must contain a state object");
@@ -90,7 +97,7 @@ public class JsonReader {
             }
             stateMap.put(type, new UnitStateImpl(code, reason));
         }
-        return new SetRequestData(stateMap, condition);
+        return new SetRequestData(stateMap, condition, responseWait);
     }
 
 }
diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
index 9a8fc084d60..fcdf3214c45 100644
--- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
+++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java
@@ -58,18 +58,20 @@ public class RestApiHandler implements HttpRequestHandler {
                 });
                 return new JsonHttpResult().setJson(jsonWriter.createJson(data));
             } else {
-                final JsonReader.SetRequestData setRequestdata = jsonReader.getStateRequestData(request);
+                final JsonReader.SetRequestData setRequestData = jsonReader.getStateRequestData(request);
                 SetResponse setResponse = restApi.setUnitState(new SetUnitStateRequest() {
                     @Override
                     public Map<String, UnitState> getNewState() {
-                        return setRequestdata.stateMap;
+                        return setRequestData.stateMap;
                     }
                     @Override
                     public String[] getUnitPath() {
                         return unitPath;
                     }
                     @Override
-                    public Condition getCondition() { return setRequestdata.condition; }
+                    public Condition getCondition() { return setRequestData.condition; }
+                    @Override
+                    public ResponseWait getResponseWait() { return setRequestData.responseWait; }
                 });
                 return new JsonHttpResult().setJson(jsonWriter.createJson(setResponse));
             }