Make ClusterPolicyOverride available to ClusterApiImpl

11 files changed, 98 insertions, 14 deletions

diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
index 49fab7522ba..20269951a1e 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
@@ -84,6 +84,8 @@ public class OrchestratorImpl implements Orchestrator {
              Clock.systemUTC(),
              new ApplicationApiFactory(configServerConfig.zookeeperserver().size(),
                                        orchestratorConfig.numProxies(),
+                                       orchestratorConfig.numProxiesAllowedDown(),
+                                       orchestratorConfig.numProxiesAllowedDownRatio(),
                                        Clock.systemUTC()),
              orchestratorConfig.serviceMonitorConvergenceLatencySeconds());
     }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java
index 0ad236c5a23..832b00f2294 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java
@@ -18,8 +18,10 @@ public class ApplicationApiFactory {
     private final OrchestrationParams orchestrationParams;
     private final Clock clock;
 
-    public ApplicationApiFactory(int numberOfConfigServers, int numProxies, Clock clock) {
-        this.orchestrationParams = HostedVespaOrchestration.create(numberOfConfigServers, numProxies);
+    public ApplicationApiFactory(int numberOfConfigServers, int numProxies, int numProxiesAllowedDown,
+                                 double numProxiesAllowedDownRatio, Clock clock) {
+        this.orchestrationParams = HostedVespaOrchestration.create(numberOfConfigServers, numProxies, numProxiesAllowedDown,
+                                                                   numProxiesAllowedDownRatio);
         this.clock = clock;
     }
 
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
index 024a3bc58db..736b909a82f 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
@@ -41,6 +41,7 @@ class ClusterApiImpl implements ClusterApi {
     private final Clock clock;
     private final Set<ServiceInstance> servicesInGroup;
     private final Set<ServiceInstance> servicesNotInGroup;
+    private final ClusterPolicyOverride clusterPolicyOverride;
 
     /** Lazily initialized in servicesDownAndNotInGroup(), do not access directly. */
     private Set<ServiceInstance> servicesDownAndNotInGroup = null;
@@ -71,6 +72,10 @@ class ClusterApiImpl implements ClusterApi {
         this.hostInfos = hostInfos;
         this.clusterControllerClientFactory = clusterControllerClientFactory;
         this.clock = clock;
+        this.clusterPolicyOverride = new ClusterPolicyOverride(serviceCluster.serviceInstances().size(),
+                                                               clusterParams.size(),
+                                                               clusterParams.allowedDown(),
+                                                               clusterParams.allowedDownRatio());
 
         Map<Boolean, Set<ServiceInstance>> serviceInstancesByLocality =
                 serviceCluster.serviceInstances().stream()
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterPolicyOverride.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterPolicyOverride.java
new file mode 100644
index 00000000000..2bdfa1a6659
--- /dev/null
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterPolicyOverride.java
@@ -0,0 +1,37 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.model;
+
+import java.util.OptionalDouble;
+import java.util.OptionalInt;
+
+/**
+ * @author hakonhall
+ */
+public record ClusterPolicyOverride(int deployedSize, OptionalInt expectedSize, OptionalInt allowedDown, OptionalDouble allowedDownRatio) {
+    public ClusterPolicyOverride {
+        if (deployedSize <= 0)
+            throw new IllegalArgumentException("deployedSize must be positive");
+
+        if (expectedSize.isPresent() && expectedSize.getAsInt() <= 0)
+            throw new IllegalArgumentException("expectedSize must be positive");
+
+        if (allowedDown.isPresent()) {
+            if (allowedDown.getAsInt() <= 0)
+                throw new IllegalArgumentException("allowedDown must be positive: " + allowedDown.getAsInt());
+            if (expectedSize.isPresent() && allowedDown.getAsInt() > expectedSize.getAsInt())
+                throw new IllegalArgumentException("allowedDown must be less than or equal to expectedSize (" + expectedSize.getAsInt() +
+                                                   "): " + allowedDown.getAsInt());
+        }
+
+        if (allowedDownRatio.isPresent() && (allowedDownRatio.getAsDouble() < 0.0 || allowedDownRatio.getAsDouble() > 1.0))
+            throw new IllegalArgumentException("allowedDownRatio must be between 0.0 and 1.0: " + allowedDownRatio.getAsDouble());
+
+    }
+
+    public OptionalInt allowedDownPercentage() {
+        return allowedDownRatio.isPresent() ?
+               OptionalInt.of((int) Math.round(allowedDownRatio.getAsDouble() * 100.0)) :
+               OptionalInt.empty();
+    }
+
+}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterParams.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterParams.java
index ab010da50ad..4048a27fda0 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterParams.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterParams.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.orchestrator.policy;
 
 import java.util.Objects;
+import java.util.OptionalDouble;
 import java.util.OptionalInt;
 
 /**
@@ -14,9 +15,13 @@ public class ClusterParams {
     private static final ClusterParams DEFAULT = new ClusterParams.Builder().build();
 
     private final int size;
+    private final int allowedDown;
+    private final double allowedDownRatio;
 
     public static class Builder {
-        private int size = 0;
+        private int size = -1;
+        private int allowedDown = -1;
+        private double allowedDownRatio = -1.0;
 
         public Builder() {}
 
@@ -25,8 +30,18 @@ public class ClusterParams {
             return this;
         }
 
+        public Builder setAllowedDown(int allowedDown) {
+            this.allowedDown = allowedDown;
+            return this;
+        }
+
+        public Builder setAllowedDownRatio(double allowedDownRatio) {
+            this.allowedDownRatio = allowedDownRatio;
+            return this;
+        }
+
         public ClusterParams build() {
-            return new ClusterParams(size);
+            return new ClusterParams(size, allowedDown, allowedDownRatio);
         }
     }
 
@@ -34,8 +49,10 @@ public class ClusterParams {
         return DEFAULT;
     }
 
-    private ClusterParams(int size) {
+    private ClusterParams(int size, int allowedDown, double allowedDownRatio) {
         this.size = size;
+        this.allowedDown = allowedDown;
+        this.allowedDownRatio = allowedDownRatio;
     }
 
     /**
@@ -46,16 +63,28 @@ public class ClusterParams {
         return size > 0 ? OptionalInt.of(size) : OptionalInt.empty();
     }
 
+    /** The number of services that are allowed to be down. */
+    public OptionalInt allowedDown() {
+        return allowedDown > 0 ? OptionalInt.of(allowedDown) : OptionalInt.empty();
+    }
+
+    /** The ratio of services that are allowed to be down. */
+    public OptionalDouble allowedDownRatio() {
+        return 0.0 <= allowedDownRatio && allowedDownRatio <= 1.0 ?
+               OptionalDouble.of(allowedDownRatio) :
+               OptionalDouble.empty();
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         ClusterParams that = (ClusterParams) o;
-        return size == that.size;
+        return size == that.size && allowedDown == that.allowedDown && Double.compare(that.allowedDownRatio, allowedDownRatio) == 0;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(size);
+        return Objects.hash(size, allowedDown, allowedDownRatio);
     }
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaOrchestration.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaOrchestration.java
index eb2b863aefb..54d440f751f 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaOrchestration.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaOrchestration.java
@@ -11,7 +11,8 @@ import com.yahoo.vespa.applicationmodel.ServiceType;
  * @author hakonhall
  */
 public class HostedVespaOrchestration {
-    public static OrchestrationParams create(int numConfigServers, int numProxies) {
+    public static OrchestrationParams create(int numConfigServers, int numProxies, int numProxiesAllowedDown,
+                                             double numProxiesAllowedDownRatio) {
         // We'll create parameters for both the controller and config server applications, even though
         // only one of them is present, as (a) no harm is done by having the extra parameters, and
         // (b) it leads to simpler code below.
@@ -75,6 +76,8 @@ public class HostedVespaOrchestration {
                                                    new ClusterParams
                                                            .Builder()
                                                            .setSize(numProxies)
+                                                           .setAllowedDown(numProxiesAllowedDown)
+                                                           .setAllowedDownRatio(numProxiesAllowedDownRatio)
                                                            .build())
                                               .build())
 
@@ -87,6 +90,8 @@ public class HostedVespaOrchestration {
                                                    new ClusterParams
                                                            .Builder()
                                                            .setSize(numProxies)
+                                                           .setAllowedDown(numProxiesAllowedDown)
+                                                           .setAllowedDownRatio(numProxiesAllowedDownRatio)
                                                            .build())
                                               .build())
 
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
index 70a8381c9ac..3eb6bf6bc5e 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
@@ -77,7 +77,7 @@ public class OrchestratorImplTest {
     private static final Zone zone = Zone.defaultZone();
 
     private final ManualClock clock = new ManualClock();
-    private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, clock);
+    private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, 1, 0.1, clock);
     private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
     private final MockCurator curator = new MockCurator();
     private final ZkStatusService statusService = new ZkStatusService(
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
index 73774321ffb..7ca1e1ebad6 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
@@ -57,7 +57,7 @@ public class OrchestratorTest {
         var flagSource = new InMemoryFlagSource();
         var timer = new TestTimer();
         var clustercontroller = new ClusterControllerClientFactoryMock();
-        var applicationApiFactory = new ApplicationApiFactory(3, 5, timer.toUtcClock());
+        var applicationApiFactory = new ApplicationApiFactory(3, 5, 1, 0.1, timer.toUtcClock());
         var clusterPolicy = new HostedVespaClusterPolicy(flagSource, zone);
         var policy = new HostedVespaPolicy(clusterPolicy, clustercontroller, applicationApiFactory, flagSource);
         var zone = new Zone(SystemName.cd, Environment.prod, RegionName.from("cd-us-east-1"));
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
index f2e2972ae9f..32aeebb2c74 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
@@ -64,8 +64,11 @@ class ModelTestUtils {
 
     public static final int NUMBER_OF_CONFIG_SERVERS = 3;
     public static final int NUMBER_OF_PROXIES = 5;
+    public static final int NUMBER_OF_PROXIES_ALLOWED_DOWN = 1;
+    public static final double NUMBER_OF_PROXIES_ALLOWED_DOWN_RATIO = 0.1;
     public static final OrchestrationParams ORCHESTRATION_PARAMS =
-            HostedVespaOrchestration.create(NUMBER_OF_CONFIG_SERVERS, NUMBER_OF_PROXIES);
+            HostedVespaOrchestration.create(NUMBER_OF_CONFIG_SERVERS, NUMBER_OF_PROXIES, NUMBER_OF_PROXIES_ALLOWED_DOWN,
+                                            NUMBER_OF_PROXIES_ALLOWED_DOWN_RATIO);
     public static final ApplicationParams APPLICATION_PARAMS = ORCHESTRATION_PARAMS
             .getApplicationParams(OrchestratorUtil.toApplicationId(
                     new ApplicationInstanceReference(TENANT_ID, APPLICATION_INSTANCE_ID)));
@@ -93,7 +96,8 @@ class ModelTestUtils {
     private final ManualClock clock = new ManualClock();
 
     ApplicationApiFactory applicationApiFactory() {
-        return new ApplicationApiFactory(NUMBER_OF_CONFIG_SERVERS, NUMBER_OF_PROXIES, clock);
+        return new ApplicationApiFactory(NUMBER_OF_CONFIG_SERVERS, NUMBER_OF_PROXIES, NUMBER_OF_PROXIES_ALLOWED_DOWN,
+                                         NUMBER_OF_PROXIES_ALLOWED_DOWN_RATIO, clock);
     }
 
     HostInfos getHostInfos() {
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
index a622142b873..97adeea9835 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
@@ -37,7 +37,7 @@ public class HostedVespaPolicyTest {
     private final ClusterControllerClientFactory clientFactory = mock(ClusterControllerClientFactory.class);
     private final ClusterControllerClient client = mock(ClusterControllerClient.class);
     private final ManualClock clock = new ManualClock();
-    private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, clock);
+    private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, 1, 0.1, clock);
     private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
 
     @Before
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java
index b9dab4b3aeb..465804a9406 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java
@@ -79,7 +79,7 @@ class HostRequestHandlerTest {
     private static final ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
     private static final StatusService EVERY_HOST_IS_UP_HOST_STATUS_SERVICE = new ZkStatusService(
             new MockCurator(), mock(Metric.class), new TestTimer(), new DummyAntiServiceMonitor());
-    private static final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, clock);
+    private static final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, 1, 0.1, clock);
 
     static {
         when(serviceMonitor.getApplication(any(HostName.class)))