Use orchestration override if present

8 files changed, 148 insertions, 57 deletions

diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java
index cb2a2fe5f62..7fa3bd45b4c 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java
@@ -30,10 +30,19 @@ public interface ClusterApi {
 
     boolean noServicesOutsideGroupIsDown() throws HostStateChangeDeniedException;
 
-    int percentageOfServicesDownOutsideGroup();
-    int percentageOfServicesDownIfGroupIsAllowedToBeDown();
+    /** Returns the number of services currently in the cluster, plus the number of missing services. */
+    int size();
+
+    int servicesDownOutsideGroup();
+    default int percentageOfServicesDownOutsideGroup() { return sizePercentageOf(servicesDownOutsideGroup()); }
+    int servicesDownIfGroupIsAllowedToBeDown();
+    default int percentageOfServicesDownIfGroupIsAllowedToBeDown() { return sizePercentageOf(servicesDownIfGroupIsAllowedToBeDown()); }
+
+    ClusterPolicyOverride clusterPolicyOverride();
 
     Optional<StorageNode> storageNodeInGroup();
 
     String downDescription();
+
+    private int sizePercentageOf(int count) { return (int) Math.round(100.0 * count / size()); }
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
index 736b909a82f..6240761dd6b 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
@@ -173,15 +173,21 @@ class ClusterApiImpl implements ClusterApi {
     }
 
     @Override
-    public int percentageOfServicesDownOutsideGroup() {
-        int numberOfServicesDown = servicesDownAndNotInGroup().size() + missingServices;
-        return numberOfServicesDown * 100 / (serviceCluster.serviceInstances().size() + missingServices);
+    public int size() { return serviceCluster.serviceInstances().size() + missingServices; }
+
+    @Override
+    public int servicesDownOutsideGroup() {
+        return servicesDownAndNotInGroup().size() + missingServices;
+    }
+
+    @Override
+    public int servicesDownIfGroupIsAllowedToBeDown() {
+        return servicesDownAndNotInGroup().size() + servicesInGroup.size() + missingServices;
     }
 
     @Override
-    public int percentageOfServicesDownIfGroupIsAllowedToBeDown() {
-        int numberOfServicesDown = servicesDownAndNotInGroup().size() + missingServices + servicesInGroup.size();
-        return numberOfServicesDown * 100 / (serviceCluster.serviceInstances().size() + missingServices);
+    public ClusterPolicyOverride clusterPolicyOverride() {
+        return clusterPolicyOverride;
     }
 
     /**
@@ -206,7 +212,7 @@ class ClusterApiImpl implements ClusterApi {
             if (suspended.size() > nodeLimit) {
                 description.append(" and " + (suspended.size() - nodeLimit) + " others");
             }
-            description.append(" are suspended.");
+            description.append(" " + isOrAre(suspended.size()) + " suspended.");
         }
 
         Set<ServiceInstance> downElsewhere = servicesDownAndNotInGroup().stream()
@@ -228,12 +234,14 @@ class ClusterApiImpl implements ClusterApi {
             if (downElsewhereTotal > serviceLimit) {
                 description.append(" and " + (downElsewhereTotal - serviceLimit) + " others");
             }
-            description.append(" are down.");
+            description.append(" " + isOrAre(downElsewhereTotal) + " down.");
         }
 
         return description.toString();
     }
 
+    private static String isOrAre(int count) { return count == 1 ? "is" : "are"; }
+
     private Optional<StorageNode> storageNodeInGroup(Predicate<ServiceInstance> storageServicePredicate) {
         if (!VespaModelUtil.isStorage(serviceCluster)) {
             return Optional.empty();
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterPolicyOverride.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterPolicyOverride.java
index 2bdfa1a6659..f724a4da9cb 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterPolicyOverride.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterPolicyOverride.java
@@ -1,6 +1,9 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.orchestrator.model;
 
+import com.yahoo.vespa.orchestrator.policy.SuspensionLimit;
+
+import java.util.Optional;
 import java.util.OptionalDouble;
 import java.util.OptionalInt;
 
@@ -28,6 +31,16 @@ public record ClusterPolicyOverride(int deployedSize, OptionalInt expectedSize,
 
     }
 
+    public static ClusterPolicyOverride fromDeployedSize(int deployedSize) {
+        return new ClusterPolicyOverride(deployedSize, OptionalInt.empty(), OptionalInt.empty(), OptionalDouble.empty());
+    }
+
+    public Optional<SuspensionLimit> getSuspensionLimit() {
+        return allowedDown.isPresent() || allowedDownRatio.isPresent() ?
+               Optional.of(new SuspensionLimit(allowedDown.orElse(0), allowedDownRatio.orElse(0.0))) :
+               Optional.empty();
+    }
+
     public OptionalInt allowedDownPercentage() {
         return allowedDownRatio.isPresent() ?
                OptionalInt.of((int) Math.round(allowedDownRatio.getAsDouble() * 100.0)) :
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java
index 5d553c86c50..88b339e15f3 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java
@@ -37,10 +37,11 @@ public class HostedVespaClusterPolicy implements ClusterPolicy {
             return SuspensionReasons.nothingNoteworthy();
         }
 
-        int percentageOfServicesAllowedToBeDown = getConcurrentSuspensionLimit(clusterApi).asPercentage();
-        if (clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() <= percentageOfServicesAllowedToBeDown) {
+        SuspensionLimit limit = getConcurrentSuspensionLimit(clusterApi);
+        if (clusterApi.servicesDownIfGroupIsAllowedToBeDown() <= limit.allowedDown())
+            return SuspensionReasons.nothingNoteworthy();
+        if (clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() <= limit.allowedDownPercentage())
             return SuspensionReasons.nothingNoteworthy();
-        }
 
         // Be a bit more cautious when removing nodes permanently
         if (!permanent) {
@@ -50,19 +51,39 @@ public class HostedVespaClusterPolicy implements ClusterPolicy {
             }
         }
 
-        String message = percentageOfServicesAllowedToBeDown <= 0
-                ? clusterApi.percentageOfServicesDownOutsideGroup() + "% of the " + clusterApi.serviceDescription(true)
-                  + " are down or suspended already:" + clusterApi.downDescription()
-                : "The percentage of downed or suspended " + clusterApi.serviceDescription(true)
-                  + " would increase from " + clusterApi.percentageOfServicesDownOutsideGroup() + "% to "
-                  + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "% (limit is "
-                  + percentageOfServicesAllowedToBeDown + "%):" + clusterApi.downDescription();
+        final String message;
+        if (limit.allowedDownPercentage() > 0) {
+            final String numberDescription;
+            final String fromDescription;
+            final String toDescription;
+            final String limitDescription;
+            if (limit.allowedDown() > 1) {
+                numberDescription = "number (percentage)";
+                fromDescription = clusterApi.servicesDownOutsideGroup() + " (" + clusterApi.percentageOfServicesDownOutsideGroup() + "%)";
+                toDescription = clusterApi.servicesDownIfGroupIsAllowedToBeDown() + " (" + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "%)";
+                limitDescription = limit.allowedDown() + " (" + limit.allowedDownPercentage() + "%)";
+            } else {
+                numberDescription = "percentage";
+                fromDescription = clusterApi.percentageOfServicesDownOutsideGroup() + "%";
+                toDescription = clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "%";
+                limitDescription = limit.allowedDownPercentage() + "%";
+            }
 
-        throw new HostStateChangeDeniedException(clusterApi.getNodeGroup(), ENOUGH_SERVICES_UP_CONSTRAINT, message);
+            message = "The %s of %s that are down would increase from %s to %s which is beyond the limit of %s"
+                    .formatted(numberDescription, clusterApi.serviceDescription(true), fromDescription, toDescription, limitDescription);
+        } else {
+            message = "%d %s %s already down".formatted(clusterApi.servicesDownOutsideGroup(),
+                                                        clusterApi.serviceDescription(false),
+                                                        clusterApi.servicesDownOutsideGroup() == 1 ? "is" : "are");
+        }
+
+        throw new HostStateChangeDeniedException(clusterApi.getNodeGroup(),
+                                                 ENOUGH_SERVICES_UP_CONSTRAINT,
+                                                 message + ":" + clusterApi.downDescription());
     }
 
     // Non-private for testing purposes
-    ConcurrentSuspensionLimitForCluster getConcurrentSuspensionLimit(ClusterApi clusterApi) {
+    SuspensionLimit getConcurrentSuspensionLimit(ClusterApi clusterApi) {
         // Possible service clusters on a node as of 2021-01-22:
         //
         //       CLUSTER ID           SERVICE TYPE                  HEALTH       ASSOCIATION
@@ -102,45 +123,50 @@ public class HostedVespaClusterPolicy implements ClusterPolicy {
         //   H  proxy (same as B)
         //   I  proxy host
 
+        Optional<SuspensionLimit> override = clusterApi.clusterPolicyOverride().getSuspensionLimit();
+        if (override.isPresent()) {
+            return override.get();
+        }
+
         if (clusterApi.serviceType().equals(ServiceType.CLUSTER_CONTROLLER)) {
-            return ConcurrentSuspensionLimitForCluster.ONE_NODE;
+            return SuspensionLimit.fromAllowedDown(1);
         }
 
         if (Set.of(ServiceType.STORAGE, ServiceType.SEARCH, ServiceType.DISTRIBUTOR, ServiceType.TRANSACTION_LOG_SERVER)
                 .contains(clusterApi.serviceType())) {
             // Delegate to the cluster controller
-            return ConcurrentSuspensionLimitForCluster.ALL_NODES;
+            return SuspensionLimit.fromAllowedDownRatio(1);
         }
 
         if (clusterApi.serviceType().equals(ServiceType.CONTAINER)) {
-            return ConcurrentSuspensionLimitForCluster.TEN_PERCENT;
+            return SuspensionLimit.fromAllowedDownRatio(0.1);
         }
 
         if (VespaModelUtil.ADMIN_CLUSTER_ID.equals(clusterApi.clusterId())) {
             if (ServiceType.SLOBROK.equals(clusterApi.serviceType())) {
-                return ConcurrentSuspensionLimitForCluster.ONE_NODE;
+                return SuspensionLimit.fromAllowedDown(1);
             }
 
-            return ConcurrentSuspensionLimitForCluster.ALL_NODES;
+            return SuspensionLimit.fromAllowedDownRatio(1);
         } else if (ServiceType.METRICS_PROXY.equals(clusterApi.serviceType())) {
-            return ConcurrentSuspensionLimitForCluster.ALL_NODES;
+            return SuspensionLimit.fromAllowedDownRatio(1);
         }
 
         if (Set.of(ServiceType.CONFIG_SERVER, ServiceType.CONTROLLER).contains(clusterApi.serviceType())) {
-            return ConcurrentSuspensionLimitForCluster.ONE_NODE;
+            return SuspensionLimit.fromAllowedDown(1);
         }
 
         if (clusterApi.serviceType().equals(ServiceType.HOST_ADMIN)) {
             if (Set.of(ClusterId.CONFIG_SERVER_HOST, ClusterId.CONTROLLER_HOST).contains(clusterApi.clusterId())) {
-                return ConcurrentSuspensionLimitForCluster.ONE_NODE;
+                return SuspensionLimit.fromAllowedDown(1);
             }
 
             return zone.system().isCd()
-                    ? ConcurrentSuspensionLimitForCluster.FIFTY_PERCENT
-                    : ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT;
+                    ? SuspensionLimit.fromAllowedDownRatio(0.5)
+                    : SuspensionLimit.fromAllowedDownRatio(0.2);
         }
 
         // The above should cover all cases, but if not we'll return a reasonable default:
-        return ConcurrentSuspensionLimitForCluster.TEN_PERCENT;
+        return SuspensionLimit.fromAllowedDownRatio(0.1);
     }
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/SuspensionLimit.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/SuspensionLimit.java
new file mode 100644
index 00000000000..8a3d62dcc9c
--- /dev/null
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/SuspensionLimit.java
@@ -0,0 +1,29 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.policy;
+
+/**
+ * @author hakonhall
+ *
+ * @param allowedDown      the maximum number of services (nodes) that are allowed to be down.
+ * @param allowedDownRatio the maximum ratio of services (nodes) that are allowed to be down.
+ */
+public record SuspensionLimit(int allowedDown, double allowedDownRatio) {
+    public SuspensionLimit {
+        if (allowedDown < 0)
+            throw new IllegalArgumentException("allowedDown cannot be negative: " + allowedDown);
+        if (allowedDownRatio < 0.0 || allowedDownRatio > 1.0)
+            throw new IllegalArgumentException("allowedDownRatio must be between 0.0 and 1.0: " + allowedDownRatio);
+    }
+
+    public static SuspensionLimit fromAllowedDown(int allowedDown) {
+        return new SuspensionLimit(allowedDown, 0);
+    }
+
+    public static SuspensionLimit fromAllowedDownRatio(double allowedDownRatio) {
+        return new SuspensionLimit(0, allowedDownRatio);
+    }
+
+    public int allowedDownPercentage() {
+        return (int) Math.round(allowedDownRatio * 100.0);
+    }
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
index 7ca1e1ebad6..ee62ffabd30 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
@@ -103,7 +103,7 @@ public class OrchestratorTest {
             fail();
         } catch (HostStateChangeDeniedException e) {
             assertTrue(e.getMessage().contains("Changing the state of cfg2 would violate enough-services-up"));
-            assertTrue(e.getMessage().contains("[cfg1] are suspended."));
+            assertTrue(e.getMessage().contains("[cfg1] is suspended."));
         }
 
         // cfg1 is removed from the application
@@ -115,7 +115,7 @@ public class OrchestratorTest {
             fail();
         } catch (HostStateChangeDeniedException e) {
             assertTrue(e.getMessage().contains("Changing the state of cfg2 would violate enough-services-up"));
-            assertTrue(e.getMessage().contains("[1 missing config server] are down."));
+            assertTrue(e.getMessage().contains("[1 missing config server] is down."));
         }
 
         // cfg1 is reprovisioned, added to the node repo, and activated
@@ -130,7 +130,7 @@ public class OrchestratorTest {
             fail();
         } catch (HostStateChangeDeniedException e) {
             assertTrue(e.getMessage().contains("Changing the state of cfg1 would violate enough-services-up"));
-            assertTrue(e.getMessage().contains("[cfg2] are suspended"));
+            assertTrue(e.getMessage().contains("[cfg2] is suspended"));
         }
 
         // etc (should be the same as for cfg1)
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
index b073f546cce..49978f824c4 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
@@ -33,7 +33,6 @@ import java.util.EnumSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
-import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import static org.junit.Assert.assertEquals;
@@ -95,7 +94,7 @@ public class ClusterApiImplTest {
         assertFalse(clusterApi.isStorageCluster());
         assertEquals(" [host3, host4] are suspended. [ServiceInstance{configId=service-2, hostName=host2, " +
                      "serviceStatus=ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}] " +
-                     "are down.",
+                     "is down.",
                 clusterApi.downDescription());
         assertEquals(60, clusterApi.percentageOfServicesDownOutsideGroup());
         assertEquals(80, clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown());
@@ -178,8 +177,9 @@ public class ClusterApiImplTest {
             policy.verifyGroupGoingDownIsFine(clusterApi);
             fail();
         } catch (HostStateChangeDeniedException e) {
-            assertTrue(e.getMessage().contains("Changing the state of cfg1 would violate enough-services-up: 33% of the config " +
-                    "servers are down or suspended already: [1 missing config server] are down."));
+            assertEquals("Changing the state of cfg1 would violate enough-services-up: 1 config server is already down: " +
+                         "[1 missing config server] is down.",
+                         e.getMessage());
         }
     }
 
@@ -197,8 +197,9 @@ public class ClusterApiImplTest {
             policy.verifyGroupGoingDownIsFine(clusterApi);
             fail();
         } catch (HostStateChangeDeniedException e) {
-            assertTrue(e.getMessage().contains("Changing the state of cfg1 would violate enough-services-up: 33% of the config " +
-                    "server hosts are down or suspended already: [1 missing config server host] are down."));
+            assertEquals("Changing the state of cfg1 would violate enough-services-up: 1 config server host is already down: " +
+                         "[1 missing config server host] is down.",
+                         e.getMessage());
         }
     }
 
@@ -212,8 +213,9 @@ public class ClusterApiImplTest {
             policy.verifyGroupGoingDownIsFine(clusterApi);
             fail();
         } catch (HostStateChangeDeniedException e) {
-            assertTrue(e.getMessage().contains("Changing the state of cfg1 would violate enough-services-up: 33% of the config " +
-                    "servers are down or suspended already: [1 missing config server] are down."));
+            assertEquals("Changing the state of cfg1 would violate enough-services-up: 1 config server is already down: " +
+                         "[1 missing config server] is down.",
+                         e.getMessage());
         }
     }
 
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java
index eb70d809855..47bdcd4e68e 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.orchestrator.model.ApplicationApi;
 import com.yahoo.vespa.orchestrator.model.ClusterApi;
+import com.yahoo.vespa.orchestrator.model.ClusterPolicyOverride;
 import com.yahoo.vespa.orchestrator.model.NodeGroup;
 import com.yahoo.vespa.orchestrator.model.VespaModelUtil;
 import org.junit.Before;
@@ -36,6 +37,7 @@ public class HostedVespaClusterPolicyTest {
     @Before
     public void setUp() {
         when(clusterApi.getApplication()).thenReturn(applicationApi);
+        when(clusterApi.clusterPolicyOverride()).thenReturn(ClusterPolicyOverride.fromDeployedSize(3));
         when(zone.system()).thenReturn(SystemName.main);
 
         NodeGroup nodeGroup = mock(NodeGroup.class);
@@ -62,24 +64,24 @@ public class HostedVespaClusterPolicyTest {
     public void testSlobrokSuspensionLimit() {
         when(clusterApi.clusterId()).thenReturn(VespaModelUtil.ADMIN_CLUSTER_ID);
         when(clusterApi.serviceType()).thenReturn(ServiceType.SLOBROK);
-        assertEquals(ConcurrentSuspensionLimitForCluster.ONE_NODE,
-                policy.getConcurrentSuspensionLimit(clusterApi));
+        assertEquals(SuspensionLimit.fromAllowedDown(1),
+                     policy.getConcurrentSuspensionLimit(clusterApi));
     }
 
     @Test
     public void testAdminSuspensionLimit() {
         when(clusterApi.clusterId()).thenReturn(VespaModelUtil.ADMIN_CLUSTER_ID);
         when(clusterApi.serviceType()).thenReturn(new ServiceType("non-slobrok-service-type"));
-        assertEquals(ConcurrentSuspensionLimitForCluster.ALL_NODES,
-                policy.getConcurrentSuspensionLimit(clusterApi));
+        assertEquals(SuspensionLimit.fromAllowedDownRatio(1.0),
+                     policy.getConcurrentSuspensionLimit(clusterApi));
     }
 
     @Test
     public void testStorageSuspensionLimit() {
         when(clusterApi.serviceType()).thenReturn(ServiceType.STORAGE);
         when(clusterApi.clusterId()).thenReturn(new ClusterId("some-cluster-id"));
-        assertEquals(ConcurrentSuspensionLimitForCluster.ALL_NODES,
-                policy.getConcurrentSuspensionLimit(clusterApi));
+        assertEquals(SuspensionLimit.fromAllowedDownRatio(1.0),
+                     policy.getConcurrentSuspensionLimit(clusterApi));
     }
 
     @Test
@@ -87,12 +89,12 @@ public class HostedVespaClusterPolicyTest {
         when(applicationApi.applicationId()).thenReturn(VespaModelUtil.TENANT_HOST_APPLICATION_ID);
         when(clusterApi.clusterId()).thenReturn(ClusterId.TENANT_HOST);
         when(clusterApi.serviceType()).thenReturn(ServiceType.HOST_ADMIN);
-        assertEquals(ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT,
+        assertEquals(SuspensionLimit.fromAllowedDownRatio(0.2),
                      policy.getConcurrentSuspensionLimit(clusterApi));
 
 
         when(zone.system()).thenReturn(SystemName.cd);
-        assertEquals(ConcurrentSuspensionLimitForCluster.FIFTY_PERCENT,
+        assertEquals(SuspensionLimit.fromAllowedDownRatio(0.5),
                      policy.getConcurrentSuspensionLimit(clusterApi));
     }
 
@@ -101,7 +103,7 @@ public class HostedVespaClusterPolicyTest {
         when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("a:b:c"));
         when(clusterApi.clusterId()).thenReturn(new ClusterId("some-cluster-id"));
         when(clusterApi.serviceType()).thenReturn(new ServiceType("some-service-type"));
-        assertEquals(ConcurrentSuspensionLimitForCluster.TEN_PERCENT,
+        assertEquals(SuspensionLimit.fromAllowedDownRatio(0.1),
                      policy.getConcurrentSuspensionLimit(clusterApi));
     }
 
@@ -132,12 +134,14 @@ public class HostedVespaClusterPolicyTest {
                                             boolean expectSuccess) throws HostStateChangeDeniedException {
         when(clusterApi.noServicesOutsideGroupIsDown()).thenReturn(noServicesOutsideGroupIsDown);
         when(clusterApi.allServicesDown()).thenReturn(noServicesInGroupIsUp);
+        when(clusterApi.servicesDownIfGroupIsAllowedToBeDown()).thenReturn(20);
         when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(20);
-        doReturn(ConcurrentSuspensionLimitForCluster.TEN_PERCENT).when(policy).getConcurrentSuspensionLimit(clusterApi);
+        doReturn(SuspensionLimit.fromAllowedDownRatio(0.1)).when(policy).getConcurrentSuspensionLimit(clusterApi);
 
         when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("a:b:c"));
         when(clusterApi.serviceType()).thenReturn(new ServiceType("service-type"));
         when(clusterApi.serviceDescription(true)).thenReturn("services of {service-type,cluster-id}");
+        when(clusterApi.servicesDownOutsideGroup()).thenReturn(5);
         when(clusterApi.percentageOfServicesDownOutsideGroup()).thenReturn(5);
         when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(percentageOfServicesDownIfGroupIsAllowedToBeDown);
         when(clusterApi.downDescription()).thenReturn(" Down description");
@@ -153,9 +157,9 @@ public class HostedVespaClusterPolicyTest {
             }
         } catch (HostStateChangeDeniedException e) {
             if (!expectSuccess) {
-                assertEquals("Changing the state of node-group would violate enough-services-up: The percentage of downed " +
-                             "or suspended services of {service-type,cluster-id} would increase from 5% to 13% (limit is 10%): " +
-                             "Down description",
+                assertEquals("Changing the state of node-group would violate enough-services-up: The percentage of " +
+                             "services of {service-type,cluster-id} that are down would increase from 5% to 13% " +
+                             "which is beyond the limit of 10%: Down description",
                              e.getMessage());
                 assertEquals("enough-services-up", e.getConstraintName());
             }