Allow fifty percent of host-admin cluster nodes down in cd-like systems

9 files changed, 60 insertions, 21 deletions

diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
index a901ecf4e76..6d50002669d 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
@@ -5,6 +5,7 @@ import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.google.inject.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.ClusterId;
@@ -74,9 +75,10 @@ public class OrchestratorImpl implements Orchestrator {
                             OrchestratorConfig orchestratorConfig,
                             ServiceMonitor serviceMonitor,
                             ConfigserverConfig configServerConfig,
-                            FlagSource flagSource)
+                            FlagSource flagSource,
+                            Zone zone)
     {
-        this(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource),
+        this(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource, zone),
                                    clusterControllerClientFactory,
                                    new ApplicationApiFactory(configServerConfig.zookeeperserver().size(), Clock.systemUTC())),
                 clusterControllerClientFactory,
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java
index 2e7a63ddb2f..78373282df8 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.orchestrator.model;
 
+import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.orchestrator.policy.SuspensionReasons;
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java
index bd0075b403c..e3989fd86d4 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.orchestrator.policy;
 
+import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.flags.BooleanFlag;
@@ -18,10 +19,12 @@ import static com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy.ENOUGH_SERVI
 public class HostedVespaClusterPolicy implements ClusterPolicy {
 
     private final BooleanFlag groupSuspensionFlag;
+    private final Zone zone;
 
-    public HostedVespaClusterPolicy(FlagSource flagSource) {
+    public HostedVespaClusterPolicy(FlagSource flagSource, Zone zone) {
         // Note that the "group" in this flag refers to hierarchical groups of a content cluster.
         this.groupSuspensionFlag = Flags.GROUP_SUSPENSION.bindTo(flagSource);
+        this.zone = zone;
     }
 
     @Override
@@ -155,7 +158,9 @@ public class HostedVespaClusterPolicy implements ClusterPolicy {
                     return ConcurrentSuspensionLimitForCluster.ONE_NODE;
                 }
 
-                return ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT;
+                return zone.system().isCd()
+                        ? ConcurrentSuspensionLimitForCluster.FIFTY_PERCENT
+                        : ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT;
             }
 
             // The above should cover all cases, but if not we'll return a reasonable default:
@@ -183,7 +188,9 @@ public class HostedVespaClusterPolicy implements ClusterPolicy {
             }
 
             if (clusterApi.getApplication().applicationId().equals(VespaModelUtil.TENANT_HOST_APPLICATION_ID)) {
-                return ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT;
+                return zone.system().isCd()
+                        ? ConcurrentSuspensionLimitForCluster.FIFTY_PERCENT
+                        : ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT;
             }
 
             return ConcurrentSuspensionLimitForCluster.TEN_PERCENT;
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
index 0dc81904582..7d1b69d7171 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
@@ -1,7 +1,8 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.orchestrator;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Zone;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.test.TestTimer;
 import com.yahoo.test.ManualClock;
@@ -77,6 +78,8 @@ import static org.mockito.internal.verification.VerificationModeFactory.atLeastO
  */
 public class OrchestratorImplTest {
 
+    private static final Zone zone = Zone.defaultZone();
+
     private final ManualClock clock = new ManualClock();
     private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, clock);
     private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
@@ -104,7 +107,9 @@ public class OrchestratorImplTest {
         app2 = OrchestratorUtil.toApplicationId(iterator.next().reference());
 
         clustercontroller = new ClusterControllerClientFactoryMock();
-        orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource), clustercontroller, applicationApiFactory),
+        orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource, zone),
+                                                                  clustercontroller,
+                                                                  applicationApiFactory),
                                             clustercontroller,
                                             statusService,
                                             new DummyServiceMonitor(),
@@ -448,7 +453,9 @@ public class OrchestratorImplTest {
         when(clusterControllerClientFactory.createClient(List.of(ccHost), "foo")).thenReturn(fooClient);
         when(clusterControllerClientFactory.createClient(List.of(ccHost), "bar")).thenReturn(barClient);
 
-        orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource), clusterControllerClientFactory, applicationApiFactory),
+        orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource, zone),
+                                                                  clusterControllerClientFactory,
+                                                                  applicationApiFactory),
                                             clusterControllerClientFactory,
                                             statusService,
                                             serviceMonitor,
@@ -507,7 +514,9 @@ public class OrchestratorImplTest {
 
         ServiceMonitor serviceMonitor = () -> new ServiceModel(Map.of(reference, applicationInstance));
 
-        orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource), clusterControllerClientFactory, applicationApiFactory),
+        orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource, zone),
+                                                                  clusterControllerClientFactory,
+                                                                  applicationApiFactory),
                                             clusterControllerClientFactory,
                                             statusService,
                                             serviceMonitor,
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
index 953c8d1043e..193798d35fc 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
@@ -43,6 +43,9 @@ import static org.mockito.Mockito.when;
  * @author hakon
  */
 public class OrchestratorTest {
+
+    private static final Zone zone = Zone.defaultZone();
+
     private final InMemoryStatusService statusService = new InMemoryStatusService();
     private DuperModelManager duperModelManager;
     private MySuperModelProvider superModelManager;
@@ -55,7 +58,7 @@ public class OrchestratorTest {
         var timer = new TestTimer();
         var clustercontroller = new ClusterControllerClientFactoryMock();
         var applicationApiFactory = new ApplicationApiFactory(3, timer.toUtcClock());
-        var policy = new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource), clustercontroller, applicationApiFactory);
+        var policy = new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource, zone), clustercontroller, applicationApiFactory);
         var zone = new Zone(SystemName.cd, Environment.prod, RegionName.from("cd-us-east-1"));
         this.superModelManager = new MySuperModelProvider();
         var duperModel = new DuperModel();
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
index 6d7c79176fb..d40ef0fb283 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.orchestrator.model;
 
 import com.yahoo.config.provision.Zone;
@@ -49,6 +49,8 @@ import static org.mockito.Mockito.when;
  */
 public class ClusterApiImplTest {
 
+    private static final Zone zone = Zone.defaultZone();
+
     private final ApplicationApi applicationApi = mock(ApplicationApi.class);
     private final ModelTestUtils modelUtils = new ModelTestUtils();
     private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(1600436659));
@@ -107,7 +109,7 @@ public class ClusterApiImplTest {
     public void testCfg1SuspensionFailsWithMissingCfg3() {
         ClusterApiImpl clusterApi = makeCfg1ClusterApi(ServiceStatus.UP, ServiceStatus.UP);
 
-        HostedVespaClusterPolicy policy = new HostedVespaClusterPolicy(flagSource);
+        HostedVespaClusterPolicy policy = new HostedVespaClusterPolicy(flagSource, zone);
 
         try {
             policy.verifyGroupGoingDownIsFine(clusterApi);
@@ -139,7 +141,7 @@ public class ClusterApiImplTest {
                 ServiceStatus.UP,
                 ServiceStatus.UP);
 
-        HostedVespaClusterPolicy policy = new HostedVespaClusterPolicy(flagSource);
+        HostedVespaClusterPolicy policy = new HostedVespaClusterPolicy(flagSource, zone);
 
         try {
             policy.verifyGroupGoingDownIsFine(clusterApi);
@@ -167,7 +169,7 @@ public class ClusterApiImplTest {
     public void testCfg1SuspendsIfDownWithMissingCfg3() throws HostStateChangeDeniedException {
         ClusterApiImpl clusterApi = makeCfg1ClusterApi(ServiceStatus.DOWN, ServiceStatus.UP);
 
-        HostedVespaClusterPolicy policy = new HostedVespaClusterPolicy(flagSource);
+        HostedVespaClusterPolicy policy = new HostedVespaClusterPolicy(flagSource, zone);
 
         policy.verifyGroupGoingDownIsFine(clusterApi);
     }
@@ -180,7 +182,7 @@ public class ClusterApiImplTest {
                 ServiceStatus.DOWN,
                 ServiceStatus.UP);
 
-        HostedVespaClusterPolicy policy = new HostedVespaClusterPolicy(flagSource);
+        HostedVespaClusterPolicy policy = new HostedVespaClusterPolicy(flagSource, zone);
 
         policy.verifyGroupGoingDownIsFine(clusterApi);
     }
@@ -189,7 +191,7 @@ public class ClusterApiImplTest {
     public void testSingleConfigServerCanSuspend() {
         for (var status : EnumSet.of(ServiceStatus.UP, ServiceStatus.DOWN)) {
             var clusterApi = makeConfigClusterApi(1, status);
-            var policy = new HostedVespaClusterPolicy(flagSource);
+            var policy = new HostedVespaClusterPolicy(flagSource, zone);
             try {
                 policy.verifyGroupGoingDownIsFine(clusterApi);
             } catch (HostStateChangeDeniedException e) {
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
index 0f71def5d2e..7cb20350845 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.orchestrator.model;
 
+import com.yahoo.config.provision.Zone;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.test.TestTimer;
 import com.yahoo.test.ManualClock;
@@ -65,7 +66,9 @@ class ModelTestUtils {
             mock(Metric.class),
             new TestTimer(),
             new DummyAntiServiceMonitor());
-    private final Orchestrator orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource), clusterControllerClientFactory, applicationApiFactory()),
+    private final Orchestrator orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource, Zone.defaultZone()),
+                                                                                         clusterControllerClientFactory,
+                                                                                         applicationApiFactory()),
                                                                    clusterControllerClientFactory,
                                                                    statusService,
                                                                    serviceMonitor,
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java
index 97c1c7ab5de..a100b3efc52 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java
@@ -2,6 +2,8 @@
 package com.yahoo.vespa.orchestrator.policy;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceType;
@@ -23,14 +25,17 @@ import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 
 public class HostedVespaClusterPolicyTest {
-    private ApplicationApi applicationApi = mock(ApplicationApi.class);
-    private ClusterApi clusterApi = mock(ClusterApi.class);
+
+    private final ApplicationApi applicationApi = mock(ApplicationApi.class);
+    private final ClusterApi clusterApi = mock(ClusterApi.class);
+    private final Zone zone = mock(Zone.class);
     private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
-    private HostedVespaClusterPolicy policy = spy(new HostedVespaClusterPolicy(flagSource));
+    private final HostedVespaClusterPolicy policy = spy(new HostedVespaClusterPolicy(flagSource, zone));
 
     @Before
     public void setUp() {
         when(clusterApi.getApplication()).thenReturn(applicationApi);
+        when(zone.system()).thenReturn(SystemName.main);
     }
 
     @Test
@@ -72,6 +77,11 @@ public class HostedVespaClusterPolicyTest {
         when(clusterApi.isStorageCluster()).thenReturn(false);
         assertEquals(ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT,
                 policy.getConcurrentSuspensionLimit(clusterApi, false));
+
+
+        when(zone.system()).thenReturn(SystemName.cd);
+        assertEquals(ConcurrentSuspensionLimitForCluster.FIFTY_PERCENT,
+                     policy.getConcurrentSuspensionLimit(clusterApi, false));
     }
 
     @Test
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionRequestHandlerTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionRequestHandlerTest.java
index 864516acbc5..76d00943e07 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionRequestHandlerTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionRequestHandlerTest.java
@@ -2,6 +2,7 @@ package com.yahoo.vespa.orchestrator.resources;// Copyright Verizon Media. Licen
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.HttpRequestBuilder;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.jdisc.core.SystemTimer;
@@ -48,7 +49,8 @@ class ApplicationSuspensionRequestHandlerTest {
                 new OrchestratorConfig(new OrchestratorConfig.Builder()),
                 serviceMonitor,
                 new ConfigserverConfig(new ConfigserverConfig.Builder()),
-                new InMemoryFlagSource());
+                new InMemoryFlagSource(),
+                Zone.defaultZone());
         var handler = new ApplicationSuspensionRequestHandler(RestApiTestDriver.createHandlerTestContext(), orchestrator);
         testDriver = RestApiTestDriver.newBuilder(handler).build();
     }