Test orchestration of config server reprovisioning

4 files changed, 329 insertions, 1 deletions

diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
index 810bc87964c..0dc81904582 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
@@ -81,7 +81,7 @@ public class OrchestratorImplTest {
     private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, clock);
     private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
     private final MockCurator curator = new MockCurator();
-    private ZkStatusService statusService = new ZkStatusService(
+    private final ZkStatusService statusService = new ZkStatusService(
             curator,
             mock(Metric.class),
             new TestTimer(),
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
new file mode 100644
index 00000000000..953c8d1043e
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
@@ -0,0 +1,166 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator;
+
+import com.yahoo.config.model.api.SuperModel;
+import com.yahoo.config.model.api.SuperModelListener;
+import com.yahoo.config.model.api.SuperModelProvider;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceStatus;
+import com.yahoo.vespa.applicationmodel.ServiceStatusInfo;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
+import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory;
+import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
+import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy;
+import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy;
+import com.yahoo.vespa.orchestrator.status.InMemoryStatusService;
+import com.yahoo.vespa.service.duper.ConfigServerApplication;
+import com.yahoo.vespa.service.duper.DuperModel;
+import com.yahoo.vespa.service.duper.DuperModelManager;
+import com.yahoo.vespa.service.manager.UnionMonitorManager;
+import com.yahoo.vespa.service.model.ServiceMonitorImpl;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author hakon
+ */
+public class OrchestratorTest {
+    private final InMemoryStatusService statusService = new InMemoryStatusService();
+    private DuperModelManager duperModelManager;
+    private MySuperModelProvider superModelManager;
+    private UnionMonitorManager monitorManager;
+    private OrchestratorImpl orchestrator;
+
+    @Before
+    public void setUp() {
+        var flagSource = new InMemoryFlagSource();
+        var timer = new TestTimer();
+        var clustercontroller = new ClusterControllerClientFactoryMock();
+        var applicationApiFactory = new ApplicationApiFactory(3, timer.toUtcClock());
+        var policy = new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource), clustercontroller, applicationApiFactory);
+        var zone = new Zone(SystemName.cd, Environment.prod, RegionName.from("cd-us-east-1"));
+        this.superModelManager = new MySuperModelProvider();
+        var duperModel = new DuperModel();
+        this.duperModelManager = new DuperModelManager(true, false, superModelManager, duperModel, flagSource, zone.system());
+        this.monitorManager = mock(UnionMonitorManager.class);
+        var metric = mock(Metric.class);
+        var serviceMonitor = new ServiceMonitorImpl(duperModelManager, monitorManager, metric, timer, zone);
+
+        this.orchestrator = new OrchestratorImpl(policy,
+                clustercontroller,
+                statusService,
+                serviceMonitor,
+                0,
+                timer.toUtcClock(),
+                applicationApiFactory,
+                flagSource);
+    }
+
+    @Test
+    public void simulate_config_server_reprovision() throws OrchestrationException {
+        // All services are healthy at all times
+        when(monitorManager.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(ServiceStatus.UP));
+
+        // There are no ordinary applications
+        superModelManager.markAsComplete();
+
+        // There is one config server application with 3 nodes
+        ApplicationId applicationId = new ConfigServerApplication().getApplicationId();
+        var cfg1 = com.yahoo.config.provision.HostName.from("cfg1");
+        var cfg2 = com.yahoo.config.provision.HostName.from("cfg2");
+        var cfg3 = com.yahoo.config.provision.HostName.from("cfg3");
+        duperModelManager.infraApplicationActivated(applicationId, List.of(cfg1, cfg2, cfg3));
+        duperModelManager.infraApplicationsIsNowComplete();
+
+        // cfg1 completes retirement
+        orchestrator.acquirePermissionToRemove(toApplicationModelHostName(cfg1));
+
+        // No other cfg is allowed to go permanently down BEFORE cfg1 is removed from the app
+        try {
+            orchestrator.acquirePermissionToRemove(toApplicationModelHostName(cfg2));
+            fail();
+        } catch (HostStateChangeDeniedException e) {
+            assertThat(e.getMessage(), containsString("Changing the state of cfg2 would violate enough-services-up"));
+            assertThat(e.getMessage(), containsString("Suspended hosts: [cfg1]"));
+        }
+
+        // cfg1 is removed from the application
+        duperModelManager.infraApplicationActivated(applicationId, List.of(cfg2, cfg3));
+
+        // No other cfg is allowed to go permanently down AFTER cfg1 is removed from the app
+        try {
+            orchestrator.acquirePermissionToRemove(toApplicationModelHostName(cfg2));
+            fail();
+        } catch (HostStateChangeDeniedException e) {
+            assertThat(e.getMessage(), containsString("Changing the state of cfg2 would violate enough-services-up"));
+            assertThat(e.getMessage(), containsString("Services down on resumed hosts: [1 missing config server]"));
+        }
+
+        // cfg1 is reprovisioned, added to the node repo, and activated
+        duperModelManager.infraApplicationActivated(applicationId, List.of(cfg1, cfg2, cfg3));
+
+        // cfg2 is allowed to be removed
+        orchestrator.acquirePermissionToRemove(toApplicationModelHostName(cfg2));
+
+        // No other cfg is allowed to go permanently down BEFORE cfg2 is removed from the app
+        try {
+            orchestrator.acquirePermissionToRemove(toApplicationModelHostName(cfg1));
+            fail();
+        } catch (HostStateChangeDeniedException e) {
+            assertThat(e.getMessage(), containsString("Changing the state of cfg1 would violate enough-services-up"));
+            assertThat(e.getMessage(), containsString("Suspended hosts: [cfg2]"));
+        }
+
+        // etc (should be the same as for cfg1)
+    }
+
+    private HostName toApplicationModelHostName(com.yahoo.config.provision.HostName hostname) {
+        return new HostName(hostname.value());
+    }
+
+    private static class MySuperModelProvider implements SuperModelProvider {
+        private boolean complete = false;
+        private SuperModelListener listener = null;
+
+        @Override
+        public void registerListener(SuperModelListener listener) {
+            if (this.listener != null) {
+                throw new IllegalStateException("This instance already has a listener");
+            }
+
+            this.listener = listener;
+        }
+
+        public void markAsComplete() {
+            complete = true;
+
+            if (listener == null) {
+                throw new IllegalStateException("This instance has no listener");
+            }
+            listener.notifyOfCompleteness(getSuperModel());
+        }
+
+        @Override
+        public SuperModel getSuperModel() {
+            return new SuperModel(Map.of(), complete);
+        }
+    }
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
index 1799b95c65a..6f34817930c 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
@@ -146,4 +146,55 @@ public class HostedVespaPolicyTest {
 
         order.verifyNoMoreInteractions();
     }
+
+    @Test
+    public void testAcquirePermissionToRemoveConfigServer() throws OrchestrationException {
+        final HostedVespaClusterPolicy clusterPolicy = mock(HostedVespaClusterPolicy.class);
+        final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory, applicationApiFactory);
+        final ApplicationApi applicationApi = mock(ApplicationApi.class);
+        when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("tenant:app:default"));
+
+        ClusterApi clusterApi1 = mock(ClusterApi.class);
+        ClusterApi clusterApi2 = mock(ClusterApi.class);
+        ClusterApi clusterApi3 = mock(ClusterApi.class);
+        List<ClusterApi> clusterApis = Arrays.asList(clusterApi1, clusterApi2, clusterApi3);
+        when(applicationApi.getClusters()).thenReturn(clusterApis);
+
+        StorageNode storageNode1 = mock(StorageNode.class);
+        HostName hostName1 = new HostName("storage-1");
+        when(storageNode1.hostName()).thenReturn(hostName1);
+
+        HostName hostName2 = new HostName("host-2");
+
+        StorageNode storageNode3 = mock(StorageNode.class);
+        HostName hostName3 = new HostName("storage-3");
+        when(storageNode1.hostName()).thenReturn(hostName3);
+
+        List<StorageNode> upStorageNodes = Arrays.asList(storageNode1, storageNode3);
+        when(applicationApi.getStorageNodesInGroupInClusterOrder()).thenReturn(upStorageNodes);
+
+        List<HostName> noRemarksHostNames = Arrays.asList(hostName1, hostName2, hostName3);
+        when(applicationApi.getNodesInGroupWith(any())).thenReturn(noRemarksHostNames);
+
+        InOrder order = inOrder(applicationApi, clusterPolicy, storageNode1, storageNode3);
+
+        OrchestratorContext context = mock(OrchestratorContext.class);
+        policy.acquirePermissionToRemove(context, applicationApi);
+
+        order.verify(applicationApi).getClusters();
+        order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi1);
+        order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi2);
+        order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi3);
+
+        order.verify(applicationApi).getStorageNodesInGroupInClusterOrder();
+        order.verify(storageNode1).setNodeState(context, ClusterControllerNodeState.DOWN);
+        order.verify(storageNode3).setNodeState(context, ClusterControllerNodeState.DOWN);
+
+        order.verify(applicationApi).getNodesInGroupWith(any());
+        order.verify(applicationApi).setHostState(context, hostName1, HostStatus.PERMANENTLY_DOWN);
+        order.verify(applicationApi).setHostState(context, hostName2, HostStatus.PERMANENTLY_DOWN);
+        order.verify(applicationApi).setHostState(context, hostName3, HostStatus.PERMANENTLY_DOWN);
+
+        order.verifyNoMoreInteractions();
+    }
 }
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java
new file mode 100644
index 00000000000..834ee6ba7b2
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java
@@ -0,0 +1,111 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.status;
+
+import com.google.common.util.concurrent.UncheckedTimeoutException;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.orchestrator.OrchestratorContext;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Function;
+
+/**
+ * In-memory implementation of StatusService.
+ *
+ * @author hakon
+ */
+public class InMemoryStatusService implements StatusService {
+
+    private final ConcurrentHashMap<ApplicationInstanceReference, ReentrantLock> locks = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<ApplicationInstanceReference, Map<HostName, HostInfo>> hostInfos = new ConcurrentHashMap<>();
+
+    @Override
+    public ApplicationLock lockApplication(OrchestratorContext context, ApplicationInstanceReference reference) throws UncheckedTimeoutException {
+        ReentrantLock lock = locks.computeIfAbsent(reference, (ignored) -> new ReentrantLock());
+
+        try {
+            if (!lock.tryLock(context.getTimeLeft().toMillis(), TimeUnit.MILLISECONDS)) {
+                throw new UncheckedTimeoutException("Timed out trying to acquire the lock on " + reference);
+            }
+        } catch (InterruptedException e) {
+            throw new UncheckedTimeoutException("Interrupted", e);
+        }
+
+        return new ApplicationLock() {
+            @Override
+            public ApplicationInstanceReference getApplicationInstanceReference() {
+                return reference;
+            }
+
+            @Override
+            public HostInfos getHostInfos() {
+                return getHostInfosByApplicationResolver().apply(reference);
+            }
+
+            @Override
+            public void setHostState(HostName hostName, HostStatus status) {
+                if (status == HostStatus.NO_REMARKS) {
+                    applicationHostInfo(reference).remove(hostName);
+                } else {
+                    applicationHostInfo(reference).put(hostName, HostInfo.createSuspended(status, Instant.EPOCH));
+                }
+            }
+
+            @Override
+            public ApplicationInstanceStatus getApplicationInstanceStatus() {
+                return ApplicationInstanceStatus.NO_REMARKS;
+            }
+
+            @Override
+            public void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus) {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public void close() {
+                lock.unlock();
+            }
+        };
+    }
+
+    private Map<HostName, HostInfo> applicationHostInfo(ApplicationInstanceReference reference) {
+        return hostInfos.getOrDefault(reference, Map.of());
+    }
+
+    @Override
+    public Set<ApplicationInstanceReference> getAllSuspendedApplications() {
+        return Set.of();
+    }
+
+    @Override
+    public Function<ApplicationInstanceReference, HostInfos> getHostInfosByApplicationResolver() {
+        return reference -> new HostInfos(Map.copyOf(applicationHostInfo(reference)));
+    }
+
+    @Override
+    public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference application) {
+        return ApplicationInstanceStatus.NO_REMARKS;
+    }
+
+    @Override
+    public HostInfo getHostInfo(ApplicationInstanceReference reference, HostName hostName) {
+        return getHostInfosByApplicationResolver().apply(reference).getOrNoRemarks(hostName);
+    }
+
+    @Override
+    public void onApplicationActivate(ApplicationInstanceReference reference, Set<HostName> hostnames) {
+        Map<HostName, HostInfo> currentHostInfos = hostInfos.computeIfAbsent(reference, (ignored) -> new HashMap<>());
+        hostnames.forEach(currentHostInfos::remove);
+    }
+
+    @Override
+    public void onApplicationRemove(ApplicationInstanceReference reference) {
+        hostInfos.remove(reference);
+    }
+}