diff options
author | Håkon Hallingstad <hakon@verizonmedia.com> | 2021-03-12 16:40:59 +0100 |
---|---|---|
committer | Håkon Hallingstad <hakon@verizonmedia.com> | 2021-03-12 16:40:59 +0100 |
commit | 15f24bf17bba11225197c70482b7fcaea77977e2 (patch) | |
tree | 042f6f8696c0de110b9758aa11f4eca1e76443f6 /orchestrator | |
parent | 2508edfa1802b3962d7f3a4a904a88c5b09380f3 (diff) |
Test orchestration of config server reprovisioning
Diffstat (limited to 'orchestrator')
4 files changed, 329 insertions, 1 deletions
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index 810bc87964c..0dc81904582 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -81,7 +81,7 @@ public class OrchestratorImplTest { private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, clock); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final MockCurator curator = new MockCurator(); - private ZkStatusService statusService = new ZkStatusService( + private final ZkStatusService statusService = new ZkStatusService( curator, mock(Metric.class), new TestTimer(), diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java new file mode 100644 index 00000000000..953c8d1043e --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java @@ -0,0 +1,166 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator; + +import com.yahoo.config.model.api.SuperModel; +import com.yahoo.config.model.api.SuperModelListener; +import com.yahoo.config.model.api.SuperModelProvider; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; +import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory; +import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; +import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; +import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; +import com.yahoo.vespa.orchestrator.status.InMemoryStatusService; +import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.service.duper.DuperModel; +import com.yahoo.vespa.service.duper.DuperModelManager; +import com.yahoo.vespa.service.manager.UnionMonitorManager; +import com.yahoo.vespa.service.model.ServiceMonitorImpl; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author hakon + */ +public class OrchestratorTest { + private final InMemoryStatusService statusService = new InMemoryStatusService(); + private DuperModelManager duperModelManager; + private MySuperModelProvider superModelManager; + private UnionMonitorManager monitorManager; + private OrchestratorImpl orchestrator; + + @Before + public void setUp() { + var flagSource = new InMemoryFlagSource(); + var timer = new TestTimer(); + var clustercontroller = new ClusterControllerClientFactoryMock(); + var applicationApiFactory = new ApplicationApiFactory(3, timer.toUtcClock()); + var policy = new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource), clustercontroller, applicationApiFactory); + var zone = new Zone(SystemName.cd, Environment.prod, RegionName.from("cd-us-east-1")); + this.superModelManager = new MySuperModelProvider(); + var duperModel = new DuperModel(); + this.duperModelManager = new DuperModelManager(true, false, superModelManager, duperModel, flagSource, zone.system()); + this.monitorManager = mock(UnionMonitorManager.class); + var metric = mock(Metric.class); + var serviceMonitor = new ServiceMonitorImpl(duperModelManager, monitorManager, metric, timer, zone); + + this.orchestrator = new OrchestratorImpl(policy, + clustercontroller, + statusService, + serviceMonitor, + 0, + timer.toUtcClock(), + applicationApiFactory, + flagSource); + } + + @Test + public void simulate_config_server_reprovision() throws OrchestrationException { + // All services are healthy at all times + when(monitorManager.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(ServiceStatus.UP)); + + // There are no ordinary applications + superModelManager.markAsComplete(); + + // There is one config server application with 3 nodes + ApplicationId applicationId = new ConfigServerApplication().getApplicationId(); + var cfg1 = com.yahoo.config.provision.HostName.from("cfg1"); + var cfg2 = com.yahoo.config.provision.HostName.from("cfg2"); + var cfg3 = com.yahoo.config.provision.HostName.from("cfg3"); + duperModelManager.infraApplicationActivated(applicationId, List.of(cfg1, cfg2, cfg3)); + duperModelManager.infraApplicationsIsNowComplete(); + + // cfg1 completes retirement + orchestrator.acquirePermissionToRemove(toApplicationModelHostName(cfg1)); + + // No other cfg is allowed to go permanently down BEFORE cfg1 is removed from the app + try { + orchestrator.acquirePermissionToRemove(toApplicationModelHostName(cfg2)); + fail(); + } catch (HostStateChangeDeniedException e) { + assertThat(e.getMessage(), containsString("Changing the state of cfg2 would violate enough-services-up")); + assertThat(e.getMessage(), containsString("Suspended hosts: [cfg1]")); + } + + // cfg1 is removed from the application + duperModelManager.infraApplicationActivated(applicationId, List.of(cfg2, cfg3)); + + // No other cfg is allowed to go permanently down AFTER cfg1 is removed from the app + try { + orchestrator.acquirePermissionToRemove(toApplicationModelHostName(cfg2)); + fail(); + } catch (HostStateChangeDeniedException e) { + assertThat(e.getMessage(), containsString("Changing the state of cfg2 would violate enough-services-up")); + assertThat(e.getMessage(), containsString("Services down on resumed hosts: [1 missing config server]")); + } + + // cfg1 is reprovisioned, added to the node repo, and activated + duperModelManager.infraApplicationActivated(applicationId, List.of(cfg1, cfg2, cfg3)); + + // cfg2 is allowed to be removed + orchestrator.acquirePermissionToRemove(toApplicationModelHostName(cfg2)); + + // No other cfg is allowed to go permanently down BEFORE cfg2 is removed from the app + try { + orchestrator.acquirePermissionToRemove(toApplicationModelHostName(cfg1)); + fail(); + } catch (HostStateChangeDeniedException e) { + assertThat(e.getMessage(), containsString("Changing the state of cfg1 would violate enough-services-up")); + assertThat(e.getMessage(), containsString("Suspended hosts: [cfg2]")); + } + + // etc (should be the same as for cfg1) + } + + private HostName toApplicationModelHostName(com.yahoo.config.provision.HostName hostname) { + return new HostName(hostname.value()); + } + + private static class MySuperModelProvider implements SuperModelProvider { + private boolean complete = false; + private SuperModelListener listener = null; + + @Override + public void registerListener(SuperModelListener listener) { + if (this.listener != null) { + throw new IllegalStateException("This instance already has a listener"); + } + + this.listener = listener; + } + + public void markAsComplete() { + complete = true; + + if (listener == null) { + throw new IllegalStateException("This instance has no listener"); + } + listener.notifyOfCompleteness(getSuperModel()); + } + + @Override + public SuperModel getSuperModel() { + return new SuperModel(Map.of(), complete); + } + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java index 1799b95c65a..6f34817930c 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java @@ -146,4 +146,55 @@ public class HostedVespaPolicyTest { order.verifyNoMoreInteractions(); } + + @Test + public void testAcquirePermissionToRemoveConfigServer() throws OrchestrationException { + final HostedVespaClusterPolicy clusterPolicy = mock(HostedVespaClusterPolicy.class); + final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory, applicationApiFactory); + final ApplicationApi applicationApi = mock(ApplicationApi.class); + when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("tenant:app:default")); + + ClusterApi clusterApi1 = mock(ClusterApi.class); + ClusterApi clusterApi2 = mock(ClusterApi.class); + ClusterApi clusterApi3 = mock(ClusterApi.class); + List<ClusterApi> clusterApis = Arrays.asList(clusterApi1, clusterApi2, clusterApi3); + when(applicationApi.getClusters()).thenReturn(clusterApis); + + StorageNode storageNode1 = mock(StorageNode.class); + HostName hostName1 = new HostName("storage-1"); + when(storageNode1.hostName()).thenReturn(hostName1); + + HostName hostName2 = new HostName("host-2"); + + StorageNode storageNode3 = mock(StorageNode.class); + HostName hostName3 = new HostName("storage-3"); + when(storageNode1.hostName()).thenReturn(hostName3); + + List<StorageNode> upStorageNodes = Arrays.asList(storageNode1, storageNode3); + when(applicationApi.getStorageNodesInGroupInClusterOrder()).thenReturn(upStorageNodes); + + List<HostName> noRemarksHostNames = Arrays.asList(hostName1, hostName2, hostName3); + when(applicationApi.getNodesInGroupWith(any())).thenReturn(noRemarksHostNames); + + InOrder order = inOrder(applicationApi, clusterPolicy, storageNode1, storageNode3); + + OrchestratorContext context = mock(OrchestratorContext.class); + policy.acquirePermissionToRemove(context, applicationApi); + + order.verify(applicationApi).getClusters(); + order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi1); + order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi2); + order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi3); + + order.verify(applicationApi).getStorageNodesInGroupInClusterOrder(); + order.verify(storageNode1).setNodeState(context, ClusterControllerNodeState.DOWN); + order.verify(storageNode3).setNodeState(context, ClusterControllerNodeState.DOWN); + + order.verify(applicationApi).getNodesInGroupWith(any()); + order.verify(applicationApi).setHostState(context, hostName1, HostStatus.PERMANENTLY_DOWN); + order.verify(applicationApi).setHostState(context, hostName2, HostStatus.PERMANENTLY_DOWN); + order.verify(applicationApi).setHostState(context, hostName3, HostStatus.PERMANENTLY_DOWN); + + order.verifyNoMoreInteractions(); + } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java new file mode 100644 index 00000000000..834ee6ba7b2 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java @@ -0,0 +1,111 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status; + +import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.orchestrator.OrchestratorContext; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; + +/** + * In-memory implementation of StatusService. + * + * @author hakon + */ +public class InMemoryStatusService implements StatusService { + + private final ConcurrentHashMap<ApplicationInstanceReference, ReentrantLock> locks = new ConcurrentHashMap<>(); + private final ConcurrentHashMap<ApplicationInstanceReference, Map<HostName, HostInfo>> hostInfos = new ConcurrentHashMap<>(); + + @Override + public ApplicationLock lockApplication(OrchestratorContext context, ApplicationInstanceReference reference) throws UncheckedTimeoutException { + ReentrantLock lock = locks.computeIfAbsent(reference, (ignored) -> new ReentrantLock()); + + try { + if (!lock.tryLock(context.getTimeLeft().toMillis(), TimeUnit.MILLISECONDS)) { + throw new UncheckedTimeoutException("Timed out trying to acquire the lock on " + reference); + } + } catch (InterruptedException e) { + throw new UncheckedTimeoutException("Interrupted", e); + } + + return new ApplicationLock() { + @Override + public ApplicationInstanceReference getApplicationInstanceReference() { + return reference; + } + + @Override + public HostInfos getHostInfos() { + return getHostInfosByApplicationResolver().apply(reference); + } + + @Override + public void setHostState(HostName hostName, HostStatus status) { + if (status == HostStatus.NO_REMARKS) { + applicationHostInfo(reference).remove(hostName); + } else { + applicationHostInfo(reference).put(hostName, HostInfo.createSuspended(status, Instant.EPOCH)); + } + } + + @Override + public ApplicationInstanceStatus getApplicationInstanceStatus() { + return ApplicationInstanceStatus.NO_REMARKS; + } + + @Override + public void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus) { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + lock.unlock(); + } + }; + } + + private Map<HostName, HostInfo> applicationHostInfo(ApplicationInstanceReference reference) { + return hostInfos.getOrDefault(reference, Map.of()); + } + + @Override + public Set<ApplicationInstanceReference> getAllSuspendedApplications() { + return Set.of(); + } + + @Override + public Function<ApplicationInstanceReference, HostInfos> getHostInfosByApplicationResolver() { + return reference -> new HostInfos(Map.copyOf(applicationHostInfo(reference))); + } + + @Override + public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference application) { + return ApplicationInstanceStatus.NO_REMARKS; + } + + @Override + public HostInfo getHostInfo(ApplicationInstanceReference reference, HostName hostName) { + return getHostInfosByApplicationResolver().apply(reference).getOrNoRemarks(hostName); + } + + @Override + public void onApplicationActivate(ApplicationInstanceReference reference, Set<HostName> hostnames) { + Map<HostName, HostInfo> currentHostInfos = hostInfos.computeIfAbsent(reference, (ignored) -> new HashMap<>()); + hostnames.forEach(currentHostInfos::remove); + } + + @Override + public void onApplicationRemove(ApplicationInstanceReference reference) { + hostInfos.remove(reference); + } +} |