From b4d42768c75fd78b26a0999cb5eeb376d0cf8da8 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Fri, 26 May 2017 17:36:03 +0200 Subject: Complete retirement early in dev CD --- .../vespa/orchestrator/OrchestratorImplTest.java | 10 +- .../ClusterControllerClientFactoryMock.java | 10 +- .../controller/ClusterControllerClientTest.java | 2 +- ...InstanceClusterControllerClientFactoryTest.java | 4 +- .../policy/HostedVespaClusterPolicyTest.java | 113 ++++ .../orchestrator/policy/HostedVespaPolicyTest.java | 744 ++------------------- .../orchestrator/resources/HostResourceTest.java | 11 +- 7 files changed, 174 insertions(+), 720 deletions(-) create mode 100644 orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java (limited to 'orchestrator/src/test/java') diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index 09aa37f9610..ccee70630af 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; @@ -228,11 +227,10 @@ public class OrchestratorImplTest { Throwable supensionFailure = new HostStateChangeDeniedException( DummyInstanceLookupService.TEST6_HOST_NAME, "some-constraint", - new ServiceType("foo"), "error message"); doThrow(supensionFailure).when(orchestrator).suspendGroup(DummyInstanceLookupService.TEST6_NODE_GROUP); - doThrow(new HostStateChangeDeniedException(DummyInstanceLookupService.TEST1_HOST_NAME, "foo1-constraint", new ServiceType("foo1-service"), "foo1-message")) + doThrow(new HostStateChangeDeniedException(DummyInstanceLookupService.TEST1_HOST_NAME, "foo1-constraint", "foo1-message")) .when(orchestrator).resume(DummyInstanceLookupService.TEST1_HOST_NAME); doNothing().when(orchestrator).resume(DummyInstanceLookupService.TEST6_HOST_NAME); doNothing().when(orchestrator).resume(DummyInstanceLookupService.TEST3_HOST_NAME); @@ -251,10 +249,10 @@ public class OrchestratorImplTest { assertEquals("Failed to suspend NodeGroup{application=tenant-id-3:application-instance-3:prod:utopia-1:default, " + "hostNames=[test6.prod.us-east-1.vespahosted.ne1.yahoo.com]} with parent host parentHostname: " + "Changing the state of test6.prod.us-east-1.vespahosted.ne1.yahoo.com would violate " + - "some-constraint for service type foo: error message; With suppressed throwable " + + "some-constraint: error message; With suppressed throwable " + "com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException: " + - "Changing the state of test1.prod.utpoia-1.vespahosted.ut1.yahoo.com would violate " + - "foo1-constraint for service type foo1-service: foo1-message", + "Changing the state of test1.prod.utpoia-1.vespahosted.ut1.yahoo.com " + + "would violate foo1-constraint: foo1-message", e.getMessage()); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java index 93dcb5815c6..a36a468a345 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java @@ -22,14 +22,14 @@ import java.util.Set; * @author smorgrav */ public class ClusterControllerClientFactoryMock implements ClusterControllerClientFactory { - Map nodes = new HashMap<>(); + Map nodes = new HashMap<>(); public boolean isInMaintenance(ApplicationInstance appInstance, HostName hostName) { try { ClusterId clusterName = VespaModelUtil.getContentClusterName(appInstance, hostName); int storageNodeIndex = VespaModelUtil.getStorageNodeIndex(appInstance, hostName); String globalMapKey = clusterName.s() + storageNodeIndex; - return nodes.getOrDefault(globalMapKey, ClusterControllerState.UP) == ClusterControllerState.MAINTENANCE; + return nodes.getOrDefault(globalMapKey, ClusterControllerNodeState.UP) == ClusterControllerNodeState.MAINTENANCE; } catch (Exception e) { //Catch all - meant to catch cases where the node is not part of a storage cluster return false; @@ -43,7 +43,7 @@ public class ClusterControllerClientFactoryMock implements ClusterControllerClie ClusterId clusterName = VespaModelUtil.getContentClusterName(app, host); int storageNodeIndex = VespaModelUtil.getStorageNodeIndex(app, host); String globalMapKey = clusterName.s() + storageNodeIndex; - nodes.put(globalMapKey, ClusterControllerState.UP); + nodes.put(globalMapKey, ClusterControllerNodeState.UP); } } } @@ -53,13 +53,13 @@ public class ClusterControllerClientFactoryMock implements ClusterControllerClie return new ClusterControllerClient() { @Override - public ClusterControllerStateResponse setNodeState(int storageNodeIndex, ClusterControllerState wantedState) throws IOException { + public ClusterControllerStateResponse setNodeState(int storageNodeIndex, ClusterControllerNodeState wantedState) throws IOException { nodes.put(clusterName + storageNodeIndex, wantedState); return new ClusterControllerStateResponse(true, "Yes"); } @Override - public ClusterControllerStateResponse setApplicationState(ClusterControllerState wantedState) throws IOException { + public ClusterControllerStateResponse setApplicationState(ClusterControllerNodeState wantedState) throws IOException { Set keyCopy = new HashSet<>(nodes.keySet()); for (String s : keyCopy) { if (s.startsWith(clusterName)) { diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java index 68df4dce393..2841677ec04 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java @@ -22,7 +22,7 @@ public class ClusterControllerClientTest { strategyMock, CLUSTER_NAME); - final ClusterControllerState wantedState = ClusterControllerState.MAINTENANCE; + final ClusterControllerNodeState wantedState = ClusterControllerNodeState.MAINTENANCE; clusterControllerClient.setNodeState(STORAGE_NODE_INDEX, wantedState); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java index cde644e7e8b..1081858812b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java @@ -64,7 +64,7 @@ public class SingleInstanceClusterControllerClientFactoryTest { final List clusterControllers = Arrays.asList(HOST_NAME_1); clientFactory.createClient(clusterControllers, "clusterName") - .setNodeState(0, ClusterControllerState.MAINTENANCE); + .setNodeState(0, ClusterControllerNodeState.MAINTENANCE); verify(jaxRsClientFactory).createClient( ClusterControllerJaxRsApi.class, @@ -90,7 +90,7 @@ public class SingleInstanceClusterControllerClientFactoryTest { final List clusterControllers = Arrays.asList(HOST_NAME_1, HOST_NAME_2, HOST_NAME_3); clientFactory.createClient(clusterControllers, "clusterName") - .setNodeState(0, ClusterControllerState.MAINTENANCE); + .setNodeState(0, ClusterControllerNodeState.MAINTENANCE); verify(jaxRsClientFactory).createClient( eq(ClusterControllerJaxRsApi.class), diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java new file mode 100644 index 00000000000..e6e60900906 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java @@ -0,0 +1,113 @@ +package com.yahoo.vespa.orchestrator.policy; + +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.orchestrator.model.ClusterApi; +import com.yahoo.vespa.orchestrator.model.NodeGroup; +import com.yahoo.vespa.orchestrator.model.VespaModelUtil; +import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class HostedVespaClusterPolicyTest { + private ClusterApi clusterApi = mock(ClusterApi.class); + private HostedVespaClusterPolicy policy = spy(new HostedVespaClusterPolicy()); + + @Test + public void testSlobrokSuspensionLimit() { + when(clusterApi.clusterId()).thenReturn(VespaModelUtil.ADMIN_CLUSTER_ID); + when(clusterApi.serviceType()).thenReturn(VespaModelUtil.SLOBROK_SERVICE_TYPE); + assertEquals(ConcurrentSuspensionLimitForCluster.ONE_NODE, + policy.getConcurrentSuspensionLimit(clusterApi)); + } + + @Test + public void testAdminSuspensionLimit() { + when(clusterApi.clusterId()).thenReturn(VespaModelUtil.ADMIN_CLUSTER_ID); + when(clusterApi.serviceType()).thenReturn(new ServiceType("non-slobrok-service-type")); + assertEquals(ConcurrentSuspensionLimitForCluster.ALL_NODES, + policy.getConcurrentSuspensionLimit(clusterApi)); + } + + @Test + public void testStorageSuspensionLimit() { + when(clusterApi.clusterId()).thenReturn(new ClusterId("some-cluster-id")); + when(clusterApi.isStorageCluster()).thenReturn(true); + assertEquals(ConcurrentSuspensionLimitForCluster.ONE_NODE, + policy.getConcurrentSuspensionLimit(clusterApi)); + } + + @Test + public void testDefaultSuspensionLimit() { + when(clusterApi.clusterId()).thenReturn(new ClusterId("some-cluster-id")); + when(clusterApi.isStorageCluster()).thenReturn(false); + assertEquals(ConcurrentSuspensionLimitForCluster.TEN_PERCENT, + policy.getConcurrentSuspensionLimit(clusterApi)); + } + + @Test + public void verifyGroupGoingDownIsFine_noServicesOutsideGroupIsDownIsFine() { + verifyGroupGoingDownIsFine(true, false, 13, true); + } + + @Test + public void verifyGroupGoingDownIsFine_noServicesInGroupIsUp() { + verifyGroupGoingDownIsFine(false, true, 13, true); + } + + @Test + public void verifyGroupGoingDownIsFine_percentageIsFine() { + verifyGroupGoingDownIsFine(false, false, 9, true); + } + + @Test + public void verifyGroupGoingDownIsFine_fails() { + verifyGroupGoingDownIsFine(false, false, 13, false); + } + + private void verifyGroupGoingDownIsFine(boolean noServicesOutsideGroupIsDown, + boolean noServicesInGroupIsUp, + int percentageOfServicesDownIfGroupIsAllowedToBeDown, + boolean expectSuccess) { + when(clusterApi.noServicesOutsideGroupIsDown()).thenReturn(noServicesOutsideGroupIsDown); + when(clusterApi.noServicesInGroupIsUp()).thenReturn(noServicesInGroupIsUp); + when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(20); + doReturn(ConcurrentSuspensionLimitForCluster.TEN_PERCENT).when(policy).getConcurrentSuspensionLimit(clusterApi); + + when(clusterApi.serviceType()).thenReturn(new ServiceType("service-type")); + when(clusterApi.percentageOfServicesDown()).thenReturn(5); + when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(percentageOfServicesDownIfGroupIsAllowedToBeDown); + when(clusterApi.servicesDownAndNotInGroupDescription()).thenReturn("services-down-and-not-in-group"); + when(clusterApi.nodesAllowedToBeDownNotInGroupDescription()).thenReturn("allowed-to-be-down"); + + NodeGroup nodeGroup = mock(NodeGroup.class); + when(clusterApi.getNodeGroup()).thenReturn(nodeGroup); + when(nodeGroup.toCommaSeparatedString()).thenReturn("node-group"); + + when(clusterApi.noServicesInGroupIsUp()).thenReturn(false); + try { + policy.verifyGroupGoingDownIsFine(clusterApi); + if (!expectSuccess) { + fail(); + } + } catch (HostStateChangeDeniedException e) { + if (!expectSuccess) { + assertEquals("Changing the state of node-group would violate enough-services-up: " + + "Suspension percentage for service type service-type would increase from " + + "5% to 13%, over the limit of 10%. These instances may be down: " + + "services-down-and-not-in-group and these hosts are allowed to be down: " + + "allowed-to-be-down", e.getMessage()); + assertEquals("enough-services-up", e.getConstraintName()); + } + } + } +} \ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java index 5cba6375717..fe2d9a60acb 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java @@ -2,53 +2,25 @@ package com.yahoo.vespa.orchestrator.policy; -import com.yahoo.vespa.applicationmodel.ApplicationInstance; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; -import com.yahoo.vespa.applicationmodel.ClusterId; -import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceCluster; -import com.yahoo.vespa.applicationmodel.ServiceInstance; -import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.applicationmodel.TenantId; -import com.yahoo.vespa.orchestrator.TestUtil; +import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; -import com.yahoo.vespa.orchestrator.controller.ClusterControllerState; -import com.yahoo.vespa.orchestrator.controller.ClusterControllerStateResponse; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState; import com.yahoo.vespa.orchestrator.model.ApplicationApi; import com.yahoo.vespa.orchestrator.model.ClusterApi; import com.yahoo.vespa.orchestrator.model.StorageNode; -import com.yahoo.vespa.orchestrator.model.VespaModelUtil; import com.yahoo.vespa.orchestrator.status.HostStatus; -import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; -import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; import org.junit.Before; import org.junit.Test; import org.mockito.InOrder; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet; -import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceInstanceSet; -import static com.yahoo.vespa.service.monitor.ServiceMonitorStatus.DOWN; -import static com.yahoo.vespa.service.monitor.ServiceMonitorStatus.NOT_CHECKED; -import static com.yahoo.vespa.service.monitor.ServiceMonitorStatus.UP; -import static org.fest.assertions.Assertions.assertThat; -import static org.junit.Assert.fail; + import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.eq; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -105,8 +77,8 @@ public class HostedVespaPolicyTest { order.verify(clusterPolicy).verifyGroupGoingDownIsFine(clusterApi3); order.verify(applicationApi).getUpStorageNodesInGroupInClusterOrder(); - order.verify(storageNode1).setNodeState(ClusterControllerState.MAINTENANCE); - order.verify(storageNode3).setNodeState(ClusterControllerState.MAINTENANCE); + order.verify(storageNode1).setNodeState(ClusterControllerNodeState.MAINTENANCE); + order.verify(storageNode3).setNodeState(ClusterControllerNodeState.MAINTENANCE); order.verify(applicationApi).getNodesInGroupWithStatus(HostStatus.NO_REMARKS); order.verify(applicationApi).setHostState(hostName1, HostStatus.ALLOWED_TO_BE_DOWN); @@ -116,689 +88,53 @@ public class HostedVespaPolicyTest { order.verifyNoMoreInteractions(); } - private static final TenantId TENANT_ID = new TenantId("tenantId"); - private static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("applicationId"); - private static final HostName HOST_NAME_1 = new HostName("host-1"); - private static final HostName HOST_NAME_2 = new HostName("host-2"); - private static final HostName HOST_NAME_3 = new HostName("host-3"); - private static final HostName HOST_NAME_4 = new HostName("host-4"); - private static final HostName HOST_NAME_5 = new HostName("host-5"); - private static final ServiceType SERVICE_TYPE_1 = new ServiceType("service-1"); - private static final ServiceType SERVICE_TYPE_2 = new ServiceType("service-2"); - - // TODO: Replace HostedVespaClusterPolicy with ClusterPolicy mock - private final HostedVespaPolicy policy = - new HostedVespaPolicy(new HostedVespaClusterPolicy(), clientFactory); - - private final MutableStatusRegistry mutablestatusRegistry = mock(MutableStatusRegistry.class); - { - when(mutablestatusRegistry.getHostStatus(any())).thenReturn(HostStatus.NO_REMARKS); - } - - @Test - public void test_policy_everyone_agrees_everything_is_up() throws Exception { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP)) - .build(); - - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_1, - mutablestatusRegistry - ); - - verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_1, HostStatus.ALLOWED_TO_BE_DOWN); - } - - private void grantWithAdminCluster( - ServiceMonitorStatus statusParallelInstanceOtherHost, - ServiceMonitorStatus statusInstanceOtherHost, - ServiceType serviceType, - boolean expectGranted) throws HostStateChangeDeniedException { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, statusInstanceOtherHost) - .instance(HOST_NAME_3, UP)) - .addCluster(new ClusterBuilder(VespaModelUtil.ADMIN_CLUSTER_ID, serviceType) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, statusParallelInstanceOtherHost)) - .build(); - - if (expectGranted) { - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_1, - mutablestatusRegistry - ); - - verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_1, HostStatus.ALLOWED_TO_BE_DOWN); - } else { - try { - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_1, - mutablestatusRegistry); - fail(); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - } - - verify(mutablestatusRegistry, never()).setHostState(any(), any()); - } - } - - @Test - public void test_parallel_cluster_down_is_ok() throws Exception { - grantWithAdminCluster(DOWN, UP, new ServiceType("some-service-type"), true); - } - - @Test - public void test_slobrok_cluster_down_is_not_ok() throws Exception { - grantWithAdminCluster(DOWN, UP, VespaModelUtil.SLOBROK_SERVICE_TYPE, false); - } - - @Test - public void test_other_cluster_instance_down_is_not_ok() throws Exception { - grantWithAdminCluster(DOWN, DOWN, new ServiceType("some-service-type"), false); - } - - @Test - public void test_all_up_is_ok() throws Exception { - grantWithAdminCluster(UP, UP, new ServiceType("some-service-type"), true); - } - - @Test - public void test_policy_other_host_allowed_to_be_down() { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_5, UP)) - .build(); - - when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - - try { - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_3, - mutablestatusRegistry); - fail(); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_1); - } - - verify(mutablestatusRegistry, never()).setHostState(any(), any()); - } - - @Test - public void test_policy_this_host_allowed_to_be_down() throws Exception { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_5, UP)) - .build(); - - when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_3))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - - verify(mutablestatusRegistry, times(0)).setHostState(HOST_NAME_3, HostStatus.ALLOWED_TO_BE_DOWN); - } - - @Test - public void from_five_to_ten_percent_suspension() throws Exception { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP)) - .build(); - - when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_3, - mutablestatusRegistry); - - verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_3, HostStatus.ALLOWED_TO_BE_DOWN); - } - - @Test - public void from_ten_to_fifteen_percent_suspension() { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP)) - .build(); - - when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - - try { - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_3, - mutablestatusRegistry); - fail(); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_1); - } - - verify(mutablestatusRegistry, never()).setHostState(any(), any()); - } - - @Test - public void from_five_to_fifteen_percent_suspension() { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP)) - .build(); - - when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - - try { - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_3, - mutablestatusRegistry); - fail(); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_1); - } - - verify(mutablestatusRegistry, never()).setHostState(any(), any()); - } - @Test - public void test_policy_no_services() throws Exception { - final ApplicationInstance applicationInstance = new AppBuilder().build(); - - HostName hostName = new HostName("test-hostname"); - policy.grantSuspensionRequest( - applicationInstance, - hostName, - mutablestatusRegistry - ); - - verify(mutablestatusRegistry, times(1)).setHostState(hostName, HostStatus.ALLOWED_TO_BE_DOWN); - } - - // The cluster name happens to be the cluster id of any of the content service clusters. - private static final String CONTENT_CLUSTER_NAME = "content-cluster-id"; - private static final HostName CLUSTER_CONTROLLER_HOST = new HostName("controller-0"); - private static final HostName STORAGE_NODE_HOST = new HostName("storage-2"); - private static final int STORAGE_NODE_INDEX = 2; - - private static final ServiceCluster CLUSTER_CONTROLLER_SERVICE_CLUSTER = new ServiceCluster<>( - new ClusterId("cluster-0"), - VespaModelUtil.CLUSTER_CONTROLLER_SERVICE_TYPE, - makeServiceInstanceSet( - new ServiceInstance<>( - TestUtil.clusterControllerConfigId(CONTENT_CLUSTER_NAME, 0), - CLUSTER_CONTROLLER_HOST, - UP))); - - private static final ServiceCluster DISTRIBUTOR_SERVICE_CLUSTER = new ServiceCluster<>( - new ClusterId(CONTENT_CLUSTER_NAME), - VespaModelUtil.DISTRIBUTOR_SERVICE_TYPE, - makeServiceInstanceSet( - new ServiceInstance<>( - new ConfigId("distributor-id-1"), - new HostName("distributor-1"), - UP))); - - private static final ServiceCluster STORAGE_SERVICE_CLUSTER = new ServiceCluster<>( - new ClusterId(CONTENT_CLUSTER_NAME), - VespaModelUtil.STORAGENODE_SERVICE_TYPE, - makeServiceInstanceSet( - new ServiceInstance<>( - TestUtil.storageNodeConfigId(CONTENT_CLUSTER_NAME, STORAGE_NODE_INDEX), - STORAGE_NODE_HOST, - UP))); - - private static final ApplicationInstance APPLICATION_INSTANCE = - new ApplicationInstance<>( - TENANT_ID, - APPLICATION_INSTANCE_ID, - makeServiceClusterSet( - CLUSTER_CONTROLLER_SERVICE_CLUSTER, - DISTRIBUTOR_SERVICE_CLUSTER, - STORAGE_SERVICE_CLUSTER)); - - // The grantSuspensionRequest() and releaseSuspensionGrant() functions happen to have similar signature, - // which allows us to reuse test code for testing both functions. The actual call to one of these two functions - // is encapsulated into the following functional interface. - interface PolicyFunction { - void grant( - final HostedVespaPolicy policy, - final ApplicationInstance applicationInstance, - final HostName hostName, - final MutableStatusRegistry hostStatusRegistry) throws HostStateChangeDeniedException; - } - - /** - * Since grantSuspensionRequest and releaseSuspensionGrant is quite similar, this test util contains the bulk - * of the test code used to test their common functionality. - * - * @param grantFunction Encapsulates the grant function to call - * @param currentHostStatus The current HostStatus of the host - * @param expectedNodeStateSentToClusterController The NodeState the test expects to be sent to the controller, - * or null if no CC request is expected to be sent. - * @param expectedHostStateSetOnHostStatusService The HostState the test expects to be set on the host service. - */ - private void testCommonGrantFunctionality( - PolicyFunction grantFunction, - ApplicationInstance application, - HostStatus currentHostStatus, - Optional expectedNodeStateSentToClusterController, - HostStatus expectedHostStateSetOnHostStatusService) throws Exception { - // There is only one service running on the host, which is a storage node. - // Therefore, the corresponding cluster controller will have to be contacted - // to ask for permission. - - ClusterControllerStateResponse response = new ClusterControllerStateResponse(true, "ok"); - // MOTD: anyInt() MUST be used for an int field, otherwise a NullPointerException is thrown! - // In general, special anyX() must be used for primitive fields. - when(client.setNodeState(anyInt(), any())).thenReturn(response); - - when(mutablestatusRegistry.getHostStatus(any())).thenReturn(currentHostStatus); - - // Execution phase. - grantFunction.grant(policy, application, STORAGE_NODE_HOST, mutablestatusRegistry); - - // Verification phase. - - if (expectedNodeStateSentToClusterController.isPresent()) { - List hostNames = CLUSTER_CONTROLLER_SERVICE_CLUSTER.serviceInstances().stream() - .map(service -> service.hostName()) - .collect(Collectors.toList()); - verify(clientFactory, times(1)) - .createClient( - hostNames, - CONTENT_CLUSTER_NAME); - verify(client, times(1)) - .setNodeState( - STORAGE_NODE_INDEX, - expectedNodeStateSentToClusterController.get()); - } else { - verify(client, never()).setNodeState(anyInt(), any()); - } - - verify(mutablestatusRegistry, times(1)) - .setHostState( - STORAGE_NODE_HOST, - expectedHostStateSetOnHostStatusService); - } - - @Test - public void test_defer_to_controller() throws Exception { - HostStatus currentHostStatus = HostStatus.NO_REMARKS; - ClusterControllerState expectedNodeStateSentToClusterController = ClusterControllerState.MAINTENANCE; - HostStatus expectedHostStateSetOnHostStatusService = HostStatus.ALLOWED_TO_BE_DOWN; - testCommonGrantFunctionality( - HostedVespaPolicy::grantSuspensionRequest, - APPLICATION_INSTANCE, - currentHostStatus, - Optional.of(expectedNodeStateSentToClusterController), - expectedHostStateSetOnHostStatusService); - } - - @Test - public void test_release_suspension_grant_gives_no_remarks() throws Exception { - HostStatus currentHostStatus = HostStatus.ALLOWED_TO_BE_DOWN; - ClusterControllerState expectedNodeStateSentToClusterController = ClusterControllerState.UP; - HostStatus expectedHostStateSetOnHostStatusService = HostStatus.NO_REMARKS; - testCommonGrantFunctionality( - HostedVespaPolicy::releaseSuspensionGrant, - APPLICATION_INSTANCE, - currentHostStatus, - Optional.of(expectedNodeStateSentToClusterController), - expectedHostStateSetOnHostStatusService); - } - - @Test - public void okToSuspendHostWithNoConfiguredServices() throws Exception { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, DOWN) - .instance(HOST_NAME_2, DOWN)) - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(HOST_NAME_4, DOWN) - .instance(HOST_NAME_5, DOWN)) - .build(); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - } - - @Test - public void okToSuspendHostWithAllItsServicesDownEvenIfOthersAreDownToo() throws Exception { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, DOWN) - .instance(HOST_NAME_3, DOWN)) - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(HOST_NAME_3, DOWN) - .instance(HOST_NAME_4, DOWN) - .instance(HOST_NAME_5, UP)) - .build(); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - } - - @Test - public void okToSuspendStorageNodeWhenStorageIsDown() throws Exception { - ServiceMonitorStatus storageNodeStatus = DOWN; - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new StorageClusterBuilder() - // DOWN storage service => ok to suspend and no cluster controller call - .instance(STORAGE_NODE_HOST, DOWN, STORAGE_NODE_INDEX) - .instance(HOST_NAME_2, DOWN, STORAGE_NODE_INDEX + 1) - .instance(HOST_NAME_3, DOWN, STORAGE_NODE_INDEX + 2)) - .addCluster(CLUSTER_CONTROLLER_SERVICE_CLUSTER) - .addCluster(DISTRIBUTOR_SERVICE_CLUSTER) - // This service has one down service on another host, which should - // not block us from suspension because STORAGE_NODE_HOST down too. - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(STORAGE_NODE_HOST, DOWN) - .instance(HOST_NAME_4, DOWN) - .instance(HOST_NAME_5, UP)) - .build(); - - HostStatus currentHostStatus = HostStatus.NO_REMARKS; - Optional dontExpectAnyCallsToClusterController = Optional.empty(); - testCommonGrantFunctionality( - HostedVespaPolicy::grantSuspensionRequest, - applicationInstance, - currentHostStatus, - dontExpectAnyCallsToClusterController, - HostStatus.ALLOWED_TO_BE_DOWN); - } - - @Test - public void denySuspendOfStorageIfOthersAreDown() throws Exception { - // If the storage service is up, but other hosts' storage services are down, - // we should be denied permission to suspend. This behavior is common for - // storage service and non-storage service (they differ when it comes to - // the cluster controller). - ServiceMonitorStatus storageNodeStatus = UP; - - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new StorageClusterBuilder() - .instance(STORAGE_NODE_HOST, storageNodeStatus, STORAGE_NODE_INDEX) - .instance(HOST_NAME_2, DOWN, STORAGE_NODE_INDEX + 1) - .instance(HOST_NAME_3, DOWN, STORAGE_NODE_INDEX + 2)) - .addCluster(CLUSTER_CONTROLLER_SERVICE_CLUSTER) - .addCluster(DISTRIBUTOR_SERVICE_CLUSTER) - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(STORAGE_NODE_HOST, DOWN) - .instance(HOST_NAME_4, DOWN) - .instance(HOST_NAME_5, UP)) - .build(); - - when(mutablestatusRegistry.getHostStatus(any())).thenReturn(HostStatus.NO_REMARKS); - - try { - policy.grantSuspensionRequest(applicationInstance, STORAGE_NODE_HOST, mutablestatusRegistry); - fail(); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - assertThat(e.getServiceType()).isEqualTo(VespaModelUtil.STORAGENODE_SERVICE_TYPE); - } - } - - // In this test we verify the storage service cluster suspend policy of allowing at most 1 - // storage service to be effectively down. The normal policy (and the one used previously for storage) - // is to allow 10%. Therefore, the test verifies we disallow suspending 2 hosts = 5% (some random number <10%). - // - // Since the Orchestrator doesn't allow suspending the host, the Orchestrator doesn't even bother calling the - // Cluster Controller. The CC also has a policy of max 1, so it's just an optimization and safety guard. - @Test - public void dontBotherCallingClusterControllerIfOtherStorageNodesAreDown() throws Exception { - StorageClusterBuilder clusterBuilder = new StorageClusterBuilder(); - for (int i = 0; i < 40; ++i) { - clusterBuilder.instance(new HostName("host-" + i), UP, i); - } - ApplicationInstance applicationInstance = - new AppBuilder().addCluster(clusterBuilder).build(); - - HostName host_1 = new HostName("host-1"); - when(mutablestatusRegistry.getHostStatus(eq(host_1))).thenReturn(HostStatus.NO_REMARKS); - - HostName host_2 = new HostName("host-2"); - when(mutablestatusRegistry.getHostStatus(eq(host_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - - try { - policy.grantSuspensionRequest(applicationInstance, host_1, mutablestatusRegistry); - fail(); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()) - .isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - assertThat(e.getServiceType()).isEqualTo(VespaModelUtil.STORAGENODE_SERVICE_TYPE); - } - - verify(mutablestatusRegistry, never()).setHostState(any(), any()); - } - - @Test - public void ownServiceInstanceDown() throws Exception { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, DOWN) - .instance(HOST_NAME_3, DOWN)) - .build(); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - } - - @Test - public void ownServiceInstanceDown_otherServiceIsAllNotChecked() throws Exception { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, DOWN) - .instance(HOST_NAME_3, DOWN)) - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(HOST_NAME_3, NOT_CHECKED) - .instance(HOST_NAME_4, NOT_CHECKED) - .instance(HOST_NAME_5, NOT_CHECKED)) - .build(); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - } - - @Test - public void ownServiceInstanceDown_otherServiceIsAllNotChecked_oneHostDown() throws Exception { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, DOWN) - .instance(HOST_NAME_3, DOWN)) - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(HOST_NAME_3, NOT_CHECKED) - .instance(HOST_NAME_4, NOT_CHECKED) - .instance(HOST_NAME_5, NOT_CHECKED)) - .build(); - - when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_4))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - try { - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - fail("Should not be allowed to set " + HOST_NAME_3 + " down when " + HOST_NAME_4 + " is already down."); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_2); - } - } + public void testAcquirePermissionToRemove() throws OrchestrationException { + final HostedVespaClusterPolicy clusterPolicy = mock(HostedVespaClusterPolicy.class); + final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory); + final ApplicationApi applicationApi = mock(ApplicationApi.class); + when(applicationApi.applicationInfo()).thenReturn("tenant:app"); - @Test - public void ownServiceInstanceDown_otherServiceIsAllUp() throws Exception { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, DOWN) - .instance(HOST_NAME_3, DOWN)) - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_5, UP)) - .build(); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - } + ClusterApi clusterApi1 = mock(ClusterApi.class); + ClusterApi clusterApi2 = mock(ClusterApi.class); + ClusterApi clusterApi3 = mock(ClusterApi.class); + List clusterApis = Arrays.asList(clusterApi1, clusterApi2, clusterApi3); + when(applicationApi.getClusters()).thenReturn(clusterApis); - @Test - public void hostHasTwoInstances_oneDownOneUp() throws Exception { - final ApplicationInstance applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_3, DOWN)) - .build(); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - } + StorageNode storageNode1 = mock(StorageNode.class); + HostName hostName1 = new HostName("storage-1"); + when(storageNode1.hostName()).thenReturn(hostName1); - // Helper classes for terseness. + HostName hostName2 = new HostName("host-2"); - private static class AppBuilder { - private final Set> serviceClusters = new HashSet<>(); + StorageNode storageNode3 = mock(StorageNode.class); + HostName hostName3 = new HostName("storage-3"); + when(storageNode1.hostName()).thenReturn(hostName3); - public AppBuilder addCluster(final ServiceCluster cluster) { - serviceClusters.add(cluster); - return this; - } + List upStorageNodes = Arrays.asList(storageNode1, storageNode3); + when(applicationApi.getStorageNodesInGroupInClusterOrder()).thenReturn(upStorageNodes); - public AppBuilder addCluster(final ClusterBuilder clusterBuilder) { - serviceClusters.add(clusterBuilder.build()); - return this; - } + List noRemarksHostNames = Arrays.asList(hostName1, hostName2, hostName3); + when(applicationApi.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)).thenReturn(noRemarksHostNames); - public AppBuilder addCluster(final StorageClusterBuilder clusterBuilder) { - serviceClusters.add(clusterBuilder.build()); - return this; - } + InOrder order = inOrder(applicationApi, clusterPolicy, storageNode1, storageNode3); - public ApplicationInstance build() { - return new ApplicationInstance<>( - TENANT_ID, - APPLICATION_INSTANCE_ID, - serviceClusters); - } - } + policy.acquirePermissionToRemove(applicationApi); - private static class ClusterBuilder { - private final ServiceType serviceType; - private final Set> instances = new HashSet<>(); - private final ClusterId clusterId; - private int instanceIndex = 0; - - public ClusterBuilder(final ClusterId clusterId, final ServiceType serviceType) { - this.clusterId = clusterId; - this.serviceType = serviceType; - } - - public ClusterBuilder(final ServiceType serviceType) { - this.clusterId = new ClusterId("clusterId"); - this.serviceType = serviceType; - } - - public ClusterBuilder instance(final HostName hostName, final ServiceMonitorStatus status) { - instances.add(new ServiceInstance<>(new ConfigId("configId-" + instanceIndex), hostName, status)); - ++instanceIndex; - return this; - } - - public ServiceCluster build() { - return new ServiceCluster<>(clusterId, serviceType, instances); - } - } + order.verify(applicationApi).getClusters(); + order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi1); + order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi2); + order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi3); - private static class StorageClusterBuilder { - private final Set> instances = new HashSet<>(); + order.verify(applicationApi).getStorageNodesInGroupInClusterOrder(); + order.verify(storageNode1).setNodeState(ClusterControllerNodeState.DOWN); + order.verify(storageNode3).setNodeState(ClusterControllerNodeState.DOWN); - public StorageClusterBuilder instance(final HostName hostName, final ServiceMonitorStatus status, int index) { - instances.add(new ServiceInstance<>(TestUtil.storageNodeConfigId("content", index), hostName, status)); - return this; - } + order.verify(applicationApi).getNodesInGroupWithStatus(HostStatus.NO_REMARKS); + order.verify(applicationApi).setHostState(hostName1, HostStatus.ALLOWED_TO_BE_DOWN); + order.verify(applicationApi).setHostState(hostName2, HostStatus.ALLOWED_TO_BE_DOWN); + order.verify(applicationApi).setHostState(hostName3, HostStatus.ALLOWED_TO_BE_DOWN); - public ServiceCluster build() { - return new ServiceCluster<>(new ClusterId(CONTENT_CLUSTER_NAME), VespaModelUtil.STORAGENODE_SERVICE_TYPE, instances); - } + order.verifyNoMoreInteractions(); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index 793b73ec5d8..3d3117f9e07 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.orchestrator.InstanceLookupService; import com.yahoo.vespa.orchestrator.OrchestratorImpl; @@ -103,6 +102,10 @@ public class HostResourceTest { public void releaseSuspensionGrant(ApplicationApi application) throws HostStateChangeDeniedException { } + @Override + public void acquirePermissionToRemove(ApplicationApi applicationApi) throws HostStateChangeDeniedException { + } + @Override public void releaseSuspensionGrant( ApplicationInstance applicationInstance, @@ -202,6 +205,11 @@ public class HostResourceTest { doThrow(); } + @Override + public void acquirePermissionToRemove(ApplicationApi applicationApi) throws HostStateChangeDeniedException { + doThrow(); + } + @Override public void releaseSuspensionGrant( ApplicationInstance applicationInstance, @@ -214,7 +222,6 @@ public class HostResourceTest { throw new HostStateChangeDeniedException( new HostName("some-host"), "impossible-policy", - new ServiceType("silly-service"), "This policy rejects all requests"); } } -- cgit v1.2.3