diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /orchestrator/src/test/java/com |
Publish
Diffstat (limited to 'orchestrator/src/test/java/com')
13 files changed, 2459 insertions, 0 deletions
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java new file mode 100644 index 00000000000..e1b73c1fe65 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java @@ -0,0 +1,166 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator; + + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A hardcoded set of applications with one storage cluster with two nodes each. + * + * @author oyving + * @author smorgrav + */ +public class DummyInstanceLookupService implements InstanceLookupService { + + public static final HostName TEST1_HOST_NAME = new HostName("test1.prod.utpoia-1.vespahosted.ut1.yahoo.com"); + public static final HostName TEST3_HOST_NAME = new HostName("test3.prod.utpoia-1.vespahosted.ut1.yahoo.com"); + public static final HostName TEST6_HOST_NAME = new HostName("test6.prod.us-east-1.vespahosted.ne1.yahoo.com"); + + private static final Set<ApplicationInstance<ServiceMonitorStatus>> apps = new HashSet<>(); + + + static { + apps.add(new ApplicationInstance<>( + new TenantId("test-tenant-id"), + new ApplicationInstanceId("application:instance"), + TestUtil.makeServiceClusterSet( + new ServiceCluster<>( + new ClusterId("test-cluster-id-1"), + new ServiceType("storagenode"), + TestUtil.makeServiceInstanceSet( + new ServiceInstance<>( + new ConfigId("storage/storage/1"), + TEST1_HOST_NAME, + ServiceMonitorStatus.UP), + new ServiceInstance<>( + new ConfigId("storage/storage/2"), + new HostName("test2.prod.utpoia-1.vespahosted.ut1.yahoo.com"), + ServiceMonitorStatus.UP))), + new ServiceCluster<>( + new ClusterId("clustercontroller"), + new ServiceType("container-clustercontroller"), + TestUtil.makeServiceInstanceSet( + new ServiceInstance<>( + new ConfigId("clustercontroller-1"), + new HostName("myclustercontroller.prod.utopia-1.vespahosted.ut1.yahoo.com"), + ServiceMonitorStatus.UP))) + + ) + )); + + apps.add(new ApplicationInstance<>( + new TenantId("mediasearch"), + new ApplicationInstanceId("imagesearch:default"), + TestUtil.makeServiceClusterSet( + new ServiceCluster<>( + new ClusterId("image"), + new ServiceType("storagenode"), + TestUtil.makeServiceInstanceSet( + new ServiceInstance<>( + new ConfigId("storage/storage/3"), + TEST3_HOST_NAME, + ServiceMonitorStatus.UP), + new ServiceInstance<>( + new ConfigId("storage/storage/4"), + new HostName("test4.prod.utpoia-1.vespahosted.ut1.yahoo.com"), + ServiceMonitorStatus.UP))), + new ServiceCluster<>( + new ClusterId("clustercontroller"), + new ServiceType("container-clustercontroller"), + TestUtil.makeServiceInstanceSet( + new ServiceInstance<>( + new ConfigId("clustercontroller-1"), + new HostName("myclustercontroller2.prod.utopia-1.vespahosted.ut1.yahoo.com"), + ServiceMonitorStatus.UP))) + ) + ) + ); + + apps.add(new ApplicationInstance<>( + new TenantId("tenant-id-3"), + new ApplicationInstanceId("application-instance-3:default"), + TestUtil.makeServiceClusterSet( + new ServiceCluster<>( + new ClusterId("cluster-id-3"), + new ServiceType("storagenode"), + TestUtil.makeServiceInstanceSet( + new ServiceInstance<>( + new ConfigId("storage/storage/1"), + TEST6_HOST_NAME, + ServiceMonitorStatus.UP), + new ServiceInstance<>( + new ConfigId("storage/storage/4"), + new HostName("test4.prod.utpoia-1.vespahosted.ut1.yahoo.com"), + ServiceMonitorStatus.UP))), + new ServiceCluster<>( + new ClusterId("clustercontroller"), + new ServiceType("container-clustercontroller"), + TestUtil.makeServiceInstanceSet( + new ServiceInstance<>( + new ConfigId("clustercontroller-1"), + new HostName("myclustercontroller3.prod.utopia-1.vespahosted.ut1.yahoo.com"), + ServiceMonitorStatus.UP))) + ) + )); + } + + + @Override + public Optional<ApplicationInstance<ServiceMonitorStatus>> findInstanceById( + final ApplicationInstanceReference applicationInstanceReference) { + for (ApplicationInstance<ServiceMonitorStatus> app : apps) { + if (app.reference().equals(applicationInstanceReference)) return Optional.of(app); + } + return Optional.empty(); + } + + @Override + public Optional<ApplicationInstance<ServiceMonitorStatus>> findInstanceByHost(HostName hostName) { + for (ApplicationInstance<ServiceMonitorStatus> app : apps) { + for (ServiceCluster<ServiceMonitorStatus> cluster : app.serviceClusters()) { + for (ServiceInstance<ServiceMonitorStatus> service : cluster.serviceInstances()) { + if (hostName.equals(service.hostName())) return Optional.of(app); + } + } + } + return Optional.empty(); + } + + @Override + public Set<ApplicationInstanceReference> knownInstances() { + return apps.stream().map(a -> + new ApplicationInstanceReference(a.tenantId(),a.applicationInstanceId())).collect(Collectors.toSet()); + + } + + public static Set<HostName> getContentHosts(ApplicationInstanceReference appRef) { + Set<HostName> hosts = apps.stream() + .filter(application -> application.reference().equals(appRef)) + .flatMap(appReference -> appReference.serviceClusters().stream()) + .filter(VespaModelUtil::isContent) + .flatMap(serviceCluster -> serviceCluster.serviceInstances().stream()) + .map(ServiceInstance::hostName) + .collect(Collectors.toSet()); + + return hosts; + } + + public static Set<ApplicationInstance<ServiceMonitorStatus>> getApplications() { + return apps; + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java new file mode 100644 index 00000000000..5c1d9bd45be --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -0,0 +1,300 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; +import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; +import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; +import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.InMemoryStatusService; +import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InOrder; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import static com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN; +import static com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus.NO_REMARKS; +import static org.hamcrest.collection.IsEmptyCollection.empty; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsCollectionContaining.hasItem; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.spy; + +/** + * Test Orchestrator with a mock backend (the InMemoryStatusService) + * + * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + */ +public class OrchestratorImplTest { + + private ApplicationId app1; + private ApplicationId app2; + private HostName app1_host1; + + private OrchestratorImpl orchestrator; + private ClusterControllerClientFactoryMock clustercontroller; + + @Before + public void setUp() throws Exception { + // Extract applications and hosts from dummy instance lookup service + Iterator<ApplicationInstance<ServiceMonitorStatus>> iterator = DummyInstanceLookupService.getApplications().iterator(); + ApplicationInstanceReference app1_ref = iterator.next().reference(); + app1 = OrchestratorUtil.toApplicationId(app1_ref); + app1_host1 = DummyInstanceLookupService.getContentHosts(app1_ref).iterator().next(); + app2 = OrchestratorUtil.toApplicationId(iterator.next().reference()); + + clustercontroller = new ClusterControllerClientFactoryMock(); + orchestrator = new OrchestratorImpl( + clustercontroller, + new InMemoryStatusService(), + new OrchestratorConfig(new OrchestratorConfig.Builder()), + new DummyInstanceLookupService()); + + clustercontroller.setAllDummyNodesAsUp(); + } + + @After + public void tearDown() throws Exception { + orchestrator = null; + clustercontroller = null; + } + + @Test + public void application_has_initially_no_remarks() throws Exception { + assertThat(orchestrator.getApplicationInstanceStatus(app1), is(NO_REMARKS)); + } + + @Test + public void application_can_be_set_in_suspend() throws Exception { + orchestrator.suspend(app1); + assertThat(orchestrator.getApplicationInstanceStatus(app1), is(ALLOWED_TO_BE_DOWN)); + } + + @Test + public void application_can_be_removed_from_suspend() throws Exception { + orchestrator.suspend(app1); + orchestrator.resume(app1); + assertThat(orchestrator.getApplicationInstanceStatus(app1), is(NO_REMARKS)); + } + + @Test + public void appliations_list_returns_empty_initially() throws Exception { + assertThat(orchestrator.getAllSuspendedApplications(), is(empty())); + } + + @Test + public void appliations_list_returns_suspended_apps() throws Exception { + // One suspended app + orchestrator.suspend(app1); + assertThat(orchestrator.getAllSuspendedApplications().size(), is(1)); + assertThat(orchestrator.getAllSuspendedApplications(), hasItem(app1)); + + // Two suspended apps + orchestrator.suspend(app2); + assertThat(orchestrator.getAllSuspendedApplications().size(), is(2)); + assertThat(orchestrator.getAllSuspendedApplications(), hasItem(app1)); + assertThat(orchestrator.getAllSuspendedApplications(), hasItem(app2)); + + // Back to one when resetting one app to no_remarks + orchestrator.resume(app1); + assertThat(orchestrator.getAllSuspendedApplications().size(), is(1)); + assertThat(orchestrator.getAllSuspendedApplications(), hasItem(app2)); + } + + + @Test + public void application_operations_are_idempotent() throws Exception { + // Two suspends + orchestrator.suspend(app1); + orchestrator.suspend(app1); + assertThat(orchestrator.getApplicationInstanceStatus(app1), is(ALLOWED_TO_BE_DOWN)); + assertThat(orchestrator.getApplicationInstanceStatus(app2), is(NO_REMARKS)); + + // Three no_remarks + orchestrator.resume(app1); + orchestrator.resume(app1); + orchestrator.resume(app1); + assertThat(orchestrator.getApplicationInstanceStatus(app1), is(NO_REMARKS)); + assertThat(orchestrator.getApplicationInstanceStatus(app2), is(NO_REMARKS)); + + // Two suspends and two on two applications interleaved + orchestrator.suspend(app2); + orchestrator.resume(app1); + orchestrator.suspend(app2); + orchestrator.resume(app1); + assertThat(orchestrator.getApplicationInstanceStatus(app1), is(NO_REMARKS)); + assertThat(orchestrator.getApplicationInstanceStatus(app2), is(ALLOWED_TO_BE_DOWN)); + } + + + @Test + public void application_suspend_sets_application_nodes_in_maintenance_and_allowed_to_be_down() throws Exception { + // Pre condition + assertEquals(NO_REMARKS, orchestrator.getApplicationInstanceStatus(app1)); + assertEquals(HostStatus.NO_REMARKS, orchestrator.getNodeStatus(app1_host1)); + assertFalse(isInMaintenance(app1, app1_host1)); + + orchestrator.suspend(app1); + + assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1)); + assertTrue(isInMaintenance(app1, app1_host1)); + } + + @Test + public void node_suspend_while_app_is_resumed_set_allowed_to_be_down_and_set_it_in_maintenance() throws Exception { + // Pre condition + assertEquals(NO_REMARKS, orchestrator.getApplicationInstanceStatus(app1)); + assertEquals(HostStatus.NO_REMARKS, orchestrator.getNodeStatus(app1_host1)); + assertFalse(isInMaintenance(app1, app1_host1)); + + orchestrator.suspend(app1_host1); + + assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1)); + assertTrue(isInMaintenance(app1, app1_host1)); + } + + @Test + public void node_suspend_while_app_is_suspended_does_nothing() throws Exception { + // Pre condition + orchestrator.suspend(app1); + assertEquals(ALLOWED_TO_BE_DOWN, orchestrator.getApplicationInstanceStatus(app1)); + assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1)); + assertTrue(isInMaintenance(app1, app1_host1)); + + orchestrator.suspend(app1_host1); + + // Should not change anything + assertEquals(ALLOWED_TO_BE_DOWN, orchestrator.getApplicationInstanceStatus(app1)); + assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1)); + assertTrue(isInMaintenance(app1, app1_host1)); + } + + @Test + public void node_resume_after_app_is_resumed_removes_allowed_be_down_and_set_it_up() throws Exception { + // Pre condition + orchestrator.suspend(app1); + assertEquals(ALLOWED_TO_BE_DOWN, orchestrator.getApplicationInstanceStatus(app1)); + assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1)); + assertTrue(isInMaintenance(app1, app1_host1)); + + orchestrator.resume(app1); + orchestrator.resume(app1_host1); + + assertEquals(HostStatus.NO_REMARKS, orchestrator.getNodeStatus(app1_host1)); + assertFalse(isInMaintenance(app1, app1_host1)); + } + + @Test + public void node_resume_while_app_is_suspended_does_nothing() throws Exception { + orchestrator.suspend(app1_host1); + orchestrator.suspend(app1); + + orchestrator.resume(app1_host1); + + assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1)); + assertTrue(isInMaintenance(app1, app1_host1)); + } + + @Test + public void applicationReferenceHasTenantAndAppInstance() { + InstanceLookupService service = new DummyInstanceLookupService(); + String applicationInstanceId = service.findInstanceByHost(DummyInstanceLookupService.TEST1_HOST_NAME).get() + .reference().toString(); + assertEquals("test-tenant-id:application:instance", applicationInstanceId); + } + + @Test + public void sortHostNamesForSuspend() throws Exception { + HostName parentHostName = new HostName("parentHostName"); + List<HostName> expectedOrder = Arrays.asList( + DummyInstanceLookupService.TEST3_HOST_NAME, + DummyInstanceLookupService.TEST1_HOST_NAME); + + assertEquals(expectedOrder, orchestrator.sortHostNamesForSuspend(Arrays.asList( + DummyInstanceLookupService.TEST1_HOST_NAME, + DummyInstanceLookupService.TEST3_HOST_NAME))); + + assertEquals(expectedOrder, orchestrator.sortHostNamesForSuspend(Arrays.asList( + DummyInstanceLookupService.TEST3_HOST_NAME, + DummyInstanceLookupService.TEST1_HOST_NAME))); + } + + @Test + public void rollbackWorks() throws Exception { + // A spy is preferential because suspendAll() relies on delegating the hard work to suspend() and resume(). + OrchestratorImpl orchestrator = spy(this.orchestrator); + + doNothing().when(orchestrator).suspend(DummyInstanceLookupService.TEST3_HOST_NAME); + + Throwable supensionFailure = new HostStateChangeDeniedException( + DummyInstanceLookupService.TEST6_HOST_NAME, + "some-constraint", + new ServiceType("foo"), + "error message"); + doThrow(supensionFailure).when(orchestrator).suspend(DummyInstanceLookupService.TEST1_HOST_NAME); + + doThrow(new HostStateChangeDeniedException(DummyInstanceLookupService.TEST1_HOST_NAME, "foo1-constraint", new ServiceType("foo1-service"), "foo1-message")) + .when(orchestrator).resume(DummyInstanceLookupService.TEST1_HOST_NAME); + doNothing().when(orchestrator).resume(DummyInstanceLookupService.TEST6_HOST_NAME); + doNothing().when(orchestrator).resume(DummyInstanceLookupService.TEST3_HOST_NAME); + + try { + orchestrator.suspendAll( + new HostName("parentHostname"), + Arrays.asList( + DummyInstanceLookupService.TEST1_HOST_NAME, + DummyInstanceLookupService.TEST3_HOST_NAME, + DummyInstanceLookupService.TEST6_HOST_NAME)); + fail(); + } catch (BatchHostStateChangeDeniedException e) { + assertEquals(e.getSuppressed().length, 1); + assertEquals("Failed to suspend [test3.prod.utpoia-1.vespahosted.ut1.yahoo.com, " + + "test6.prod.us-east-1.vespahosted.ne1.yahoo.com, test1.prod.utpoia-1.vespahosted.ut1.yahoo.com] " + + "with parent host parentHostname: Changing the state of host " + + "test6.prod.us-east-1.vespahosted.ne1.yahoo.com would violate some-constraint for " + + "service type foo: error message; " + + "With suppressed throwable com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException: " + + "Changing the state of host test1.prod.utpoia-1.vespahosted.ut1.yahoo.com would violate " + + "foo1-constraint for service type foo1-service: foo1-message", e.getMessage()); + } + + InOrder order = inOrder(orchestrator); + order.verify(orchestrator).suspend(DummyInstanceLookupService.TEST3_HOST_NAME); + order.verify(orchestrator).suspend(DummyInstanceLookupService.TEST6_HOST_NAME); + + // As of 2016-06-07: + // TEST1_HOST_NAME: test-tenant-id:application:instance + // TEST3_HOST_NAME: mediasearch:imagesearch:default + // TEST6_HOST_NAME: tenant-id-3:application-instance-3:default + // Meaning the order is 3, 6, then 1. For rollback/resume the order is reversed. + order.verify(orchestrator).resume(DummyInstanceLookupService.TEST1_HOST_NAME); + order.verify(orchestrator).resume(DummyInstanceLookupService.TEST6_HOST_NAME); + order.verify(orchestrator).resume(DummyInstanceLookupService.TEST3_HOST_NAME); + order.verifyNoMoreInteractions(); + } + + private boolean isInMaintenance(ApplicationId appId, HostName hostName) throws ApplicationIdNotFoundException { + for (ApplicationInstance<ServiceMonitorStatus> app : DummyInstanceLookupService.getApplications()) { + if (app.reference().equals(OrchestratorUtil.toApplicationInstanceReference(appId))) { + return clustercontroller.isInMaintenance(app, hostName); + } + } + throw new ApplicationIdNotFoundException(); + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java new file mode 100644 index 00000000000..0626bf72e60 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.TenantId; +import org.junit.Assert; +import org.junit.Test; + +/** + * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + */ +public class OrchestratorUtilTest { + + private static final ApplicationId APPID_1 = ApplicationId.from( + TenantName.from("mediasearch"), + ApplicationName.from("tumblr-search"), + InstanceName.defaultName()); + + private static final ApplicationInstanceReference APPREF_1 = new ApplicationInstanceReference( + new TenantId("test-tenant"), + new ApplicationInstanceId("test-application:test-environment:test-region:test-instance-key")); + + /** + * Here we don't care how the internal of the different application + * id/reference look like as long as we get back to exactly where we + * started from a round trip. I.e I'm not testing validity of the + * different representations. + */ + @Test + public void applicationid_conversion_are_symmetric() throws Exception { + + // From appId to appRef and back + ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(APPID_1); + ApplicationId appIdRoundTrip = OrchestratorUtil.toApplicationId(appRef); + + Assert.assertEquals(APPID_1, appIdRoundTrip); + + // From appRef to appId and back + ApplicationId appId = OrchestratorUtil.toApplicationId(APPREF_1); + ApplicationInstanceReference appRefRoundTrip = OrchestratorUtil.toApplicationInstanceReference(appId); + + Assert.assertEquals(APPREF_1, appRefRoundTrip); + } +}
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestIds.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestIds.java new file mode 100644 index 00000000000..e9d8b498f32 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestIds.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator; + +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.TenantId; + +/** + * @author tonytv + */ +public class TestIds { + public static final ApplicationInstanceReference APPLICATION_INSTANCE_REFERENCE = + new ApplicationInstanceReference( + new TenantId("test-tenant"), + new ApplicationInstanceId("test-application:test-environment:test-region:test-instance-key")); + + public static final ApplicationInstanceReference APPLICATION_INSTANCE_REFERENCE2 = + new ApplicationInstanceReference( + new TenantId("test-tenant2"), + new ApplicationInstanceId("test-application2:test-environment:test-region:test-instance-key")); + + public static final HostName HOST_NAME1 = new HostName("host1.test.corp.yahoo.com"); +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestUtil.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestUtil.java new file mode 100644 index 00000000000..244ddfc858d --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestUtil.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator; + +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceInstance; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * Utility methods for creating test setups. + * + * @author <a href="mailto:bakksjo@yahoo-inc.com">Oyvind Bakksjo</a> + */ +public class TestUtil { + @SafeVarargs + public static <S> Set<ServiceInstance<S>> makeServiceInstanceSet( + final ServiceInstance<S>... serviceInstances) { + return new HashSet<>(Arrays.asList(serviceInstances)); + } + + @SafeVarargs + public static <S> Set<ServiceCluster<S>> makeServiceClusterSet( + final ServiceCluster<S>... serviceClusters) { + return new HashSet<>(Arrays.asList(serviceClusters)); + } + + public static ConfigId storageNodeConfigId(int index) { + return new ConfigId("storage/storage/" + index); + } + + public static ConfigId clusterControllerConfigId(int index) { + return new ConfigId("admin/cluster-controllers/" + index); + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/VespaModelUtilTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/VespaModelUtilTest.java new file mode 100644 index 00000000000..ab533528cd4 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/VespaModelUtilTest.java @@ -0,0 +1,223 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator; + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; +import org.junit.Test; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.Sets.newHashSet; +import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet; +import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceInstanceSet; +import static junit.framework.TestCase.assertFalse; +import static junit.framework.TestCase.assertTrue; +import static org.fest.assertions.Assertions.assertThat; + +/** + * @author hakon + */ +public class VespaModelUtilTest { + // Cluster Controller Service Cluster + + private static final ClusterId CONTENT_CLUSTER_ID = new ClusterId("content-cluster-0"); + + public static final HostName controller0Host = new HostName("controller-0"); + + private static final ServiceInstance<ServiceMonitorStatus> controller0 = new ServiceInstance<>( + TestUtil.clusterControllerConfigId(0), + controller0Host, + ServiceMonitorStatus.UP); + private static final ServiceInstance<ServiceMonitorStatus> controller1 = new ServiceInstance<>( + TestUtil.clusterControllerConfigId(1), + new HostName("controller-1"), + ServiceMonitorStatus.UP); + + private static final ServiceCluster<ServiceMonitorStatus> controllerCluster = + new ServiceCluster<>( + new ClusterId(CONTENT_CLUSTER_ID.s() + "-controller"), + VespaModelUtil.CLUSTER_CONTROLLER_SERVICE_TYPE, + makeServiceInstanceSet(controller1, controller0)); + + // Distributor Service Cluster + + private static final ServiceInstance<ServiceMonitorStatus> distributor0 = new ServiceInstance<>( + new ConfigId("distributor-config-id"), + new HostName("distributor-0"), + ServiceMonitorStatus.UP); + + + private static final ServiceCluster<ServiceMonitorStatus> distributorCluster = + new ServiceCluster<>( + CONTENT_CLUSTER_ID, + VespaModelUtil.DISTRIBUTOR_SERVICE_TYPE, + makeServiceInstanceSet(distributor0)); + + // Storage Node Service Cluster + + public static final HostName storage0Host = new HostName("storage-0"); + private static final ServiceInstance<ServiceMonitorStatus> storage0 = new ServiceInstance<>( + new ConfigId("storage-config-id"), + storage0Host, + ServiceMonitorStatus.UP); + + private static final ServiceCluster<ServiceMonitorStatus> storageCluster = + new ServiceCluster<>( + CONTENT_CLUSTER_ID, + VespaModelUtil.STORAGENODE_SERVICE_TYPE, + makeServiceInstanceSet(storage0)); + + // Secondary Distributor Service Cluster + + private static final ServiceInstance<ServiceMonitorStatus> secondaryDistributor0 = new ServiceInstance<>( + new ConfigId("secondary-distributor-config-id"), + new HostName("secondary-distributor-0"), + ServiceMonitorStatus.UP); + + private static final ClusterId SECONDARY_CONTENT_CLUSTER_ID = new ClusterId("secondary-content-cluster-0"); + private static final ServiceCluster<ServiceMonitorStatus> secondaryDistributorCluster = + new ServiceCluster<>( + SECONDARY_CONTENT_CLUSTER_ID, + VespaModelUtil.DISTRIBUTOR_SERVICE_TYPE, + makeServiceInstanceSet(secondaryDistributor0)); + + // Secondary Storage Node Service Cluster + + public static final HostName secondaryStorage0Host = new HostName("secondary-storage-0"); + private static final ServiceInstance<ServiceMonitorStatus> secondaryStorage0 = new ServiceInstance<>( + new ConfigId("secondary-storage-config-id"), + secondaryStorage0Host, + ServiceMonitorStatus.UP); + + private static final ServiceCluster<ServiceMonitorStatus> secondaryStorageCluster = + new ServiceCluster<>( + SECONDARY_CONTENT_CLUSTER_ID, + VespaModelUtil.STORAGENODE_SERVICE_TYPE, + makeServiceInstanceSet(secondaryStorage0)); + + // The Application Instance + + public static final ApplicationInstance<ServiceMonitorStatus> application = + new ApplicationInstance<>( + new TenantId("tenant-0"), + new ApplicationInstanceId("application-0"), + makeServiceClusterSet( + controllerCluster, + distributorCluster, + storageCluster, + secondaryDistributorCluster, + secondaryStorageCluster)); + + private ServiceCluster<?> createServiceCluster(ServiceType serviceType) { + return new ServiceCluster<ServiceMonitorStatus>( + new ClusterId("cluster-id"), + serviceType, + new HashSet<>()); + } + + @Test + public void verifyControllerClusterIsRecognized() { + ServiceCluster<?> cluster = createServiceCluster(VespaModelUtil.CLUSTER_CONTROLLER_SERVICE_TYPE); + assertTrue(VespaModelUtil.isClusterController(cluster)); + } + + @Test + public void verifyNonControllerClusterIsNotRecognized() { + ServiceCluster<?> cluster = createServiceCluster(new ServiceType("foo")); + assertFalse(VespaModelUtil.isClusterController(cluster)); + } + + @Test + public void verifyStorageClusterIsRecognized() { + ServiceCluster<?> cluster = createServiceCluster(VespaModelUtil.STORAGENODE_SERVICE_TYPE); + assertTrue(VespaModelUtil.isStorage(cluster)); + cluster = createServiceCluster(VespaModelUtil.STORAGENODE_SERVICE_TYPE); + assertTrue(VespaModelUtil.isStorage(cluster)); + } + + @Test + public void verifyNonStorageClusterIsNotRecognized() { + ServiceCluster<?> cluster = createServiceCluster(new ServiceType("foo")); + assertFalse(VespaModelUtil.isStorage(cluster)); + } + + @Test + public void verifyContentClusterIsRecognized() { + ServiceCluster<?> cluster = createServiceCluster(VespaModelUtil.DISTRIBUTOR_SERVICE_TYPE); + assertTrue(VespaModelUtil.isContent(cluster)); + cluster = createServiceCluster(VespaModelUtil.STORAGENODE_SERVICE_TYPE); + assertTrue(VespaModelUtil.isContent(cluster)); + cluster = createServiceCluster(VespaModelUtil.SEARCHNODE_SERVICE_TYPE); + assertTrue(VespaModelUtil.isContent(cluster)); + } + + @Test + public void verifyNonContentClusterIsNotRecognized() { + ServiceCluster<?> cluster = createServiceCluster(new ServiceType("foo")); + assertFalse(VespaModelUtil.isContent(cluster)); + } + + @Test + public void testGettingClusterControllerInstances() { + Set<ServiceInstance<?>> controllers = + new HashSet<>(VespaModelUtil.getClusterControllerInstances(application, CONTENT_CLUSTER_ID)); + Set<ServiceInstance<ServiceMonitorStatus>> expectedControllers = newHashSet(controller0, controller1); + + assertThat(controllers).isEqualTo(expectedControllers); + } + + @Test + public void testGetControllerHostName() { + HostName host = VespaModelUtil.getControllerHostName(application, CONTENT_CLUSTER_ID); + assertThat(host).isEqualTo(controller0Host); + } + + @Test + public void testGetContentClusterName() { + ClusterId contentClusterName = VespaModelUtil.getContentClusterName(application, distributor0.hostName()); + assertThat(CONTENT_CLUSTER_ID).isEqualTo(contentClusterName); + } + + @Test + public void testGetContentClusterNameForSecondaryContentCluster() { + ClusterId contentClusterName = VespaModelUtil.getContentClusterName(application, secondaryDistributor0.hostName()); + assertThat(SECONDARY_CONTENT_CLUSTER_ID).isEqualTo(contentClusterName); + } + + @Test + public void testGetStorageNodeAtHost() { + Optional<ServiceInstance<ServiceMonitorStatus>> service = + VespaModelUtil.getStorageNodeAtHost(application, storage0Host); + assertTrue(service.isPresent()); + assertThat(service.get()).isEqualTo(storage0); + } + + @Test + public void testGetStorageNodeAtHostWithUnknownHost() { + Optional<ServiceInstance<ServiceMonitorStatus>> service = + VespaModelUtil.getStorageNodeAtHost(application, new HostName("storage-1")); + assertFalse(service.isPresent()); + } + + @Test + public void testGetClusterControllerIndex() { + ConfigId configId = new ConfigId("admin/cluster-controllers/2"); + assertThat(VespaModelUtil.getClusterControllerIndex(configId)).isEqualTo(2); + } + + @Test + public void testGetStorageNodeIndex() { + ConfigId configId = TestUtil.storageNodeConfigId(3); + assertThat(VespaModelUtil.getStorageNodeIndex(configId)).isEqualTo(3); + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java new file mode 100644 index 00000000000..a682bfe8856 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.controller; + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.orchestrator.DummyInstanceLookupService; +import com.yahoo.vespa.orchestrator.VespaModelUtil; +import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Mock implementation of ClusterControllerClient + * <p> + * + * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + */ +public class ClusterControllerClientFactoryMock implements ClusterControllerClientFactory { + Map<String, ClusterControllerState> nodes = new HashMap<>(); + + public boolean isInMaintenance(ApplicationInstance<ServiceMonitorStatus> appInstance, HostName hostName) { + try { + ClusterId clusterName = VespaModelUtil.getContentClusterName(appInstance, hostName); + int storageNodeIndex = VespaModelUtil.getStorageNodeIndex(appInstance, hostName); + String globalMapKey = clusterName.s() + storageNodeIndex; + return nodes.getOrDefault(globalMapKey, ClusterControllerState.UP) == ClusterControllerState.MAINTENANCE; + } catch (Exception e) { + //Catch all - meant to catch cases where the node is not part of a storage cluster + return false; + } + } + + public void setAllDummyNodesAsUp() { + for (ApplicationInstance<ServiceMonitorStatus> app : DummyInstanceLookupService.getApplications()) { + Set<HostName> hosts = DummyInstanceLookupService.getContentHosts(app.reference()); + for (HostName host : hosts) { + ClusterId clusterName = VespaModelUtil.getContentClusterName(app, host); + int storageNodeIndex = VespaModelUtil.getStorageNodeIndex(app, host); + String globalMapKey = clusterName.s() + storageNodeIndex; + nodes.put(globalMapKey, ClusterControllerState.UP); + } + } + } + + @Override + public ClusterControllerClient createClient(Collection<? extends ServiceInstance<?>> clusterControllers, String clusterName) { + return new ClusterControllerClient() { + + @Override + public ClusterControllerStateResponse setNodeState(int storageNodeIndex, ClusterControllerState wantedState) throws IOException { + nodes.put(clusterName + storageNodeIndex, wantedState); + return new ClusterControllerStateResponse(true, "Yes"); + } + + @Override + public ClusterControllerStateResponse setApplicationState(ClusterControllerState wantedState) throws IOException { + Set<String> keyCopy = new HashSet<>(nodes.keySet()); + for (String s : keyCopy) { + if (s.startsWith(clusterName)) { + nodes.put(s, wantedState); + } + } + return new ClusterControllerStateResponse(true, "It works"); + } + }; + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java new file mode 100644 index 00000000000..68df4dce393 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.controller; + +import com.yahoo.vespa.jaxrs.client.JaxRsStrategy; +import com.yahoo.vespa.jaxrs.client.LocalPassThroughJaxRsStrategy; +import org.junit.Test; + +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class ClusterControllerClientTest { + private static final String CLUSTER_NAME = "clusterName"; + private static final int STORAGE_NODE_INDEX = 0; + + @Test + public void correctParametersArePassedThrough() throws Exception { + final ClusterControllerJaxRsApi clusterControllerApi = mock(ClusterControllerJaxRsApi.class); + final JaxRsStrategy<ClusterControllerJaxRsApi> strategyMock = new LocalPassThroughJaxRsStrategy<>(clusterControllerApi); + final ClusterControllerClient clusterControllerClient = new ClusterControllerClientImpl( + strategyMock, + CLUSTER_NAME); + + final ClusterControllerState wantedState = ClusterControllerState.MAINTENANCE; + + clusterControllerClient.setNodeState(STORAGE_NODE_INDEX, wantedState); + + final ClusterControllerStateRequest expectedNodeStateRequest = new ClusterControllerStateRequest( + new ClusterControllerStateRequest.State(wantedState, ClusterControllerClientImpl.REQUEST_REASON), + ClusterControllerStateRequest.Condition.SAFE); + verify(clusterControllerApi, times(1)) + .setNodeState( + eq(CLUSTER_NAME), + eq(STORAGE_NODE_INDEX), + eq(expectedNodeStateRequest)); + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java new file mode 100644 index 00000000000..cc217885047 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.controller; + +import com.yahoo.vespa.jaxrs.client.JaxRsClientFactory; +import com.yahoo.vespa.orchestrator.TestUtil; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collection; +import java.util.Collections; + +import static org.hamcrest.CoreMatchers.anyOf; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SingleInstanceClusterControllerClientFactoryTest { + private static final int PORT = SingleInstanceClusterControllerClientFactory.CLUSTERCONTROLLER_HARDCODED_PORT; + private static final String PATH = SingleInstanceClusterControllerClientFactory.CLUSTERCONTROLLER_API_PATH; + + private static final HostName HOST_NAME_1 = new HostName("host1"); + private static final HostName HOST_NAME_2 = new HostName("host2"); + private static final HostName HOST_NAME_3 = new HostName("host3"); + + private final ClusterControllerJaxRsApi mockApi = mock(ClusterControllerJaxRsApi.class); + private final JaxRsClientFactory jaxRsClientFactory = mock(JaxRsClientFactory.class); + private final ClusterControllerClientFactory clientFactory + = new SingleInstanceClusterControllerClientFactory(jaxRsClientFactory); + + @Before + public void setup() { + when( + jaxRsClientFactory.createClient( + eq(ClusterControllerJaxRsApi.class), + any(HostName.class), + anyInt(), + anyString())) + .thenReturn(mockApi); + } + + @Test + public void testCreateClientWithNoClusterControllerInstances() throws Exception { + final Collection<ServiceInstance<ServiceMonitorStatus>> clusterControllers = Collections.emptySet(); + + try { + clientFactory.createClient(clusterControllers, "clusterName"); + fail(); + } catch (IllegalArgumentException e) { + // As expected. + } + } + + @Test + public void testCreateClientWithSingleClusterControllerInstance() throws Exception { + final Collection<ServiceInstance<ServiceMonitorStatus>> clusterControllers = Collections.singleton( + new ServiceInstance<>(clusterControllerConfigId(1), HOST_NAME_1, ServiceMonitorStatus.UP)); + + clientFactory.createClient(clusterControllers, "clusterName") + .setNodeState(0, ClusterControllerState.MAINTENANCE); + + verify(jaxRsClientFactory).createClient( + ClusterControllerJaxRsApi.class, + HOST_NAME_1, + PORT, + PATH); + } + + @Test + public void testCreateClientWithTwoNonClusterControllerInstances() throws Exception { + final Collection<ServiceInstance<ServiceMonitorStatus>> clusterControllers = TestUtil.makeServiceInstanceSet( + new ServiceInstance<>(new ConfigId("not-a-cluster-controller-1"), HOST_NAME_1, ServiceMonitorStatus.UP), + new ServiceInstance<>(new ConfigId("not-a-cluster-controller-2"), HOST_NAME_2, ServiceMonitorStatus.UP)); + + try { + clientFactory.createClient(clusterControllers, "clusterName"); + fail(); + } catch (IllegalArgumentException e) { + // As expected. + } + } + + @Test + public void testCreateClientWithThreeClusterControllerInstances() throws Exception { + final Collection<ServiceInstance<ServiceMonitorStatus>> clusterControllers = TestUtil.makeServiceInstanceSet( + new ServiceInstance<>(clusterControllerConfigId(1), HOST_NAME_1, ServiceMonitorStatus.UP), + new ServiceInstance<>(clusterControllerConfigId(2), HOST_NAME_2, ServiceMonitorStatus.UP), + new ServiceInstance<>(clusterControllerConfigId(3), HOST_NAME_3, ServiceMonitorStatus.UP)); + + clientFactory.createClient(clusterControllers, "clusterName") + .setNodeState(0, ClusterControllerState.MAINTENANCE); + + verify(jaxRsClientFactory).createClient( + eq(ClusterControllerJaxRsApi.class), + argThat(is(anyOf( + equalTo(HOST_NAME_1), + equalTo(HOST_NAME_2), + equalTo(HOST_NAME_3)))), + eq(PORT), + eq(PATH)); + } + + private static ConfigId clusterControllerConfigId(final int index) { + return new ConfigId("admin/cluster-controllers/" + index); + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java new file mode 100644 index 00000000000..45d605b6d8a --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java @@ -0,0 +1,738 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.policy; + + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.orchestrator.TestUtil; +import com.yahoo.vespa.orchestrator.VespaModelUtil; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerState; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerStateResponse; +import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; +import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; +import org.junit.Test; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet; +import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceInstanceSet; +import static com.yahoo.vespa.service.monitor.ServiceMonitorStatus.DOWN; +import static com.yahoo.vespa.service.monitor.ServiceMonitorStatus.NOT_CHECKED; +import static com.yahoo.vespa.service.monitor.ServiceMonitorStatus.UP; +import static org.fest.assertions.Assertions.assertThat; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author oyving + * @author bakksjo + */ +public class HostedVespaPolicyTest { + private static final TenantId TENANT_ID = new TenantId("tenantId"); + private static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("applicationId"); + private static final HostName HOST_NAME_1 = new HostName("host-1"); + private static final HostName HOST_NAME_2 = new HostName("host-2"); + private static final HostName HOST_NAME_3 = new HostName("host-3"); + private static final HostName HOST_NAME_4 = new HostName("host-4"); + private static final HostName HOST_NAME_5 = new HostName("host-5"); + private static final ServiceType SERVICE_TYPE_1 = new ServiceType("service-1"); + private static final ServiceType SERVICE_TYPE_2 = new ServiceType("service-2"); + + private final ClusterControllerClientFactory clusterControllerClientFactory + = mock(ClusterControllerClientFactory.class); + private final ClusterControllerClient client = mock(ClusterControllerClient.class); + { + when(clusterControllerClientFactory.createClient(any(), any())).thenReturn(client); + } + + private final HostedVespaPolicy policy + = new HostedVespaPolicy(clusterControllerClientFactory); + + private final MutableStatusRegistry mutablestatusRegistry = mock(MutableStatusRegistry.class); + { + when(mutablestatusRegistry.getHostStatus(any())).thenReturn(HostStatus.NO_REMARKS); + } + + @Test + public void test_policy_everyone_agrees_everything_is_up() throws Exception { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, UP) + .instance(HOST_NAME_3, UP)) + .build(); + + policy.grantSuspensionRequest( + applicationInstance, + HOST_NAME_1, + mutablestatusRegistry + ); + + verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_1, HostStatus.ALLOWED_TO_BE_DOWN); + } + + private void grantWithAdminCluster( + ServiceMonitorStatus statusParallelInstanceOtherHost, + ServiceMonitorStatus statusInstanceOtherHost, + ServiceType serviceType, + boolean expectGranted) throws HostStateChangeDeniedException { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, statusInstanceOtherHost) + .instance(HOST_NAME_3, UP)) + .addCluster(new ClusterBuilder(VespaModelUtil.ADMIN_CLUSTER_ID, serviceType) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, statusParallelInstanceOtherHost)) + .build(); + + if (expectGranted) { + policy.grantSuspensionRequest( + applicationInstance, + HOST_NAME_1, + mutablestatusRegistry + ); + + verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_1, HostStatus.ALLOWED_TO_BE_DOWN); + } else { + try { + policy.grantSuspensionRequest( + applicationInstance, + HOST_NAME_1, + mutablestatusRegistry); + fail(); + } catch (HostStateChangeDeniedException e) { + // As expected. + assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); + } + + verify(mutablestatusRegistry, never()).setHostState(any(), any()); + } + } + + @Test + public void test_parallel_cluster_down_is_ok() throws Exception { + grantWithAdminCluster(DOWN, UP, new ServiceType("some-service-type"), true); + } + + @Test + public void test_slobrok_cluster_down_is_not_ok() throws Exception { + grantWithAdminCluster(DOWN, UP, VespaModelUtil.SLOBROK_SERVICE_TYPE, false); + } + + @Test + public void test_other_cluster_instance_down_is_not_ok() throws Exception { + grantWithAdminCluster(DOWN, DOWN, new ServiceType("some-service-type"), false); + } + + @Test + public void test_all_up_is_ok() throws Exception { + grantWithAdminCluster(UP, UP, new ServiceType("some-service-type"), true); + } + + @Test + public void test_policy_other_host_allowed_to_be_down() { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, UP) + .instance(HOST_NAME_3, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_5, UP)) + .build(); + + when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); + + try { + policy.grantSuspensionRequest( + applicationInstance, + HOST_NAME_3, + mutablestatusRegistry); + fail(); + } catch (HostStateChangeDeniedException e) { + // As expected. + assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); + assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_1); + } + + verify(mutablestatusRegistry, never()).setHostState(any(), any()); + } + + @Test + public void test_policy_this_host_allowed_to_be_down() throws Exception { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, UP) + .instance(HOST_NAME_3, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_5, UP)) + .build(); + + when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_3))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); + + policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); + + verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_3, HostStatus.ALLOWED_TO_BE_DOWN); + } + + @Test + public void from_five_to_ten_percent_suspension() throws Exception { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, UP) + .instance(HOST_NAME_3, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP)) + .build(); + + when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); + + policy.grantSuspensionRequest( + applicationInstance, + HOST_NAME_3, + mutablestatusRegistry); + + verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_3, HostStatus.ALLOWED_TO_BE_DOWN); + } + + @Test + public void from_ten_to_fifteen_percent_suspension() { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, UP) + .instance(HOST_NAME_2, UP) + .instance(HOST_NAME_3, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP)) + .build(); + + when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); + + try { + policy.grantSuspensionRequest( + applicationInstance, + HOST_NAME_3, + mutablestatusRegistry); + fail(); + } catch (HostStateChangeDeniedException e) { + // As expected. + assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); + assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_1); + } + + verify(mutablestatusRegistry, never()).setHostState(any(), any()); + } + + @Test + public void from_five_to_fifteen_percent_suspension() { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, UP) + .instance(HOST_NAME_3, UP) + .instance(HOST_NAME_3, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP) + .instance(HOST_NAME_5, UP)) + .build(); + + when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); + + try { + policy.grantSuspensionRequest( + applicationInstance, + HOST_NAME_3, + mutablestatusRegistry); + fail(); + } catch (HostStateChangeDeniedException e) { + // As expected. + assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); + assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_1); + } + + verify(mutablestatusRegistry, never()).setHostState(any(), any()); + } + + @Test + public void test_policy_no_services() throws Exception { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder().build(); + + HostName hostName = new HostName("test-hostname"); + policy.grantSuspensionRequest( + applicationInstance, + hostName, + mutablestatusRegistry + ); + + verify(mutablestatusRegistry, times(1)).setHostState(hostName, HostStatus.ALLOWED_TO_BE_DOWN); + } + + // The cluster name happens to be the cluster id of any of the content service clusters. + private static final String CONTENT_CLUSTER_NAME = "content-cluster-id"; + private static final HostName CLUSTER_CONTROLLER_HOST = new HostName("controller-0"); + private static final HostName STORAGE_NODE_HOST = new HostName("storage-2"); + private static final int STORAGE_NODE_INDEX = 2; + + private static final ServiceCluster<ServiceMonitorStatus> CLUSTER_CONTROLLER_SERVICE_CLUSTER = new ServiceCluster<>( + new ClusterId("cluster-0"), + VespaModelUtil.CLUSTER_CONTROLLER_SERVICE_TYPE, + makeServiceInstanceSet( + new ServiceInstance<>( + TestUtil.clusterControllerConfigId(0), + CLUSTER_CONTROLLER_HOST, + UP))); + + private static final ServiceCluster<ServiceMonitorStatus> DISTRIBUTOR_SERVICE_CLUSTER = new ServiceCluster<>( + new ClusterId(CONTENT_CLUSTER_NAME), + VespaModelUtil.DISTRIBUTOR_SERVICE_TYPE, + makeServiceInstanceSet( + new ServiceInstance<>( + new ConfigId("distributor-id-1"), + new HostName("distributor-1"), + UP))); + + private static final ServiceCluster<ServiceMonitorStatus> STORAGE_SERVICE_CLUSTER = new ServiceCluster<>( + new ClusterId(CONTENT_CLUSTER_NAME), + VespaModelUtil.STORAGENODE_SERVICE_TYPE, + makeServiceInstanceSet( + new ServiceInstance<>( + TestUtil.storageNodeConfigId(STORAGE_NODE_INDEX), + STORAGE_NODE_HOST, + UP))); + + private static final ApplicationInstance<ServiceMonitorStatus> APPLICATION_INSTANCE = + new ApplicationInstance<>( + TENANT_ID, + APPLICATION_INSTANCE_ID, + makeServiceClusterSet( + CLUSTER_CONTROLLER_SERVICE_CLUSTER, + DISTRIBUTOR_SERVICE_CLUSTER, + STORAGE_SERVICE_CLUSTER)); + + // The grantSuspensionRequest() and releaseSuspensionGrant() functions happen to have similar signature, + // which allows us to reuse test code for testing both functions. The actual call to one of these two functions + // is encapsulated into the following functional interface. + interface PolicyFunction { + void grant( + final HostedVespaPolicy policy, + final ApplicationInstance<ServiceMonitorStatus> applicationInstance, + final HostName hostName, + final MutableStatusRegistry hostStatusRegistry) throws HostStateChangeDeniedException; + } + + /** + * Since grantSuspensionRequest and releaseSuspensionGrant is quite similar, this test util contains the bulk + * of the test code used to test their common functionality. + * + * @param grantFunction Encapsulates the grant function to call + * @param currentHostStatus The current HostStatus of the host + * @param expectedNodeStateSentToClusterController The NodeState the test expects to be sent to the controller, + * or null if no CC request is expected to be sent. + * @param expectedHostStateSetOnHostStatusService The HostState the test expects to be set on the host service. + */ + private void testCommonGrantFunctionality( + PolicyFunction grantFunction, + ApplicationInstance<ServiceMonitorStatus> application, + HostStatus currentHostStatus, + Optional<ClusterControllerState> expectedNodeStateSentToClusterController, + HostStatus expectedHostStateSetOnHostStatusService) throws Exception { + // There is only one service running on the host, which is a storage node. + // Therefore, the corresponding cluster controller will have to be contacted + // to ask for permission. + + ClusterControllerStateResponse response = new ClusterControllerStateResponse(true, "ok"); + // MOTD: anyInt() MUST be used for an int field, otherwise a NullPointerException is thrown! + // In general, special anyX() must be used for primitive fields. + when(client.setNodeState(anyInt(), any())).thenReturn(response); + + when(mutablestatusRegistry.getHostStatus(any())).thenReturn(currentHostStatus); + + // Execution phase. + grantFunction.grant(policy, application, STORAGE_NODE_HOST, mutablestatusRegistry); + + // Verification phase. + + if (expectedNodeStateSentToClusterController.isPresent()) { + verify(clusterControllerClientFactory, times(1)) + .createClient( + CLUSTER_CONTROLLER_SERVICE_CLUSTER.serviceInstances(), + CONTENT_CLUSTER_NAME); + verify(client, times(1)) + .setNodeState( + STORAGE_NODE_INDEX, + expectedNodeStateSentToClusterController.get()); + } else { + verify(client, never()).setNodeState(anyInt(), any()); + } + + verify(mutablestatusRegistry, times(1)) + .setHostState( + STORAGE_NODE_HOST, + expectedHostStateSetOnHostStatusService); + } + + @Test + public void test_defer_to_controller() throws Exception { + HostStatus currentHostStatus = HostStatus.NO_REMARKS; + ClusterControllerState expectedNodeStateSentToClusterController = ClusterControllerState.MAINTENANCE; + HostStatus expectedHostStateSetOnHostStatusService = HostStatus.ALLOWED_TO_BE_DOWN; + testCommonGrantFunctionality( + HostedVespaPolicy::grantSuspensionRequest, + APPLICATION_INSTANCE, + currentHostStatus, + Optional.of(expectedNodeStateSentToClusterController), + expectedHostStateSetOnHostStatusService); + } + + @Test + public void test_release_suspension_grant_gives_no_remarks() throws Exception { + HostStatus currentHostStatus = HostStatus.ALLOWED_TO_BE_DOWN; + ClusterControllerState expectedNodeStateSentToClusterController = ClusterControllerState.UP; + HostStatus expectedHostStateSetOnHostStatusService = HostStatus.NO_REMARKS; + testCommonGrantFunctionality( + HostedVespaPolicy::releaseSuspensionGrant, + APPLICATION_INSTANCE, + currentHostStatus, + Optional.of(expectedNodeStateSentToClusterController), + expectedHostStateSetOnHostStatusService); + } + + @Test + public void okToSuspendHostWithNoConfiguredServices() throws Exception { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, DOWN) + .instance(HOST_NAME_2, DOWN)) + .addCluster(new ClusterBuilder(SERVICE_TYPE_2) + .instance(HOST_NAME_4, DOWN) + .instance(HOST_NAME_5, DOWN)) + .build(); + + policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); + } + + @Test + public void okToSuspendHostWithAllItsServicesDownEvenIfOthersAreDownToo() throws Exception { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, DOWN) + .instance(HOST_NAME_3, DOWN)) + .addCluster(new ClusterBuilder(SERVICE_TYPE_2) + .instance(HOST_NAME_3, DOWN) + .instance(HOST_NAME_4, DOWN) + .instance(HOST_NAME_5, UP)) + .build(); + + policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); + } + + @Test + public void okToSuspendStorageNodeWhenStorageIsDown() throws Exception { + ServiceMonitorStatus storageNodeStatus = DOWN; + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new StorageClusterBuilder() + // DOWN storage service => ok to suspend and no cluster controller call + .instance(STORAGE_NODE_HOST, DOWN, STORAGE_NODE_INDEX) + .instance(HOST_NAME_2, DOWN, STORAGE_NODE_INDEX + 1) + .instance(HOST_NAME_3, DOWN, STORAGE_NODE_INDEX + 2)) + .addCluster(CLUSTER_CONTROLLER_SERVICE_CLUSTER) + .addCluster(DISTRIBUTOR_SERVICE_CLUSTER) + // This service has one down service on another host, which should + // not block us from suspension because STORAGE_NODE_HOST down too. + .addCluster(new ClusterBuilder(SERVICE_TYPE_2) + .instance(STORAGE_NODE_HOST, DOWN) + .instance(HOST_NAME_4, DOWN) + .instance(HOST_NAME_5, UP)) + .build(); + + HostStatus currentHostStatus = HostStatus.NO_REMARKS; + Optional<ClusterControllerState> dontExpectAnyCallsToClusterController = Optional.empty(); + testCommonGrantFunctionality( + HostedVespaPolicy::grantSuspensionRequest, + applicationInstance, + currentHostStatus, + dontExpectAnyCallsToClusterController, + HostStatus.ALLOWED_TO_BE_DOWN); + } + + @Test + public void denySuspendOfStorageIfOthersAreDown() throws Exception { + // If the storage service is up, but other hosts' storage services are down, + // we should be denied permission to suspend. This behavior is common for + // storage service and non-storage service (they differ when it comes to + // the cluster controller). + ServiceMonitorStatus storageNodeStatus = UP; + + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new StorageClusterBuilder() + .instance(STORAGE_NODE_HOST, storageNodeStatus, STORAGE_NODE_INDEX) + .instance(HOST_NAME_2, DOWN, STORAGE_NODE_INDEX + 1) + .instance(HOST_NAME_3, DOWN, STORAGE_NODE_INDEX + 2)) + .addCluster(CLUSTER_CONTROLLER_SERVICE_CLUSTER) + .addCluster(DISTRIBUTOR_SERVICE_CLUSTER) + .addCluster(new ClusterBuilder(SERVICE_TYPE_2) + .instance(STORAGE_NODE_HOST, DOWN) + .instance(HOST_NAME_4, DOWN) + .instance(HOST_NAME_5, UP)) + .build(); + + when(mutablestatusRegistry.getHostStatus(any())).thenReturn(HostStatus.NO_REMARKS); + + try { + policy.grantSuspensionRequest(applicationInstance, STORAGE_NODE_HOST, mutablestatusRegistry); + fail(); + } catch (HostStateChangeDeniedException e) { + // As expected. + assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); + assertThat(e.getServiceType()).isEqualTo(VespaModelUtil.STORAGENODE_SERVICE_TYPE); + } + } + + // In this test we verify the storage service cluster suspend policy of allowing at most 1 + // storage service to be effectively down. The normal policy (and the one used previously for storage) + // is to allow 10%. Therefore, the test verifies we disallow suspending 2 hosts = 5% (some random number <10%). + // + // Since the Orchestrator doesn't allow suspending the host, the Orchestrator doesn't even bother calling the + // Cluster Controller. The CC also has a policy of max 1, so it's just an optimization and safety guard. + @Test + public void dontBotherCallingClusterControllerIfOtherStorageNodesAreDown() throws Exception { + StorageClusterBuilder clusterBuilder = new StorageClusterBuilder(); + for (int i = 0; i < 40; ++i) { + clusterBuilder.instance(new HostName("host-" + i), UP, i); + } + ApplicationInstance<ServiceMonitorStatus> applicationInstance = + new AppBuilder().addCluster(clusterBuilder).build(); + + HostName host_1 = new HostName("host-1"); + when(mutablestatusRegistry.getHostStatus(eq(host_1))).thenReturn(HostStatus.NO_REMARKS); + + HostName host_2 = new HostName("host-2"); + when(mutablestatusRegistry.getHostStatus(eq(host_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); + + try { + policy.grantSuspensionRequest(applicationInstance, host_1, mutablestatusRegistry); + fail(); + } catch (HostStateChangeDeniedException e) { + // As expected. + assertThat(e.getConstraintName()) + .isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); + assertThat(e.getServiceType()).isEqualTo(VespaModelUtil.STORAGENODE_SERVICE_TYPE); + } + + verify(mutablestatusRegistry, never()).setHostState(any(), any()); + } + + @Test + public void ownServiceInstanceDown() throws Exception { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, DOWN) + .instance(HOST_NAME_3, DOWN)) + .build(); + + policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); + } + + @Test + public void ownServiceInstanceDown_otherServiceIsAllNotChecked() throws Exception { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, DOWN) + .instance(HOST_NAME_3, DOWN)) + .addCluster(new ClusterBuilder(SERVICE_TYPE_2) + .instance(HOST_NAME_3, NOT_CHECKED) + .instance(HOST_NAME_4, NOT_CHECKED) + .instance(HOST_NAME_5, NOT_CHECKED)) + .build(); + + policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); + } + + @Test + public void ownServiceInstanceDown_otherServiceIsAllNotChecked_oneHostDown() throws Exception { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, DOWN) + .instance(HOST_NAME_3, DOWN)) + .addCluster(new ClusterBuilder(SERVICE_TYPE_2) + .instance(HOST_NAME_3, NOT_CHECKED) + .instance(HOST_NAME_4, NOT_CHECKED) + .instance(HOST_NAME_5, NOT_CHECKED)) + .build(); + + when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_4))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); + try { + policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); + fail("Should not be allowed to set " + HOST_NAME_3 + " down when " + HOST_NAME_4 + " is already down."); + } catch (HostStateChangeDeniedException e) { + // As expected. + assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); + assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_2); + } + } + + @Test + public void ownServiceInstanceDown_otherServiceIsAllUp() throws Exception { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, DOWN) + .instance(HOST_NAME_3, DOWN)) + .addCluster(new ClusterBuilder(SERVICE_TYPE_2) + .instance(HOST_NAME_3, UP) + .instance(HOST_NAME_4, UP) + .instance(HOST_NAME_5, UP)) + .build(); + + policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); + } + + @Test + public void hostHasTwoInstances_oneDownOneUp() throws Exception { + final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() + .addCluster(new ClusterBuilder(SERVICE_TYPE_1) + .instance(HOST_NAME_1, UP) + .instance(HOST_NAME_2, UP) + .instance(HOST_NAME_3, UP) + .instance(HOST_NAME_3, DOWN)) + .build(); + + policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); + } + + // Helper classes for terseness. + + private static class AppBuilder { + private final Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters = new HashSet<>(); + + public AppBuilder addCluster(final ServiceCluster<ServiceMonitorStatus> cluster) { + serviceClusters.add(cluster); + return this; + } + + public AppBuilder addCluster(final ClusterBuilder clusterBuilder) { + serviceClusters.add(clusterBuilder.build()); + return this; + } + + public AppBuilder addCluster(final StorageClusterBuilder clusterBuilder) { + serviceClusters.add(clusterBuilder.build()); + return this; + } + + public ApplicationInstance<ServiceMonitorStatus> build() { + return new ApplicationInstance<>( + TENANT_ID, + APPLICATION_INSTANCE_ID, + serviceClusters); + } + } + + private static class ClusterBuilder { + private final ServiceType serviceType; + private final Set<ServiceInstance<ServiceMonitorStatus>> instances = new HashSet<>(); + private final ClusterId clusterId; + private int instanceIndex = 0; + + public ClusterBuilder(final ClusterId clusterId, final ServiceType serviceType) { + this.clusterId = clusterId; + this.serviceType = serviceType; + } + + public ClusterBuilder(final ServiceType serviceType) { + this.clusterId = new ClusterId("clusterId"); + this.serviceType = serviceType; + } + + public ClusterBuilder instance(final HostName hostName, final ServiceMonitorStatus status) { + instances.add(new ServiceInstance<>(new ConfigId("configId-" + instanceIndex), hostName, status)); + ++instanceIndex; + return this; + } + + public ServiceCluster<ServiceMonitorStatus> build() { + return new ServiceCluster<>(clusterId, serviceType, instances); + } + } + + private static class StorageClusterBuilder { + private final Set<ServiceInstance<ServiceMonitorStatus>> instances = new HashSet<>(); + + public StorageClusterBuilder instance(final HostName hostName, final ServiceMonitorStatus status, int index) { + instances.add(new ServiceInstance<>(TestUtil.storageNodeConfigId(index), hostName, status)); + return this; + } + + public ServiceCluster<ServiceMonitorStatus> build() { + return new ServiceCluster<>(new ClusterId(CONTENT_CLUSTER_NAME), VespaModelUtil.STORAGENODE_SERVICE_TYPE, instances); + } + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java new file mode 100644 index 00000000000..b8396a51e1e --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java @@ -0,0 +1,143 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.resources; + +import com.yahoo.application.Application; +import com.yahoo.application.Networking; +import com.yahoo.container.Container; +import com.yahoo.jdisc.http.server.jetty.JettyHttpServer; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.GenericType; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.net.URI; +import java.nio.file.Paths; +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +/** + * Tests the implementation of the orchestrators ApplicationAPI. + * + * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + */ +public class ApplicationSuspensionResourceTest { + + static final String BASE_PATH = "/orchestrator/v1/suspensions/applications"; + static final String RESOURCE_1 = "mediasearch:imagesearch:default"; + static final String RESOURCE_2 = "test-tenant-id:application:instance"; + static final String INVALID_RESOURCE_NAME = "something_without_colons"; + + Application jdiscApplication; + WebTarget webTarget; + + @Before + public void setup() throws Exception { + jdiscApplication = Application.fromApplicationPackage(Paths.get("src/test/application"), + Networking.enable); + Client client = ClientBuilder.newClient(); + + JettyHttpServer serverProvider = (JettyHttpServer) Container.get().getServerProviderRegistry().allComponents().get(0); + String url = "http://localhost:" + serverProvider.getListenPort() + BASE_PATH; + webTarget = client.target(new URI(url)); + } + + @After + public void teardown() throws Exception { + jdiscApplication.close(); + webTarget = null; + } + + @Ignore + @Test + public void run_application_locally_for_manual_browser_testing() throws Exception { + System.out.println(webTarget.getUri()); + Thread.sleep(3600 * 1000); + } + + @Test + public void get_all_suspended_applications_return_empty_list_initially() throws Exception { + Response reply = webTarget.request().get(); + assertEquals(200, reply.getStatus()); + assertEquals("[]", reply.readEntity(String.class)); + } + + @Test + public void invalid_application_id_throws_http_400() throws Exception { + Response reply = webTarget.request().post(Entity.entity(INVALID_RESOURCE_NAME,MediaType.APPLICATION_JSON_TYPE)); + assertEquals(400, reply.getStatus()); + } + + @Test + public void get_application_status_returns_404_for_notsuspended_and_204_for_suspended() throws Exception { + // Get on application that is not suspended + Response reply = webTarget.path(RESOURCE_1).request().get(); + assertEquals(404, reply.getStatus()); + + // Post application + reply = webTarget.request().post(Entity.entity(RESOURCE_1,MediaType.APPLICATION_JSON_TYPE)); + assertEquals(204, reply.getStatus()); + + // Get on the application that now should be in suspended + reply = webTarget.path(RESOURCE_1).request().get(); + assertEquals(204, reply.getStatus()); + } + + + @Test + public void delete_works_on_suspended_and_not_suspended_applications() throws Exception { + // Delete an application that is not suspended + Response reply = webTarget.path(RESOURCE_1).request().delete(); + assertEquals(204, reply.getStatus()); + + // Put application in suspend + reply = webTarget.request().post(Entity.entity(RESOURCE_1,MediaType.APPLICATION_JSON_TYPE)); + assertEquals(204, reply.getStatus()); + + // Check that it is in suspend + reply = webTarget.path(RESOURCE_1).request(MediaType.APPLICATION_JSON).get(); + assertEquals(204, reply.getStatus()); + + // Delete it + reply = webTarget.path(RESOURCE_1).request().delete(); + assertEquals(204, reply.getStatus()); + + // Check that it is not in suspend anymore + reply = webTarget.path(RESOURCE_1).request(MediaType.APPLICATION_JSON).get(); + assertEquals(404, reply.getStatus()); + } + + @Test + public void list_applications_returns_the_correct_list_of_suspended_applications() throws Exception { + // Test that initially we have the empty set + Response reply = webTarget.request(MediaType.APPLICATION_JSON).get(); + assertEquals(200, reply.getStatus()); + assertEquals("[]", reply.readEntity(String.class)); + + // Add a couple of applications to maintenance + webTarget.request().post(Entity.entity(RESOURCE_1,MediaType.APPLICATION_JSON_TYPE)); + webTarget.request().post(Entity.entity(RESOURCE_2,MediaType.APPLICATION_JSON_TYPE)); + assertEquals(200, reply.getStatus()); + + // Test that we get them back + Set<String> responses = webTarget.request(MediaType.APPLICATION_JSON_TYPE) + .get(new GenericType<Set<String>>() {}); + assertEquals(2, responses.size()); + + // Remove suspend for the first resource + webTarget.path(RESOURCE_1).request().delete(); + + // Test that we are back to the start with the empty set + responses = webTarget.request(MediaType.APPLICATION_JSON_TYPE) + .get(new GenericType<Set<String>>() {}); + assertEquals(1, responses.size()); + assertEquals(RESOURCE_2, responses.iterator().next()); + } +}
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java new file mode 100644 index 00000000000..fb046f978c7 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -0,0 +1,227 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.resources; + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.orchestrator.InstanceLookupService; +import com.yahoo.vespa.orchestrator.OrchestratorImpl; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; +import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; +import com.yahoo.vespa.orchestrator.policy.Policy; +import com.yahoo.vespa.orchestrator.restapi.wire.BatchHostSuspendRequest; +import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; +import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; +import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; +import com.yahoo.vespa.orchestrator.status.StatusService; +import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; +import org.junit.Test; + +import javax.ws.rs.WebApplicationException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + +import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet; +import static org.fest.assertions.Assertions.assertThat; +import static org.fest.assertions.Fail.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class HostResourceTest { + private static final int SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS = 0; + private static final TenantId TENANT_ID = new TenantId("tenantId"); + private static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("applicationId"); + private static final ApplicationInstanceReference APPLICATION_INSTANCE_REFERENCE = + new ApplicationInstanceReference(TENANT_ID, APPLICATION_INSTANCE_ID); + + private static final StatusService EVERY_HOST_IS_UP_HOST_STATUS_SERVICE = mock(StatusService.class); + private static final MutableStatusRegistry EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY = mock(MutableStatusRegistry.class); + static { + when(EVERY_HOST_IS_UP_HOST_STATUS_SERVICE.forApplicationInstance(eq(APPLICATION_INSTANCE_REFERENCE))) + .thenReturn(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY); + when(EVERY_HOST_IS_UP_HOST_STATUS_SERVICE.lockApplicationInstance_forCurrentThreadOnly(eq(APPLICATION_INSTANCE_REFERENCE))) + .thenReturn(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY); + when(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY.getHostStatus(any())) + .thenReturn(HostStatus.NO_REMARKS); + when(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY.getApplicationInstanceStatus()) + .thenReturn(ApplicationInstanceStatus.NO_REMARKS); + } + + private static final InstanceLookupService mockInstanceLookupService = mock(InstanceLookupService.class); + static { + when(mockInstanceLookupService.findInstanceByHost(any())) + .thenReturn(Optional.of( + new ApplicationInstance<>( + TENANT_ID, + APPLICATION_INSTANCE_ID, + makeServiceClusterSet()))); + } + + + private static final InstanceLookupService alwaysEmptyInstanceLookUpService = new InstanceLookupService() { + @Override + public Optional<ApplicationInstance<ServiceMonitorStatus>> findInstanceById( + final ApplicationInstanceReference applicationInstanceReference) { + return Optional.empty(); + } + + @Override + public Optional<ApplicationInstance<ServiceMonitorStatus>> findInstanceByHost(final HostName hostName) { + return Optional.empty(); + } + + @Override + public Set<ApplicationInstanceReference> knownInstances() { + return Collections.emptySet(); + } + }; + + private static class AlwaysAllowPolicy implements Policy { + @Override + public void grantSuspensionRequest( + ApplicationInstance<ServiceMonitorStatus> applicationInstance, + HostName hostName, + MutableStatusRegistry hostStatusRegistry) { + } + @Override + public void releaseSuspensionGrant( + ApplicationInstance<ServiceMonitorStatus> applicationInstance, + HostName hostName, + MutableStatusRegistry hostStatusRegistry) { + } + } + + private static final OrchestratorImpl alwaysAllowOrchestrator = new OrchestratorImpl( + new AlwaysAllowPolicy(), + new ClusterControllerClientFactoryMock(), + EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, mockInstanceLookupService, + SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS + ); + + private static final OrchestratorImpl hostNotFoundOrchestrator = new OrchestratorImpl( + new AlwaysAllowPolicy(), + new ClusterControllerClientFactoryMock(), + EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, alwaysEmptyInstanceLookUpService, + SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS + ); + + @Test + public void returns_200_on_success() throws Exception { + HostResource hostResource = + new HostResource(alwaysAllowOrchestrator); + + final String hostName = "hostname"; + + UpdateHostResponse response = hostResource.suspend(hostName); + + assertThat(response.hostname()).isEqualTo(hostName); + } + + @Test + public void returns_200_on_success_batch() throws Exception { + HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator); + BatchHostSuspendRequest request = + new BatchHostSuspendRequest("parentHostname", Arrays.asList("hostname1", "hostname2")); + BatchOperationResult response = hostSuspensionResource.suspendAll(request); + assertThat(response.success()); + } + + @Test + public void throws_404_when_host_unknown() throws Exception { + try { + HostResource hostResource = + new HostResource(hostNotFoundOrchestrator); + hostResource.suspend("hostname"); + fail(); + } catch (WebApplicationException w) { + assertThat(w.getResponse().getStatus()).isEqualTo(404); + } + } + + // Note: Missing host is 404 for a single-host, but 400 for multi-host (batch). + // This is so because the hostname is part of the URL path for single-host, while the + // hostnames are part of the request body for multi-host. + @Test + public void throws_400_when_host_unknown_for_batch() throws Exception { + try { + HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(hostNotFoundOrchestrator); + BatchHostSuspendRequest request = + new BatchHostSuspendRequest("parentHostname", Arrays.asList("hostname1", "hostname2")); + hostSuspensionResource.suspendAll(request); + fail(); + } catch (WebApplicationException w) { + assertThat(w.getResponse().getStatus()).isEqualTo(400); + } + } + + private static class AlwaysFailPolicy implements Policy { + @Override + public void grantSuspensionRequest( + ApplicationInstance<ServiceMonitorStatus> applicationInstance, + HostName hostName, + MutableStatusRegistry hostStatusRegistry) throws HostStateChangeDeniedException { + doThrow(); + } + @Override + public void releaseSuspensionGrant( + ApplicationInstance<ServiceMonitorStatus> applicationInstance, + HostName hostName, + MutableStatusRegistry hostStatusRegistry) throws HostStateChangeDeniedException { + doThrow(); + } + + private static void doThrow() throws HostStateChangeDeniedException { + throw new HostStateChangeDeniedException( + new HostName("some-host"), + "impossible-policy", + new ServiceType("silly-service"), + "This policy rejects all requests"); + } + } + + @Test + public void throws_409_when_request_rejected_by_policies() throws Exception { + final OrchestratorImpl alwaysRejectResolver = new OrchestratorImpl( + new AlwaysFailPolicy(), + new ClusterControllerClientFactoryMock(), + EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,mockInstanceLookupService, + SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS); + + try { + HostResource hostResource = new HostResource(alwaysRejectResolver); + hostResource.suspend("hostname"); + fail(); + } catch (WebApplicationException w) { + assertThat(w.getResponse().getStatus()).isEqualTo(409); + } + } + + @Test + public void throws_409_when_request_rejected_by_policies_for_batch() throws Exception { + final OrchestratorImpl alwaysRejectResolver = new OrchestratorImpl( + new AlwaysFailPolicy(), + new ClusterControllerClientFactoryMock(), + EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, + mockInstanceLookupService, + SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS); + + try { + HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysRejectResolver); + BatchHostSuspendRequest request = + new BatchHostSuspendRequest("parentHostname", Arrays.asList("hostname1", "hostname2")); + hostSuspensionResource.suspendAll(request); + fail(); + } catch (WebApplicationException w) { + assertThat(w.getResponse().getStatus()).isEqualTo(409); + } + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java new file mode 100644 index 00000000000..ac73e264477 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java @@ -0,0 +1,323 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.orchestrator.TestIds; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.curator.SessionFailRetryLoop.SessionFailedException; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.test.KillSession; +import org.apache.curator.test.TestingServer; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Logger; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsCollectionContaining.hasItem; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +public class ZookeeperStatusServiceTest { + private TestingServer testingServer; + private ZookeeperStatusService zookeeperStatusService; + private CuratorFramework curatorFramework; + + @Before + public void setUp() throws Exception { + Logger.getLogger("").setLevel(LogLevel.WARNING); + + testingServer = new TestingServer(); + curatorFramework = createConnectedCuratorFramework(testingServer); + zookeeperStatusService = new ZookeeperStatusService(curatorFramework); + } + + private static CuratorFramework createConnectedCuratorFramework(TestingServer server) throws InterruptedException { + CuratorFramework curatorFramework = CuratorFrameworkFactory.builder() + .connectString(server.getConnectString()) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + + curatorFramework.start(); + curatorFramework.blockUntilConnected(1, TimeUnit.MINUTES); + return curatorFramework; + } + + @After + public void tearDown() throws Exception { + if (curatorFramework != null) { //teardown is called even if setUp fails. + curatorFramework.close(); + } + if (testingServer != null) { + testingServer.close(); + } + } + + @Test + public void host_state_for_unknown_hosts_is_no_remarks() { + assertThat( + zookeeperStatusService.forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE) + .getHostStatus(TestIds.HOST_NAME1), + is(HostStatus.NO_REMARKS)); + } + + @Test + public void setting_host_state_is_idempotent() { + try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly( + TestIds.APPLICATION_INSTANCE_REFERENCE)) { + + //shuffling to catch "clean database" failures for all cases. + for (HostStatus hostStatus: shuffledList(HostStatus.values())) { + doTimes(2, () -> { + statusRegistry.setHostState( + TestIds.HOST_NAME1, + hostStatus); + + assertThat(statusRegistry.getHostStatus( + TestIds.HOST_NAME1), + is(hostStatus)); + }); + } + } + } + + @Test + public void locks_are_exclusive() throws Exception { + try (CuratorFramework curatorFramework2 = createConnectedCuratorFramework(testingServer)) { + ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curatorFramework2); + + final CompletableFuture<Void> lockedSuccessfullyFuture; + try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly( + TestIds.APPLICATION_INSTANCE_REFERENCE)) { + + lockedSuccessfullyFuture = CompletableFuture.runAsync(() -> { + try (MutableStatusRegistry statusRegistry2 = zookeeperStatusService2 + .lockApplicationInstance_forCurrentThreadOnly(TestIds.APPLICATION_INSTANCE_REFERENCE)) + { + } + }); + + try { + lockedSuccessfullyFuture.get(3, TimeUnit.SECONDS); + fail("Both zookeeper host status services locked simultaneously for the same application instance"); + } catch (TimeoutException ignored) { + } + } + + lockedSuccessfullyFuture.get(1, TimeUnit.MINUTES); + } + } + + @Test + public void session_expiry_when_holding_lock_causes_operations_to_fail() throws Exception { + try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly( + TestIds.APPLICATION_INSTANCE_REFERENCE)) { + + KillSession.kill(curatorFramework.getZookeeperClient().getZooKeeper(), testingServer.getConnectString()); + + assertSessionFailed(() -> + statusRegistry.setHostState( + TestIds.HOST_NAME1, + HostStatus.ALLOWED_TO_BE_DOWN)); + + + assertSessionFailed(() -> + statusRegistry.getHostStatus( + TestIds.HOST_NAME1)); + + } + } + + @Test + public void failing_to_get_lock_closes_SessionFailRetryLoop() throws Exception { + try (CuratorFramework curatorFramework2 = createConnectedCuratorFramework(testingServer)) { + ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curatorFramework2); + + try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly( + TestIds.APPLICATION_INSTANCE_REFERENCE)) { + + //must run in separate thread, since having 2 locks in the same thread fails + CompletableFuture<Void> resultOfZkOperationAfterLockFailure = CompletableFuture.runAsync(() -> { + try { + zookeeperStatusService2.lockApplicationInstance_forCurrentThreadOnly( + TestIds.APPLICATION_INSTANCE_REFERENCE, + 1, TimeUnit.SECONDS); + fail("Both zookeeper host status services locked simultaneously for the same application instance"); + } catch (RuntimeException e) { + } + + killSession(curatorFramework2, testingServer); + + //Throws SessionFailedException if the SessionFailRetryLoop has not been closed. + zookeeperStatusService2.forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE) + .getHostStatus(TestIds.HOST_NAME1); + }); + + assertThat(resultOfZkOperationAfterLockFailure, notHoldsException()); + } + } + } + + //IsNot does not delegate to matcher.describeMismatch. See the related issue + //https://code.google.com/p/hamcrest/issues/detail?id=107 Confusing failure description when using negation + //Creating not(holdsException) directly instead. + private Matcher<Future<?>> notHoldsException() { + return new TypeSafeMatcher<Future<?>>() { + @Override + protected boolean matchesSafely(Future<?> item) { + return !getException(item).isPresent(); + } + + private Optional<Throwable> getException(Future<?> item) { + try { + item.get(); + return Optional.empty(); + } catch (ExecutionException e) { + return Optional.of(e.getCause()); + } catch (InterruptedException e) { + return Optional.of(e); + } + } + + @Override + public void describeTo(Description description) { + description.appendText("notHoldsException()"); + } + + @Override + protected void describeMismatchSafely(Future<?> item, Description mismatchDescription) { + getException(item).ifPresent( throwable -> + mismatchDescription + .appendText("Got exception: ") + .appendText(ExceptionUtils.getMessage(throwable)) + .appendText(ExceptionUtils.getFullStackTrace(throwable))); + } + }; + } + + private static void killSession(CuratorFramework curatorFramework, TestingServer testingServer) { + try { + KillSession.kill(curatorFramework.getZookeeperClient().getZooKeeper(), testingServer.getConnectString()); + } catch (Exception e) { + throw new RuntimeException("Failed killing session. ", e); + } + } + + /** + * This requirement is due to limitations in SessionFailRetryLoop + */ + @Test(expected = AssertionError.class) + public void multiple_locks_in_a_single_thread_gives_error() throws InterruptedException { + try (CuratorFramework curatorFramework2 = createConnectedCuratorFramework(testingServer)) { + ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curatorFramework2); + + try (MutableStatusRegistry statusRegistry1 = zookeeperStatusService + .lockApplicationInstance_forCurrentThreadOnly(TestIds.APPLICATION_INSTANCE_REFERENCE); + MutableStatusRegistry statusRegistry2 = zookeeperStatusService2 + .lockApplicationInstance_forCurrentThreadOnly(TestIds.APPLICATION_INSTANCE_REFERENCE2)) + { + } + } + } + + @Test + public void suspend_and_resume_application_works_and_is_symmetric() { + + // Initial state is NO_REMARK + assertThat( + zookeeperStatusService + .forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE) + .getApplicationInstanceStatus(), + is(ApplicationInstanceStatus.NO_REMARKS)); + + // Suspend + try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly( + TestIds.APPLICATION_INSTANCE_REFERENCE)) { + statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN); + } + + assertThat( + zookeeperStatusService + .forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE) + .getApplicationInstanceStatus(), + is(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN)); + + // Resume + try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly( + TestIds.APPLICATION_INSTANCE_REFERENCE)) { + statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.NO_REMARKS); + } + + assertThat( + zookeeperStatusService + .forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE) + .getApplicationInstanceStatus(), + is(ApplicationInstanceStatus.NO_REMARKS)); + } + + @Test + public void suspending_two_applications_returns_two_applications() { + Set<ApplicationInstanceReference> suspendedApps + = zookeeperStatusService.getAllSuspendedApplications(); + assertThat(suspendedApps.size(), is(0)); + + try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly( + TestIds.APPLICATION_INSTANCE_REFERENCE)) { + statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN); + } + + try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly( + TestIds.APPLICATION_INSTANCE_REFERENCE2)) { + statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN); + } + + suspendedApps = zookeeperStatusService.getAllSuspendedApplications(); + assertThat(suspendedApps.size(), is(2)); + assertThat(suspendedApps, hasItem(TestIds.APPLICATION_INSTANCE_REFERENCE)); + assertThat(suspendedApps, hasItem(TestIds.APPLICATION_INSTANCE_REFERENCE2)); + } + + private static void assertSessionFailed(Runnable statusServiceOperations) { + try { + statusServiceOperations.run(); + fail("Expected session expired exception"); + } catch (RuntimeException e) { + if (!(e.getCause() instanceof SessionFailedException)) { + throw e; + } + } + } + + //TODO: move to vespajlib + private static <T> List<T> shuffledList(T[] values) { + //new ArrayList necessary to avoid "write through" behaviour + List<T> list = new ArrayList<>(Arrays.asList(values)); + Collections.shuffle(list); + return list; + } + + //TODO: move to vespajlib + private static void doTimes(int numberOfIterations, Runnable runnable) { + for (int i = 0; i < numberOfIterations; i++) { + runnable.run(); + } + } +} |