summaryrefslogtreecommitdiffstats
path: root/orchestrator/src/test/java/com
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
committerJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
commit72231250ed81e10d66bfe70701e64fa5fe50f712 (patch)
tree2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /orchestrator/src/test/java/com
Publish
Diffstat (limited to 'orchestrator/src/test/java/com')
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java166
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java300
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java49
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestIds.java24
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestUtil.java37
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/VespaModelUtilTest.java223
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java74
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java38
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java117
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java738
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java143
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java227
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java323
13 files changed, 2459 insertions, 0 deletions
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java
new file mode 100644
index 00000000000..e1b73c1fe65
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java
@@ -0,0 +1,166 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator;
+
+
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.ClusterId;
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceCluster;
+import com.yahoo.vespa.applicationmodel.ServiceInstance;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.applicationmodel.TenantId;
+import com.yahoo.vespa.service.monitor.ServiceMonitorStatus;
+
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * A hardcoded set of applications with one storage cluster with two nodes each.
+ *
+ * @author oyving
+ * @author smorgrav
+ */
+public class DummyInstanceLookupService implements InstanceLookupService {
+
+ public static final HostName TEST1_HOST_NAME = new HostName("test1.prod.utpoia-1.vespahosted.ut1.yahoo.com");
+ public static final HostName TEST3_HOST_NAME = new HostName("test3.prod.utpoia-1.vespahosted.ut1.yahoo.com");
+ public static final HostName TEST6_HOST_NAME = new HostName("test6.prod.us-east-1.vespahosted.ne1.yahoo.com");
+
+ private static final Set<ApplicationInstance<ServiceMonitorStatus>> apps = new HashSet<>();
+
+
+ static {
+ apps.add(new ApplicationInstance<>(
+ new TenantId("test-tenant-id"),
+ new ApplicationInstanceId("application:instance"),
+ TestUtil.makeServiceClusterSet(
+ new ServiceCluster<>(
+ new ClusterId("test-cluster-id-1"),
+ new ServiceType("storagenode"),
+ TestUtil.makeServiceInstanceSet(
+ new ServiceInstance<>(
+ new ConfigId("storage/storage/1"),
+ TEST1_HOST_NAME,
+ ServiceMonitorStatus.UP),
+ new ServiceInstance<>(
+ new ConfigId("storage/storage/2"),
+ new HostName("test2.prod.utpoia-1.vespahosted.ut1.yahoo.com"),
+ ServiceMonitorStatus.UP))),
+ new ServiceCluster<>(
+ new ClusterId("clustercontroller"),
+ new ServiceType("container-clustercontroller"),
+ TestUtil.makeServiceInstanceSet(
+ new ServiceInstance<>(
+ new ConfigId("clustercontroller-1"),
+ new HostName("myclustercontroller.prod.utopia-1.vespahosted.ut1.yahoo.com"),
+ ServiceMonitorStatus.UP)))
+
+ )
+ ));
+
+ apps.add(new ApplicationInstance<>(
+ new TenantId("mediasearch"),
+ new ApplicationInstanceId("imagesearch:default"),
+ TestUtil.makeServiceClusterSet(
+ new ServiceCluster<>(
+ new ClusterId("image"),
+ new ServiceType("storagenode"),
+ TestUtil.makeServiceInstanceSet(
+ new ServiceInstance<>(
+ new ConfigId("storage/storage/3"),
+ TEST3_HOST_NAME,
+ ServiceMonitorStatus.UP),
+ new ServiceInstance<>(
+ new ConfigId("storage/storage/4"),
+ new HostName("test4.prod.utpoia-1.vespahosted.ut1.yahoo.com"),
+ ServiceMonitorStatus.UP))),
+ new ServiceCluster<>(
+ new ClusterId("clustercontroller"),
+ new ServiceType("container-clustercontroller"),
+ TestUtil.makeServiceInstanceSet(
+ new ServiceInstance<>(
+ new ConfigId("clustercontroller-1"),
+ new HostName("myclustercontroller2.prod.utopia-1.vespahosted.ut1.yahoo.com"),
+ ServiceMonitorStatus.UP)))
+ )
+ )
+ );
+
+ apps.add(new ApplicationInstance<>(
+ new TenantId("tenant-id-3"),
+ new ApplicationInstanceId("application-instance-3:default"),
+ TestUtil.makeServiceClusterSet(
+ new ServiceCluster<>(
+ new ClusterId("cluster-id-3"),
+ new ServiceType("storagenode"),
+ TestUtil.makeServiceInstanceSet(
+ new ServiceInstance<>(
+ new ConfigId("storage/storage/1"),
+ TEST6_HOST_NAME,
+ ServiceMonitorStatus.UP),
+ new ServiceInstance<>(
+ new ConfigId("storage/storage/4"),
+ new HostName("test4.prod.utpoia-1.vespahosted.ut1.yahoo.com"),
+ ServiceMonitorStatus.UP))),
+ new ServiceCluster<>(
+ new ClusterId("clustercontroller"),
+ new ServiceType("container-clustercontroller"),
+ TestUtil.makeServiceInstanceSet(
+ new ServiceInstance<>(
+ new ConfigId("clustercontroller-1"),
+ new HostName("myclustercontroller3.prod.utopia-1.vespahosted.ut1.yahoo.com"),
+ ServiceMonitorStatus.UP)))
+ )
+ ));
+ }
+
+
+ @Override
+ public Optional<ApplicationInstance<ServiceMonitorStatus>> findInstanceById(
+ final ApplicationInstanceReference applicationInstanceReference) {
+ for (ApplicationInstance<ServiceMonitorStatus> app : apps) {
+ if (app.reference().equals(applicationInstanceReference)) return Optional.of(app);
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<ApplicationInstance<ServiceMonitorStatus>> findInstanceByHost(HostName hostName) {
+ for (ApplicationInstance<ServiceMonitorStatus> app : apps) {
+ for (ServiceCluster<ServiceMonitorStatus> cluster : app.serviceClusters()) {
+ for (ServiceInstance<ServiceMonitorStatus> service : cluster.serviceInstances()) {
+ if (hostName.equals(service.hostName())) return Optional.of(app);
+ }
+ }
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ public Set<ApplicationInstanceReference> knownInstances() {
+ return apps.stream().map(a ->
+ new ApplicationInstanceReference(a.tenantId(),a.applicationInstanceId())).collect(Collectors.toSet());
+
+ }
+
+ public static Set<HostName> getContentHosts(ApplicationInstanceReference appRef) {
+ Set<HostName> hosts = apps.stream()
+ .filter(application -> application.reference().equals(appRef))
+ .flatMap(appReference -> appReference.serviceClusters().stream())
+ .filter(VespaModelUtil::isContent)
+ .flatMap(serviceCluster -> serviceCluster.serviceInstances().stream())
+ .map(ServiceInstance::hostName)
+ .collect(Collectors.toSet());
+
+ return hosts;
+ }
+
+ public static Set<ApplicationInstance<ServiceMonitorStatus>> getApplications() {
+ return apps;
+ }
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
new file mode 100644
index 00000000000..5c1d9bd45be
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
@@ -0,0 +1,300 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
+import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException;
+import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
+import com.yahoo.vespa.orchestrator.status.HostStatus;
+import com.yahoo.vespa.orchestrator.status.InMemoryStatusService;
+import com.yahoo.vespa.service.monitor.ServiceMonitorStatus;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.InOrder;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+import static com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN;
+import static com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus.NO_REMARKS;
+import static org.hamcrest.collection.IsEmptyCollection.empty;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsCollectionContaining.hasItem;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.spy;
+
+/**
+ * Test Orchestrator with a mock backend (the InMemoryStatusService)
+ *
+ * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a>
+ */
+public class OrchestratorImplTest {
+
+ private ApplicationId app1;
+ private ApplicationId app2;
+ private HostName app1_host1;
+
+ private OrchestratorImpl orchestrator;
+ private ClusterControllerClientFactoryMock clustercontroller;
+
+ @Before
+ public void setUp() throws Exception {
+ // Extract applications and hosts from dummy instance lookup service
+ Iterator<ApplicationInstance<ServiceMonitorStatus>> iterator = DummyInstanceLookupService.getApplications().iterator();
+ ApplicationInstanceReference app1_ref = iterator.next().reference();
+ app1 = OrchestratorUtil.toApplicationId(app1_ref);
+ app1_host1 = DummyInstanceLookupService.getContentHosts(app1_ref).iterator().next();
+ app2 = OrchestratorUtil.toApplicationId(iterator.next().reference());
+
+ clustercontroller = new ClusterControllerClientFactoryMock();
+ orchestrator = new OrchestratorImpl(
+ clustercontroller,
+ new InMemoryStatusService(),
+ new OrchestratorConfig(new OrchestratorConfig.Builder()),
+ new DummyInstanceLookupService());
+
+ clustercontroller.setAllDummyNodesAsUp();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ orchestrator = null;
+ clustercontroller = null;
+ }
+
+ @Test
+ public void application_has_initially_no_remarks() throws Exception {
+ assertThat(orchestrator.getApplicationInstanceStatus(app1), is(NO_REMARKS));
+ }
+
+ @Test
+ public void application_can_be_set_in_suspend() throws Exception {
+ orchestrator.suspend(app1);
+ assertThat(orchestrator.getApplicationInstanceStatus(app1), is(ALLOWED_TO_BE_DOWN));
+ }
+
+ @Test
+ public void application_can_be_removed_from_suspend() throws Exception {
+ orchestrator.suspend(app1);
+ orchestrator.resume(app1);
+ assertThat(orchestrator.getApplicationInstanceStatus(app1), is(NO_REMARKS));
+ }
+
+ @Test
+ public void appliations_list_returns_empty_initially() throws Exception {
+ assertThat(orchestrator.getAllSuspendedApplications(), is(empty()));
+ }
+
+ @Test
+ public void appliations_list_returns_suspended_apps() throws Exception {
+ // One suspended app
+ orchestrator.suspend(app1);
+ assertThat(orchestrator.getAllSuspendedApplications().size(), is(1));
+ assertThat(orchestrator.getAllSuspendedApplications(), hasItem(app1));
+
+ // Two suspended apps
+ orchestrator.suspend(app2);
+ assertThat(orchestrator.getAllSuspendedApplications().size(), is(2));
+ assertThat(orchestrator.getAllSuspendedApplications(), hasItem(app1));
+ assertThat(orchestrator.getAllSuspendedApplications(), hasItem(app2));
+
+ // Back to one when resetting one app to no_remarks
+ orchestrator.resume(app1);
+ assertThat(orchestrator.getAllSuspendedApplications().size(), is(1));
+ assertThat(orchestrator.getAllSuspendedApplications(), hasItem(app2));
+ }
+
+
+ @Test
+ public void application_operations_are_idempotent() throws Exception {
+ // Two suspends
+ orchestrator.suspend(app1);
+ orchestrator.suspend(app1);
+ assertThat(orchestrator.getApplicationInstanceStatus(app1), is(ALLOWED_TO_BE_DOWN));
+ assertThat(orchestrator.getApplicationInstanceStatus(app2), is(NO_REMARKS));
+
+ // Three no_remarks
+ orchestrator.resume(app1);
+ orchestrator.resume(app1);
+ orchestrator.resume(app1);
+ assertThat(orchestrator.getApplicationInstanceStatus(app1), is(NO_REMARKS));
+ assertThat(orchestrator.getApplicationInstanceStatus(app2), is(NO_REMARKS));
+
+ // Two suspends and two on two applications interleaved
+ orchestrator.suspend(app2);
+ orchestrator.resume(app1);
+ orchestrator.suspend(app2);
+ orchestrator.resume(app1);
+ assertThat(orchestrator.getApplicationInstanceStatus(app1), is(NO_REMARKS));
+ assertThat(orchestrator.getApplicationInstanceStatus(app2), is(ALLOWED_TO_BE_DOWN));
+ }
+
+
+ @Test
+ public void application_suspend_sets_application_nodes_in_maintenance_and_allowed_to_be_down() throws Exception {
+ // Pre condition
+ assertEquals(NO_REMARKS, orchestrator.getApplicationInstanceStatus(app1));
+ assertEquals(HostStatus.NO_REMARKS, orchestrator.getNodeStatus(app1_host1));
+ assertFalse(isInMaintenance(app1, app1_host1));
+
+ orchestrator.suspend(app1);
+
+ assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1));
+ assertTrue(isInMaintenance(app1, app1_host1));
+ }
+
+ @Test
+ public void node_suspend_while_app_is_resumed_set_allowed_to_be_down_and_set_it_in_maintenance() throws Exception {
+ // Pre condition
+ assertEquals(NO_REMARKS, orchestrator.getApplicationInstanceStatus(app1));
+ assertEquals(HostStatus.NO_REMARKS, orchestrator.getNodeStatus(app1_host1));
+ assertFalse(isInMaintenance(app1, app1_host1));
+
+ orchestrator.suspend(app1_host1);
+
+ assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1));
+ assertTrue(isInMaintenance(app1, app1_host1));
+ }
+
+ @Test
+ public void node_suspend_while_app_is_suspended_does_nothing() throws Exception {
+ // Pre condition
+ orchestrator.suspend(app1);
+ assertEquals(ALLOWED_TO_BE_DOWN, orchestrator.getApplicationInstanceStatus(app1));
+ assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1));
+ assertTrue(isInMaintenance(app1, app1_host1));
+
+ orchestrator.suspend(app1_host1);
+
+ // Should not change anything
+ assertEquals(ALLOWED_TO_BE_DOWN, orchestrator.getApplicationInstanceStatus(app1));
+ assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1));
+ assertTrue(isInMaintenance(app1, app1_host1));
+ }
+
+ @Test
+ public void node_resume_after_app_is_resumed_removes_allowed_be_down_and_set_it_up() throws Exception {
+ // Pre condition
+ orchestrator.suspend(app1);
+ assertEquals(ALLOWED_TO_BE_DOWN, orchestrator.getApplicationInstanceStatus(app1));
+ assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1));
+ assertTrue(isInMaintenance(app1, app1_host1));
+
+ orchestrator.resume(app1);
+ orchestrator.resume(app1_host1);
+
+ assertEquals(HostStatus.NO_REMARKS, orchestrator.getNodeStatus(app1_host1));
+ assertFalse(isInMaintenance(app1, app1_host1));
+ }
+
+ @Test
+ public void node_resume_while_app_is_suspended_does_nothing() throws Exception {
+ orchestrator.suspend(app1_host1);
+ orchestrator.suspend(app1);
+
+ orchestrator.resume(app1_host1);
+
+ assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, orchestrator.getNodeStatus(app1_host1));
+ assertTrue(isInMaintenance(app1, app1_host1));
+ }
+
+ @Test
+ public void applicationReferenceHasTenantAndAppInstance() {
+ InstanceLookupService service = new DummyInstanceLookupService();
+ String applicationInstanceId = service.findInstanceByHost(DummyInstanceLookupService.TEST1_HOST_NAME).get()
+ .reference().toString();
+ assertEquals("test-tenant-id:application:instance", applicationInstanceId);
+ }
+
+ @Test
+ public void sortHostNamesForSuspend() throws Exception {
+ HostName parentHostName = new HostName("parentHostName");
+ List<HostName> expectedOrder = Arrays.asList(
+ DummyInstanceLookupService.TEST3_HOST_NAME,
+ DummyInstanceLookupService.TEST1_HOST_NAME);
+
+ assertEquals(expectedOrder, orchestrator.sortHostNamesForSuspend(Arrays.asList(
+ DummyInstanceLookupService.TEST1_HOST_NAME,
+ DummyInstanceLookupService.TEST3_HOST_NAME)));
+
+ assertEquals(expectedOrder, orchestrator.sortHostNamesForSuspend(Arrays.asList(
+ DummyInstanceLookupService.TEST3_HOST_NAME,
+ DummyInstanceLookupService.TEST1_HOST_NAME)));
+ }
+
+ @Test
+ public void rollbackWorks() throws Exception {
+ // A spy is preferential because suspendAll() relies on delegating the hard work to suspend() and resume().
+ OrchestratorImpl orchestrator = spy(this.orchestrator);
+
+ doNothing().when(orchestrator).suspend(DummyInstanceLookupService.TEST3_HOST_NAME);
+
+ Throwable supensionFailure = new HostStateChangeDeniedException(
+ DummyInstanceLookupService.TEST6_HOST_NAME,
+ "some-constraint",
+ new ServiceType("foo"),
+ "error message");
+ doThrow(supensionFailure).when(orchestrator).suspend(DummyInstanceLookupService.TEST1_HOST_NAME);
+
+ doThrow(new HostStateChangeDeniedException(DummyInstanceLookupService.TEST1_HOST_NAME, "foo1-constraint", new ServiceType("foo1-service"), "foo1-message"))
+ .when(orchestrator).resume(DummyInstanceLookupService.TEST1_HOST_NAME);
+ doNothing().when(orchestrator).resume(DummyInstanceLookupService.TEST6_HOST_NAME);
+ doNothing().when(orchestrator).resume(DummyInstanceLookupService.TEST3_HOST_NAME);
+
+ try {
+ orchestrator.suspendAll(
+ new HostName("parentHostname"),
+ Arrays.asList(
+ DummyInstanceLookupService.TEST1_HOST_NAME,
+ DummyInstanceLookupService.TEST3_HOST_NAME,
+ DummyInstanceLookupService.TEST6_HOST_NAME));
+ fail();
+ } catch (BatchHostStateChangeDeniedException e) {
+ assertEquals(e.getSuppressed().length, 1);
+ assertEquals("Failed to suspend [test3.prod.utpoia-1.vespahosted.ut1.yahoo.com, " +
+ "test6.prod.us-east-1.vespahosted.ne1.yahoo.com, test1.prod.utpoia-1.vespahosted.ut1.yahoo.com] " +
+ "with parent host parentHostname: Changing the state of host " +
+ "test6.prod.us-east-1.vespahosted.ne1.yahoo.com would violate some-constraint for " +
+ "service type foo: error message; " +
+ "With suppressed throwable com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException: " +
+ "Changing the state of host test1.prod.utpoia-1.vespahosted.ut1.yahoo.com would violate " +
+ "foo1-constraint for service type foo1-service: foo1-message", e.getMessage());
+ }
+
+ InOrder order = inOrder(orchestrator);
+ order.verify(orchestrator).suspend(DummyInstanceLookupService.TEST3_HOST_NAME);
+ order.verify(orchestrator).suspend(DummyInstanceLookupService.TEST6_HOST_NAME);
+
+ // As of 2016-06-07:
+ // TEST1_HOST_NAME: test-tenant-id:application:instance
+ // TEST3_HOST_NAME: mediasearch:imagesearch:default
+ // TEST6_HOST_NAME: tenant-id-3:application-instance-3:default
+ // Meaning the order is 3, 6, then 1. For rollback/resume the order is reversed.
+ order.verify(orchestrator).resume(DummyInstanceLookupService.TEST1_HOST_NAME);
+ order.verify(orchestrator).resume(DummyInstanceLookupService.TEST6_HOST_NAME);
+ order.verify(orchestrator).resume(DummyInstanceLookupService.TEST3_HOST_NAME);
+ order.verifyNoMoreInteractions();
+ }
+
+ private boolean isInMaintenance(ApplicationId appId, HostName hostName) throws ApplicationIdNotFoundException {
+ for (ApplicationInstance<ServiceMonitorStatus> app : DummyInstanceLookupService.getApplications()) {
+ if (app.reference().equals(OrchestratorUtil.toApplicationInstanceReference(appId))) {
+ return clustercontroller.isInMaintenance(app, hostName);
+ }
+ }
+ throw new ApplicationIdNotFoundException();
+ }
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java
new file mode 100644
index 00000000000..0626bf72e60
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.TenantId;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a>
+ */
+public class OrchestratorUtilTest {
+
+ private static final ApplicationId APPID_1 = ApplicationId.from(
+ TenantName.from("mediasearch"),
+ ApplicationName.from("tumblr-search"),
+ InstanceName.defaultName());
+
+ private static final ApplicationInstanceReference APPREF_1 = new ApplicationInstanceReference(
+ new TenantId("test-tenant"),
+ new ApplicationInstanceId("test-application:test-environment:test-region:test-instance-key"));
+
+ /**
+ * Here we don't care how the internal of the different application
+ * id/reference look like as long as we get back to exactly where we
+ * started from a round trip. I.e I'm not testing validity of the
+ * different representations.
+ */
+ @Test
+ public void applicationid_conversion_are_symmetric() throws Exception {
+
+ // From appId to appRef and back
+ ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(APPID_1);
+ ApplicationId appIdRoundTrip = OrchestratorUtil.toApplicationId(appRef);
+
+ Assert.assertEquals(APPID_1, appIdRoundTrip);
+
+ // From appRef to appId and back
+ ApplicationId appId = OrchestratorUtil.toApplicationId(APPREF_1);
+ ApplicationInstanceReference appRefRoundTrip = OrchestratorUtil.toApplicationInstanceReference(appId);
+
+ Assert.assertEquals(APPREF_1, appRefRoundTrip);
+ }
+} \ No newline at end of file
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestIds.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestIds.java
new file mode 100644
index 00000000000..e9d8b498f32
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestIds.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator;
+
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.TenantId;
+
+/**
+ * @author tonytv
+ */
+public class TestIds {
+ public static final ApplicationInstanceReference APPLICATION_INSTANCE_REFERENCE =
+ new ApplicationInstanceReference(
+ new TenantId("test-tenant"),
+ new ApplicationInstanceId("test-application:test-environment:test-region:test-instance-key"));
+
+ public static final ApplicationInstanceReference APPLICATION_INSTANCE_REFERENCE2 =
+ new ApplicationInstanceReference(
+ new TenantId("test-tenant2"),
+ new ApplicationInstanceId("test-application2:test-environment:test-region:test-instance-key"));
+
+ public static final HostName HOST_NAME1 = new HostName("host1.test.corp.yahoo.com");
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestUtil.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestUtil.java
new file mode 100644
index 00000000000..244ddfc858d
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/TestUtil.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator;
+
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.ServiceCluster;
+import com.yahoo.vespa.applicationmodel.ServiceInstance;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Utility methods for creating test setups.
+ *
+ * @author <a href="mailto:bakksjo@yahoo-inc.com">Oyvind Bakksjo</a>
+ */
+public class TestUtil {
+ @SafeVarargs
+ public static <S> Set<ServiceInstance<S>> makeServiceInstanceSet(
+ final ServiceInstance<S>... serviceInstances) {
+ return new HashSet<>(Arrays.asList(serviceInstances));
+ }
+
+ @SafeVarargs
+ public static <S> Set<ServiceCluster<S>> makeServiceClusterSet(
+ final ServiceCluster<S>... serviceClusters) {
+ return new HashSet<>(Arrays.asList(serviceClusters));
+ }
+
+ public static ConfigId storageNodeConfigId(int index) {
+ return new ConfigId("storage/storage/" + index);
+ }
+
+ public static ConfigId clusterControllerConfigId(int index) {
+ return new ConfigId("admin/cluster-controllers/" + index);
+ }
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/VespaModelUtilTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/VespaModelUtilTest.java
new file mode 100644
index 00000000000..ab533528cd4
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/VespaModelUtilTest.java
@@ -0,0 +1,223 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator;
+
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ClusterId;
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceCluster;
+import com.yahoo.vespa.applicationmodel.ServiceInstance;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.applicationmodel.TenantId;
+import com.yahoo.vespa.service.monitor.ServiceMonitorStatus;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+import static com.google.common.collect.Sets.newHashSet;
+import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet;
+import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceInstanceSet;
+import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertTrue;
+import static org.fest.assertions.Assertions.assertThat;
+
+/**
+ * @author hakon
+ */
+public class VespaModelUtilTest {
+ // Cluster Controller Service Cluster
+
+ private static final ClusterId CONTENT_CLUSTER_ID = new ClusterId("content-cluster-0");
+
+ public static final HostName controller0Host = new HostName("controller-0");
+
+ private static final ServiceInstance<ServiceMonitorStatus> controller0 = new ServiceInstance<>(
+ TestUtil.clusterControllerConfigId(0),
+ controller0Host,
+ ServiceMonitorStatus.UP);
+ private static final ServiceInstance<ServiceMonitorStatus> controller1 = new ServiceInstance<>(
+ TestUtil.clusterControllerConfigId(1),
+ new HostName("controller-1"),
+ ServiceMonitorStatus.UP);
+
+ private static final ServiceCluster<ServiceMonitorStatus> controllerCluster =
+ new ServiceCluster<>(
+ new ClusterId(CONTENT_CLUSTER_ID.s() + "-controller"),
+ VespaModelUtil.CLUSTER_CONTROLLER_SERVICE_TYPE,
+ makeServiceInstanceSet(controller1, controller0));
+
+ // Distributor Service Cluster
+
+ private static final ServiceInstance<ServiceMonitorStatus> distributor0 = new ServiceInstance<>(
+ new ConfigId("distributor-config-id"),
+ new HostName("distributor-0"),
+ ServiceMonitorStatus.UP);
+
+
+ private static final ServiceCluster<ServiceMonitorStatus> distributorCluster =
+ new ServiceCluster<>(
+ CONTENT_CLUSTER_ID,
+ VespaModelUtil.DISTRIBUTOR_SERVICE_TYPE,
+ makeServiceInstanceSet(distributor0));
+
+ // Storage Node Service Cluster
+
+ public static final HostName storage0Host = new HostName("storage-0");
+ private static final ServiceInstance<ServiceMonitorStatus> storage0 = new ServiceInstance<>(
+ new ConfigId("storage-config-id"),
+ storage0Host,
+ ServiceMonitorStatus.UP);
+
+ private static final ServiceCluster<ServiceMonitorStatus> storageCluster =
+ new ServiceCluster<>(
+ CONTENT_CLUSTER_ID,
+ VespaModelUtil.STORAGENODE_SERVICE_TYPE,
+ makeServiceInstanceSet(storage0));
+
+ // Secondary Distributor Service Cluster
+
+ private static final ServiceInstance<ServiceMonitorStatus> secondaryDistributor0 = new ServiceInstance<>(
+ new ConfigId("secondary-distributor-config-id"),
+ new HostName("secondary-distributor-0"),
+ ServiceMonitorStatus.UP);
+
+ private static final ClusterId SECONDARY_CONTENT_CLUSTER_ID = new ClusterId("secondary-content-cluster-0");
+ private static final ServiceCluster<ServiceMonitorStatus> secondaryDistributorCluster =
+ new ServiceCluster<>(
+ SECONDARY_CONTENT_CLUSTER_ID,
+ VespaModelUtil.DISTRIBUTOR_SERVICE_TYPE,
+ makeServiceInstanceSet(secondaryDistributor0));
+
+ // Secondary Storage Node Service Cluster
+
+ public static final HostName secondaryStorage0Host = new HostName("secondary-storage-0");
+ private static final ServiceInstance<ServiceMonitorStatus> secondaryStorage0 = new ServiceInstance<>(
+ new ConfigId("secondary-storage-config-id"),
+ secondaryStorage0Host,
+ ServiceMonitorStatus.UP);
+
+ private static final ServiceCluster<ServiceMonitorStatus> secondaryStorageCluster =
+ new ServiceCluster<>(
+ SECONDARY_CONTENT_CLUSTER_ID,
+ VespaModelUtil.STORAGENODE_SERVICE_TYPE,
+ makeServiceInstanceSet(secondaryStorage0));
+
+ // The Application Instance
+
+ public static final ApplicationInstance<ServiceMonitorStatus> application =
+ new ApplicationInstance<>(
+ new TenantId("tenant-0"),
+ new ApplicationInstanceId("application-0"),
+ makeServiceClusterSet(
+ controllerCluster,
+ distributorCluster,
+ storageCluster,
+ secondaryDistributorCluster,
+ secondaryStorageCluster));
+
+ private ServiceCluster<?> createServiceCluster(ServiceType serviceType) {
+ return new ServiceCluster<ServiceMonitorStatus>(
+ new ClusterId("cluster-id"),
+ serviceType,
+ new HashSet<>());
+ }
+
+ @Test
+ public void verifyControllerClusterIsRecognized() {
+ ServiceCluster<?> cluster = createServiceCluster(VespaModelUtil.CLUSTER_CONTROLLER_SERVICE_TYPE);
+ assertTrue(VespaModelUtil.isClusterController(cluster));
+ }
+
+ @Test
+ public void verifyNonControllerClusterIsNotRecognized() {
+ ServiceCluster<?> cluster = createServiceCluster(new ServiceType("foo"));
+ assertFalse(VespaModelUtil.isClusterController(cluster));
+ }
+
+ @Test
+ public void verifyStorageClusterIsRecognized() {
+ ServiceCluster<?> cluster = createServiceCluster(VespaModelUtil.STORAGENODE_SERVICE_TYPE);
+ assertTrue(VespaModelUtil.isStorage(cluster));
+ cluster = createServiceCluster(VespaModelUtil.STORAGENODE_SERVICE_TYPE);
+ assertTrue(VespaModelUtil.isStorage(cluster));
+ }
+
+ @Test
+ public void verifyNonStorageClusterIsNotRecognized() {
+ ServiceCluster<?> cluster = createServiceCluster(new ServiceType("foo"));
+ assertFalse(VespaModelUtil.isStorage(cluster));
+ }
+
+ @Test
+ public void verifyContentClusterIsRecognized() {
+ ServiceCluster<?> cluster = createServiceCluster(VespaModelUtil.DISTRIBUTOR_SERVICE_TYPE);
+ assertTrue(VespaModelUtil.isContent(cluster));
+ cluster = createServiceCluster(VespaModelUtil.STORAGENODE_SERVICE_TYPE);
+ assertTrue(VespaModelUtil.isContent(cluster));
+ cluster = createServiceCluster(VespaModelUtil.SEARCHNODE_SERVICE_TYPE);
+ assertTrue(VespaModelUtil.isContent(cluster));
+ }
+
+ @Test
+ public void verifyNonContentClusterIsNotRecognized() {
+ ServiceCluster<?> cluster = createServiceCluster(new ServiceType("foo"));
+ assertFalse(VespaModelUtil.isContent(cluster));
+ }
+
+ @Test
+ public void testGettingClusterControllerInstances() {
+ Set<ServiceInstance<?>> controllers =
+ new HashSet<>(VespaModelUtil.getClusterControllerInstances(application, CONTENT_CLUSTER_ID));
+ Set<ServiceInstance<ServiceMonitorStatus>> expectedControllers = newHashSet(controller0, controller1);
+
+ assertThat(controllers).isEqualTo(expectedControllers);
+ }
+
+ @Test
+ public void testGetControllerHostName() {
+ HostName host = VespaModelUtil.getControllerHostName(application, CONTENT_CLUSTER_ID);
+ assertThat(host).isEqualTo(controller0Host);
+ }
+
+ @Test
+ public void testGetContentClusterName() {
+ ClusterId contentClusterName = VespaModelUtil.getContentClusterName(application, distributor0.hostName());
+ assertThat(CONTENT_CLUSTER_ID).isEqualTo(contentClusterName);
+ }
+
+ @Test
+ public void testGetContentClusterNameForSecondaryContentCluster() {
+ ClusterId contentClusterName = VespaModelUtil.getContentClusterName(application, secondaryDistributor0.hostName());
+ assertThat(SECONDARY_CONTENT_CLUSTER_ID).isEqualTo(contentClusterName);
+ }
+
+ @Test
+ public void testGetStorageNodeAtHost() {
+ Optional<ServiceInstance<ServiceMonitorStatus>> service =
+ VespaModelUtil.getStorageNodeAtHost(application, storage0Host);
+ assertTrue(service.isPresent());
+ assertThat(service.get()).isEqualTo(storage0);
+ }
+
+ @Test
+ public void testGetStorageNodeAtHostWithUnknownHost() {
+ Optional<ServiceInstance<ServiceMonitorStatus>> service =
+ VespaModelUtil.getStorageNodeAtHost(application, new HostName("storage-1"));
+ assertFalse(service.isPresent());
+ }
+
+ @Test
+ public void testGetClusterControllerIndex() {
+ ConfigId configId = new ConfigId("admin/cluster-controllers/2");
+ assertThat(VespaModelUtil.getClusterControllerIndex(configId)).isEqualTo(2);
+ }
+
+ @Test
+ public void testGetStorageNodeIndex() {
+ ConfigId configId = TestUtil.storageNodeConfigId(3);
+ assertThat(VespaModelUtil.getStorageNodeIndex(configId)).isEqualTo(3);
+ }
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java
new file mode 100644
index 00000000000..a682bfe8856
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.controller;
+
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ClusterId;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceInstance;
+import com.yahoo.vespa.orchestrator.DummyInstanceLookupService;
+import com.yahoo.vespa.orchestrator.VespaModelUtil;
+import com.yahoo.vespa.service.monitor.ServiceMonitorStatus;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Mock implementation of ClusterControllerClient
+ * <p>
+ *
+ * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a>
+ */
+public class ClusterControllerClientFactoryMock implements ClusterControllerClientFactory {
+ Map<String, ClusterControllerState> nodes = new HashMap<>();
+
+ public boolean isInMaintenance(ApplicationInstance<ServiceMonitorStatus> appInstance, HostName hostName) {
+ try {
+ ClusterId clusterName = VespaModelUtil.getContentClusterName(appInstance, hostName);
+ int storageNodeIndex = VespaModelUtil.getStorageNodeIndex(appInstance, hostName);
+ String globalMapKey = clusterName.s() + storageNodeIndex;
+ return nodes.getOrDefault(globalMapKey, ClusterControllerState.UP) == ClusterControllerState.MAINTENANCE;
+ } catch (Exception e) {
+ //Catch all - meant to catch cases where the node is not part of a storage cluster
+ return false;
+ }
+ }
+
+ public void setAllDummyNodesAsUp() {
+ for (ApplicationInstance<ServiceMonitorStatus> app : DummyInstanceLookupService.getApplications()) {
+ Set<HostName> hosts = DummyInstanceLookupService.getContentHosts(app.reference());
+ for (HostName host : hosts) {
+ ClusterId clusterName = VespaModelUtil.getContentClusterName(app, host);
+ int storageNodeIndex = VespaModelUtil.getStorageNodeIndex(app, host);
+ String globalMapKey = clusterName.s() + storageNodeIndex;
+ nodes.put(globalMapKey, ClusterControllerState.UP);
+ }
+ }
+ }
+
+ @Override
+ public ClusterControllerClient createClient(Collection<? extends ServiceInstance<?>> clusterControllers, String clusterName) {
+ return new ClusterControllerClient() {
+
+ @Override
+ public ClusterControllerStateResponse setNodeState(int storageNodeIndex, ClusterControllerState wantedState) throws IOException {
+ nodes.put(clusterName + storageNodeIndex, wantedState);
+ return new ClusterControllerStateResponse(true, "Yes");
+ }
+
+ @Override
+ public ClusterControllerStateResponse setApplicationState(ClusterControllerState wantedState) throws IOException {
+ Set<String> keyCopy = new HashSet<>(nodes.keySet());
+ for (String s : keyCopy) {
+ if (s.startsWith(clusterName)) {
+ nodes.put(s, wantedState);
+ }
+ }
+ return new ClusterControllerStateResponse(true, "It works");
+ }
+ };
+ }
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java
new file mode 100644
index 00000000000..68df4dce393
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.controller;
+
+import com.yahoo.vespa.jaxrs.client.JaxRsStrategy;
+import com.yahoo.vespa.jaxrs.client.LocalPassThroughJaxRsStrategy;
+import org.junit.Test;
+
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class ClusterControllerClientTest {
+ private static final String CLUSTER_NAME = "clusterName";
+ private static final int STORAGE_NODE_INDEX = 0;
+
+ @Test
+ public void correctParametersArePassedThrough() throws Exception {
+ final ClusterControllerJaxRsApi clusterControllerApi = mock(ClusterControllerJaxRsApi.class);
+ final JaxRsStrategy<ClusterControllerJaxRsApi> strategyMock = new LocalPassThroughJaxRsStrategy<>(clusterControllerApi);
+ final ClusterControllerClient clusterControllerClient = new ClusterControllerClientImpl(
+ strategyMock,
+ CLUSTER_NAME);
+
+ final ClusterControllerState wantedState = ClusterControllerState.MAINTENANCE;
+
+ clusterControllerClient.setNodeState(STORAGE_NODE_INDEX, wantedState);
+
+ final ClusterControllerStateRequest expectedNodeStateRequest = new ClusterControllerStateRequest(
+ new ClusterControllerStateRequest.State(wantedState, ClusterControllerClientImpl.REQUEST_REASON),
+ ClusterControllerStateRequest.Condition.SAFE);
+ verify(clusterControllerApi, times(1))
+ .setNodeState(
+ eq(CLUSTER_NAME),
+ eq(STORAGE_NODE_INDEX),
+ eq(expectedNodeStateRequest));
+ }
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java
new file mode 100644
index 00000000000..cc217885047
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java
@@ -0,0 +1,117 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.controller;
+
+import com.yahoo.vespa.jaxrs.client.JaxRsClientFactory;
+import com.yahoo.vespa.orchestrator.TestUtil;
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceInstance;
+import com.yahoo.vespa.service.monitor.ServiceMonitorStatus;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.hamcrest.CoreMatchers.anyOf;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class SingleInstanceClusterControllerClientFactoryTest {
+ private static final int PORT = SingleInstanceClusterControllerClientFactory.CLUSTERCONTROLLER_HARDCODED_PORT;
+ private static final String PATH = SingleInstanceClusterControllerClientFactory.CLUSTERCONTROLLER_API_PATH;
+
+ private static final HostName HOST_NAME_1 = new HostName("host1");
+ private static final HostName HOST_NAME_2 = new HostName("host2");
+ private static final HostName HOST_NAME_3 = new HostName("host3");
+
+ private final ClusterControllerJaxRsApi mockApi = mock(ClusterControllerJaxRsApi.class);
+ private final JaxRsClientFactory jaxRsClientFactory = mock(JaxRsClientFactory.class);
+ private final ClusterControllerClientFactory clientFactory
+ = new SingleInstanceClusterControllerClientFactory(jaxRsClientFactory);
+
+ @Before
+ public void setup() {
+ when(
+ jaxRsClientFactory.createClient(
+ eq(ClusterControllerJaxRsApi.class),
+ any(HostName.class),
+ anyInt(),
+ anyString()))
+ .thenReturn(mockApi);
+ }
+
+ @Test
+ public void testCreateClientWithNoClusterControllerInstances() throws Exception {
+ final Collection<ServiceInstance<ServiceMonitorStatus>> clusterControllers = Collections.emptySet();
+
+ try {
+ clientFactory.createClient(clusterControllers, "clusterName");
+ fail();
+ } catch (IllegalArgumentException e) {
+ // As expected.
+ }
+ }
+
+ @Test
+ public void testCreateClientWithSingleClusterControllerInstance() throws Exception {
+ final Collection<ServiceInstance<ServiceMonitorStatus>> clusterControllers = Collections.singleton(
+ new ServiceInstance<>(clusterControllerConfigId(1), HOST_NAME_1, ServiceMonitorStatus.UP));
+
+ clientFactory.createClient(clusterControllers, "clusterName")
+ .setNodeState(0, ClusterControllerState.MAINTENANCE);
+
+ verify(jaxRsClientFactory).createClient(
+ ClusterControllerJaxRsApi.class,
+ HOST_NAME_1,
+ PORT,
+ PATH);
+ }
+
+ @Test
+ public void testCreateClientWithTwoNonClusterControllerInstances() throws Exception {
+ final Collection<ServiceInstance<ServiceMonitorStatus>> clusterControllers = TestUtil.makeServiceInstanceSet(
+ new ServiceInstance<>(new ConfigId("not-a-cluster-controller-1"), HOST_NAME_1, ServiceMonitorStatus.UP),
+ new ServiceInstance<>(new ConfigId("not-a-cluster-controller-2"), HOST_NAME_2, ServiceMonitorStatus.UP));
+
+ try {
+ clientFactory.createClient(clusterControllers, "clusterName");
+ fail();
+ } catch (IllegalArgumentException e) {
+ // As expected.
+ }
+ }
+
+ @Test
+ public void testCreateClientWithThreeClusterControllerInstances() throws Exception {
+ final Collection<ServiceInstance<ServiceMonitorStatus>> clusterControllers = TestUtil.makeServiceInstanceSet(
+ new ServiceInstance<>(clusterControllerConfigId(1), HOST_NAME_1, ServiceMonitorStatus.UP),
+ new ServiceInstance<>(clusterControllerConfigId(2), HOST_NAME_2, ServiceMonitorStatus.UP),
+ new ServiceInstance<>(clusterControllerConfigId(3), HOST_NAME_3, ServiceMonitorStatus.UP));
+
+ clientFactory.createClient(clusterControllers, "clusterName")
+ .setNodeState(0, ClusterControllerState.MAINTENANCE);
+
+ verify(jaxRsClientFactory).createClient(
+ eq(ClusterControllerJaxRsApi.class),
+ argThat(is(anyOf(
+ equalTo(HOST_NAME_1),
+ equalTo(HOST_NAME_2),
+ equalTo(HOST_NAME_3)))),
+ eq(PORT),
+ eq(PATH));
+ }
+
+ private static ConfigId clusterControllerConfigId(final int index) {
+ return new ConfigId("admin/cluster-controllers/" + index);
+ }
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
new file mode 100644
index 00000000000..45d605b6d8a
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
@@ -0,0 +1,738 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.policy;
+
+
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ClusterId;
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceCluster;
+import com.yahoo.vespa.applicationmodel.ServiceInstance;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.applicationmodel.TenantId;
+import com.yahoo.vespa.orchestrator.TestUtil;
+import com.yahoo.vespa.orchestrator.VespaModelUtil;
+import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient;
+import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
+import com.yahoo.vespa.orchestrator.controller.ClusterControllerState;
+import com.yahoo.vespa.orchestrator.controller.ClusterControllerStateResponse;
+import com.yahoo.vespa.orchestrator.status.HostStatus;
+import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.service.monitor.ServiceMonitorStatus;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet;
+import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceInstanceSet;
+import static com.yahoo.vespa.service.monitor.ServiceMonitorStatus.DOWN;
+import static com.yahoo.vespa.service.monitor.ServiceMonitorStatus.NOT_CHECKED;
+import static com.yahoo.vespa.service.monitor.ServiceMonitorStatus.UP;
+import static org.fest.assertions.Assertions.assertThat;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author oyving
+ * @author bakksjo
+ */
+public class HostedVespaPolicyTest {
+ private static final TenantId TENANT_ID = new TenantId("tenantId");
+ private static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("applicationId");
+ private static final HostName HOST_NAME_1 = new HostName("host-1");
+ private static final HostName HOST_NAME_2 = new HostName("host-2");
+ private static final HostName HOST_NAME_3 = new HostName("host-3");
+ private static final HostName HOST_NAME_4 = new HostName("host-4");
+ private static final HostName HOST_NAME_5 = new HostName("host-5");
+ private static final ServiceType SERVICE_TYPE_1 = new ServiceType("service-1");
+ private static final ServiceType SERVICE_TYPE_2 = new ServiceType("service-2");
+
+ private final ClusterControllerClientFactory clusterControllerClientFactory
+ = mock(ClusterControllerClientFactory.class);
+ private final ClusterControllerClient client = mock(ClusterControllerClient.class);
+ {
+ when(clusterControllerClientFactory.createClient(any(), any())).thenReturn(client);
+ }
+
+ private final HostedVespaPolicy policy
+ = new HostedVespaPolicy(clusterControllerClientFactory);
+
+ private final MutableStatusRegistry mutablestatusRegistry = mock(MutableStatusRegistry.class);
+ {
+ when(mutablestatusRegistry.getHostStatus(any())).thenReturn(HostStatus.NO_REMARKS);
+ }
+
+ @Test
+ public void test_policy_everyone_agrees_everything_is_up() throws Exception {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, UP)
+ .instance(HOST_NAME_3, UP))
+ .build();
+
+ policy.grantSuspensionRequest(
+ applicationInstance,
+ HOST_NAME_1,
+ mutablestatusRegistry
+ );
+
+ verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_1, HostStatus.ALLOWED_TO_BE_DOWN);
+ }
+
+ private void grantWithAdminCluster(
+ ServiceMonitorStatus statusParallelInstanceOtherHost,
+ ServiceMonitorStatus statusInstanceOtherHost,
+ ServiceType serviceType,
+ boolean expectGranted) throws HostStateChangeDeniedException {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, statusInstanceOtherHost)
+ .instance(HOST_NAME_3, UP))
+ .addCluster(new ClusterBuilder(VespaModelUtil.ADMIN_CLUSTER_ID, serviceType)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, statusParallelInstanceOtherHost))
+ .build();
+
+ if (expectGranted) {
+ policy.grantSuspensionRequest(
+ applicationInstance,
+ HOST_NAME_1,
+ mutablestatusRegistry
+ );
+
+ verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_1, HostStatus.ALLOWED_TO_BE_DOWN);
+ } else {
+ try {
+ policy.grantSuspensionRequest(
+ applicationInstance,
+ HOST_NAME_1,
+ mutablestatusRegistry);
+ fail();
+ } catch (HostStateChangeDeniedException e) {
+ // As expected.
+ assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT);
+ }
+
+ verify(mutablestatusRegistry, never()).setHostState(any(), any());
+ }
+ }
+
+ @Test
+ public void test_parallel_cluster_down_is_ok() throws Exception {
+ grantWithAdminCluster(DOWN, UP, new ServiceType("some-service-type"), true);
+ }
+
+ @Test
+ public void test_slobrok_cluster_down_is_not_ok() throws Exception {
+ grantWithAdminCluster(DOWN, UP, VespaModelUtil.SLOBROK_SERVICE_TYPE, false);
+ }
+
+ @Test
+ public void test_other_cluster_instance_down_is_not_ok() throws Exception {
+ grantWithAdminCluster(DOWN, DOWN, new ServiceType("some-service-type"), false);
+ }
+
+ @Test
+ public void test_all_up_is_ok() throws Exception {
+ grantWithAdminCluster(UP, UP, new ServiceType("some-service-type"), true);
+ }
+
+ @Test
+ public void test_policy_other_host_allowed_to_be_down() {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, UP)
+ .instance(HOST_NAME_3, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_5, UP))
+ .build();
+
+ when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN);
+
+ try {
+ policy.grantSuspensionRequest(
+ applicationInstance,
+ HOST_NAME_3,
+ mutablestatusRegistry);
+ fail();
+ } catch (HostStateChangeDeniedException e) {
+ // As expected.
+ assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT);
+ assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_1);
+ }
+
+ verify(mutablestatusRegistry, never()).setHostState(any(), any());
+ }
+
+ @Test
+ public void test_policy_this_host_allowed_to_be_down() throws Exception {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, UP)
+ .instance(HOST_NAME_3, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_5, UP))
+ .build();
+
+ when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_3))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN);
+
+ policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry);
+
+ verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_3, HostStatus.ALLOWED_TO_BE_DOWN);
+ }
+
+ @Test
+ public void from_five_to_ten_percent_suspension() throws Exception {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, UP)
+ .instance(HOST_NAME_3, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP))
+ .build();
+
+ when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN);
+
+ policy.grantSuspensionRequest(
+ applicationInstance,
+ HOST_NAME_3,
+ mutablestatusRegistry);
+
+ verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_3, HostStatus.ALLOWED_TO_BE_DOWN);
+ }
+
+ @Test
+ public void from_ten_to_fifteen_percent_suspension() {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, UP)
+ .instance(HOST_NAME_2, UP)
+ .instance(HOST_NAME_3, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP))
+ .build();
+
+ when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN);
+
+ try {
+ policy.grantSuspensionRequest(
+ applicationInstance,
+ HOST_NAME_3,
+ mutablestatusRegistry);
+ fail();
+ } catch (HostStateChangeDeniedException e) {
+ // As expected.
+ assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT);
+ assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_1);
+ }
+
+ verify(mutablestatusRegistry, never()).setHostState(any(), any());
+ }
+
+ @Test
+ public void from_five_to_fifteen_percent_suspension() {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, UP)
+ .instance(HOST_NAME_3, UP)
+ .instance(HOST_NAME_3, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP)
+ .instance(HOST_NAME_5, UP))
+ .build();
+
+ when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN);
+
+ try {
+ policy.grantSuspensionRequest(
+ applicationInstance,
+ HOST_NAME_3,
+ mutablestatusRegistry);
+ fail();
+ } catch (HostStateChangeDeniedException e) {
+ // As expected.
+ assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT);
+ assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_1);
+ }
+
+ verify(mutablestatusRegistry, never()).setHostState(any(), any());
+ }
+
+ @Test
+ public void test_policy_no_services() throws Exception {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder().build();
+
+ HostName hostName = new HostName("test-hostname");
+ policy.grantSuspensionRequest(
+ applicationInstance,
+ hostName,
+ mutablestatusRegistry
+ );
+
+ verify(mutablestatusRegistry, times(1)).setHostState(hostName, HostStatus.ALLOWED_TO_BE_DOWN);
+ }
+
+ // The cluster name happens to be the cluster id of any of the content service clusters.
+ private static final String CONTENT_CLUSTER_NAME = "content-cluster-id";
+ private static final HostName CLUSTER_CONTROLLER_HOST = new HostName("controller-0");
+ private static final HostName STORAGE_NODE_HOST = new HostName("storage-2");
+ private static final int STORAGE_NODE_INDEX = 2;
+
+ private static final ServiceCluster<ServiceMonitorStatus> CLUSTER_CONTROLLER_SERVICE_CLUSTER = new ServiceCluster<>(
+ new ClusterId("cluster-0"),
+ VespaModelUtil.CLUSTER_CONTROLLER_SERVICE_TYPE,
+ makeServiceInstanceSet(
+ new ServiceInstance<>(
+ TestUtil.clusterControllerConfigId(0),
+ CLUSTER_CONTROLLER_HOST,
+ UP)));
+
+ private static final ServiceCluster<ServiceMonitorStatus> DISTRIBUTOR_SERVICE_CLUSTER = new ServiceCluster<>(
+ new ClusterId(CONTENT_CLUSTER_NAME),
+ VespaModelUtil.DISTRIBUTOR_SERVICE_TYPE,
+ makeServiceInstanceSet(
+ new ServiceInstance<>(
+ new ConfigId("distributor-id-1"),
+ new HostName("distributor-1"),
+ UP)));
+
+ private static final ServiceCluster<ServiceMonitorStatus> STORAGE_SERVICE_CLUSTER = new ServiceCluster<>(
+ new ClusterId(CONTENT_CLUSTER_NAME),
+ VespaModelUtil.STORAGENODE_SERVICE_TYPE,
+ makeServiceInstanceSet(
+ new ServiceInstance<>(
+ TestUtil.storageNodeConfigId(STORAGE_NODE_INDEX),
+ STORAGE_NODE_HOST,
+ UP)));
+
+ private static final ApplicationInstance<ServiceMonitorStatus> APPLICATION_INSTANCE =
+ new ApplicationInstance<>(
+ TENANT_ID,
+ APPLICATION_INSTANCE_ID,
+ makeServiceClusterSet(
+ CLUSTER_CONTROLLER_SERVICE_CLUSTER,
+ DISTRIBUTOR_SERVICE_CLUSTER,
+ STORAGE_SERVICE_CLUSTER));
+
+ // The grantSuspensionRequest() and releaseSuspensionGrant() functions happen to have similar signature,
+ // which allows us to reuse test code for testing both functions. The actual call to one of these two functions
+ // is encapsulated into the following functional interface.
+ interface PolicyFunction {
+ void grant(
+ final HostedVespaPolicy policy,
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance,
+ final HostName hostName,
+ final MutableStatusRegistry hostStatusRegistry) throws HostStateChangeDeniedException;
+ }
+
+ /**
+ * Since grantSuspensionRequest and releaseSuspensionGrant is quite similar, this test util contains the bulk
+ * of the test code used to test their common functionality.
+ *
+ * @param grantFunction Encapsulates the grant function to call
+ * @param currentHostStatus The current HostStatus of the host
+ * @param expectedNodeStateSentToClusterController The NodeState the test expects to be sent to the controller,
+ * or null if no CC request is expected to be sent.
+ * @param expectedHostStateSetOnHostStatusService The HostState the test expects to be set on the host service.
+ */
+ private void testCommonGrantFunctionality(
+ PolicyFunction grantFunction,
+ ApplicationInstance<ServiceMonitorStatus> application,
+ HostStatus currentHostStatus,
+ Optional<ClusterControllerState> expectedNodeStateSentToClusterController,
+ HostStatus expectedHostStateSetOnHostStatusService) throws Exception {
+ // There is only one service running on the host, which is a storage node.
+ // Therefore, the corresponding cluster controller will have to be contacted
+ // to ask for permission.
+
+ ClusterControllerStateResponse response = new ClusterControllerStateResponse(true, "ok");
+ // MOTD: anyInt() MUST be used for an int field, otherwise a NullPointerException is thrown!
+ // In general, special anyX() must be used for primitive fields.
+ when(client.setNodeState(anyInt(), any())).thenReturn(response);
+
+ when(mutablestatusRegistry.getHostStatus(any())).thenReturn(currentHostStatus);
+
+ // Execution phase.
+ grantFunction.grant(policy, application, STORAGE_NODE_HOST, mutablestatusRegistry);
+
+ // Verification phase.
+
+ if (expectedNodeStateSentToClusterController.isPresent()) {
+ verify(clusterControllerClientFactory, times(1))
+ .createClient(
+ CLUSTER_CONTROLLER_SERVICE_CLUSTER.serviceInstances(),
+ CONTENT_CLUSTER_NAME);
+ verify(client, times(1))
+ .setNodeState(
+ STORAGE_NODE_INDEX,
+ expectedNodeStateSentToClusterController.get());
+ } else {
+ verify(client, never()).setNodeState(anyInt(), any());
+ }
+
+ verify(mutablestatusRegistry, times(1))
+ .setHostState(
+ STORAGE_NODE_HOST,
+ expectedHostStateSetOnHostStatusService);
+ }
+
+ @Test
+ public void test_defer_to_controller() throws Exception {
+ HostStatus currentHostStatus = HostStatus.NO_REMARKS;
+ ClusterControllerState expectedNodeStateSentToClusterController = ClusterControllerState.MAINTENANCE;
+ HostStatus expectedHostStateSetOnHostStatusService = HostStatus.ALLOWED_TO_BE_DOWN;
+ testCommonGrantFunctionality(
+ HostedVespaPolicy::grantSuspensionRequest,
+ APPLICATION_INSTANCE,
+ currentHostStatus,
+ Optional.of(expectedNodeStateSentToClusterController),
+ expectedHostStateSetOnHostStatusService);
+ }
+
+ @Test
+ public void test_release_suspension_grant_gives_no_remarks() throws Exception {
+ HostStatus currentHostStatus = HostStatus.ALLOWED_TO_BE_DOWN;
+ ClusterControllerState expectedNodeStateSentToClusterController = ClusterControllerState.UP;
+ HostStatus expectedHostStateSetOnHostStatusService = HostStatus.NO_REMARKS;
+ testCommonGrantFunctionality(
+ HostedVespaPolicy::releaseSuspensionGrant,
+ APPLICATION_INSTANCE,
+ currentHostStatus,
+ Optional.of(expectedNodeStateSentToClusterController),
+ expectedHostStateSetOnHostStatusService);
+ }
+
+ @Test
+ public void okToSuspendHostWithNoConfiguredServices() throws Exception {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, DOWN)
+ .instance(HOST_NAME_2, DOWN))
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_2)
+ .instance(HOST_NAME_4, DOWN)
+ .instance(HOST_NAME_5, DOWN))
+ .build();
+
+ policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry);
+ }
+
+ @Test
+ public void okToSuspendHostWithAllItsServicesDownEvenIfOthersAreDownToo() throws Exception {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, DOWN)
+ .instance(HOST_NAME_3, DOWN))
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_2)
+ .instance(HOST_NAME_3, DOWN)
+ .instance(HOST_NAME_4, DOWN)
+ .instance(HOST_NAME_5, UP))
+ .build();
+
+ policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry);
+ }
+
+ @Test
+ public void okToSuspendStorageNodeWhenStorageIsDown() throws Exception {
+ ServiceMonitorStatus storageNodeStatus = DOWN;
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new StorageClusterBuilder()
+ // DOWN storage service => ok to suspend and no cluster controller call
+ .instance(STORAGE_NODE_HOST, DOWN, STORAGE_NODE_INDEX)
+ .instance(HOST_NAME_2, DOWN, STORAGE_NODE_INDEX + 1)
+ .instance(HOST_NAME_3, DOWN, STORAGE_NODE_INDEX + 2))
+ .addCluster(CLUSTER_CONTROLLER_SERVICE_CLUSTER)
+ .addCluster(DISTRIBUTOR_SERVICE_CLUSTER)
+ // This service has one down service on another host, which should
+ // not block us from suspension because STORAGE_NODE_HOST down too.
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_2)
+ .instance(STORAGE_NODE_HOST, DOWN)
+ .instance(HOST_NAME_4, DOWN)
+ .instance(HOST_NAME_5, UP))
+ .build();
+
+ HostStatus currentHostStatus = HostStatus.NO_REMARKS;
+ Optional<ClusterControllerState> dontExpectAnyCallsToClusterController = Optional.empty();
+ testCommonGrantFunctionality(
+ HostedVespaPolicy::grantSuspensionRequest,
+ applicationInstance,
+ currentHostStatus,
+ dontExpectAnyCallsToClusterController,
+ HostStatus.ALLOWED_TO_BE_DOWN);
+ }
+
+ @Test
+ public void denySuspendOfStorageIfOthersAreDown() throws Exception {
+ // If the storage service is up, but other hosts' storage services are down,
+ // we should be denied permission to suspend. This behavior is common for
+ // storage service and non-storage service (they differ when it comes to
+ // the cluster controller).
+ ServiceMonitorStatus storageNodeStatus = UP;
+
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new StorageClusterBuilder()
+ .instance(STORAGE_NODE_HOST, storageNodeStatus, STORAGE_NODE_INDEX)
+ .instance(HOST_NAME_2, DOWN, STORAGE_NODE_INDEX + 1)
+ .instance(HOST_NAME_3, DOWN, STORAGE_NODE_INDEX + 2))
+ .addCluster(CLUSTER_CONTROLLER_SERVICE_CLUSTER)
+ .addCluster(DISTRIBUTOR_SERVICE_CLUSTER)
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_2)
+ .instance(STORAGE_NODE_HOST, DOWN)
+ .instance(HOST_NAME_4, DOWN)
+ .instance(HOST_NAME_5, UP))
+ .build();
+
+ when(mutablestatusRegistry.getHostStatus(any())).thenReturn(HostStatus.NO_REMARKS);
+
+ try {
+ policy.grantSuspensionRequest(applicationInstance, STORAGE_NODE_HOST, mutablestatusRegistry);
+ fail();
+ } catch (HostStateChangeDeniedException e) {
+ // As expected.
+ assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT);
+ assertThat(e.getServiceType()).isEqualTo(VespaModelUtil.STORAGENODE_SERVICE_TYPE);
+ }
+ }
+
+ // In this test we verify the storage service cluster suspend policy of allowing at most 1
+ // storage service to be effectively down. The normal policy (and the one used previously for storage)
+ // is to allow 10%. Therefore, the test verifies we disallow suspending 2 hosts = 5% (some random number <10%).
+ //
+ // Since the Orchestrator doesn't allow suspending the host, the Orchestrator doesn't even bother calling the
+ // Cluster Controller. The CC also has a policy of max 1, so it's just an optimization and safety guard.
+ @Test
+ public void dontBotherCallingClusterControllerIfOtherStorageNodesAreDown() throws Exception {
+ StorageClusterBuilder clusterBuilder = new StorageClusterBuilder();
+ for (int i = 0; i < 40; ++i) {
+ clusterBuilder.instance(new HostName("host-" + i), UP, i);
+ }
+ ApplicationInstance<ServiceMonitorStatus> applicationInstance =
+ new AppBuilder().addCluster(clusterBuilder).build();
+
+ HostName host_1 = new HostName("host-1");
+ when(mutablestatusRegistry.getHostStatus(eq(host_1))).thenReturn(HostStatus.NO_REMARKS);
+
+ HostName host_2 = new HostName("host-2");
+ when(mutablestatusRegistry.getHostStatus(eq(host_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN);
+
+ try {
+ policy.grantSuspensionRequest(applicationInstance, host_1, mutablestatusRegistry);
+ fail();
+ } catch (HostStateChangeDeniedException e) {
+ // As expected.
+ assertThat(e.getConstraintName())
+ .isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT);
+ assertThat(e.getServiceType()).isEqualTo(VespaModelUtil.STORAGENODE_SERVICE_TYPE);
+ }
+
+ verify(mutablestatusRegistry, never()).setHostState(any(), any());
+ }
+
+ @Test
+ public void ownServiceInstanceDown() throws Exception {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, DOWN)
+ .instance(HOST_NAME_3, DOWN))
+ .build();
+
+ policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry);
+ }
+
+ @Test
+ public void ownServiceInstanceDown_otherServiceIsAllNotChecked() throws Exception {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, DOWN)
+ .instance(HOST_NAME_3, DOWN))
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_2)
+ .instance(HOST_NAME_3, NOT_CHECKED)
+ .instance(HOST_NAME_4, NOT_CHECKED)
+ .instance(HOST_NAME_5, NOT_CHECKED))
+ .build();
+
+ policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry);
+ }
+
+ @Test
+ public void ownServiceInstanceDown_otherServiceIsAllNotChecked_oneHostDown() throws Exception {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, DOWN)
+ .instance(HOST_NAME_3, DOWN))
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_2)
+ .instance(HOST_NAME_3, NOT_CHECKED)
+ .instance(HOST_NAME_4, NOT_CHECKED)
+ .instance(HOST_NAME_5, NOT_CHECKED))
+ .build();
+
+ when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_4))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN);
+ try {
+ policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry);
+ fail("Should not be allowed to set " + HOST_NAME_3 + " down when " + HOST_NAME_4 + " is already down.");
+ } catch (HostStateChangeDeniedException e) {
+ // As expected.
+ assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT);
+ assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_2);
+ }
+ }
+
+ @Test
+ public void ownServiceInstanceDown_otherServiceIsAllUp() throws Exception {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, DOWN)
+ .instance(HOST_NAME_3, DOWN))
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_2)
+ .instance(HOST_NAME_3, UP)
+ .instance(HOST_NAME_4, UP)
+ .instance(HOST_NAME_5, UP))
+ .build();
+
+ policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry);
+ }
+
+ @Test
+ public void hostHasTwoInstances_oneDownOneUp() throws Exception {
+ final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder()
+ .addCluster(new ClusterBuilder(SERVICE_TYPE_1)
+ .instance(HOST_NAME_1, UP)
+ .instance(HOST_NAME_2, UP)
+ .instance(HOST_NAME_3, UP)
+ .instance(HOST_NAME_3, DOWN))
+ .build();
+
+ policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry);
+ }
+
+ // Helper classes for terseness.
+
+ private static class AppBuilder {
+ private final Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters = new HashSet<>();
+
+ public AppBuilder addCluster(final ServiceCluster<ServiceMonitorStatus> cluster) {
+ serviceClusters.add(cluster);
+ return this;
+ }
+
+ public AppBuilder addCluster(final ClusterBuilder clusterBuilder) {
+ serviceClusters.add(clusterBuilder.build());
+ return this;
+ }
+
+ public AppBuilder addCluster(final StorageClusterBuilder clusterBuilder) {
+ serviceClusters.add(clusterBuilder.build());
+ return this;
+ }
+
+ public ApplicationInstance<ServiceMonitorStatus> build() {
+ return new ApplicationInstance<>(
+ TENANT_ID,
+ APPLICATION_INSTANCE_ID,
+ serviceClusters);
+ }
+ }
+
+ private static class ClusterBuilder {
+ private final ServiceType serviceType;
+ private final Set<ServiceInstance<ServiceMonitorStatus>> instances = new HashSet<>();
+ private final ClusterId clusterId;
+ private int instanceIndex = 0;
+
+ public ClusterBuilder(final ClusterId clusterId, final ServiceType serviceType) {
+ this.clusterId = clusterId;
+ this.serviceType = serviceType;
+ }
+
+ public ClusterBuilder(final ServiceType serviceType) {
+ this.clusterId = new ClusterId("clusterId");
+ this.serviceType = serviceType;
+ }
+
+ public ClusterBuilder instance(final HostName hostName, final ServiceMonitorStatus status) {
+ instances.add(new ServiceInstance<>(new ConfigId("configId-" + instanceIndex), hostName, status));
+ ++instanceIndex;
+ return this;
+ }
+
+ public ServiceCluster<ServiceMonitorStatus> build() {
+ return new ServiceCluster<>(clusterId, serviceType, instances);
+ }
+ }
+
+ private static class StorageClusterBuilder {
+ private final Set<ServiceInstance<ServiceMonitorStatus>> instances = new HashSet<>();
+
+ public StorageClusterBuilder instance(final HostName hostName, final ServiceMonitorStatus status, int index) {
+ instances.add(new ServiceInstance<>(TestUtil.storageNodeConfigId(index), hostName, status));
+ return this;
+ }
+
+ public ServiceCluster<ServiceMonitorStatus> build() {
+ return new ServiceCluster<>(new ClusterId(CONTENT_CLUSTER_NAME), VespaModelUtil.STORAGENODE_SERVICE_TYPE, instances);
+ }
+ }
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java
new file mode 100644
index 00000000000..b8396a51e1e
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java
@@ -0,0 +1,143 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.resources;
+
+import com.yahoo.application.Application;
+import com.yahoo.application.Networking;
+import com.yahoo.container.Container;
+import com.yahoo.jdisc.http.server.jetty.JettyHttpServer;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.GenericType;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.net.URI;
+import java.nio.file.Paths;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests the implementation of the orchestrators ApplicationAPI.
+ *
+ * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a>
+ */
+public class ApplicationSuspensionResourceTest {
+
+ static final String BASE_PATH = "/orchestrator/v1/suspensions/applications";
+ static final String RESOURCE_1 = "mediasearch:imagesearch:default";
+ static final String RESOURCE_2 = "test-tenant-id:application:instance";
+ static final String INVALID_RESOURCE_NAME = "something_without_colons";
+
+ Application jdiscApplication;
+ WebTarget webTarget;
+
+ @Before
+ public void setup() throws Exception {
+ jdiscApplication = Application.fromApplicationPackage(Paths.get("src/test/application"),
+ Networking.enable);
+ Client client = ClientBuilder.newClient();
+
+ JettyHttpServer serverProvider = (JettyHttpServer) Container.get().getServerProviderRegistry().allComponents().get(0);
+ String url = "http://localhost:" + serverProvider.getListenPort() + BASE_PATH;
+ webTarget = client.target(new URI(url));
+ }
+
+ @After
+ public void teardown() throws Exception {
+ jdiscApplication.close();
+ webTarget = null;
+ }
+
+ @Ignore
+ @Test
+ public void run_application_locally_for_manual_browser_testing() throws Exception {
+ System.out.println(webTarget.getUri());
+ Thread.sleep(3600 * 1000);
+ }
+
+ @Test
+ public void get_all_suspended_applications_return_empty_list_initially() throws Exception {
+ Response reply = webTarget.request().get();
+ assertEquals(200, reply.getStatus());
+ assertEquals("[]", reply.readEntity(String.class));
+ }
+
+ @Test
+ public void invalid_application_id_throws_http_400() throws Exception {
+ Response reply = webTarget.request().post(Entity.entity(INVALID_RESOURCE_NAME,MediaType.APPLICATION_JSON_TYPE));
+ assertEquals(400, reply.getStatus());
+ }
+
+ @Test
+ public void get_application_status_returns_404_for_notsuspended_and_204_for_suspended() throws Exception {
+ // Get on application that is not suspended
+ Response reply = webTarget.path(RESOURCE_1).request().get();
+ assertEquals(404, reply.getStatus());
+
+ // Post application
+ reply = webTarget.request().post(Entity.entity(RESOURCE_1,MediaType.APPLICATION_JSON_TYPE));
+ assertEquals(204, reply.getStatus());
+
+ // Get on the application that now should be in suspended
+ reply = webTarget.path(RESOURCE_1).request().get();
+ assertEquals(204, reply.getStatus());
+ }
+
+
+ @Test
+ public void delete_works_on_suspended_and_not_suspended_applications() throws Exception {
+ // Delete an application that is not suspended
+ Response reply = webTarget.path(RESOURCE_1).request().delete();
+ assertEquals(204, reply.getStatus());
+
+ // Put application in suspend
+ reply = webTarget.request().post(Entity.entity(RESOURCE_1,MediaType.APPLICATION_JSON_TYPE));
+ assertEquals(204, reply.getStatus());
+
+ // Check that it is in suspend
+ reply = webTarget.path(RESOURCE_1).request(MediaType.APPLICATION_JSON).get();
+ assertEquals(204, reply.getStatus());
+
+ // Delete it
+ reply = webTarget.path(RESOURCE_1).request().delete();
+ assertEquals(204, reply.getStatus());
+
+ // Check that it is not in suspend anymore
+ reply = webTarget.path(RESOURCE_1).request(MediaType.APPLICATION_JSON).get();
+ assertEquals(404, reply.getStatus());
+ }
+
+ @Test
+ public void list_applications_returns_the_correct_list_of_suspended_applications() throws Exception {
+ // Test that initially we have the empty set
+ Response reply = webTarget.request(MediaType.APPLICATION_JSON).get();
+ assertEquals(200, reply.getStatus());
+ assertEquals("[]", reply.readEntity(String.class));
+
+ // Add a couple of applications to maintenance
+ webTarget.request().post(Entity.entity(RESOURCE_1,MediaType.APPLICATION_JSON_TYPE));
+ webTarget.request().post(Entity.entity(RESOURCE_2,MediaType.APPLICATION_JSON_TYPE));
+ assertEquals(200, reply.getStatus());
+
+ // Test that we get them back
+ Set<String> responses = webTarget.request(MediaType.APPLICATION_JSON_TYPE)
+ .get(new GenericType<Set<String>>() {});
+ assertEquals(2, responses.size());
+
+ // Remove suspend for the first resource
+ webTarget.path(RESOURCE_1).request().delete();
+
+ // Test that we are back to the start with the empty set
+ responses = webTarget.request(MediaType.APPLICATION_JSON_TYPE)
+ .get(new GenericType<Set<String>>() {});
+ assertEquals(1, responses.size());
+ assertEquals(RESOURCE_2, responses.iterator().next());
+ }
+} \ No newline at end of file
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
new file mode 100644
index 00000000000..fb046f978c7
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
@@ -0,0 +1,227 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.resources;
+
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.applicationmodel.TenantId;
+import com.yahoo.vespa.orchestrator.InstanceLookupService;
+import com.yahoo.vespa.orchestrator.OrchestratorImpl;
+import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
+import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
+import com.yahoo.vespa.orchestrator.policy.Policy;
+import com.yahoo.vespa.orchestrator.restapi.wire.BatchHostSuspendRequest;
+import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult;
+import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
+import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
+import com.yahoo.vespa.orchestrator.status.HostStatus;
+import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.orchestrator.status.StatusService;
+import com.yahoo.vespa.service.monitor.ServiceMonitorStatus;
+import org.junit.Test;
+
+import javax.ws.rs.WebApplicationException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.Set;
+
+import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet;
+import static org.fest.assertions.Assertions.assertThat;
+import static org.fest.assertions.Fail.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class HostResourceTest {
+ private static final int SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS = 0;
+ private static final TenantId TENANT_ID = new TenantId("tenantId");
+ private static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("applicationId");
+ private static final ApplicationInstanceReference APPLICATION_INSTANCE_REFERENCE =
+ new ApplicationInstanceReference(TENANT_ID, APPLICATION_INSTANCE_ID);
+
+ private static final StatusService EVERY_HOST_IS_UP_HOST_STATUS_SERVICE = mock(StatusService.class);
+ private static final MutableStatusRegistry EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY = mock(MutableStatusRegistry.class);
+ static {
+ when(EVERY_HOST_IS_UP_HOST_STATUS_SERVICE.forApplicationInstance(eq(APPLICATION_INSTANCE_REFERENCE)))
+ .thenReturn(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY);
+ when(EVERY_HOST_IS_UP_HOST_STATUS_SERVICE.lockApplicationInstance_forCurrentThreadOnly(eq(APPLICATION_INSTANCE_REFERENCE)))
+ .thenReturn(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY);
+ when(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY.getHostStatus(any()))
+ .thenReturn(HostStatus.NO_REMARKS);
+ when(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY.getApplicationInstanceStatus())
+ .thenReturn(ApplicationInstanceStatus.NO_REMARKS);
+ }
+
+ private static final InstanceLookupService mockInstanceLookupService = mock(InstanceLookupService.class);
+ static {
+ when(mockInstanceLookupService.findInstanceByHost(any()))
+ .thenReturn(Optional.of(
+ new ApplicationInstance<>(
+ TENANT_ID,
+ APPLICATION_INSTANCE_ID,
+ makeServiceClusterSet())));
+ }
+
+
+ private static final InstanceLookupService alwaysEmptyInstanceLookUpService = new InstanceLookupService() {
+ @Override
+ public Optional<ApplicationInstance<ServiceMonitorStatus>> findInstanceById(
+ final ApplicationInstanceReference applicationInstanceReference) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<ApplicationInstance<ServiceMonitorStatus>> findInstanceByHost(final HostName hostName) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Set<ApplicationInstanceReference> knownInstances() {
+ return Collections.emptySet();
+ }
+ };
+
+ private static class AlwaysAllowPolicy implements Policy {
+ @Override
+ public void grantSuspensionRequest(
+ ApplicationInstance<ServiceMonitorStatus> applicationInstance,
+ HostName hostName,
+ MutableStatusRegistry hostStatusRegistry) {
+ }
+ @Override
+ public void releaseSuspensionGrant(
+ ApplicationInstance<ServiceMonitorStatus> applicationInstance,
+ HostName hostName,
+ MutableStatusRegistry hostStatusRegistry) {
+ }
+ }
+
+ private static final OrchestratorImpl alwaysAllowOrchestrator = new OrchestratorImpl(
+ new AlwaysAllowPolicy(),
+ new ClusterControllerClientFactoryMock(),
+ EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, mockInstanceLookupService,
+ SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS
+ );
+
+ private static final OrchestratorImpl hostNotFoundOrchestrator = new OrchestratorImpl(
+ new AlwaysAllowPolicy(),
+ new ClusterControllerClientFactoryMock(),
+ EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, alwaysEmptyInstanceLookUpService,
+ SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS
+ );
+
+ @Test
+ public void returns_200_on_success() throws Exception {
+ HostResource hostResource =
+ new HostResource(alwaysAllowOrchestrator);
+
+ final String hostName = "hostname";
+
+ UpdateHostResponse response = hostResource.suspend(hostName);
+
+ assertThat(response.hostname()).isEqualTo(hostName);
+ }
+
+ @Test
+ public void returns_200_on_success_batch() throws Exception {
+ HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator);
+ BatchHostSuspendRequest request =
+ new BatchHostSuspendRequest("parentHostname", Arrays.asList("hostname1", "hostname2"));
+ BatchOperationResult response = hostSuspensionResource.suspendAll(request);
+ assertThat(response.success());
+ }
+
+ @Test
+ public void throws_404_when_host_unknown() throws Exception {
+ try {
+ HostResource hostResource =
+ new HostResource(hostNotFoundOrchestrator);
+ hostResource.suspend("hostname");
+ fail();
+ } catch (WebApplicationException w) {
+ assertThat(w.getResponse().getStatus()).isEqualTo(404);
+ }
+ }
+
+ // Note: Missing host is 404 for a single-host, but 400 for multi-host (batch).
+ // This is so because the hostname is part of the URL path for single-host, while the
+ // hostnames are part of the request body for multi-host.
+ @Test
+ public void throws_400_when_host_unknown_for_batch() throws Exception {
+ try {
+ HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(hostNotFoundOrchestrator);
+ BatchHostSuspendRequest request =
+ new BatchHostSuspendRequest("parentHostname", Arrays.asList("hostname1", "hostname2"));
+ hostSuspensionResource.suspendAll(request);
+ fail();
+ } catch (WebApplicationException w) {
+ assertThat(w.getResponse().getStatus()).isEqualTo(400);
+ }
+ }
+
+ private static class AlwaysFailPolicy implements Policy {
+ @Override
+ public void grantSuspensionRequest(
+ ApplicationInstance<ServiceMonitorStatus> applicationInstance,
+ HostName hostName,
+ MutableStatusRegistry hostStatusRegistry) throws HostStateChangeDeniedException {
+ doThrow();
+ }
+ @Override
+ public void releaseSuspensionGrant(
+ ApplicationInstance<ServiceMonitorStatus> applicationInstance,
+ HostName hostName,
+ MutableStatusRegistry hostStatusRegistry) throws HostStateChangeDeniedException {
+ doThrow();
+ }
+
+ private static void doThrow() throws HostStateChangeDeniedException {
+ throw new HostStateChangeDeniedException(
+ new HostName("some-host"),
+ "impossible-policy",
+ new ServiceType("silly-service"),
+ "This policy rejects all requests");
+ }
+ }
+
+ @Test
+ public void throws_409_when_request_rejected_by_policies() throws Exception {
+ final OrchestratorImpl alwaysRejectResolver = new OrchestratorImpl(
+ new AlwaysFailPolicy(),
+ new ClusterControllerClientFactoryMock(),
+ EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,mockInstanceLookupService,
+ SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS);
+
+ try {
+ HostResource hostResource = new HostResource(alwaysRejectResolver);
+ hostResource.suspend("hostname");
+ fail();
+ } catch (WebApplicationException w) {
+ assertThat(w.getResponse().getStatus()).isEqualTo(409);
+ }
+ }
+
+ @Test
+ public void throws_409_when_request_rejected_by_policies_for_batch() throws Exception {
+ final OrchestratorImpl alwaysRejectResolver = new OrchestratorImpl(
+ new AlwaysFailPolicy(),
+ new ClusterControllerClientFactoryMock(),
+ EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,
+ mockInstanceLookupService,
+ SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS);
+
+ try {
+ HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysRejectResolver);
+ BatchHostSuspendRequest request =
+ new BatchHostSuspendRequest("parentHostname", Arrays.asList("hostname1", "hostname2"));
+ hostSuspensionResource.suspendAll(request);
+ fail();
+ } catch (WebApplicationException w) {
+ assertThat(w.getResponse().getStatus()).isEqualTo(409);
+ }
+ }
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java
new file mode 100644
index 00000000000..ac73e264477
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java
@@ -0,0 +1,323 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.status;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.orchestrator.TestIds;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.curator.SessionFailRetryLoop.SessionFailedException;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.curator.test.KillSession;
+import org.apache.curator.test.TestingServer;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
+import org.hamcrest.TypeSafeMatcher;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.logging.Logger;
+
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsCollectionContaining.hasItem;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+public class ZookeeperStatusServiceTest {
+ private TestingServer testingServer;
+ private ZookeeperStatusService zookeeperStatusService;
+ private CuratorFramework curatorFramework;
+
+ @Before
+ public void setUp() throws Exception {
+ Logger.getLogger("").setLevel(LogLevel.WARNING);
+
+ testingServer = new TestingServer();
+ curatorFramework = createConnectedCuratorFramework(testingServer);
+ zookeeperStatusService = new ZookeeperStatusService(curatorFramework);
+ }
+
+ private static CuratorFramework createConnectedCuratorFramework(TestingServer server) throws InterruptedException {
+ CuratorFramework curatorFramework = CuratorFrameworkFactory.builder()
+ .connectString(server.getConnectString())
+ .retryPolicy(new ExponentialBackoffRetry(1000, 3))
+ .build();
+
+ curatorFramework.start();
+ curatorFramework.blockUntilConnected(1, TimeUnit.MINUTES);
+ return curatorFramework;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (curatorFramework != null) { //teardown is called even if setUp fails.
+ curatorFramework.close();
+ }
+ if (testingServer != null) {
+ testingServer.close();
+ }
+ }
+
+ @Test
+ public void host_state_for_unknown_hosts_is_no_remarks() {
+ assertThat(
+ zookeeperStatusService.forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE)
+ .getHostStatus(TestIds.HOST_NAME1),
+ is(HostStatus.NO_REMARKS));
+ }
+
+ @Test
+ public void setting_host_state_is_idempotent() {
+ try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
+ TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+
+ //shuffling to catch "clean database" failures for all cases.
+ for (HostStatus hostStatus: shuffledList(HostStatus.values())) {
+ doTimes(2, () -> {
+ statusRegistry.setHostState(
+ TestIds.HOST_NAME1,
+ hostStatus);
+
+ assertThat(statusRegistry.getHostStatus(
+ TestIds.HOST_NAME1),
+ is(hostStatus));
+ });
+ }
+ }
+ }
+
+ @Test
+ public void locks_are_exclusive() throws Exception {
+ try (CuratorFramework curatorFramework2 = createConnectedCuratorFramework(testingServer)) {
+ ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curatorFramework2);
+
+ final CompletableFuture<Void> lockedSuccessfullyFuture;
+ try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
+ TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+
+ lockedSuccessfullyFuture = CompletableFuture.runAsync(() -> {
+ try (MutableStatusRegistry statusRegistry2 = zookeeperStatusService2
+ .lockApplicationInstance_forCurrentThreadOnly(TestIds.APPLICATION_INSTANCE_REFERENCE))
+ {
+ }
+ });
+
+ try {
+ lockedSuccessfullyFuture.get(3, TimeUnit.SECONDS);
+ fail("Both zookeeper host status services locked simultaneously for the same application instance");
+ } catch (TimeoutException ignored) {
+ }
+ }
+
+ lockedSuccessfullyFuture.get(1, TimeUnit.MINUTES);
+ }
+ }
+
+ @Test
+ public void session_expiry_when_holding_lock_causes_operations_to_fail() throws Exception {
+ try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
+ TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+
+ KillSession.kill(curatorFramework.getZookeeperClient().getZooKeeper(), testingServer.getConnectString());
+
+ assertSessionFailed(() ->
+ statusRegistry.setHostState(
+ TestIds.HOST_NAME1,
+ HostStatus.ALLOWED_TO_BE_DOWN));
+
+
+ assertSessionFailed(() ->
+ statusRegistry.getHostStatus(
+ TestIds.HOST_NAME1));
+
+ }
+ }
+
+ @Test
+ public void failing_to_get_lock_closes_SessionFailRetryLoop() throws Exception {
+ try (CuratorFramework curatorFramework2 = createConnectedCuratorFramework(testingServer)) {
+ ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curatorFramework2);
+
+ try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
+ TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+
+ //must run in separate thread, since having 2 locks in the same thread fails
+ CompletableFuture<Void> resultOfZkOperationAfterLockFailure = CompletableFuture.runAsync(() -> {
+ try {
+ zookeeperStatusService2.lockApplicationInstance_forCurrentThreadOnly(
+ TestIds.APPLICATION_INSTANCE_REFERENCE,
+ 1, TimeUnit.SECONDS);
+ fail("Both zookeeper host status services locked simultaneously for the same application instance");
+ } catch (RuntimeException e) {
+ }
+
+ killSession(curatorFramework2, testingServer);
+
+ //Throws SessionFailedException if the SessionFailRetryLoop has not been closed.
+ zookeeperStatusService2.forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE)
+ .getHostStatus(TestIds.HOST_NAME1);
+ });
+
+ assertThat(resultOfZkOperationAfterLockFailure, notHoldsException());
+ }
+ }
+ }
+
+ //IsNot does not delegate to matcher.describeMismatch. See the related issue
+ //https://code.google.com/p/hamcrest/issues/detail?id=107 Confusing failure description when using negation
+ //Creating not(holdsException) directly instead.
+ private Matcher<Future<?>> notHoldsException() {
+ return new TypeSafeMatcher<Future<?>>() {
+ @Override
+ protected boolean matchesSafely(Future<?> item) {
+ return !getException(item).isPresent();
+ }
+
+ private Optional<Throwable> getException(Future<?> item) {
+ try {
+ item.get();
+ return Optional.empty();
+ } catch (ExecutionException e) {
+ return Optional.of(e.getCause());
+ } catch (InterruptedException e) {
+ return Optional.of(e);
+ }
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ description.appendText("notHoldsException()");
+ }
+
+ @Override
+ protected void describeMismatchSafely(Future<?> item, Description mismatchDescription) {
+ getException(item).ifPresent( throwable ->
+ mismatchDescription
+ .appendText("Got exception: ")
+ .appendText(ExceptionUtils.getMessage(throwable))
+ .appendText(ExceptionUtils.getFullStackTrace(throwable)));
+ }
+ };
+ }
+
+ private static void killSession(CuratorFramework curatorFramework, TestingServer testingServer) {
+ try {
+ KillSession.kill(curatorFramework.getZookeeperClient().getZooKeeper(), testingServer.getConnectString());
+ } catch (Exception e) {
+ throw new RuntimeException("Failed killing session. ", e);
+ }
+ }
+
+ /**
+ * This requirement is due to limitations in SessionFailRetryLoop
+ */
+ @Test(expected = AssertionError.class)
+ public void multiple_locks_in_a_single_thread_gives_error() throws InterruptedException {
+ try (CuratorFramework curatorFramework2 = createConnectedCuratorFramework(testingServer)) {
+ ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curatorFramework2);
+
+ try (MutableStatusRegistry statusRegistry1 = zookeeperStatusService
+ .lockApplicationInstance_forCurrentThreadOnly(TestIds.APPLICATION_INSTANCE_REFERENCE);
+ MutableStatusRegistry statusRegistry2 = zookeeperStatusService2
+ .lockApplicationInstance_forCurrentThreadOnly(TestIds.APPLICATION_INSTANCE_REFERENCE2))
+ {
+ }
+ }
+ }
+
+ @Test
+ public void suspend_and_resume_application_works_and_is_symmetric() {
+
+ // Initial state is NO_REMARK
+ assertThat(
+ zookeeperStatusService
+ .forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE)
+ .getApplicationInstanceStatus(),
+ is(ApplicationInstanceStatus.NO_REMARKS));
+
+ // Suspend
+ try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
+ TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+ statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
+ }
+
+ assertThat(
+ zookeeperStatusService
+ .forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE)
+ .getApplicationInstanceStatus(),
+ is(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN));
+
+ // Resume
+ try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
+ TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+ statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.NO_REMARKS);
+ }
+
+ assertThat(
+ zookeeperStatusService
+ .forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE)
+ .getApplicationInstanceStatus(),
+ is(ApplicationInstanceStatus.NO_REMARKS));
+ }
+
+ @Test
+ public void suspending_two_applications_returns_two_applications() {
+ Set<ApplicationInstanceReference> suspendedApps
+ = zookeeperStatusService.getAllSuspendedApplications();
+ assertThat(suspendedApps.size(), is(0));
+
+ try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
+ TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+ statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
+ }
+
+ try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
+ TestIds.APPLICATION_INSTANCE_REFERENCE2)) {
+ statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
+ }
+
+ suspendedApps = zookeeperStatusService.getAllSuspendedApplications();
+ assertThat(suspendedApps.size(), is(2));
+ assertThat(suspendedApps, hasItem(TestIds.APPLICATION_INSTANCE_REFERENCE));
+ assertThat(suspendedApps, hasItem(TestIds.APPLICATION_INSTANCE_REFERENCE2));
+ }
+
+ private static void assertSessionFailed(Runnable statusServiceOperations) {
+ try {
+ statusServiceOperations.run();
+ fail("Expected session expired exception");
+ } catch (RuntimeException e) {
+ if (!(e.getCause() instanceof SessionFailedException)) {
+ throw e;
+ }
+ }
+ }
+
+ //TODO: move to vespajlib
+ private static <T> List<T> shuffledList(T[] values) {
+ //new ArrayList necessary to avoid "write through" behaviour
+ List<T> list = new ArrayList<>(Arrays.asList(values));
+ Collections.shuffle(list);
+ return list;
+ }
+
+ //TODO: move to vespajlib
+ private static void doTimes(int numberOfIterations, Runnable runnable) {
+ for (int i = 0; i < numberOfIterations; i++) {
+ runnable.run();
+ }
+ }
+}