Added multi docker container test

4 files changed, 174 insertions, 7 deletions

diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
index 81de3783bb2..b6165a8d545 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
@@ -102,7 +102,9 @@ public class DockerMock implements Docker {
 
     @Override
     public void deleteApplicationStorage(ContainerName containerName) throws IOException {
-
+        synchronized (monitor) {
+            requests.append("deleteApplicationStorage with ContainerName: ").append(containerName).append("\n");
+        }
     }
 
     @Override
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
new file mode 100644
index 00000000000..c61c690bb98
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
@@ -0,0 +1,151 @@
+package com.yahoo.vespa.hosted.node.admin.integrationTests;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
+import com.yahoo.vespa.hosted.node.admin.NodeAdmin;
+import com.yahoo.vespa.hosted.node.admin.NodeAdminImpl;
+import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater;
+import com.yahoo.vespa.hosted.node.admin.NodeAgent;
+import com.yahoo.vespa.hosted.node.admin.NodeAgentImpl;
+import com.yahoo.vespa.hosted.node.admin.docker.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Optional;
+import java.util.function.Function;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author valerijf
+ */
+public class MultiDockerTest {
+    private NodeRepoMock nodeRepositoryMock;
+    private DockerMock dockerMock;
+    private NodeAdmin nodeAdmin;
+    private NodeAdminStateUpdater updater;
+
+    @Before
+    public void before() throws InterruptedException {
+        try {
+            OrchestratorMock.semaphore.acquire();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+
+        OrchestratorMock.reset();
+        NodeRepoMock.reset();
+        DockerMock.reset();
+
+        OrchestratorMock orchestratorMock = new OrchestratorMock();
+        nodeRepositoryMock = new NodeRepoMock();
+        dockerMock = new DockerMock();
+
+        Function<HostName, NodeAgent> nodeAgentFactory = (hostName) ->
+                new NodeAgentImpl(hostName, dockerMock, nodeRepositoryMock, orchestratorMock);
+        nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory);
+        updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, 1, 1, orchestratorMock, "basehostname");
+    }
+
+    @After
+    public void after() {
+        updater.deconstruct();
+        OrchestratorMock.semaphore.release();
+    }
+
+    @Test
+    public void test() throws InterruptedException, IOException {
+        addAndWaitForNode(new HostName("host1"), new ContainerName("container1"), Optional.of(new DockerImage("image1")));
+        ContainerNodeSpec containerNodeSpec2 =
+                addAndWaitForNode(new HostName("host2"), new ContainerName("container2"), Optional.of(new DockerImage("image2")));
+
+
+        NodeRepoMock.updateContainerNodeSpec(
+                containerNodeSpec2.hostname,
+                containerNodeSpec2.wantedDockerImage,
+                containerNodeSpec2.containerName,
+                NodeState.DIRTY,
+                containerNodeSpec2.wantedRestartGeneration,
+                containerNodeSpec2.currentRestartGeneration,
+                containerNodeSpec2.minCpuCores,
+                containerNodeSpec2.minMainMemoryAvailableGb,
+                containerNodeSpec2.minDiskAvailableGb);
+
+        // Wait until it is marked ready
+        Optional<ContainerNodeSpec> tempContainerNodeSpec;
+        while ((tempContainerNodeSpec = nodeRepositoryMock.getContainerNodeSpec(containerNodeSpec2.hostname)).isPresent()
+                && tempContainerNodeSpec.get().nodeState != NodeState.READY) {
+            Thread.sleep(10);
+        }
+
+
+        addAndWaitForNode(new HostName("host3"), new ContainerName("container3"), Optional.of(new DockerImage("image1")));
+
+        assertThat(DockerMock.getRequests(), is(
+                "startContainer with DockerImage: DockerImage { imageId=image1 }, HostName: host1, ContainerName: ContainerName { name=container1 }, " +
+                "minCpuCores: 1.0, minDiskAvailableGb: 1.0, minMainMemoryAvailableGb: 1.0\n" +
+                "executeInContainer with ContainerName: ContainerName { name=container1 }, args: [/usr/bin/env, test, -x, /opt/vespa/bin/vespa-nodectl]\n" +
+                "executeInContainer with ContainerName: ContainerName { name=container1 }, args: [/opt/vespa/bin/vespa-nodectl, resume]\n" +
+
+                "startContainer with DockerImage: DockerImage { imageId=image2 }, HostName: host2, ContainerName: ContainerName { name=container2 }, " +
+                "minCpuCores: 1.0, minDiskAvailableGb: 1.0, minMainMemoryAvailableGb: 1.0\n" +
+                "executeInContainer with ContainerName: ContainerName { name=container2 }, args: [/usr/bin/env, test, -x, /opt/vespa/bin/vespa-nodectl]\n" +
+                "executeInContainer with ContainerName: ContainerName { name=container2 }, args: [/opt/vespa/bin/vespa-nodectl, resume]\n" +
+
+                "stopContainer with ContainerName: ContainerName { name=container2 }\n" +
+                "deleteContainer with ContainerName: ContainerName { name=container2 }\n" +
+                "deleteApplicationStorage with ContainerName: ContainerName { name=container2 }\n" +
+
+                "startContainer with DockerImage: DockerImage { imageId=image1 }, HostName: host3, ContainerName: ContainerName { name=container3 }, " +
+                "minCpuCores: 1.0, minDiskAvailableGb: 1.0, minMainMemoryAvailableGb: 1.0\n" +
+                "executeInContainer with ContainerName: ContainerName { name=container3 }, args: [/usr/bin/env, test, -x, /opt/vespa/bin/vespa-nodectl]\n" +
+                "executeInContainer with ContainerName: ContainerName { name=container3 }, args: [/opt/vespa/bin/vespa-nodectl, resume]\n"));
+
+
+        String nodeRepoExpectedRequests =
+                "updateNodeAttributes with HostName: host1, restartGeneration: 1, DockerImage: DockerImage { imageId=image1 }, containerVespaVersion: null\n" +
+                "updateNodeAttributes with HostName: host2, restartGeneration: 1, DockerImage: DockerImage { imageId=image2 }, containerVespaVersion: null\n" +
+                "markAsReady with HostName: host2\n" +
+                "updateNodeAttributes with HostName: host3, restartGeneration: 1, DockerImage: DockerImage { imageId=image1 }, containerVespaVersion: null\n";
+
+        while (!NodeRepoMock.getRequests().equals(nodeRepoExpectedRequests)) {
+            Thread.sleep(10);
+        }
+
+        assertThat(NodeRepoMock.getRequests(), is(nodeRepoExpectedRequests));
+    }
+
+    private ContainerNodeSpec addAndWaitForNode(HostName hostName, ContainerName containerName, Optional<DockerImage> dockerImage) throws InterruptedException {
+        ContainerNodeSpec containerNodeSpec = new ContainerNodeSpec(
+                hostName,
+                dockerImage,
+                containerName,
+                NodeState.ACTIVE,
+                Optional.of(1L),
+                Optional.of(1L),
+                Optional.of(1d),
+                Optional.of(1d),
+                Optional.of(1d));
+        NodeRepoMock.addContainerNodeSpec(containerNodeSpec);
+
+        // Wait for node admin to be notified with node repo state and the docker container has been started
+        while (nodeAdmin.getListOfHosts().size() != NodeRepoMock.getNumberOfContainerSpecs()) {
+            Thread.sleep(10);
+        }
+
+        while (!DockerMock.getRequests().endsWith("startContainer with DockerImage: " + dockerImage.get() + ", " +
+                "HostName: " + hostName + ", ContainerName: " + containerName + ", minCpuCores: 1.0, minDiskAvailableGb: 1.0, " +
+                "minMainMemoryAvailableGb: 1.0\nexecuteInContainer with ContainerName: " + containerName + ", " +
+                "args: [/usr/bin/env, test, -x, /opt/vespa/bin/vespa-nodectl]\nexecuteInContainer with ContainerName: " +
+                containerName + ", args: [/opt/vespa/bin/vespa-nodectl, resume]\n")) {
+            Thread.sleep(10);
+        }
+
+        return containerNodeSpec;
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java
index d79079261cb..bffd6ddd22a 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java
@@ -92,11 +92,8 @@ public class NodeRepoMock implements NodeRepository {
     }
 
     public static void addContainerNodeSpec(ContainerNodeSpec containerNodeSpec) {
+        removeContainerNodeSpec(containerNodeSpec.hostname);
         synchronized (monitor) {
-            containerNodeSpecs = containerNodeSpecs.stream()
-                    .filter(c -> !c.hostname.equals(containerNodeSpec.hostname))
-                    .collect(Collectors.toList());
-
             containerNodeSpecs.add(containerNodeSpec);
         }
     }
@@ -107,6 +104,20 @@ public class NodeRepoMock implements NodeRepository {
         }
     }
 
+    public static void removeContainerNodeSpec(HostName hostName) {
+        synchronized (monitor) {
+            containerNodeSpecs = containerNodeSpecs.stream()
+                    .filter(c -> !c.hostname.equals(hostName))
+                    .collect(Collectors.toList());
+        }
+    }
+
+    public static int getNumberOfContainerSpecs() {
+        synchronized (monitor) {
+            return containerNodeSpecs.size();
+        }
+    }
+
     public static String getRequests() {
         synchronized (monitor) {
             return requests.toString();
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
index dba84f81cd3..3ced257b60a 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
@@ -117,11 +117,14 @@ public class NodeStateTest {
 
 
         // Wait until docker receives deleteContainer request
-        while (!DockerMock.getRequests().endsWith("deleteContainer with ContainerName: ContainerName { name=container }\n")) {
+        String expectedDockerRequests = "stopContainer with ContainerName: ContainerName { name=container }\n" +
+                "deleteContainer with ContainerName: ContainerName { name=container }\n" +
+                "deleteApplicationStorage with ContainerName: ContainerName { name=container }\n";
+        while (!DockerMock.getRequests().endsWith(expectedDockerRequests)) {
             Thread.sleep(10);
         }
 
-        assertThat(DockerMock.getRequests(), endsWith("deleteContainer with ContainerName: ContainerName { name=container }\n"));
+        assertThat(DockerMock.getRequests(), endsWith(expectedDockerRequests));
     }