diff options
author | Haakon Dybdahl <dybis@users.noreply.github.com> | 2016-06-28 14:38:18 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-06-28 14:38:18 +0200 |
commit | 95c79e22adc07931d4a935ecf6df2da0c3662ea0 (patch) | |
tree | b669bccb233d2a21a7e017c15f1379339164fa91 /node-admin | |
parent | 194ecb413e2a9876e1df499f102e282c98a93a8b (diff) | |
parent | 4968db1792c5ab8cf863f082c93307f7f3fea653 (diff) |
Merge pull request #241 from yahoo/freva/node-admin-integration-tests-3
Freva/Node-Admin integration tests 3
Diffstat (limited to 'node-admin')
5 files changed, 281 insertions, 7 deletions
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java new file mode 100644 index 00000000000..9f0227e7b38 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java @@ -0,0 +1,107 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.NodeAdmin; +import com.yahoo.vespa.hosted.node.admin.NodeAdminImpl; +import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.NodeAgent; +import com.yahoo.vespa.hosted.node.admin.NodeAgentImpl; +import com.yahoo.vespa.hosted.node.admin.docker.ContainerName; +import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Optional; +import java.util.function.Function; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author valerijf + */ +public class DockerFailTest { + private NodeRepoMock nodeRepositoryMock; + private DockerMock dockerMock; + private ContainerNodeSpec initialContainerNodeSpec; + private NodeAdminStateUpdater updater; + + @Before + public void before() throws InterruptedException { + try { + OrchestratorMock.semaphore.acquire(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + OrchestratorMock.reset(); + NodeRepoMock.reset(); + DockerMock.reset(); + + OrchestratorMock orchestratorMock = new OrchestratorMock(); + nodeRepositoryMock = new NodeRepoMock(); + dockerMock = new DockerMock(); + + Function<HostName, NodeAgent> nodeAgentFactory = (hostName) -> + new NodeAgentImpl(hostName, dockerMock, nodeRepositoryMock, orchestratorMock); + NodeAdmin nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory); + + HostName hostName = new HostName("hostName"); + initialContainerNodeSpec = new ContainerNodeSpec( + hostName, + Optional.of(new DockerImage("dockerImage")), + new ContainerName("container"), + NodeState.ACTIVE, + Optional.of(1L), + Optional.of(1L), + Optional.of(1d), + Optional.of(1d), + Optional.of(1d)); + NodeRepoMock.addContainerNodeSpec(initialContainerNodeSpec); + + updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, 1, 1, orchestratorMock, "basehostname"); + + // Wait for node admin to be notified with node repo state and the docker container has been started + while (nodeAdmin.getListOfHosts().size() == 0) { + Thread.sleep(10); + } + + while (!DockerMock.getRequests().startsWith("startContainer with DockerImage: DockerImage { imageId=dockerImage }, " + + "HostName: hostName, ContainerName: ContainerName { name=container }, minCpuCores: 1.0, minDiskAvailableGb: 1.0, " + + "minMainMemoryAvailableGb: 1.0\nexecuteInContainer with ContainerName: ContainerName { name=container }, " + + "args: [/usr/bin/env, test, -x, /opt/vespa/bin/vespa-nodectl]\nexecuteInContainer with ContainerName: " + + "ContainerName { name=container }, args: [/opt/vespa/bin/vespa-nodectl, resume]\n")) { + Thread.sleep(10); + } + } + + @After + public void after() { + updater.deconstruct(); + OrchestratorMock.semaphore.release(); + } + + @Test + public void dockerFailTest() throws InterruptedException { + dockerMock.deleteContainer(initialContainerNodeSpec.containerName); + + String goal = "startContainer with DockerImage: DockerImage { imageId=dockerImage }, HostName: hostName, " + + "ContainerName: ContainerName { name=container }, minCpuCores: 1.0, minDiskAvailableGb: 1.0, minMainMemoryAvailableGb: 1.0\n" + + "executeInContainer with ContainerName: ContainerName { name=container }, args: [/usr/bin/env, test, -x, /opt/vespa/bin/vespa-nodectl]\n" + + "executeInContainer with ContainerName: ContainerName { name=container }, args: [/opt/vespa/bin/vespa-nodectl, resume]\n" + + "deleteContainer with ContainerName: ContainerName { name=container }\n" + + "startContainer with DockerImage: DockerImage { imageId=dockerImage }, HostName: hostName, ContainerName: " + + "ContainerName { name=container }, minCpuCores: 1.0, minDiskAvailableGb: 1.0, minMainMemoryAvailableGb: 1.0\n" + + "executeInContainer with ContainerName: ContainerName { name=container }, args: [/usr/bin/env, test, -x, /opt/vespa/bin/vespa-nodectl]\n" + + "executeInContainer with ContainerName: ContainerName { name=container }, args: [/opt/vespa/bin/vespa-nodectl, resume]\n"; + while (!DockerMock.getRequests().equals(goal)) { + Thread.sleep(10); + } + + assertThat(DockerMock.getRequests(), is(goal)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java index 81de3783bb2..b6165a8d545 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java @@ -102,7 +102,9 @@ public class DockerMock implements Docker { @Override public void deleteApplicationStorage(ContainerName containerName) throws IOException { - + synchronized (monitor) { + requests.append("deleteApplicationStorage with ContainerName: ").append(containerName).append("\n"); + } } @Override diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java new file mode 100644 index 00000000000..c61c690bb98 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java @@ -0,0 +1,151 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.NodeAdmin; +import com.yahoo.vespa.hosted.node.admin.NodeAdminImpl; +import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.NodeAgent; +import com.yahoo.vespa.hosted.node.admin.NodeAgentImpl; +import com.yahoo.vespa.hosted.node.admin.docker.ContainerName; +import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Optional; +import java.util.function.Function; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author valerijf + */ +public class MultiDockerTest { + private NodeRepoMock nodeRepositoryMock; + private DockerMock dockerMock; + private NodeAdmin nodeAdmin; + private NodeAdminStateUpdater updater; + + @Before + public void before() throws InterruptedException { + try { + OrchestratorMock.semaphore.acquire(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + OrchestratorMock.reset(); + NodeRepoMock.reset(); + DockerMock.reset(); + + OrchestratorMock orchestratorMock = new OrchestratorMock(); + nodeRepositoryMock = new NodeRepoMock(); + dockerMock = new DockerMock(); + + Function<HostName, NodeAgent> nodeAgentFactory = (hostName) -> + new NodeAgentImpl(hostName, dockerMock, nodeRepositoryMock, orchestratorMock); + nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory); + updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, 1, 1, orchestratorMock, "basehostname"); + } + + @After + public void after() { + updater.deconstruct(); + OrchestratorMock.semaphore.release(); + } + + @Test + public void test() throws InterruptedException, IOException { + addAndWaitForNode(new HostName("host1"), new ContainerName("container1"), Optional.of(new DockerImage("image1"))); + ContainerNodeSpec containerNodeSpec2 = + addAndWaitForNode(new HostName("host2"), new ContainerName("container2"), Optional.of(new DockerImage("image2"))); + + + NodeRepoMock.updateContainerNodeSpec( + containerNodeSpec2.hostname, + containerNodeSpec2.wantedDockerImage, + containerNodeSpec2.containerName, + NodeState.DIRTY, + containerNodeSpec2.wantedRestartGeneration, + containerNodeSpec2.currentRestartGeneration, + containerNodeSpec2.minCpuCores, + containerNodeSpec2.minMainMemoryAvailableGb, + containerNodeSpec2.minDiskAvailableGb); + + // Wait until it is marked ready + Optional<ContainerNodeSpec> tempContainerNodeSpec; + while ((tempContainerNodeSpec = nodeRepositoryMock.getContainerNodeSpec(containerNodeSpec2.hostname)).isPresent() + && tempContainerNodeSpec.get().nodeState != NodeState.READY) { + Thread.sleep(10); + } + + + addAndWaitForNode(new HostName("host3"), new ContainerName("container3"), Optional.of(new DockerImage("image1"))); + + assertThat(DockerMock.getRequests(), is( + "startContainer with DockerImage: DockerImage { imageId=image1 }, HostName: host1, ContainerName: ContainerName { name=container1 }, " + + "minCpuCores: 1.0, minDiskAvailableGb: 1.0, minMainMemoryAvailableGb: 1.0\n" + + "executeInContainer with ContainerName: ContainerName { name=container1 }, args: [/usr/bin/env, test, -x, /opt/vespa/bin/vespa-nodectl]\n" + + "executeInContainer with ContainerName: ContainerName { name=container1 }, args: [/opt/vespa/bin/vespa-nodectl, resume]\n" + + + "startContainer with DockerImage: DockerImage { imageId=image2 }, HostName: host2, ContainerName: ContainerName { name=container2 }, " + + "minCpuCores: 1.0, minDiskAvailableGb: 1.0, minMainMemoryAvailableGb: 1.0\n" + + "executeInContainer with ContainerName: ContainerName { name=container2 }, args: [/usr/bin/env, test, -x, /opt/vespa/bin/vespa-nodectl]\n" + + "executeInContainer with ContainerName: ContainerName { name=container2 }, args: [/opt/vespa/bin/vespa-nodectl, resume]\n" + + + "stopContainer with ContainerName: ContainerName { name=container2 }\n" + + "deleteContainer with ContainerName: ContainerName { name=container2 }\n" + + "deleteApplicationStorage with ContainerName: ContainerName { name=container2 }\n" + + + "startContainer with DockerImage: DockerImage { imageId=image1 }, HostName: host3, ContainerName: ContainerName { name=container3 }, " + + "minCpuCores: 1.0, minDiskAvailableGb: 1.0, minMainMemoryAvailableGb: 1.0\n" + + "executeInContainer with ContainerName: ContainerName { name=container3 }, args: [/usr/bin/env, test, -x, /opt/vespa/bin/vespa-nodectl]\n" + + "executeInContainer with ContainerName: ContainerName { name=container3 }, args: [/opt/vespa/bin/vespa-nodectl, resume]\n")); + + + String nodeRepoExpectedRequests = + "updateNodeAttributes with HostName: host1, restartGeneration: 1, DockerImage: DockerImage { imageId=image1 }, containerVespaVersion: null\n" + + "updateNodeAttributes with HostName: host2, restartGeneration: 1, DockerImage: DockerImage { imageId=image2 }, containerVespaVersion: null\n" + + "markAsReady with HostName: host2\n" + + "updateNodeAttributes with HostName: host3, restartGeneration: 1, DockerImage: DockerImage { imageId=image1 }, containerVespaVersion: null\n"; + + while (!NodeRepoMock.getRequests().equals(nodeRepoExpectedRequests)) { + Thread.sleep(10); + } + + assertThat(NodeRepoMock.getRequests(), is(nodeRepoExpectedRequests)); + } + + private ContainerNodeSpec addAndWaitForNode(HostName hostName, ContainerName containerName, Optional<DockerImage> dockerImage) throws InterruptedException { + ContainerNodeSpec containerNodeSpec = new ContainerNodeSpec( + hostName, + dockerImage, + containerName, + NodeState.ACTIVE, + Optional.of(1L), + Optional.of(1L), + Optional.of(1d), + Optional.of(1d), + Optional.of(1d)); + NodeRepoMock.addContainerNodeSpec(containerNodeSpec); + + // Wait for node admin to be notified with node repo state and the docker container has been started + while (nodeAdmin.getListOfHosts().size() != NodeRepoMock.getNumberOfContainerSpecs()) { + Thread.sleep(10); + } + + while (!DockerMock.getRequests().endsWith("startContainer with DockerImage: " + dockerImage.get() + ", " + + "HostName: " + hostName + ", ContainerName: " + containerName + ", minCpuCores: 1.0, minDiskAvailableGb: 1.0, " + + "minMainMemoryAvailableGb: 1.0\nexecuteInContainer with ContainerName: " + containerName + ", " + + "args: [/usr/bin/env, test, -x, /opt/vespa/bin/vespa-nodectl]\nexecuteInContainer with ContainerName: " + + containerName + ", args: [/opt/vespa/bin/vespa-nodectl, resume]\n")) { + Thread.sleep(10); + } + + return containerNodeSpec; + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java index d79079261cb..bffd6ddd22a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java @@ -92,11 +92,8 @@ public class NodeRepoMock implements NodeRepository { } public static void addContainerNodeSpec(ContainerNodeSpec containerNodeSpec) { + removeContainerNodeSpec(containerNodeSpec.hostname); synchronized (monitor) { - containerNodeSpecs = containerNodeSpecs.stream() - .filter(c -> !c.hostname.equals(containerNodeSpec.hostname)) - .collect(Collectors.toList()); - containerNodeSpecs.add(containerNodeSpec); } } @@ -107,6 +104,20 @@ public class NodeRepoMock implements NodeRepository { } } + public static void removeContainerNodeSpec(HostName hostName) { + synchronized (monitor) { + containerNodeSpecs = containerNodeSpecs.stream() + .filter(c -> !c.hostname.equals(hostName)) + .collect(Collectors.toList()); + } + } + + public static int getNumberOfContainerSpecs() { + synchronized (monitor) { + return containerNodeSpecs.size(); + } + } + public static String getRequests() { synchronized (monitor) { return requests.toString(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java index dba84f81cd3..3ced257b60a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java @@ -117,11 +117,14 @@ public class NodeStateTest { // Wait until docker receives deleteContainer request - while (!DockerMock.getRequests().endsWith("deleteContainer with ContainerName: ContainerName { name=container }\n")) { + String expectedDockerRequests = "stopContainer with ContainerName: ContainerName { name=container }\n" + + "deleteContainer with ContainerName: ContainerName { name=container }\n" + + "deleteApplicationStorage with ContainerName: ContainerName { name=container }\n"; + while (!DockerMock.getRequests().endsWith(expectedDockerRequests)) { Thread.sleep(10); } - assertThat(DockerMock.getRequests(), endsWith("deleteContainer with ContainerName: ContainerName { name=container }\n")); + assertThat(DockerMock.getRequests(), endsWith(expectedDockerRequests)); } |