diff options
5 files changed, 74 insertions, 820 deletions
diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index 604d0f1e6c7..9699b7bd2df 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -121,6 +121,8 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> + <source>1.8</source> + <target>1.8</target> <compilerArgs> <arg>-Xlint:rawtypes</arg> <arg>-Xlint:unchecked</arg> diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 464c4fc8cac..419c1165f4c 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -36,7 +36,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -class DockerImpl implements Docker { +public class DockerImpl implements Docker { private static final Logger logger = Logger.getLogger(DockerImpl.class.getName()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java index f6440518875..28a8e9bc1aa 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java @@ -10,15 +10,16 @@ import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; public interface DockerOperations { String getVespaVersionOrNull(ContainerName containerName); + // Returns true if container is absent on return + boolean removeContainerIfNeeded(ContainerNodeSpec nodeSpec, HostName hostname, Orchestrator orchestrator) + throws Exception; + // Returns true if started boolean startContainerIfNeeded(ContainerNodeSpec nodeSpec); // Returns false if image is already downloaded boolean shouldScheduleDownloadOfImage(DockerImage dockerImage); - boolean removeContainerIfNeeded(ContainerNodeSpec nodeSpec, HostName hostname, Orchestrator orchestrator) - throws Exception; - void scheduleDownloadOfImage(ContainerNodeSpec nodeSpec, Runnable callback); void executeResume(ContainerName containerName); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index fff9950a96f..547079205df 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -166,7 +166,7 @@ public class NodeAgentImpl implements NodeAgent { } addDebugMessage("Starting optional node program resume command"); logger.info("Starting optional node program resume command"); - dockerOperations.executeResume(nodeSpec.containerName);//, RESUME_NODE_COMMAND); + dockerOperations.executeResume(nodeSpec.containerName); containerState = RUNNING; } @@ -177,6 +177,7 @@ public class NodeAgentImpl implements NodeAgent { nodeSpec.wantedRestartGeneration.get(), nodeSpec.wantedDockerImage.get(), containerVespaVersion); + // TODO: We should only update if the new current values match the node repo's current values if (!currentAttributes.equals(lastAttributesSet)) { logger.info("Publishing new set of attributes to node repo: " + lastAttributesSet + " -> " + currentAttributes); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 1a2bd8cc3ba..5bf291c5310 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -9,25 +9,20 @@ import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; -import com.yahoo.vespa.hosted.node.admin.integrationTests.DockerMock; import com.yahoo.vespa.hosted.node.admin.maintenance.MaintenanceScheduler; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import org.junit.Ignore; +import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; import org.junit.Test; import org.mockito.InOrder; -import java.io.IOException; import java.util.Optional; -import java.util.concurrent.CompletableFuture; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.anyVararg; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; @@ -50,15 +45,14 @@ public class NodeAgentImplTest { private static final ProcessResult NODE_PROGRAM_DOESNT_EXIST = new ProcessResult(1, "", ""); private final HostName hostName = new HostName("hostname"); - private final Docker docker = mock(Docker.class); + private final Docker docker = mock(Docker.class); // TODO: Remove: Use dockerOperations only private final DockerOperations dockerOperations = mock(DockerOperations.class); private final NodeRepository nodeRepository = mock(NodeRepository.class); private final Orchestrator orchestrator = mock(Orchestrator.class); private final MaintenanceScheduler maintenanceScheduler = mock(MaintenanceScheduler.class); - private final NodeAgentImpl nodeAgent = new NodeAgentImpl(hostName, nodeRepository, orchestrator, new DockerOperationsImpl(docker), maintenanceScheduler); + private final NodeAgentImpl nodeAgent = new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, maintenanceScheduler); - @Ignore // TODO: Remove @Test public void upToDateContainerIsUntouched() throws Exception { final long restartGeneration = 1; @@ -75,37 +69,30 @@ public class NodeAgentImplTest { MIN_MAIN_MEMORY_AVAILABLE_GB, MIN_DISK_AVAILABLE_GB); final boolean isRunning = true; - final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning); final String vespaVersion = "7.8.9"; - when(dockerOperations.shouldScheduleDownloadOfImage(dockerImage)).thenReturn(false); - when(dockerOperations.removeContainerIfNeeded(eq(nodeSpec), eq(hostName), any())); - when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST); - when(docker.executeInContainer(eq(containerName), eq(DockerOperationsImpl.GET_VESPA_VERSION_COMMAND))) - .thenReturn(new ProcessResult(0, vespaVersion, "")); - + when(dockerOperations.shouldScheduleDownloadOfImage(any())).thenReturn(false); + when(dockerOperations.removeContainerIfNeeded(eq(nodeSpec), eq(hostName), any())).thenReturn(false); + when(dockerOperations.startContainerIfNeeded(eq(nodeSpec))).thenReturn(false); + when(dockerOperations.getVespaVersionOrNull(eq(containerName))).thenReturn(vespaVersion); when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)); + nodeAgent.tick(); verify(orchestrator, never()).suspend(any(HostName.class)); - verify(docker, never()).stopContainer(any(ContainerName.class)); - verify(docker, never()).deleteContainer(any(ContainerName.class)); - verify(docker, never()).createStartContainerCommand( - any(DockerImage.class), - any(ContainerName.class), - any(HostName.class)); - verify(docker, times(1)).executeInContainer(any(), anyVararg()); - final InOrder inOrder = inOrder(orchestrator, nodeRepository); + verify(dockerOperations, never()).scheduleDownloadOfImage(any(), any()); + + final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository); + // TODO: Verify this isn't run unless 1st time + inOrder.verify(dockerOperations, times(1)).executeResume(eq(containerName)); + // TODO: This should not happen when nothing is changed. Now it happens 1st time through. inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage, vespaVersion); inOrder.verify(orchestrator).resume(hostName); } - @Ignore // TODO: Remove @Test - public void newRestartGenerationCausesRestart() throws Exception { - final long wantedRestartGeneration = 2; - final long currentRestartGeneration = 1; + public void absentContainerCausesStart() throws Exception { + final long restartGeneration = 1; final DockerImage dockerImage = new DockerImage("dockerImage"); final ContainerName containerName = new ContainerName("container-name"); final ContainerNodeSpec nodeSpec = new ContainerNodeSpec( @@ -113,87 +100,31 @@ public class NodeAgentImplTest { Optional.of(dockerImage), containerName, NodeState.ACTIVE, - Optional.of(wantedRestartGeneration), - Optional.of(currentRestartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); - final boolean isRunning = true; - final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning); - final String vespaVersion = "7.8.9"; - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)).thenReturn(Optional.empty()); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST); - when(docker.executeInContainer(eq(containerName), eq(DockerOperationsImpl.GET_VESPA_VERSION_COMMAND))) - .thenReturn(new ProcessResult(0, vespaVersion, "")); - when(orchestrator.suspend(any(HostName.class))).thenReturn(true); - - nodeAgent.tick(); - - final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository); - inOrder.verify(orchestrator).suspend(hostName); - inOrder.verify(docker, times(1)).executeInContainer(any(), anyVararg()); - inOrder.verify(docker).stopContainer(containerName); - inOrder.verify(docker).deleteContainer(containerName); - inOrder.verify(docker).createStartContainerCommand( - nodeSpec.wantedDockerImage.get(), - nodeSpec.containerName, - nodeSpec.hostname); - - inOrder.verify(docker, times(1)).executeInContainer(any(), anyVararg()); - inOrder.verify(nodeRepository).updateNodeAttributes(hostName, wantedRestartGeneration, dockerImage, vespaVersion); - - inOrder.verify(orchestrator).resume(hostName); - } - - @Ignore // TODO: Remove - @Test - public void newDockerImageCausesRestart() throws Exception { - final long restartGeneration = 1; - final DockerImage currentDockerImage = new DockerImage("currentDockerImage"); - final DockerImage wantedDockerImage = new DockerImage("wantedDockerImage"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec = new ContainerNodeSpec( - hostName, - Optional.of(wantedDockerImage), - containerName, - NodeState.ACTIVE, Optional.of(restartGeneration), Optional.of(restartGeneration), MIN_CPU_CORES, MIN_MAIN_MEMORY_AVAILABLE_GB, MIN_DISK_AVAILABLE_GB); final boolean isRunning = true; - final Container existingContainer = new Container(hostName, currentDockerImage, containerName, isRunning); final String vespaVersion = "7.8.9"; - when(docker.imageIsDownloaded(wantedDockerImage)).thenReturn(true); - when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST); - when(docker.executeInContainer(eq(containerName), eq(DockerOperationsImpl.GET_VESPA_VERSION_COMMAND))) - .thenReturn(new ProcessResult(0, vespaVersion, "")); - when(orchestrator.suspend(any(HostName.class))).thenReturn(true); - - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)).thenReturn(Optional.empty()); + when(dockerOperations.shouldScheduleDownloadOfImage(any())).thenReturn(false); + when(dockerOperations.removeContainerIfNeeded(eq(nodeSpec), eq(hostName), any())).thenReturn(true); + when(dockerOperations.startContainerIfNeeded(eq(nodeSpec))).thenReturn(true); + when(dockerOperations.getVespaVersionOrNull(eq(containerName))).thenReturn(vespaVersion); when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); nodeAgent.tick(); - final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository); - inOrder.verify(orchestrator).suspend(hostName); - inOrder.verify(docker).stopContainer(containerName); - inOrder.verify(docker).deleteContainer(containerName); - inOrder.verify(docker).createStartContainerCommand( - nodeSpec.wantedDockerImage.get(), - nodeSpec.containerName, - nodeSpec.hostname); - inOrder.verify(docker, times(1)).executeInContainer(any(), anyVararg()); - inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, wantedDockerImage, vespaVersion); + verify(orchestrator, never()).suspend(any(HostName.class)); + verify(dockerOperations, never()).scheduleDownloadOfImage(any(), any()); + + final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository); + inOrder.verify(dockerOperations, times(1)).executeResume(eq(containerName)); + inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage, vespaVersion); inOrder.verify(orchestrator).resume(hostName); } - @Ignore // TODO: Remove @Test public void containerIsNotStoppedIfNewImageMustBePulled() throws Exception { final ContainerName containerName = new ContainerName("container"); @@ -211,111 +142,22 @@ public class NodeAgentImplTest { MIN_CPU_CORES, MIN_MAIN_MEMORY_AVAILABLE_GB, MIN_DISK_AVAILABLE_GB); - final Container existingContainer = new Container(hostName, oldDockerImage, containerName, true); - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)); - when(docker.imageIsDownloaded(newDockerImage)).thenReturn(false); - when(docker.pullImageAsync(newDockerImage)).thenReturn(new CompletableFuture<>()); - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - - nodeAgent.tick(); - - verify(docker, never()).stopContainer(containerName); - verify(docker).pullImageAsync(newDockerImage); - } - - @Ignore // TODO: Remove - @Test - public void stoppedContainerIsRestarted() throws Exception { - final long restartGeneration = 1; - final DockerImage dockerImage = new DockerImage("dockerImage"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec = new ContainerNodeSpec( - hostName, - Optional.of(dockerImage), - containerName, - NodeState.ACTIVE, - Optional.of(restartGeneration), - Optional.of(restartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); - final boolean isRunning = false; - final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning); - final Container existingContainer2 = new Container(hostName, dockerImage, containerName, true); - - final String vespaVersion = "7.8.9"; - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)).thenReturn(Optional.empty()).thenReturn(Optional.of(existingContainer2)); - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST); - - when(docker.executeInContainer(eq(containerName), eq(DockerOperationsImpl.GET_VESPA_VERSION_COMMAND))) - .thenReturn(new ProcessResult(0, vespaVersion, "")); - when(orchestrator.suspend(any(HostName.class))).thenReturn(true); - when(docker.createStartContainerCommand( - nodeSpec.wantedDockerImage.get(), - nodeSpec.containerName, - nodeSpec.hostname)).thenReturn(new DockerMock.StartContainerCommandMock()); - - nodeAgent.tick(); - - verify(docker, never()).stopContainer(any(ContainerName.class)); - verify(orchestrator, never()).suspend(any(HostName.class)); - verify(docker, times(1)).executeInContainer(any(), anyVararg()); - final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository); - inOrder.verify(docker).deleteContainer(containerName); - inOrder.verify(docker).createStartContainerCommand( - nodeSpec.wantedDockerImage.get(), - nodeSpec.containerName, - nodeSpec.hostname); - inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage, vespaVersion); - inOrder.verify(orchestrator).resume(hostName); - } - - @Ignore // TODO: Remove - @Test - public void missingContainerIsStarted() throws Exception { - final long restartGeneration = 1; - final DockerImage dockerImage = new DockerImage("dockerImage"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec = new ContainerNodeSpec( - hostName, - Optional.of(dockerImage), - containerName, - NodeState.ACTIVE, - Optional.of(restartGeneration), - Optional.of(restartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); final String vespaVersion = "7.8.9"; - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST); - - when(docker.executeInContainer(eq(containerName), eq(DockerOperationsImpl.GET_VESPA_VERSION_COMMAND))) - .thenReturn(new ProcessResult(0, vespaVersion, "")); - when(orchestrator.suspend(any(HostName.class))).thenReturn(true); + when(dockerOperations.shouldScheduleDownloadOfImage(any())).thenReturn(true); when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - when(docker.getContainer(hostName)).thenReturn(Optional.empty()); nodeAgent.tick(); - verify(docker, never()).stopContainer(any(ContainerName.class)); - verify(docker, never()).deleteContainer(any(ContainerName.class)); - verify(docker, times(1)).executeInContainer(any(), anyVararg()); verify(orchestrator, never()).suspend(any(HostName.class)); - final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository); - inOrder.verify(docker).createStartContainerCommand( - nodeSpec.wantedDockerImage.get(), - nodeSpec.containerName, - nodeSpec.hostname); - inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage, vespaVersion); - inOrder.verify(orchestrator).resume(hostName); + verify(orchestrator, never()).resume(any(HostName.class)); + verify(dockerOperations, never()).removeContainerIfNeeded(eq(nodeSpec), eq(hostName), any()); + + final InOrder inOrder = inOrder(dockerOperations); + inOrder.verify(dockerOperations, times(1)).shouldScheduleDownloadOfImage(eq(newDockerImage)); + inOrder.verify(dockerOperations, times(1)).scheduleDownloadOfImage(eq(nodeSpec), any()); } - @Ignore // TODO: Remove @Test public void noRestartIfOrchestratorSuspendFails() throws Exception { final long wantedRestartGeneration = 2; @@ -332,13 +174,10 @@ public class NodeAgentImplTest { MIN_CPU_CORES, MIN_MAIN_MEMORY_AVAILABLE_GB, MIN_DISK_AVAILABLE_GB); - final boolean isRunning = true; - final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning); - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)); - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); + when(dockerOperations.shouldScheduleDownloadOfImage(any())).thenReturn(false); + when(dockerOperations.removeContainerIfNeeded(eq(nodeSpec), eq(hostName), any())) + .thenThrow(new OrchestratorException("Cannot suspend")); try { nodeAgent.tick(); @@ -346,19 +185,12 @@ public class NodeAgentImplTest { } catch (Exception e) { } - verify(orchestrator).suspend(hostName); - verify(docker, never()).stopContainer(any(ContainerName.class)); - verify(docker, never()).deleteContainer(any(ContainerName.class)); - verify(docker, never()).createStartContainerCommand( - any(DockerImage.class), - any(ContainerName.class), - any(HostName.class)); + verify(dockerOperations, never()).startContainerIfNeeded(eq(nodeSpec)); verify(orchestrator, never()).resume(any(HostName.class)); verify(nodeRepository, never()).updateNodeAttributes( any(HostName.class), anyLong(), any(DockerImage.class), anyString()); } - @Ignore // TODO: Remove @Test public void failedNodeRunningContainerIsTakenDown() throws Exception { final long restartGeneration = 1; @@ -374,107 +206,17 @@ public class NodeAgentImplTest { MIN_CPU_CORES, MIN_MAIN_MEMORY_AVAILABLE_GB, MIN_DISK_AVAILABLE_GB); - final boolean isRunning = true; - final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)); - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - - nodeAgent.tick(); - - verify(orchestrator, never()).suspend(any(HostName.class)); - final InOrder inOrder = inOrder(orchestrator, docker); - inOrder.verify(docker).stopContainer(containerName); - inOrder.verify(docker).deleteContainer(containerName); - verify(docker, never()).createStartContainerCommand( - any(DockerImage.class), - any(ContainerName.class), - any(HostName.class)); - verify(maintenanceScheduler, never()).deleteContainerStorage(any(ContainerName.class)); - verify(orchestrator, never()).resume(any(HostName.class)); - verify(nodeRepository, never()).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - } - - @Ignore // TODO: Remove - @Test - public void failedNodeStoppedContainerIsTakenDown() throws Exception { - final long restartGeneration = 1; - final DockerImage dockerImage = new DockerImage("dockerImage"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec = new ContainerNodeSpec( - hostName, - Optional.of(dockerImage), - containerName, - NodeState.FAILED, - Optional.of(restartGeneration), - Optional.of(restartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); - final boolean isRunning = false; - final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)); when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); nodeAgent.tick(); - verify(orchestrator, never()).suspend(any(HostName.class)); - verify(docker, never()).stopContainer(any(ContainerName.class)); - verify(docker).deleteContainer(containerName); - verify(docker, never()).createStartContainerCommand( - any(DockerImage.class), - any(ContainerName.class), - any(HostName.class)); - verify(maintenanceScheduler, never()).deleteContainerStorage(any(ContainerName.class)); + verify(dockerOperations, times(1)).removeContainerIfNeeded(eq(nodeSpec), eq(hostName), any()); verify(orchestrator, never()).resume(any(HostName.class)); verify(nodeRepository, never()).updateNodeAttributes( any(HostName.class), anyLong(), any(DockerImage.class), anyString()); } - @Ignore // TODO: Remove - @Test - public void failedNodeNoContainerNoActionTaken() throws Exception { - final long restartGeneration = 1; - final DockerImage dockerImage = new DockerImage("dockerImage"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec = new ContainerNodeSpec( - hostName, - Optional.of(dockerImage), - containerName, - NodeState.FAILED, - Optional.of(restartGeneration), - Optional.of(restartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - - when(docker.getContainer(hostName)).thenReturn(Optional.empty()); - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - - nodeAgent.tick(); - - verify(orchestrator, never()).suspend(any(HostName.class)); - verify(docker, never()).stopContainer(any(ContainerName.class)); - verify(docker, never()).deleteContainer(containerName); - verify(docker, never()).createStartContainerCommand( - any(DockerImage.class), - any(ContainerName.class), - any(HostName.class)); - verify(maintenanceScheduler, never()).deleteContainerStorage(any(ContainerName.class)); - verify(orchestrator, never()).resume(any(HostName.class)); - verify(nodeRepository, never()).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - } - - @Ignore // TODO: Remove @Test public void inactiveNodeRunningContainerIsTakenDown() throws Exception { final long restartGeneration = 1; @@ -490,114 +232,22 @@ public class NodeAgentImplTest { MIN_CPU_CORES, MIN_MAIN_MEMORY_AVAILABLE_GB, MIN_DISK_AVAILABLE_GB); - final boolean isRunning = true; - final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning); - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)).thenReturn(Optional.empty()); when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); nodeAgent.tick(); - verify(orchestrator, never()).suspend(any(HostName.class)); - final InOrder inOrder = inOrder(orchestrator, docker); - inOrder.verify(docker).stopContainer(containerName); - inOrder.verify(docker).deleteContainer(containerName); - verify(docker, never()).createStartContainerCommand( - any(DockerImage.class), - any(ContainerName.class), - any(HostName.class)); - verify(maintenanceScheduler, never()).deleteContainerStorage(any(ContainerName.class)); - verify(orchestrator, never()).resume(any(HostName.class)); - verify(nodeRepository, never()).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - } + final InOrder inOrder = inOrder(maintenanceScheduler, dockerOperations); + inOrder.verify(maintenanceScheduler, times(1)).removeOldFilesFromNode(eq(containerName)); + inOrder.verify(dockerOperations, times(1)).removeContainerIfNeeded(eq(nodeSpec), eq(hostName), any()); - @Ignore // TODO: Remove - @Test - public void inactiveNodeStoppedContainerIsTakenDown() throws Exception { - final long restartGeneration = 1; - final DockerImage dockerImage = new DockerImage("dockerImage"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec = new ContainerNodeSpec( - hostName, - Optional.of(dockerImage), - containerName, - NodeState.INACTIVE, - Optional.of(restartGeneration), - Optional.of(restartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); - final boolean isRunning = false; - final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)).thenReturn(Optional.empty()); - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - - nodeAgent.tick(); - - verify(orchestrator, never()).suspend(any(HostName.class)); - verify(docker, never()).stopContainer(any(ContainerName.class)); - verify(docker).deleteContainer(containerName); - verify(docker, never()).createStartContainerCommand( - any(DockerImage.class), - any(ContainerName.class), - any(HostName.class)); - verify(maintenanceScheduler, never()).deleteContainerStorage(any(ContainerName.class)); - verify(orchestrator, never()).resume(any(HostName.class)); - verify(nodeRepository, never()).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - } - - @Ignore // TODO: Remove - @Test - public void inactiveNodeNoContainerNoActionTaken() throws Exception { - final long restartGeneration = 1; - final DockerImage dockerImage = new DockerImage("dockerImage"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec = new ContainerNodeSpec( - hostName, - Optional.of(dockerImage), - containerName, - NodeState.INACTIVE, - Optional.of(restartGeneration), - Optional.of(restartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - - when(docker.getContainer(hostName)).thenReturn(Optional.empty()); - - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - - nodeAgent.tick(); - - verify(orchestrator, never()).suspend(any(HostName.class)); - verify(docker, never()).stopContainer(any(ContainerName.class)); - verify(docker, never()).deleteContainer(any(ContainerName.class)); - verify(docker, never()).createStartContainerCommand( - any(DockerImage.class), - any(ContainerName.class), - any(HostName.class)); - verify(maintenanceScheduler, never()).deleteContainerStorage(any(ContainerName.class)); verify(orchestrator, never()).resume(any(HostName.class)); verify(nodeRepository, never()).updateNodeAttributes( any(HostName.class), anyLong(), any(DockerImage.class), anyString()); } - @Ignore // TODO: Remove - @Test - public void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycled() throws Exception { + private void nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState nodeState) + throws Exception { final long restartGeneration = 1; final DockerImage dockerImage = new DockerImage("dockerImage"); final ContainerName containerName = new ContainerName("container-name"); @@ -605,307 +255,39 @@ public class NodeAgentImplTest { hostName, Optional.of(dockerImage), containerName, - NodeState.DIRTY, + nodeState, Optional.of(restartGeneration), Optional.of(restartGeneration), MIN_CPU_CORES, MIN_MAIN_MEMORY_AVAILABLE_GB, MIN_DISK_AVAILABLE_GB); - final boolean isRunning = true; - final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)).thenReturn(Optional.empty()); when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); nodeAgent.tick(); - verify(orchestrator, never()).suspend(any(HostName.class)); - final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository, maintenanceScheduler); - inOrder.verify(docker).stopContainer(containerName); - inOrder.verify(docker).deleteContainer(containerName); - inOrder.verify(maintenanceScheduler).deleteContainerStorage(containerName); - inOrder.verify(nodeRepository).markAsReady(hostName); - verify(docker, never()).createStartContainerCommand( - any(DockerImage.class), - any(ContainerName.class), - any(HostName.class)); - verify(orchestrator, never()).resume(any(HostName.class)); - verify(nodeRepository, never()).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - } - - @Ignore // TODO: Remove - @Test - public void dirtyNodeStoppedContainerIsTakenDownAndCleanedAndRecycled() throws Exception { - final long restartGeneration = 1; - final DockerImage dockerImage = new DockerImage("dockerImage"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec = new ContainerNodeSpec( - hostName, - Optional.of(dockerImage), - containerName, - NodeState.DIRTY, - Optional.of(restartGeneration), - Optional.of(restartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); - final boolean isRunning = false; - final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - - when(docker.getContainer(hostName)).thenReturn(Optional.empty()); - - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)).thenReturn(Optional.empty()); + final InOrder inOrder = inOrder(maintenanceScheduler, dockerOperations, nodeRepository); + inOrder.verify(maintenanceScheduler, times(1)).removeOldFilesFromNode(eq(containerName)); + inOrder.verify(dockerOperations, times(1)).removeContainerIfNeeded(eq(nodeSpec), eq(hostName), any()); + inOrder.verify(maintenanceScheduler, times(1)).deleteContainerStorage(eq(containerName)); + inOrder.verify(nodeRepository, times(1)).markAsReady(eq(hostName)); - nodeAgent.tick(); - - verify(orchestrator, never()).suspend(any(HostName.class)); - verify(docker, never()).stopContainer(any(ContainerName.class)); - final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository, maintenanceScheduler); - inOrder.verify(docker).deleteContainer(containerName); - inOrder.verify(maintenanceScheduler).deleteContainerStorage(containerName); - inOrder.verify(nodeRepository).markAsReady(hostName); - verify(docker, never()).createStartContainerCommand( - any(DockerImage.class), - any(ContainerName.class), - any(HostName.class)); + verify(dockerOperations, never()).startContainerIfNeeded(any()); verify(orchestrator, never()).resume(any(HostName.class)); verify(nodeRepository, never()).updateNodeAttributes( any(HostName.class), anyLong(), any(DockerImage.class), anyString()); } - @Ignore // TODO: Remove @Test - public void dirtyNodeWithNoContainerIsCleanedAndRecycled() throws Exception { - final long restartGeneration = 1; - final DockerImage dockerImage = new DockerImage("dockerImage"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec = new ContainerNodeSpec( - hostName, - Optional.of(dockerImage), - containerName, - NodeState.DIRTY, - Optional.of(restartGeneration), - Optional.of(restartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - - when(docker.getContainer(hostName)).thenReturn(Optional.empty()); - - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - - nodeAgent.tick(); - - verify(orchestrator, never()).suspend(any(HostName.class)); - verify(docker, never()).stopContainer(any(ContainerName.class)); - verify(docker, never()).deleteContainer(any(ContainerName.class)); - final InOrder inOrder = inOrder(docker, nodeRepository, maintenanceScheduler); - inOrder.verify(maintenanceScheduler).deleteContainerStorage(containerName); - inOrder.verify(nodeRepository).markAsReady(hostName); - verify(docker, never()).createStartContainerCommand( - any(DockerImage.class), - any(ContainerName.class), - any(HostName.class)); - verify(orchestrator, never()).resume(any(HostName.class)); - verify(nodeRepository, never()).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); + public void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycled() throws Exception { + nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState.DIRTY); } - @Ignore // TODO: Remove @Test public void provisionedNodeWithNoContainerIsCleanedAndRecycled() throws Exception { - final long restartGeneration = 1; - final DockerImage dockerImage = new DockerImage("dockerImage"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec = new ContainerNodeSpec( - hostName, - Optional.of(dockerImage), - containerName, - NodeState.PROVISIONED, - Optional.of(restartGeneration), - Optional.of(restartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); - - when(docker.imageIsDownloaded(dockerImage)).thenReturn(true); - - - when(docker.getContainer(hostName)).thenReturn(Optional.empty()); - - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - - nodeAgent.tick(); - - verify(orchestrator, never()).suspend(any(HostName.class)); - verify(docker, never()).stopContainer(any(ContainerName.class)); - verify(docker, never()).deleteContainer(any(ContainerName.class)); - final InOrder inOrder = inOrder(docker, nodeRepository, maintenanceScheduler); - inOrder.verify(maintenanceScheduler).deleteContainerStorage(containerName); - inOrder.verify(nodeRepository).markAsReady(hostName); - verify(docker, never()).createStartContainerCommand( - any(DockerImage.class), - any(ContainerName.class), - any(HostName.class)); - verify(orchestrator, never()).resume(any(HostName.class)); - verify(nodeRepository, never()).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - } - - @Ignore // TODO: Remove - @Test - public void noRedundantNodeRepositoryCalls() throws Exception { - final long restartGeneration = 1; - final DockerImage dockerImage1 = new DockerImage("dockerImage1"); - final DockerImage dockerImage2 = new DockerImage("dockerImage2"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec1 = new ContainerNodeSpec( - hostName, - Optional.of(dockerImage1), - containerName, - NodeState.ACTIVE, - Optional.of(restartGeneration), - Optional.of(restartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); - final ContainerNodeSpec nodeSpec2 = new ContainerNodeSpec( - nodeSpec1.hostname, - Optional.of(dockerImage2), - nodeSpec1.containerName, - nodeSpec1.nodeState, - nodeSpec1.wantedRestartGeneration, - nodeSpec1.currentRestartGeneration, - nodeSpec1.minCpuCores, - nodeSpec1.minMainMemoryAvailableGb, - nodeSpec1.minDiskAvailableGb); - final boolean isRunning = true; - final Container existingContainer1 = new Container(hostName, dockerImage1, containerName, isRunning); - final Container existingContainer2 = new Container(hostName, dockerImage2, containerName, isRunning); - final String vespaVersion = "7.8.9"; - - when(docker.imageIsDownloaded(any(DockerImage.class))).thenReturn(true); - when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST); - when(docker.executeInContainer(eq(containerName), eq(DockerOperationsImpl.GET_VESPA_VERSION_COMMAND))) - .thenReturn(new ProcessResult(0, vespaVersion, "")); - when(orchestrator.suspend(any(HostName.class))).thenReturn(true); - - final InOrder inOrder = inOrder(nodeRepository, docker); - - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer1)); - - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec1)); - - nodeAgent.tick(); - - inOrder.verify(docker, times(1)).executeInContainer(any(), anyVararg()); - // Should get exactly one invocation. - inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage1, vespaVersion); - verify(nodeRepository, times(1)).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - - nodeAgent.tick(); - - inOrder.verify(docker, never()).executeInContainer(any(), anyVararg()); - // No attributes have changed; no second invocation should take place. - verify(nodeRepository, times(1)).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer1)); - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec2)); - - nodeAgent.tick(); - - inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg()); - // One attribute has changed, should cause new invocation. - inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage2, vespaVersion); - verify(nodeRepository, times(2)).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer2)); - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec2)); - - nodeAgent.tick(); - - inOrder.verify(docker, never()).executeInContainer(any(), anyVararg()); - // No attributes have changed; no new invocation should take place. - verify(nodeRepository, times(2)).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer2)); - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec1)); - nodeAgent.tick(); - - inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg()); - // Back to previous node spec should also count as new data and cause a new invocation. - inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage1, vespaVersion); - verify(nodeRepository, times(3)).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); + nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState.PROVISIONED); } - @Ignore // TODO: Remove - @Test - public void failedNodeRepositoryUpdateIsRetried() throws Exception { - final long restartGeneration = 1; - final DockerImage dockerImage1 = new DockerImage("dockerImage1"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec1 = new ContainerNodeSpec( - hostName, - Optional.of(dockerImage1), - containerName, - NodeState.ACTIVE, - Optional.of(restartGeneration), - Optional.of(restartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); - final boolean isRunning = true; - final Container existingContainer = new Container(hostName, dockerImage1, containerName, isRunning); - final String vespaVersion = "7.8.9"; - - when(docker.imageIsDownloaded(any(DockerImage.class))).thenReturn(true); - when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST); - when(docker.executeInContainer(eq(containerName), eq(DockerOperationsImpl.GET_VESPA_VERSION_COMMAND))) - .thenReturn(new ProcessResult(0, vespaVersion, "")); - when(orchestrator.suspend(any(HostName.class))).thenReturn(true); - doThrow(new IOException()).doNothing().when(nodeRepository).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - - final InOrder inOrder = inOrder(nodeRepository); - - - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)); - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec1)); - - try { - nodeAgent.tick(); - fail("Expected to throw an exception"); - } catch (Exception e) { - } - // Should get exactly one invocation. - inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage1, vespaVersion); - verify(nodeRepository, times(1)).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - - nodeAgent.tick(); - - // First attribute update failed, so it should be retried now. - inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage1, vespaVersion); - verify(nodeRepository, times(2)).updateNodeAttributes( - any(HostName.class), anyLong(), any(DockerImage.class), anyString()); - } - - @Ignore // TODO: Remove @Test public void resumeProgramRunsUntilSuccess() throws Exception { final long restartGeneration = 1; @@ -923,165 +305,33 @@ public class NodeAgentImplTest { MIN_MAIN_MEMORY_AVAILABLE_GB, MIN_DISK_AVAILABLE_GB); final String vespaVersion = "7.8.9"; - final boolean isRunning = true; - final Optional<Container> uptodateContainer = Optional.of( - new Container(hostName, wantedDockerImage, containerName, isRunning)); - when(docker.imageIsDownloaded(wantedDockerImage)).thenReturn(true); - when(docker.executeInContainer(eq(containerName), anyVararg())) - .thenReturn(new ProcessResult(0, "node program exists", "")) - .thenReturn(new ProcessResult(1, "node program fails 1st time", "")) - .thenReturn(new ProcessResult(0, "node program exists", "")) - .thenReturn(new ProcessResult(1, "node program fails 2nd time", "")) - .thenReturn(new ProcessResult(0, "node program exists", "")) - .thenReturn(new ProcessResult(0, "node program succeeds 3rd time", "")); + when(nodeRepository.getContainerNodeSpec(eq(hostName))).thenReturn(Optional.of(nodeSpec)); + when(dockerOperations.shouldScheduleDownloadOfImage(eq(wantedDockerImage))).thenReturn(false); + when(dockerOperations.removeContainerIfNeeded(eq(nodeSpec), eq(hostName), any())).thenReturn(true); + when(dockerOperations.getVespaVersionOrNull(eq(containerName))).thenReturn(vespaVersion); - when(docker.executeInContainer(eq(containerName), eq(DockerOperationsImpl.GET_VESPA_VERSION_COMMAND))) - .thenReturn(new ProcessResult(0, vespaVersion, "")); + doThrow(new RuntimeException("Failed 1st time")) + .doNothing() + .when(dockerOperations).executeResume(eq(containerName)); - final InOrder inOrder = inOrder(orchestrator, docker); + final InOrder inOrder = inOrder(orchestrator, dockerOperations, nodeRepository); // 1st try - when(docker.getContainer(hostName)).thenReturn(NO_CONTAINER); - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - try { nodeAgent.tick(); fail("Expected to throw an exception"); - } catch (Exception e) { + } catch (RuntimeException e) { } - inOrder.verify(docker).createStartContainerCommand( - nodeSpec.wantedDockerImage.get(), - nodeSpec.containerName, - nodeSpec.hostname); - inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg()); - inOrder.verifyNoMoreInteractions(); - // 2nd try - try { - nodeAgent.tick(); - fail("Expected to throw an exception"); - } catch (Exception e) { - } - - inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg()); + inOrder.verify(dockerOperations, times(1)).executeResume(any()); inOrder.verifyNoMoreInteractions(); - // 3rd try success - nodeAgent.tick(); - - inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg()); - inOrder.verify(orchestrator).resume(hostName); - inOrder.verifyNoMoreInteractions(); - - // 4th and 5th times, already started, no calls to executeInContainer - when(docker.getContainer(hostName)).thenReturn(uptodateContainer); - - nodeAgent.tick(); - - inOrder.verify(docker, never()).executeInContainer(any(), anyVararg()); - inOrder.verify(orchestrator).resume(hostName); - inOrder.verifyNoMoreInteractions(); - - nodeAgent.tick(); - - inOrder.verify(docker, never()).executeInContainer(any(), anyVararg()); - inOrder.verify(orchestrator).resume(hostName); - inOrder.verifyNoMoreInteractions(); - } - - // The suspend program can fail by returning non-zero exit code, or throw IOException. - private enum NodeProgramFailureScenario { - EXCEPTION, NODE_PROGRAM_FAILURE - } - - private void failSuspendProgram(NodeProgramFailureScenario scenario) throws Exception { - final long restartGeneration = 1; - final HostName hostName = new HostName("hostname"); - final DockerImage currentDockerImage = new DockerImage("currentDockerImage"); - final DockerImage wantedDockerImage = new DockerImage("wantedDockerImage"); - final ContainerName containerName = new ContainerName("container-name"); - final ContainerNodeSpec nodeSpec = new ContainerNodeSpec( - hostName, - Optional.of(wantedDockerImage), - containerName, - NodeState.ACTIVE, - Optional.of(restartGeneration), - Optional.of(restartGeneration), - MIN_CPU_CORES, - MIN_MAIN_MEMORY_AVAILABLE_GB, - MIN_DISK_AVAILABLE_GB); - final boolean isRunning = true; - final Container existingContainer = new Container(hostName, currentDockerImage, containerName, isRunning); - final String vespaVersion = "7.8.9"; - - when(docker.imageIsDownloaded(wantedDockerImage)).thenReturn(true); - switch (scenario) { - case EXCEPTION: - when(docker.executeInContainer(eq(containerName), anyVararg())) - .thenThrow(new RuntimeException()) // suspending node - .thenReturn(new ProcessResult(1, "", "")); // resuming node, node program doesn't exist - break; - case NODE_PROGRAM_FAILURE: - when(docker.executeInContainer(eq(containerName), anyVararg())) - .thenReturn(new ProcessResult(0, "", "")) // program exists - .thenReturn(new ProcessResult(1, "", "error")) // and program fails to suspend - .thenReturn(new ProcessResult(0, "", "")) // program exists - .thenReturn(new ProcessResult(0, "output", "")); // resuming succeeds - break; - } - when(docker.executeInContainer(eq(containerName), eq(DockerOperationsImpl.GET_VESPA_VERSION_COMMAND))) - .thenReturn(new ProcessResult(0, vespaVersion, "")); - when(orchestrator.suspend(any(HostName.class))).thenReturn(true); - - when(docker.getContainer(hostName)).thenReturn(Optional.of(existingContainer)).thenReturn(Optional.empty()); - when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - + // 2nd try nodeAgent.tick(); - - final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository); - inOrder.verify(orchestrator).suspend(hostName); - - switch (scenario) { - case EXCEPTION: - inOrder.verify(docker, times(1)).executeInContainer(any(), anyVararg()); - break; - case NODE_PROGRAM_FAILURE: - inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg()); - break; - } - - inOrder.verify(docker).stopContainer(containerName); - inOrder.verify(docker).deleteContainer(containerName); - inOrder.verify(docker).createStartContainerCommand( - nodeSpec.wantedDockerImage.get(), - nodeSpec.containerName, - nodeSpec.hostname); - - switch (scenario) { - case EXCEPTION: - inOrder.verify(docker, times(1)).executeInContainer(any(), anyVararg()); - break; - case NODE_PROGRAM_FAILURE: - inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg()); - break; - } - - inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, wantedDockerImage, vespaVersion); + inOrder.verify(dockerOperations).executeResume(any()); inOrder.verify(orchestrator).resume(hostName); inOrder.verifyNoMoreInteractions(); } - - @Ignore // TODO: Remove - @Test - public void suspendExceptionIsIgnored() throws Exception { - failSuspendProgram(NodeProgramFailureScenario.EXCEPTION); - } - - @Ignore // TODO: Remove - @Test - public void suspendFailureIsIgnored() throws Exception { - failSuspendProgram(NodeProgramFailureScenario.NODE_PROGRAM_FAILURE); - } } |