diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2018-10-15 13:41:37 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-10-15 13:41:37 +0200 |
commit | 5504d4ddf0c5d4e1aa2ac966a25363bdb37bbae0 (patch) | |
tree | cd88c643b5ccecb20061ba872a57df494d0aca48 | |
parent | b3e088d6a4cbbe629a4545f15107da56e1008ff2 (diff) | |
parent | d60a43f09ea04dee7d9881f514db1d4d27980796 (diff) |
Merge pull request #7295 from vespa-engine/freva/node-agent-context
NodeAdmin: Use NodeAgentContext in DockerOperations
21 files changed, 258 insertions, 331 deletions
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index cf168df4634..04aad562711 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; import java.net.Inet6Address; import java.net.InetAddress; +import java.nio.file.Path; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.util.ArrayList; @@ -107,15 +108,13 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { } @Override - public Docker.CreateContainerCommand withVolume(String path, String volumePath) { - assert path.indexOf(':') == -1; + public Docker.CreateContainerCommand withVolume(Path path, Path volumePath) { volumeBindSpecs.add(path + ":" + volumePath + ":Z"); return this; } @Override - public Docker.CreateContainerCommand withSharedVolume(String path, String volumePath) { - assert path.indexOf(':') == -1; + public Docker.CreateContainerCommand withSharedVolume(Path path, Path volumePath) { volumeBindSpecs.add(path + ":" + volumePath + ":z"); return this; } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java index bc2c9514709..1713b4570b8 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -2,9 +2,11 @@ package com.yahoo.vespa.hosted.dockerapi; import java.net.InetAddress; +import java.nio.file.Path; import java.time.Duration; import java.util.List; import java.util.Optional; +import java.util.OptionalLong; /** * API to simplify the com.github.dockerjava API for clients, @@ -26,19 +28,19 @@ public interface Docker { * /foo's content visible/readable/writable only inside the container which was last * started and on the host. All the other containers will get "Permission denied". * - * <p>Use {@link #withSharedVolume(String, String)} to mount a given host directory + * <p>Use {@link #withSharedVolume(Path, Path)} to mount a given host directory * into multiple containers. */ - CreateContainerCommand withVolume(String path, String volumePath); + CreateContainerCommand withVolume(Path path, Path volumePath); /** * Mounts a directory on host inside the docker container. * * <p>The bind mount content will be <b>shared</b> among multiple containers. * - * @see #withVolume(String, String) + * @see #withVolume(Path, Path) */ - CreateContainerCommand withSharedVolume(String path, String volumePath); + CreateContainerCommand withSharedVolume(Path path, Path volumePath); CreateContainerCommand withNetworkMode(String mode); CreateContainerCommand withIpAddress(InetAddress address); CreateContainerCommand withUlimit(String name, int softLimit, int hardLimit); @@ -84,34 +86,12 @@ public interface Docker { boolean deleteUnusedDockerImages(List<DockerImage> excludes, Duration minImageAgeToDelete); /** - * Execute a command in docker container as $VESPA_USER. Will block until the command is finished. - * - * @param containerName The name of the container - * @param command The command with arguments to run. - * - * @return exitcodes, stdout and stderr in the ProcessResult - */ - ProcessResult executeInContainer(ContainerName containerName, String... command); - - /** - * Execute a command in docker container as "root". Will block until the command is finished. - * - * @param containerName The name of the container - * @param command The command with arguments to run. - * - * @return exitcodes, stdout and stderr in the ProcessResult - */ - ProcessResult executeInContainerAsRoot(ContainerName containerName, String... command); - - /** - * Execute a command in docker container as "root" - * The timeout will not kill the process spawned. - * * @param containerName The name of the container + * @param user can be "username", "username:group", "uid" or "uid:gid" * @param timeoutSeconds Timeout for the process to finish in seconds or without timeout if empty - * @param command The command with arguments to run. + * @param command The command with arguments to run * * @return exitcodes, stdout and stderr in the ProcessResult */ - ProcessResult executeInContainerAsRoot(ContainerName containerName, Long timeoutSeconds, String... command); + ProcessResult executeInContainerAsUser(ContainerName containerName, String user, OptionalLong timeoutSeconds, String... command); } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 22c865f6d9e..2ac68be2b2e 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -34,6 +34,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -41,8 +42,6 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - public class DockerImpl implements Docker { private static final Logger logger = Logger.getLogger(DockerImpl.class.getName()); @@ -120,25 +119,9 @@ public class DockerImpl implements Docker { return new CreateContainerCommandImpl(dockerClient, image, containerResources, name, hostName); } - @Override - public ProcessResult executeInContainer(ContainerName containerName, String... args) { - return executeInContainerAsUser(containerName, getDefaults().vespaUser(), Optional.empty(), args); - } - - @Override - public ProcessResult executeInContainerAsRoot(ContainerName containerName, String... args) { - return executeInContainerAsUser(containerName, "root", Optional.empty(), args); - } @Override - public ProcessResult executeInContainerAsRoot(ContainerName containerName, Long timeoutSeconds, String... args) { - return executeInContainerAsUser(containerName, "root", Optional.of(timeoutSeconds), args); - } - - /** - * Execute command in container as user, "user" can be "username", "username:group", "uid" or "uid:gid" - */ - private ProcessResult executeInContainerAsUser(ContainerName containerName, String user, Optional<Long> timeoutSeconds, String... command) { + public ProcessResult executeInContainerAsUser(ContainerName containerName, String user, OptionalLong timeoutSeconds, String... command) { try { ExecCreateCmdResponse response = execCreateCmd(containerName, user, command); @@ -148,7 +131,7 @@ public class DockerImpl implements Docker { .exec(new ExecStartResultCallback(output, errors)); if (timeoutSeconds.isPresent()) { - if (!callback.awaitCompletion(timeoutSeconds.get(), TimeUnit.SECONDS)) + if (!callback.awaitCompletion(timeoutSeconds.getAsLong(), TimeUnit.SECONDS)) throw new DockerExecTimeoutException(String.format( "Command '%s' did not finish within %s seconds.", command[0], timeoutSeconds)); } else { diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java index 5ce8c6b093c..dc041b159b5 100644 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import java.net.InetAddress; import java.net.UnknownHostException; +import java.nio.file.Paths; import java.util.Optional; import java.util.stream.Stream; @@ -27,7 +28,7 @@ public class CreateContainerCommandImplTest { .withUlimit("nproc", 10, 20) .withEnvironment("env1", "val1") .withEnvironment("env2", "val2") - .withVolume("vol1", "/host/vol1") + .withVolume(Paths.get("vol1"), Paths.get("/host/vol1")) .withAddCapability("SYS_PTRACE") .withAddCapability("SYS_ADMIN") .withDropCapability("NET_ADMIN") diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java index f1a8d4ef65e..8d9ea61c1d9 100644 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java @@ -19,6 +19,8 @@ import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Matchers; +import java.util.OptionalLong; + import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; @@ -66,7 +68,8 @@ public class DockerImplTest { when(state.isRunning()).thenReturn(false); when(state.getExitCode()).thenReturn(exitCode); - final ProcessResult result = docker.executeInContainer(new ContainerName(containerId), command); + final ProcessResult result = docker.executeInContainerAsUser( + new ContainerName(containerId), "root", OptionalLong.empty(), command); assertThat(result.getExitStatus(), is(exitCode)); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java index 7abf179f616..c65d59a79dc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.ContainerStats; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; @@ -14,39 +15,39 @@ import java.util.Optional; public interface DockerOperations { - void createContainer(ContainerName containerName, NodeSpec node, ContainerData containerData); + void createContainer(NodeAgentContext context, NodeSpec node, ContainerData containerData); - void startContainer(ContainerName containerName); + void startContainer(NodeAgentContext context); - void removeContainer(Container existingContainer); + void removeContainer(NodeAgentContext context, Container container); - Optional<Container> getContainer(ContainerName containerName); + Optional<Container> getContainer(NodeAgentContext context); boolean pullImageAsyncIfNeeded(DockerImage dockerImage); - ProcessResult executeCommandInContainerAsRoot(ContainerName containerName, String... command); + ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, String... command); - ProcessResult executeCommandInContainerAsRoot(ContainerName containerName, Long timeoutSeconds, String... command); + ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, Long timeoutSeconds, String... command); ProcessResult executeCommandInNetworkNamespace(ContainerName containerName, String... command); /** Resume node. Resuming a node means that it is ready to take on traffic. */ - void resumeNode(ContainerName containerName); + void resumeNode(NodeAgentContext context); /** * Suspend node. Suspending a node means the node should be taken temporarly offline, * such that maintenance of the node can be done (upgrading, rebooting, etc). */ - void suspendNode(ContainerName containerName); + void suspendNode(NodeAgentContext context); - void restartVespa(ContainerName containerName); + void restartVespa(NodeAgentContext context); - void startServices(ContainerName containerName); + void startServices(NodeAgentContext context); - void stopServices(ContainerName containerName); + void stopServices(NodeAgentContext context); - Optional<ContainerStats> getContainerStats(ContainerName containerName); + Optional<ContainerStats> getContainerStats(NodeAgentContext context); /** * Returns the list of containers managed by node-admin diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 13ebf84228c..13ee623a74e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -13,22 +13,22 @@ import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses; -import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import java.io.IOException; import java.net.Inet6Address; import java.net.InetAddress; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; +import java.util.logging.Logger; import java.util.stream.Stream; /** @@ -38,6 +38,8 @@ import java.util.stream.Stream; */ public class DockerOperationsImpl implements DockerOperations { + private static final Logger logger = Logger.getLogger(DockerOperationsImpl.class.getName()); + private static final String MANAGER_NAME = "node-admin"; private static final String IPV6_NPT_PREFIX = "fd00::"; @@ -46,22 +48,16 @@ public class DockerOperationsImpl implements DockerOperations { private final Docker docker; private final Environment environment; private final ProcessExecuter processExecuter; - private final String nodeProgram; - private final Map<Path, Boolean> directoriesToMount; public DockerOperationsImpl(Docker docker, Environment environment, ProcessExecuter processExecuter) { this.docker = docker; this.environment = environment; this.processExecuter = processExecuter; - - this.nodeProgram = environment.pathInNodeUnderVespaHome("bin/vespa-nodectl").toString(); - this.directoriesToMount = getDirectoriesToMount(environment); } @Override - public void createContainer(ContainerName containerName, NodeSpec node, ContainerData containerData) { - PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName); - logger.info("Creating container " + containerName); + public void createContainer(NodeAgentContext context, NodeSpec node, ContainerData containerData) { + context.log(logger, "Creating container"); // IPv6 - Assume always valid Inet6Address ipV6Address = environment.getIpAddresses().getIPv6Address(node.getHostname()).orElseThrow( @@ -73,29 +69,18 @@ public class DockerOperationsImpl implements DockerOperations { Docker.CreateContainerCommand command = docker.createContainerCommand( node.getWantedDockerImage().get(), ContainerResources.from(node.getMinCpuCores(), node.getMinMainMemoryAvailableGb()), - containerName, + context.containerName(), node.getHostname()) .withManagedBy(MANAGER_NAME) .withEnvironment("VESPA_CONFIGSERVERS", configServers) .withEnvironment("CONTAINER_ENVIRONMENT_SETTINGS", - environment.getContainerEnvironmentResolver().createSettings(environment, node)) + environment.getContainerEnvironmentResolver().createSettings(environment, node)) .withUlimit("nofile", 262_144, 262_144) .withUlimit("nproc", 32_768, 409_600) .withUlimit("core", -1, -1) .withAddCapability("SYS_PTRACE") // Needed for gcore, pstack etc. .withAddCapability("SYS_ADMIN"); // Needed for perf - if (isInfrastructureHost(environment.getNodeType())) { - command.withVolume("/var/lib/sia", "/var/lib/sia"); - } - - if (environment.getNodeType() == NodeType.proxyhost) { - command.withVolume("/opt/yahoo/share/ssl/certs", "/opt/yahoo/share/ssl/certs"); - } - - if (environment.getNodeType() == NodeType.host) { - command.withSharedVolume("/var/zpe", environment.pathInNodeUnderVespaHome("var/zpe").toString()); - } DockerNetworking networking = environment.getDockerNetworking(); command.withNetworkMode(networking.getDockerNetworkMode()); @@ -116,10 +101,7 @@ public class DockerOperationsImpl implements DockerOperations { addEtcHosts(containerData, node.getHostname(), ipV4Local, ipV6Local); } - for (Path pathInNode : directoriesToMount.keySet()) { - String pathInHost = environment.pathInHostFromPathInNode(containerName, pathInNode).toString(); - command.withVolume(pathInHost, pathInNode.toString()); - } + addMounts(context, command); // TODO: Enforce disk constraints long minMainMemoryAvailableMb = (long) (node.getMinMainMemoryAvailableGb() * 1024); @@ -164,35 +146,25 @@ public class DockerOperationsImpl implements DockerOperations { } @Override - public void startContainer(ContainerName containerName) { - PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName); - logger.info("Starting container " + containerName); - - docker.startContainer(containerName); - - directoriesToMount.entrySet().stream() - .filter(Map.Entry::getValue) - .map(Map.Entry::getKey) - .forEach(path -> - docker.executeInContainerAsRoot(containerName, "chmod", "-R", "a+w", path.toString())); + public void startContainer(NodeAgentContext context) { + context.log(logger, "Starting container"); + docker.startContainer(context.containerName()); } @Override - public void removeContainer(Container existingContainer) { - final ContainerName containerName = existingContainer.name; - PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName); - if (existingContainer.state.isRunning()) { - logger.info("Stopping container " + containerName.asString()); - docker.stopContainer(containerName); + public void removeContainer(NodeAgentContext context, Container container) { + if (container.state.isRunning()) { + context.log(logger, "Stopping container"); + docker.stopContainer(context.containerName()); } - logger.info("Deleting container " + containerName.asString()); - docker.deleteContainer(containerName); + context.log(logger, "Deleting container"); + docker.deleteContainer(context.containerName()); } @Override - public Optional<Container> getContainer(ContainerName containerName) { - return docker.getContainer(containerName); + public Optional<Container> getContainer(NodeAgentContext context) { + return docker.getContainer(context.containerName()); } @Override @@ -200,88 +172,83 @@ public class DockerOperationsImpl implements DockerOperations { return docker.pullImageAsyncIfNeeded(dockerImage); } - ProcessResult executeCommandInContainer(ContainerName containerName, String... command) { - ProcessResult result = docker.executeInContainerAsRoot(containerName, command); - - if (!result.isSuccess()) { - throw new RuntimeException("Container " + containerName.asString() + - ": command " + Arrays.toString(command) + " failed: " + result); - } - return result; - } - @Override - public ProcessResult executeCommandInContainerAsRoot(ContainerName containerName, Long timeoutSeconds, String... command) { - return docker.executeInContainerAsRoot(containerName, timeoutSeconds, command); + public ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, Long timeoutSeconds, String... command) { + return docker.executeInContainerAsUser(context.containerName(), "root", OptionalLong.of(timeoutSeconds), command); } @Override - public ProcessResult executeCommandInContainerAsRoot(ContainerName containerName, String... command) { - return docker.executeInContainerAsRoot(containerName, command); + public ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, String... command) { + return docker.executeInContainerAsUser(context.containerName(), "root", OptionalLong.empty(), command); } @Override public ProcessResult executeCommandInNetworkNamespace(ContainerName containerName, String... command) { - final PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName); final Integer containerPid = docker.getContainer(containerName) .filter(container -> container.state.isRunning()) .map(container -> container.pid) .orElseThrow(() -> new RuntimeException("PID not found for container with name: " + containerName.asString())); - Path procPath = environment.getPathResolver().getPathToRootOfHost().resolve("proc"); - final String[] wrappedCommand = Stream.concat( - Stream.of("nsenter", String.format("--net=%s/%d/ns/net", procPath, containerPid), "--"), + Stream.of("nsenter", String.format("--net=/proc/%d/ns/net", containerPid), "--"), Stream.of(command)) .toArray(String[]::new); try { Pair<Integer, String> result = processExecuter.exec(wrappedCommand); if (result.getFirst() != 0) { - String msg = String.format( + throw new RuntimeException(String.format( "Failed to execute %s in network namespace for %s (PID = %d), exit code: %d, output: %s", - Arrays.toString(wrappedCommand), containerName.asString(), containerPid, result.getFirst(), result.getSecond()); - logger.error(msg); - throw new RuntimeException(msg); + Arrays.toString(wrappedCommand), containerName.asString(), containerPid, result.getFirst(), result.getSecond())); } return new ProcessResult(0, result.getSecond(), ""); } catch (IOException e) { - logger.warning(String.format("IOException while executing %s in network namespace for %s (PID = %d)", + throw new RuntimeException(String.format("IOException while executing %s in network namespace for %s (PID = %d)", Arrays.toString(wrappedCommand), containerName.asString(), containerPid), e); - throw new RuntimeException(e); } } @Override - public void resumeNode(ContainerName containerName) { - executeCommandInContainer(containerName, nodeProgram, "resume"); + public void resumeNode(NodeAgentContext context) { + executeNodeCtlInContainer(context, "resume"); } @Override - public void suspendNode(ContainerName containerName) { - executeCommandInContainer(containerName, nodeProgram, "suspend"); + public void suspendNode(NodeAgentContext context) { + executeNodeCtlInContainer(context, "suspend"); } @Override - public void restartVespa(ContainerName containerName) { - executeCommandInContainer(containerName, nodeProgram, "restart-vespa"); + public void restartVespa(NodeAgentContext context) { + executeNodeCtlInContainer(context, "restart-vespa"); } @Override - public void startServices(ContainerName containerName) { - executeCommandInContainer(containerName, nodeProgram, "start"); + public void startServices(NodeAgentContext context) { + executeNodeCtlInContainer(context, "start"); } @Override - public void stopServices(ContainerName containerName) { - executeCommandInContainer(containerName, nodeProgram, "stop"); + public void stopServices(NodeAgentContext context) { + executeNodeCtlInContainer(context, "stop"); + } + + ProcessResult executeNodeCtlInContainer(NodeAgentContext context, String program) { + String[] command = new String[] {context.pathInNodeUnderVespaHome("bin/vespa-nodectl").toString(), program}; + ProcessResult result = executeCommandInContainerAsRoot(context, command); + + if (!result.isSuccess()) { + throw new RuntimeException("Container " + context.containerName().asString() + + ": command " + Arrays.toString(command) + " failed: " + result); + } + return result; } @Override - public Optional<ContainerStats> getContainerStats(ContainerName containerName) { - return docker.getContainerStats(containerName); + public Optional<ContainerStats> getContainerStats(NodeAgentContext context) { + return docker.getContainerStats(context.containerName()); } @Override @@ -292,56 +259,71 @@ public class DockerOperationsImpl implements DockerOperations { /** * Returns map of directories to mount and whether they should be writable by everyone */ - private static Map<Path, Boolean> getDirectoriesToMount(Environment environment) { - final Map<Path, Boolean> directoriesToMount = new HashMap<>(); - directoriesToMount.put(Paths.get("/etc/yamas-agent"), true); - directoriesToMount.put(Paths.get("/etc/filebeat"), true); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/daemontools_y"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/jdisc_core"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/langdetect/"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/nginx"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/vespa"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/yca"), true); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/yck"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/yell"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/ykeykey"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/ykeykeyd"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/yms_agent"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/ysar"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/ystatus"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/zpu"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/cache"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/crash"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/db/jdisc"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/db/vespa"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/jdisc_container"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/jdisc_core"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/maven"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/mediasearch"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/run"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/scoreboards"), true); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/service"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/share"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/spool"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/vespa"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/yca"), true); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/ycore++"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/zookeeper"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("tmp"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/container-data"), false); - if (environment.getNodeType() == NodeType.proxyhost) - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/vespa-hosted/routing"), true); - if (environment.getNodeType() == NodeType.host) - directoriesToMount.put(Paths.get("/var/lib/sia"), true); - - return Collections.unmodifiableMap(directoriesToMount); + private static void addMounts(NodeAgentContext context, Docker.CreateContainerCommand command) { + final Path varLibSia = Paths.get("/var/lib/sia"); + + // Paths unique to each container + List<Path> paths = new ArrayList<>(Arrays.asList( + Paths.get("/etc/yamas-agent"), + Paths.get("/etc/filebeat"), + context.pathInNodeUnderVespaHome("logs/daemontools_y"), + context.pathInNodeUnderVespaHome("logs/jdisc_core"), + context.pathInNodeUnderVespaHome("logs/langdetect/"), + context.pathInNodeUnderVespaHome("logs/nginx"), + context.pathInNodeUnderVespaHome("logs/vespa"), + context.pathInNodeUnderVespaHome("logs/yca"), + context.pathInNodeUnderVespaHome("logs/yck"), + context.pathInNodeUnderVespaHome("logs/yell"), + context.pathInNodeUnderVespaHome("logs/ykeykey"), + context.pathInNodeUnderVespaHome("logs/ykeykeyd"), + context.pathInNodeUnderVespaHome("logs/yms_agent"), + context.pathInNodeUnderVespaHome("logs/ysar"), + context.pathInNodeUnderVespaHome("logs/ystatus"), + context.pathInNodeUnderVespaHome("logs/zpu"), + context.pathInNodeUnderVespaHome("var/cache"), + context.pathInNodeUnderVespaHome("var/crash"), + context.pathInNodeUnderVespaHome("var/db/jdisc"), + context.pathInNodeUnderVespaHome("var/db/vespa"), + context.pathInNodeUnderVespaHome("var/jdisc_container"), + context.pathInNodeUnderVespaHome("var/jdisc_core"), + context.pathInNodeUnderVespaHome("var/maven"), + context.pathInNodeUnderVespaHome("var/mediasearch"), + context.pathInNodeUnderVespaHome("var/run"), + context.pathInNodeUnderVespaHome("var/scoreboards"), + context.pathInNodeUnderVespaHome("var/service"), + context.pathInNodeUnderVespaHome("var/share"), + context.pathInNodeUnderVespaHome("var/spool"), + context.pathInNodeUnderVespaHome("var/vespa"), + context.pathInNodeUnderVespaHome("var/yca"), + context.pathInNodeUnderVespaHome("var/ycore++"), + context.pathInNodeUnderVespaHome("var/zookeeper"), + context.pathInNodeUnderVespaHome("tmp"), + context.pathInNodeUnderVespaHome("var/container-data"))); + + if (context.nodeType() == NodeType.proxyhost) + paths.add(context.pathInNodeUnderVespaHome("var/vespa-hosted/routing")); + if (context.nodeType() == NodeType.host) + paths.add(varLibSia); + + paths.forEach(path -> command.withVolume(context.pathOnHostFromPathInNode(path), path)); + + + // Shared paths + if (isInfrastructureHost(context.nodeType())) + command.withSharedVolume(varLibSia, varLibSia); + + if (context.nodeType() == NodeType.proxyhost) + command.withSharedVolume(Paths.get("/opt/yahoo/share/ssl/certs"), Paths.get("/opt/yahoo/share/ssl/certs")); + + if (context.nodeType() == NodeType.host) + command.withSharedVolume(Paths.get("/var/zpe"), context.pathInNodeUnderVespaHome("var/zpe")); } /** Returns whether given nodeType is a Docker host for infrastructure nodes */ private static boolean isInfrastructureHost(NodeType nodeType) { return nodeType == NodeType.confighost || - nodeType == NodeType.proxyhost || - nodeType == NodeType.controllerhost; + nodeType == NodeType.proxyhost || + nodeType == NodeType.controllerhost; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 3e43ed63aea..85ec3712126 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -138,7 +138,7 @@ public class StorageMaintainer { // Write config and restart yamas-agent Path yamasAgentFolder = context.pathOnHostFromPathInNode("/etc/yamas-agent"); configs.forEach(s -> uncheck(() -> s.writeTo(yamasAgentFolder))); - dockerOperations.executeCommandInContainerAsRoot(context.containerName(), "service", "yamas-agent", "restart"); + dockerOperations.executeCommandInContainerAsRoot(context, "service", "yamas-agent", "restart"); } private SecretAgentCheckConfig annotatedCheck(NodeSpec node, SecretAgentCheckConfig check) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java index d1c44a55c1b..2cddee6aa2a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java @@ -39,7 +39,7 @@ public class CoreCollector { Path readBinPathFallback(NodeAgentContext context, Path coredumpPath) { String command = gdb + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'"; String[] wrappedCommand = {"/bin/sh", "-c", command}; - ProcessResult result = docker.executeCommandInContainerAsRoot(context.containerName(), wrappedCommand); + ProcessResult result = docker.executeCommandInContainerAsRoot(context, wrappedCommand); Matcher matcher = CORE_GENERATOR_PATH_PATTERN.matcher(result.getOutput()); if (! matcher.find()) { @@ -52,7 +52,7 @@ public class CoreCollector { Path readBinPath(NodeAgentContext context, Path coredumpPath) { String[] command = {"file", coredumpPath.toString()}; try { - ProcessResult result = docker.executeCommandInContainerAsRoot(context.containerName(), command); + ProcessResult result = docker.executeCommandInContainerAsRoot(context, command); if (result.getExitStatus() != 0) { throw new RuntimeException("file command failed with " + result); } @@ -77,7 +77,7 @@ public class CoreCollector { List<String> readBacktrace(NodeAgentContext context, Path coredumpPath, Path binPath, boolean allThreads) { String threads = allThreads ? "thread apply all bt" : "bt"; String[] command = {gdb.toString(), "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()}; - ProcessResult result = docker.executeCommandInContainerAsRoot(context.containerName(), command); + ProcessResult result = docker.executeCommandInContainerAsRoot(context, command); if (result.getExitStatus() != 0) { throw new RuntimeException("Failed to read backtrace " + result + ", Command: " + Arrays.toString(command)); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 644ac587c34..d436e214266 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.concurrent.ThreadFactoryFactory; -import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper; @@ -39,7 +38,6 @@ public class NodeAdminImpl implements NodeAdmin { private final ScheduledExecutorService metricsScheduler = Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("metricsscheduler")); - private final DockerOperations dockerOperations; private final Function<String, NodeAgent> nodeAgentFactory; private final Runnable aclMaintainer; @@ -58,7 +56,13 @@ public class NodeAdminImpl implements NodeAdmin { Runnable aclMaintainer, MetricReceiverWrapper metricReceiver, Clock clock) { - this.dockerOperations = dockerOperations; + this(nodeAgentFactory, aclMaintainer, metricReceiver, clock); + } + + public NodeAdminImpl(Function<String, NodeAgent> nodeAgentFactory, + Runnable aclMaintainer, + MetricReceiverWrapper metricReceiver, + Clock clock) { this.nodeAgentFactory = nodeAgentFactory; this.aclMaintainer = aclMaintainer; @@ -203,9 +207,6 @@ public class NodeAdminImpl implements NodeAdmin { } void synchronizeNodesToNodeAgents(Set<String> hostnamesToRun) { - Map<String, Container> runningContainersByHostname = dockerOperations.getAllManagedContainers().stream() - .collect(Collectors.toMap(c -> c.hostname, c -> c)); - // Stop and remove NodeAgents that should no longer be running diff(nodeAgentsByHostname.keySet(), hostnamesToRun) .forEach(hostname -> nodeAgentsByHostname.remove(hostname).stop()); @@ -213,11 +214,6 @@ public class NodeAdminImpl implements NodeAdmin { // Start NodeAgent for hostnames that should be running, but aren't yet diff(hostnamesToRun, nodeAgentsByHostname.keySet()) .forEach(this::startNodeAgent); - - // Remove containers that are running, but have no NodeAgent managing it (and after the previous steps, - // these containers shouldn't be running, otherwise a NodeAgent would have been created) - diff(runningContainersByHostname.keySet(), nodeAgentsByHostname.keySet()) - .forEach(hostname -> dockerOperations.removeContainer(runningContainersByHostname.get(hostname))); } private void startNodeAgent(String hostname) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 3af78593a58..350ca2263ef 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -178,7 +178,7 @@ public class NodeAgentImpl implements NodeAgent { serviceRestarter = service -> { try { ProcessResult processResult = dockerOperations.executeCommandInContainerAsRoot( - context.containerName(), "service", service, "restart"); + context, "service", service, "restart"); if (!processResult.isSuccess()) { context.log(logger, LogLevel.ERROR, "Failed to restart service " + service + ": " + processResult); @@ -219,7 +219,7 @@ public class NodeAgentImpl implements NodeAgent { void startServicesIfNeeded() { if (!hasStartedServices) { context.log(logger, "Starting services"); - dockerOperations.startServices(context.containerName()); + dockerOperations.startServices(context); hasStartedServices = true; } } @@ -234,7 +234,7 @@ public class NodeAgentImpl implements NodeAgent { } context.log(logger, LogLevel.DEBUG, "Starting optional node program resume command"); - dockerOperations.resumeNode(context.containerName()); + dockerOperations.resumeNode(context); hasResumedNode = true; } } @@ -265,8 +265,8 @@ public class NodeAgentImpl implements NodeAgent { private void startContainer(NodeSpec node) { ContainerData containerData = createContainerData(context, node); - dockerOperations.createContainer(context.containerName(), node, containerData); - dockerOperations.startContainer(context.containerName()); + dockerOperations.createContainer(context, node, containerData); + dockerOperations.startContainer(context); lastCpuMetric = new CpuUsageReporter(); hasStartedServices = true; // Automatically started with the container @@ -302,7 +302,7 @@ public class NodeAgentImpl implements NodeAgent { context.log(logger, "Restarting services"); // Since we are restarting the services we need to suspend the node. orchestratorSuspendNode(); - dockerOperations.restartVespa(context.containerName()); + dockerOperations.restartVespa(context); } } @@ -312,7 +312,7 @@ public class NodeAgentImpl implements NodeAgent { if (containerState == ABSENT) return; try { hasStartedServices = hasResumedNode = false; - dockerOperations.stopServices(context.containerName()); + dockerOperations.stopServices(context); } catch (ContainerNotFoundException e) { containerState = ABSENT; } @@ -324,7 +324,7 @@ public class NodeAgentImpl implements NodeAgent { if (containerState == ABSENT) return; try { hasResumedNode = false; - dockerOperations.suspendNode(context.containerName()); + dockerOperations.suspendNode(context); } catch (ContainerNotFoundException e) { containerState = ABSENT; } catch (RuntimeException e) { @@ -379,7 +379,7 @@ public class NodeAgentImpl implements NodeAgent { } stopFilebeatSchedulerIfNeeded(); storageMaintainer.handleCoreDumpsForContainer(context, node, Optional.of(existingContainer)); - dockerOperations.removeContainer(existingContainer); + dockerOperations.removeContainer(context, existingContainer); containerState = ABSENT; context.log(logger, "Container successfully removed, new containerState is " + containerState); return Optional.empty(); @@ -600,7 +600,7 @@ public class NodeAgentImpl implements NodeAgent { final NodeSpec node = lastNode; if (node == null || containerState != UNKNOWN) return; - Optional<ContainerStats> containerStats = dockerOperations.getContainerStats(context.containerName()); + Optional<ContainerStats> containerStats = dockerOperations.getContainerStats(context); if (!containerStats.isPresent()) return; Dimensions.Builder dimensionsBuilder = new Dimensions.Builder() @@ -675,7 +675,7 @@ public class NodeAgentImpl implements NodeAgent { // Push metrics to the metrics proxy in each container - give it maximum 1 seconds to complete String[] command = {"vespa-rpc-invoke", "-t", "2", "tcp/localhost:19091", "setExtraMetrics", wrappedMetrics}; - dockerOperations.executeCommandInContainerAsRoot(context.containerName(), 5L, command); + dockerOperations.executeCommandInContainerAsRoot(context, 5L, command); } catch (DockerExecTimeoutException | JsonProcessingException e) { context.log(logger, LogLevel.WARNING, "Failed to push metrics to container", e); } @@ -683,7 +683,7 @@ public class NodeAgentImpl implements NodeAgent { private Optional<Container> getContainer() { if (containerState == ABSENT) return Optional.empty(); - Optional<Container> container = dockerOperations.getContainer(context.containerName()); + Optional<Container> container = dockerOperations.getContainer(context); if (! container.isPresent()) containerState = ABSENT; return container; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java index 7f4dad6ee36..760326f061e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java @@ -12,6 +12,8 @@ import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImplTest; import org.junit.Test; import org.mockito.InOrder; @@ -19,6 +21,7 @@ import java.io.IOException; import java.net.InetAddress; import java.nio.file.Paths; import java.util.Optional; +import java.util.OptionalLong; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; @@ -48,43 +51,41 @@ public class DockerOperationsImplTest { @Test public void processResultFromNodeProgramWhenSuccess() { - final ContainerName containerName = new ContainerName("container-name"); + final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname("container-123.domain.tld"); final ProcessResult actualResult = new ProcessResult(0, "output", "errors"); - final String programPath = "/bin/command"; - final String[] command = new String[]{programPath, "arg"}; - when(docker.executeInContainerAsRoot(any(), anyVararg())) + when(docker.executeInContainerAsUser(any(), any(), any(), anyVararg())) .thenReturn(actualResult); // output from node program - ProcessResult result = dockerOperations.executeCommandInContainer(containerName, command); + ProcessResult result = dockerOperations.executeNodeCtlInContainer(context, "start"); final InOrder inOrder = inOrder(docker); - inOrder.verify(docker, times(1)).executeInContainerAsRoot( - eq(containerName), - eq(command[0]), - eq(command[1])); + inOrder.verify(docker, times(1)).executeInContainerAsUser( + eq(context.containerName()), + eq("root"), + eq(OptionalLong.empty()), + eq("/opt/vespa/bin/vespa-nodectl"), + eq("start")); assertThat(result, is(actualResult)); } @Test(expected = RuntimeException.class) public void processResultFromNodeProgramWhenNonZeroExitCode() { - final ContainerName containerName = new ContainerName("container-name"); + final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname("container-123.domain.tld"); final ProcessResult actualResult = new ProcessResult(3, "output", "errors"); - final String programPath = "/bin/command"; - final String[] command = new String[]{programPath, "arg"}; - when(docker.executeInContainerAsRoot(any(), anyVararg())) + when(docker.executeInContainerAsUser(any(), any(), any(), anyVararg())) .thenReturn(actualResult); // output from node program - dockerOperations.executeCommandInContainer(containerName, command); + dockerOperations.executeNodeCtlInContainer(context, "start"); } @Test public void runsCommandInNetworkNamespace() throws IOException { Container container = makeContainer("container-42", Container.State.RUNNING, 42); - when(processExecuter.exec(aryEq(new String[]{"nsenter", "--net=/host/proc/42/ns/net", "--", "iptables", "-nvL"}))) + when(processExecuter.exec(aryEq(new String[]{"nsenter", "--net=/proc/42/ns/net", "--", "iptables", "-nvL"}))) .thenReturn(new Pair<>(0, "")); dockerOperations.executeCommandInNetworkNamespace(container.name, "iptables", "-nvL"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java index ec29cf0fe25..d7e0fe3d8e5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java @@ -37,14 +37,14 @@ public class DockerFailTest { dockerTester.callOrderVerifier.assertInOrder(1200, "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); + "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); dockerTester.dockerMock.deleteContainer(new ContainerName("host1")); dockerTester.callOrderVerifier.assertInOrder( "deleteContainer with ContainerName { name=host1 }", "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); + "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); } } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java index 313db1cfc47..26843b119de 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import java.net.InetAddress; +import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -17,6 +18,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; /** * Mock with some simple logic @@ -106,25 +108,9 @@ public class DockerMock implements Docker { } @Override - public ProcessResult executeInContainer(ContainerName containerName, String... args) { + public ProcessResult executeInContainerAsUser(ContainerName containerName, String user, OptionalLong timeout, String... args) { synchronized (monitor) { - callOrderVerifier.add("executeInContainer with " + containerName + ", args: " + Arrays.toString(args)); - } - return new ProcessResult(0, null, ""); - } - - @Override - public ProcessResult executeInContainerAsRoot(ContainerName containerName, String... args) { - synchronized (monitor) { - callOrderVerifier.add("executeInContainerAsRoot with " + containerName + ", args: " + Arrays.toString(args)); - } - return new ProcessResult(0, null, ""); - } - - @Override - public ProcessResult executeInContainerAsRoot(ContainerName containerName, Long timeoutSeconds, String... args) { - synchronized (monitor) { - callOrderVerifier.add("executeInContainerAsRoot with " + containerName + ", args: " + Arrays.toString(args)); + callOrderVerifier.add("executeInContainer " + containerName.asString() + " as " + user + ", args: " + Arrays.toString(args)); } return new ProcessResult(0, null, ""); } @@ -142,12 +128,12 @@ public class DockerMock implements Docker { } @Override - public CreateContainerCommand withVolume(String path, String volumePath) { + public CreateContainerCommand withVolume(Path path, Path volumePath) { return this; } @Override - public CreateContainerCommand withSharedVolume(String path, String volumePath) { + public CreateContainerCommand withSharedVolume(Path path, Path volumePath) { return this; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 3c316de94eb..ef369a21c37 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -100,7 +100,7 @@ public class DockerTester implements AutoCloseable { Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl( NodeAgentContextImplTest.nodeAgentFromHostname(fileSystem, hostName), nodeRepositoryMock, orchestratorMock, dockerOperations, storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL, athenzCredentialsMaintainer); - nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, aclMaintainer, mr, Clock.systemUTC()); + nodeAdmin = new NodeAdminImpl(nodeAgentFactory, aclMaintainer, mr, Clock.systemUTC()); nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock, storageMaintainer, nodeAdmin, DOCKER_HOST_HOSTNAME, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, Optional.of(new ClassLocking())); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java index 9c9b2afe956..e130cd0d475 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java @@ -38,16 +38,16 @@ public class MultiDockerTest { dockerTester.callOrderVerifier.assertInOrder( "createContainerCommand with DockerImage { imageId=image1 }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]", + "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]", "createContainerCommand with DockerImage { imageId=image2 }, HostName: host2.test.yahoo.com, ContainerName { name=host2 }", - "executeInContainerAsRoot with ContainerName { name=host2 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]", + "executeInContainer host2 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]", "stopContainer with ContainerName { name=host2 }", "deleteContainer with ContainerName { name=host2 }", "createContainerCommand with DockerImage { imageId=image1 }, HostName: host3.test.yahoo.com, ContainerName { name=host3 }", - "executeInContainerAsRoot with ContainerName { name=host3 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); + "executeInContainer host3 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); dockerTester.callOrderVerifier.assertInOrderWithAssertMessage( "Maintainer did not receive call to delete application storage", @@ -87,7 +87,7 @@ public class MultiDockerTest { ContainerName containerName = ContainerName.fromHostname(hostName); tester.callOrderVerifier.assertInOrder( "createContainerCommand with " + dockerImage + ", HostName: " + hostName + ", " + containerName, - "executeInContainerAsRoot with " + containerName + ", args: [" + DockerTester.NODE_PROGRAM + ", resume]"); + "executeInContainer " + containerName.asString() + " as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); return nodeSpec; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java index 3681878d728..da743c40e8b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java @@ -36,7 +36,7 @@ public class NodeStateTest { tester.callOrderVerifier.assertInOrder( "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); + "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); } @@ -59,7 +59,7 @@ public class NodeStateTest { } dockerTester.callOrderVerifier.assertInOrder( - "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", stop]", + "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", stop]", "stopContainer with ContainerName { name=host1 }", "deleteContainer with ContainerName { name=host1 }"); } @@ -102,7 +102,7 @@ public class NodeStateTest { "Node not started again after being put to active state", "deleteContainer with ContainerName { name=host1 }", "createContainerCommand with DockerImage { imageId=newDockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); + "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); } } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java index 0406934defc..2e290e014c2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java @@ -38,7 +38,7 @@ public class RestartTest { dockerTester.callOrderVerifier.assertInOrder( "Suspend for host1.test.yahoo.com", - "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", restart-vespa]"); + "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", restart-vespa]"); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java index b5950646da9..0fed9433432 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -152,7 +152,7 @@ public class CoreCollectorTest { } private void mockExec(String[] cmd, String output, String error) { - when(docker.executeCommandInContainerAsRoot(context.containerName(), cmd)) + when(docker.executeCommandInContainerAsRoot(context, cmd)) .thenReturn(new ProcessResult(error.isEmpty() ? 0 : 1, output, error)); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index 065b039c7fd..2431433b17f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import org.junit.Test; @@ -37,12 +36,11 @@ import static org.mockito.Mockito.when; public class NodeAdminImplTest { // Trick to allow mocking of typed interface without casts/warnings. private interface NodeAgentFactory extends Function<String, NodeAgent> {} - private final DockerOperations dockerOperations = mock(DockerOperations.class); private final Function<String, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class); private final Runnable aclMaintainer = mock(Runnable.class); private final ManualClock clock = new ManualClock(); - private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, aclMaintainer, + private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentFactory, aclMaintainer, new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock); @Test @@ -53,7 +51,6 @@ public class NodeAdminImplTest { final NodeAgent nodeAgent2 = mock(NodeAgentImpl.class); when(nodeAgentFactory.apply(eq(hostName1))).thenReturn(nodeAgent1); when(nodeAgentFactory.apply(eq(hostName2))).thenReturn(nodeAgent2); - when(dockerOperations.getAllManagedContainers()).thenReturn(Collections.emptyList()); final InOrder inOrder = inOrder(nodeAgentFactory, nodeAgent1, nodeAgent2); @@ -91,8 +88,6 @@ public class NodeAdminImplTest { @Test public void testSetFrozen() { - when(dockerOperations.getAllManagedContainers()).thenReturn(Collections.emptyList()); - List<NodeAgent> nodeAgents = new ArrayList<>(); Set<String> existingContainerHostnames = new HashSet<>(); for (int i = 0; i < 3; i++) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index fbb584ea1d9..b2facd3dd6d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -124,15 +124,15 @@ public class NodeAgentImplTest { nodeAgent.converge(); - verify(dockerOperations, never()).removeContainer(any()); + verify(dockerOperations, never()).removeContainer(eq(context), any()); verify(orchestrator, never()).suspend(any(String.class)); verify(dockerOperations, never()).pullImageAsyncIfNeeded(any()); verify(storageMaintainer, never()).removeOldFilesFromNode(eq(context)); final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository); // TODO: Verify this isn't run unless 1st time - inOrder.verify(dockerOperations, never()).startServices(eq(context.containerName())); - inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context.containerName())); + inOrder.verify(dockerOperations, never()).startServices(eq(context)); + inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(orchestrator).resume(hostName); } @@ -176,24 +176,24 @@ public class NodeAgentImplTest { when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(187500000000L)); nodeAgent.converge(); - inOrder.verify(dockerOperations, never()).startServices(eq(context.containerName())); - inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context.containerName())); + inOrder.verify(dockerOperations, never()).startServices(eq(context)); + inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); nodeAgent.suspend(); nodeAgent.converge(); - inOrder.verify(dockerOperations, never()).startServices(eq(context.containerName())); - inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context.containerName())); // Expect a resume, but no start services + inOrder.verify(dockerOperations, never()).startServices(eq(context)); + inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); // Expect a resume, but no start services // No new suspends/stops, so no need to resume/start nodeAgent.converge(); - inOrder.verify(dockerOperations, never()).startServices(eq(context.containerName())); - inOrder.verify(dockerOperations, never()).resumeNode(eq(context.containerName())); + inOrder.verify(dockerOperations, never()).startServices(eq(context)); + inOrder.verify(dockerOperations, never()).resumeNode(eq(context)); nodeAgent.suspend(); nodeAgent.stopServices(); nodeAgent.converge(); - inOrder.verify(dockerOperations, times(1)).startServices(eq(context.containerName())); - inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context.containerName())); + inOrder.verify(dockerOperations, times(1)).startServices(eq(context)); + inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); } @Test @@ -219,16 +219,16 @@ public class NodeAgentImplTest { nodeAgent.converge(); - verify(dockerOperations, never()).removeContainer(any()); + verify(dockerOperations, never()).removeContainer(eq(context), any()); verify(dockerOperations, never()).startServices(any()); verify(orchestrator, never()).suspend(any(String.class)); final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository, aclMaintainer); inOrder.verify(dockerOperations, times(1)).pullImageAsyncIfNeeded(eq(dockerImage)); - inOrder.verify(dockerOperations, times(1)).createContainer(eq(context.containerName()), eq(node), any()); - inOrder.verify(dockerOperations, times(1)).startContainer(eq(context.containerName())); + inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); + inOrder.verify(dockerOperations, times(1)).startContainer(eq(context)); inOrder.verify(aclMaintainer, times(1)).run(); - inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context.containerName())); + inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( hostName, new NodeAttributes() .withRestartGeneration(restartGeneration) @@ -262,7 +262,7 @@ public class NodeAgentImplTest { verify(orchestrator, never()).suspend(any(String.class)); verify(orchestrator, never()).resume(any(String.class)); - verify(dockerOperations, never()).removeContainer(any()); + verify(dockerOperations, never()).removeContainer(eq(context), any()); final InOrder inOrder = inOrder(dockerOperations); inOrder.verify(dockerOperations, times(1)).pullImageAsyncIfNeeded(eq(newDockerImage)); @@ -302,9 +302,9 @@ public class NodeAgentImplTest { inOrder.verify(orchestrator).resume(any(String.class)); inOrder.verify(orchestrator).resume(any(String.class)); inOrder.verify(orchestrator).suspend(any(String.class)); - inOrder.verify(dockerOperations).removeContainer(any()); - inOrder.verify(dockerOperations, times(1)).createContainer(eq(context.containerName()), eq(thirdSpec), any()); - inOrder.verify(dockerOperations).startContainer(eq(context.containerName())); + inOrder.verify(dockerOperations).removeContainer(eq(context), any()); + inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), eq(thirdSpec), any()); + inOrder.verify(dockerOperations).startContainer(eq(context)); inOrder.verify(orchestrator).resume(any(String.class)); } @@ -329,8 +329,8 @@ public class NodeAgentImplTest { fail("Expected to throw an exception"); } catch (Exception ignored) { } - verify(dockerOperations, never()).createContainer(eq(context.containerName()), eq(node), any()); - verify(dockerOperations, never()).startContainer(eq(context.containerName())); + verify(dockerOperations, never()).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, never()).startContainer(eq(context)); verify(orchestrator, never()).resume(any(String.class)); verify(nodeRepository, never()).updateNodeAttributes(any(String.class), any(NodeAttributes.class)); } @@ -354,7 +354,7 @@ public class NodeAgentImplTest { nodeAgent.converge(); - verify(dockerOperations, never()).removeContainer(any()); + verify(dockerOperations, never()).removeContainer(eq(context), any()); verify(orchestrator, never()).resume(any(String.class)); verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); } @@ -379,10 +379,10 @@ public class NodeAgentImplTest { nodeAgent.converge(); // Should only be called once, when we initialize - verify(dockerOperations, times(1)).getContainer(eq(context.containerName())); - verify(dockerOperations, never()).removeContainer(any()); - verify(dockerOperations, never()).createContainer(eq(context.containerName()), eq(node), any()); - verify(dockerOperations, never()).startContainer(eq(context.containerName())); + verify(dockerOperations, times(1)).getContainer(eq(context)); + verify(dockerOperations, never()).removeContainer(eq(context), any()); + verify(dockerOperations, never()).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, never()).startContainer(eq(context)); verify(orchestrator, never()).resume(any(String.class)); verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); } @@ -410,7 +410,7 @@ public class NodeAgentImplTest { nodeAgent.converge(); final InOrder inOrder = inOrder(storageMaintainer, dockerOperations); - inOrder.verify(dockerOperations, never()).removeContainer(any()); + inOrder.verify(dockerOperations, never()).removeContainer(eq(context), any()); verify(orchestrator, never()).resume(any(String.class)); verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); @@ -457,16 +457,16 @@ public class NodeAgentImplTest { nodeAgent.converge(); final InOrder inOrder = inOrder(storageMaintainer, dockerOperations, nodeRepository); - inOrder.verify(dockerOperations, times(1)).stopServices(eq(context.containerName())); + inOrder.verify(dockerOperations, times(1)).stopServices(eq(context)); inOrder.verify(storageMaintainer, times(1)).handleCoreDumpsForContainer(eq(context), eq(node), any()); - inOrder.verify(dockerOperations, times(1)).removeContainer(any()); + inOrder.verify(dockerOperations, times(1)).removeContainer(eq(context), any()); inOrder.verify(storageMaintainer, times(1)).archiveNodeStorage(eq(context)); inOrder.verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(Node.State.ready)); - verify(dockerOperations, never()).createContainer(eq(context.containerName()), any(), any()); - verify(dockerOperations, never()).startContainer(eq(context.containerName())); - verify(dockerOperations, never()).suspendNode(eq(context.containerName())); - verify(dockerOperations, times(1)).stopServices(eq(context.containerName())); + verify(dockerOperations, never()).createContainer(eq(context), any(), any()); + verify(dockerOperations, never()).startContainer(eq(context)); + verify(dockerOperations, never()).suspendNode(eq(context)); + verify(dockerOperations, times(1)).stopServices(eq(context)); verify(orchestrator, never()).resume(any(String.class)); verify(orchestrator, never()).suspend(any(String.class)); // current Docker image and vespa version should be cleared @@ -519,9 +519,9 @@ public class NodeAgentImplTest { nodeAgent.tick(); - verify(dockerOperations, times(1)).removeContainer(any()); - verify(dockerOperations, times(1)).createContainer(eq(context.containerName()), eq(node), any()); - verify(dockerOperations, times(1)).startContainer(eq(context.containerName())); + verify(dockerOperations, times(1)).removeContainer(eq(context), any()); + verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, times(1)).startContainer(eq(context)); } @Test @@ -544,7 +544,7 @@ public class NodeAgentImplTest { final InOrder inOrder = inOrder(orchestrator, dockerOperations, nodeRepository); doThrow(new RuntimeException("Failed 1st time")) .doNothing() - .when(dockerOperations).resumeNode(eq(context.containerName())); + .when(dockerOperations).resumeNode(eq(context)); // 1st try try { @@ -610,16 +610,16 @@ public class NodeAgentImplTest { when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar")); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); - doThrow(new DockerException("Failed to set up network")).doNothing().when(dockerOperations).startContainer(eq(context.containerName())); + doThrow(new DockerException("Failed to set up network")).doNothing().when(dockerOperations).startContainer(eq(context)); try { nodeAgent.converge(); fail("Expected to get DockerException"); } catch (DockerException ignored) { } - verify(dockerOperations, never()).removeContainer(any()); - verify(dockerOperations, times(1)).createContainer(eq(context.containerName()), eq(node), any()); - verify(dockerOperations, times(1)).startContainer(eq(context.containerName())); + verify(dockerOperations, never()).removeContainer(eq(context), any()); + verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, times(1)).startContainer(eq(context)); verify(nodeAgent, never()).resumeNodeIfNeeded(any()); // The docker container was actually started and is running, but subsequent exec calls to set up @@ -627,9 +627,9 @@ public class NodeAgentImplTest { mockGetContainer(dockerImage, true); nodeAgent.converge(); - verify(dockerOperations, times(1)).removeContainer(any()); - verify(dockerOperations, times(2)).createContainer(eq(context.containerName()), eq(node), any()); - verify(dockerOperations, times(2)).startContainer(eq(context.containerName())); + verify(dockerOperations, times(1)).removeContainer(eq(context), any()); + verify(dockerOperations, times(2)).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, times(2)).startContainer(eq(context)); verify(nodeAgent, times(1)).resumeNodeIfNeeded(any()); } @@ -666,7 +666,7 @@ public class NodeAgentImplTest { when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(39625000000L)); - when(dockerOperations.getContainerStats(eq(context.containerName()))) + when(dockerOperations.getContainerStats(eq(context))) .thenReturn(Optional.of(stats1)) .thenReturn(Optional.of(stats2)); when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar")); @@ -683,13 +683,13 @@ public class NodeAgentImplTest { String[] expectedCommand = {"vespa-rpc-invoke", "-t", "2", "tcp/localhost:19091", "setExtraMetrics", expectedMetrics}; doAnswer(invocation -> { - ContainerName calledContainerName = (ContainerName) invocation.getArguments()[0]; + NodeAgentContext calledContainerName = (NodeAgentContext) invocation.getArguments()[0]; long calledTimeout = (long) invocation.getArguments()[1]; String[] calledCommand = new String[invocation.getArguments().length - 2]; System.arraycopy(invocation.getArguments(), 2, calledCommand, 0, calledCommand.length); calledCommand[calledCommand.length - 1] = calledCommand[calledCommand.length - 1].replaceAll("\"timestamp\":\\d+", "\"timestamp\":0"); - assertEquals(context.containerName(), calledContainerName); + assertEquals(context, calledContainerName); assertEquals(5L, calledTimeout); assertArrayEquals(expectedCommand, calledCommand); return null; @@ -707,7 +707,7 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(null, false); when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); - when(dockerOperations.getContainerStats(eq(context.containerName()))).thenReturn(Optional.empty()); + when(dockerOperations.getContainerStats(eq(context))).thenReturn(Optional.empty()); nodeAgent.converge(); // Run the converge loop once to initialize lastNode @@ -737,15 +737,15 @@ public class NodeAgentImplTest { nodeAgent.converge(); - verify(dockerOperations, never()).removeContainer(any()); + verify(dockerOperations, never()).removeContainer(eq(context), any()); verify(orchestrator, never()).suspend(any(String.class)); final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository, aclMaintainer); inOrder.verify(dockerOperations, times(1)).pullImageAsyncIfNeeded(eq(dockerImage)); - inOrder.verify(dockerOperations, times(1)).createContainer(eq(context.containerName()), eq(node), any()); - inOrder.verify(dockerOperations, times(1)).startContainer(eq(context.containerName())); + inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); + inOrder.verify(dockerOperations, times(1)).startContainer(eq(context)); inOrder.verify(aclMaintainer, times(1)).run(); - inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context.containerName())); + inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( hostName, new NodeAttributes() .withRebootGeneration(rebootGeneration) @@ -775,6 +775,6 @@ public class NodeAgentImplTest { isRunning ? 1 : 0)) : Optional.empty(); - when(dockerOperations.getContainer(eq(context.containerName()))).thenReturn(container); + when(dockerOperations.getContainer(eq(context))).thenReturn(container); } } |