aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorValerij Fredriksen <freva@users.noreply.github.com>2018-10-15 13:41:37 +0200
committerGitHub <noreply@github.com>2018-10-15 13:41:37 +0200
commit5504d4ddf0c5d4e1aa2ac966a25363bdb37bbae0 (patch)
treecd88c643b5ccecb20061ba872a57df494d0aca48
parentb3e088d6a4cbbe629a4545f15107da56e1008ff2 (diff)
parentd60a43f09ea04dee7d9881f514db1d4d27980796 (diff)
Merge pull request #7295 from vespa-engine/freva/node-agent-context
NodeAdmin: Use NodeAgentContext in DockerOperations
-rw-r--r--docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java7
-rw-r--r--docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java38
-rw-r--r--docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java23
-rw-r--r--docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java3
-rw-r--r--docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java25
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java244
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java2
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java6
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java18
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java24
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java31
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java4
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java26
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java2
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java8
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java6
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java2
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java2
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java7
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java106
21 files changed, 258 insertions, 331 deletions
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
index cf168df4634..04aad562711 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.dockerapi.exception.DockerException;
import java.net.Inet6Address;
import java.net.InetAddress;
+import java.nio.file.Path;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.ArrayList;
@@ -107,15 +108,13 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
}
@Override
- public Docker.CreateContainerCommand withVolume(String path, String volumePath) {
- assert path.indexOf(':') == -1;
+ public Docker.CreateContainerCommand withVolume(Path path, Path volumePath) {
volumeBindSpecs.add(path + ":" + volumePath + ":Z");
return this;
}
@Override
- public Docker.CreateContainerCommand withSharedVolume(String path, String volumePath) {
- assert path.indexOf(':') == -1;
+ public Docker.CreateContainerCommand withSharedVolume(Path path, Path volumePath) {
volumeBindSpecs.add(path + ":" + volumePath + ":z");
return this;
}
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java
index bc2c9514709..1713b4570b8 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java
@@ -2,9 +2,11 @@
package com.yahoo.vespa.hosted.dockerapi;
import java.net.InetAddress;
+import java.nio.file.Path;
import java.time.Duration;
import java.util.List;
import java.util.Optional;
+import java.util.OptionalLong;
/**
* API to simplify the com.github.dockerjava API for clients,
@@ -26,19 +28,19 @@ public interface Docker {
* /foo's content visible/readable/writable only inside the container which was last
* started and on the host. All the other containers will get "Permission denied".
*
- * <p>Use {@link #withSharedVolume(String, String)} to mount a given host directory
+ * <p>Use {@link #withSharedVolume(Path, Path)} to mount a given host directory
* into multiple containers.
*/
- CreateContainerCommand withVolume(String path, String volumePath);
+ CreateContainerCommand withVolume(Path path, Path volumePath);
/**
* Mounts a directory on host inside the docker container.
*
* <p>The bind mount content will be <b>shared</b> among multiple containers.
*
- * @see #withVolume(String, String)
+ * @see #withVolume(Path, Path)
*/
- CreateContainerCommand withSharedVolume(String path, String volumePath);
+ CreateContainerCommand withSharedVolume(Path path, Path volumePath);
CreateContainerCommand withNetworkMode(String mode);
CreateContainerCommand withIpAddress(InetAddress address);
CreateContainerCommand withUlimit(String name, int softLimit, int hardLimit);
@@ -84,34 +86,12 @@ public interface Docker {
boolean deleteUnusedDockerImages(List<DockerImage> excludes, Duration minImageAgeToDelete);
/**
- * Execute a command in docker container as $VESPA_USER. Will block until the command is finished.
- *
- * @param containerName The name of the container
- * @param command The command with arguments to run.
- *
- * @return exitcodes, stdout and stderr in the ProcessResult
- */
- ProcessResult executeInContainer(ContainerName containerName, String... command);
-
- /**
- * Execute a command in docker container as "root". Will block until the command is finished.
- *
- * @param containerName The name of the container
- * @param command The command with arguments to run.
- *
- * @return exitcodes, stdout and stderr in the ProcessResult
- */
- ProcessResult executeInContainerAsRoot(ContainerName containerName, String... command);
-
- /**
- * Execute a command in docker container as "root"
- * The timeout will not kill the process spawned.
- *
* @param containerName The name of the container
+ * @param user can be "username", "username:group", "uid" or "uid:gid"
* @param timeoutSeconds Timeout for the process to finish in seconds or without timeout if empty
- * @param command The command with arguments to run.
+ * @param command The command with arguments to run
*
* @return exitcodes, stdout and stderr in the ProcessResult
*/
- ProcessResult executeInContainerAsRoot(ContainerName containerName, Long timeoutSeconds, String... command);
+ ProcessResult executeInContainerAsUser(ContainerName containerName, String user, OptionalLong timeoutSeconds, String... command);
}
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
index 22c865f6d9e..2ac68be2b2e 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
@@ -34,6 +34,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.OptionalLong;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -41,8 +42,6 @@ import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static com.yahoo.vespa.defaults.Defaults.getDefaults;
-
public class DockerImpl implements Docker {
private static final Logger logger = Logger.getLogger(DockerImpl.class.getName());
@@ -120,25 +119,9 @@ public class DockerImpl implements Docker {
return new CreateContainerCommandImpl(dockerClient, image, containerResources, name, hostName);
}
- @Override
- public ProcessResult executeInContainer(ContainerName containerName, String... args) {
- return executeInContainerAsUser(containerName, getDefaults().vespaUser(), Optional.empty(), args);
- }
-
- @Override
- public ProcessResult executeInContainerAsRoot(ContainerName containerName, String... args) {
- return executeInContainerAsUser(containerName, "root", Optional.empty(), args);
- }
@Override
- public ProcessResult executeInContainerAsRoot(ContainerName containerName, Long timeoutSeconds, String... args) {
- return executeInContainerAsUser(containerName, "root", Optional.of(timeoutSeconds), args);
- }
-
- /**
- * Execute command in container as user, "user" can be "username", "username:group", "uid" or "uid:gid"
- */
- private ProcessResult executeInContainerAsUser(ContainerName containerName, String user, Optional<Long> timeoutSeconds, String... command) {
+ public ProcessResult executeInContainerAsUser(ContainerName containerName, String user, OptionalLong timeoutSeconds, String... command) {
try {
ExecCreateCmdResponse response = execCreateCmd(containerName, user, command);
@@ -148,7 +131,7 @@ public class DockerImpl implements Docker {
.exec(new ExecStartResultCallback(output, errors));
if (timeoutSeconds.isPresent()) {
- if (!callback.awaitCompletion(timeoutSeconds.get(), TimeUnit.SECONDS))
+ if (!callback.awaitCompletion(timeoutSeconds.getAsLong(), TimeUnit.SECONDS))
throw new DockerExecTimeoutException(String.format(
"Command '%s' did not finish within %s seconds.", command[0], timeoutSeconds));
} else {
diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java
index 5ce8c6b093c..dc041b159b5 100644
--- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java
+++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java
@@ -6,6 +6,7 @@ import org.junit.Test;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.nio.file.Paths;
import java.util.Optional;
import java.util.stream.Stream;
@@ -27,7 +28,7 @@ public class CreateContainerCommandImplTest {
.withUlimit("nproc", 10, 20)
.withEnvironment("env1", "val1")
.withEnvironment("env2", "val2")
- .withVolume("vol1", "/host/vol1")
+ .withVolume(Paths.get("vol1"), Paths.get("/host/vol1"))
.withAddCapability("SYS_PTRACE")
.withAddCapability("SYS_ADMIN")
.withDropCapability("NET_ADMIN")
diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java
index f1a8d4ef65e..8d9ea61c1d9 100644
--- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java
+++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java
@@ -19,6 +19,8 @@ import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Matchers;
+import java.util.OptionalLong;
+
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
@@ -66,7 +68,8 @@ public class DockerImplTest {
when(state.isRunning()).thenReturn(false);
when(state.getExitCode()).thenReturn(exitCode);
- final ProcessResult result = docker.executeInContainer(new ContainerName(containerId), command);
+ final ProcessResult result = docker.executeInContainerAsUser(
+ new ContainerName(containerId), "root", OptionalLong.empty(), command);
assertThat(result.getExitStatus(), is(exitCode));
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java
index 7abf179f616..c65d59a79dc 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java
@@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName;
import com.yahoo.vespa.hosted.dockerapi.ContainerStats;
import com.yahoo.vespa.hosted.dockerapi.DockerImage;
import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData;
@@ -14,39 +15,39 @@ import java.util.Optional;
public interface DockerOperations {
- void createContainer(ContainerName containerName, NodeSpec node, ContainerData containerData);
+ void createContainer(NodeAgentContext context, NodeSpec node, ContainerData containerData);
- void startContainer(ContainerName containerName);
+ void startContainer(NodeAgentContext context);
- void removeContainer(Container existingContainer);
+ void removeContainer(NodeAgentContext context, Container container);
- Optional<Container> getContainer(ContainerName containerName);
+ Optional<Container> getContainer(NodeAgentContext context);
boolean pullImageAsyncIfNeeded(DockerImage dockerImage);
- ProcessResult executeCommandInContainerAsRoot(ContainerName containerName, String... command);
+ ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, String... command);
- ProcessResult executeCommandInContainerAsRoot(ContainerName containerName, Long timeoutSeconds, String... command);
+ ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, Long timeoutSeconds, String... command);
ProcessResult executeCommandInNetworkNamespace(ContainerName containerName, String... command);
/** Resume node. Resuming a node means that it is ready to take on traffic. */
- void resumeNode(ContainerName containerName);
+ void resumeNode(NodeAgentContext context);
/**
* Suspend node. Suspending a node means the node should be taken temporarly offline,
* such that maintenance of the node can be done (upgrading, rebooting, etc).
*/
- void suspendNode(ContainerName containerName);
+ void suspendNode(NodeAgentContext context);
- void restartVespa(ContainerName containerName);
+ void restartVespa(NodeAgentContext context);
- void startServices(ContainerName containerName);
+ void startServices(NodeAgentContext context);
- void stopServices(ContainerName containerName);
+ void stopServices(NodeAgentContext context);
- Optional<ContainerStats> getContainerStats(ContainerName containerName);
+ Optional<ContainerStats> getContainerStats(NodeAgentContext context);
/**
* Returns the list of containers managed by node-admin
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
index 13ebf84228c..13ee623a74e 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
@@ -13,22 +13,22 @@ import com.yahoo.vespa.hosted.dockerapi.Docker;
import com.yahoo.vespa.hosted.dockerapi.DockerImage;
import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
import com.yahoo.vespa.hosted.node.admin.component.Environment;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData;
import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses;
-import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger;
import java.io.IOException;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
+import java.util.OptionalLong;
+import java.util.logging.Logger;
import java.util.stream.Stream;
/**
@@ -38,6 +38,8 @@ import java.util.stream.Stream;
*/
public class DockerOperationsImpl implements DockerOperations {
+ private static final Logger logger = Logger.getLogger(DockerOperationsImpl.class.getName());
+
private static final String MANAGER_NAME = "node-admin";
private static final String IPV6_NPT_PREFIX = "fd00::";
@@ -46,22 +48,16 @@ public class DockerOperationsImpl implements DockerOperations {
private final Docker docker;
private final Environment environment;
private final ProcessExecuter processExecuter;
- private final String nodeProgram;
- private final Map<Path, Boolean> directoriesToMount;
public DockerOperationsImpl(Docker docker, Environment environment, ProcessExecuter processExecuter) {
this.docker = docker;
this.environment = environment;
this.processExecuter = processExecuter;
-
- this.nodeProgram = environment.pathInNodeUnderVespaHome("bin/vespa-nodectl").toString();
- this.directoriesToMount = getDirectoriesToMount(environment);
}
@Override
- public void createContainer(ContainerName containerName, NodeSpec node, ContainerData containerData) {
- PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName);
- logger.info("Creating container " + containerName);
+ public void createContainer(NodeAgentContext context, NodeSpec node, ContainerData containerData) {
+ context.log(logger, "Creating container");
// IPv6 - Assume always valid
Inet6Address ipV6Address = environment.getIpAddresses().getIPv6Address(node.getHostname()).orElseThrow(
@@ -73,29 +69,18 @@ public class DockerOperationsImpl implements DockerOperations {
Docker.CreateContainerCommand command = docker.createContainerCommand(
node.getWantedDockerImage().get(),
ContainerResources.from(node.getMinCpuCores(), node.getMinMainMemoryAvailableGb()),
- containerName,
+ context.containerName(),
node.getHostname())
.withManagedBy(MANAGER_NAME)
.withEnvironment("VESPA_CONFIGSERVERS", configServers)
.withEnvironment("CONTAINER_ENVIRONMENT_SETTINGS",
- environment.getContainerEnvironmentResolver().createSettings(environment, node))
+ environment.getContainerEnvironmentResolver().createSettings(environment, node))
.withUlimit("nofile", 262_144, 262_144)
.withUlimit("nproc", 32_768, 409_600)
.withUlimit("core", -1, -1)
.withAddCapability("SYS_PTRACE") // Needed for gcore, pstack etc.
.withAddCapability("SYS_ADMIN"); // Needed for perf
- if (isInfrastructureHost(environment.getNodeType())) {
- command.withVolume("/var/lib/sia", "/var/lib/sia");
- }
-
- if (environment.getNodeType() == NodeType.proxyhost) {
- command.withVolume("/opt/yahoo/share/ssl/certs", "/opt/yahoo/share/ssl/certs");
- }
-
- if (environment.getNodeType() == NodeType.host) {
- command.withSharedVolume("/var/zpe", environment.pathInNodeUnderVespaHome("var/zpe").toString());
- }
DockerNetworking networking = environment.getDockerNetworking();
command.withNetworkMode(networking.getDockerNetworkMode());
@@ -116,10 +101,7 @@ public class DockerOperationsImpl implements DockerOperations {
addEtcHosts(containerData, node.getHostname(), ipV4Local, ipV6Local);
}
- for (Path pathInNode : directoriesToMount.keySet()) {
- String pathInHost = environment.pathInHostFromPathInNode(containerName, pathInNode).toString();
- command.withVolume(pathInHost, pathInNode.toString());
- }
+ addMounts(context, command);
// TODO: Enforce disk constraints
long minMainMemoryAvailableMb = (long) (node.getMinMainMemoryAvailableGb() * 1024);
@@ -164,35 +146,25 @@ public class DockerOperationsImpl implements DockerOperations {
}
@Override
- public void startContainer(ContainerName containerName) {
- PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName);
- logger.info("Starting container " + containerName);
-
- docker.startContainer(containerName);
-
- directoriesToMount.entrySet().stream()
- .filter(Map.Entry::getValue)
- .map(Map.Entry::getKey)
- .forEach(path ->
- docker.executeInContainerAsRoot(containerName, "chmod", "-R", "a+w", path.toString()));
+ public void startContainer(NodeAgentContext context) {
+ context.log(logger, "Starting container");
+ docker.startContainer(context.containerName());
}
@Override
- public void removeContainer(Container existingContainer) {
- final ContainerName containerName = existingContainer.name;
- PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName);
- if (existingContainer.state.isRunning()) {
- logger.info("Stopping container " + containerName.asString());
- docker.stopContainer(containerName);
+ public void removeContainer(NodeAgentContext context, Container container) {
+ if (container.state.isRunning()) {
+ context.log(logger, "Stopping container");
+ docker.stopContainer(context.containerName());
}
- logger.info("Deleting container " + containerName.asString());
- docker.deleteContainer(containerName);
+ context.log(logger, "Deleting container");
+ docker.deleteContainer(context.containerName());
}
@Override
- public Optional<Container> getContainer(ContainerName containerName) {
- return docker.getContainer(containerName);
+ public Optional<Container> getContainer(NodeAgentContext context) {
+ return docker.getContainer(context.containerName());
}
@Override
@@ -200,88 +172,83 @@ public class DockerOperationsImpl implements DockerOperations {
return docker.pullImageAsyncIfNeeded(dockerImage);
}
- ProcessResult executeCommandInContainer(ContainerName containerName, String... command) {
- ProcessResult result = docker.executeInContainerAsRoot(containerName, command);
-
- if (!result.isSuccess()) {
- throw new RuntimeException("Container " + containerName.asString() +
- ": command " + Arrays.toString(command) + " failed: " + result);
- }
- return result;
- }
-
@Override
- public ProcessResult executeCommandInContainerAsRoot(ContainerName containerName, Long timeoutSeconds, String... command) {
- return docker.executeInContainerAsRoot(containerName, timeoutSeconds, command);
+ public ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, Long timeoutSeconds, String... command) {
+ return docker.executeInContainerAsUser(context.containerName(), "root", OptionalLong.of(timeoutSeconds), command);
}
@Override
- public ProcessResult executeCommandInContainerAsRoot(ContainerName containerName, String... command) {
- return docker.executeInContainerAsRoot(containerName, command);
+ public ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, String... command) {
+ return docker.executeInContainerAsUser(context.containerName(), "root", OptionalLong.empty(), command);
}
@Override
public ProcessResult executeCommandInNetworkNamespace(ContainerName containerName, String... command) {
- final PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName);
final Integer containerPid = docker.getContainer(containerName)
.filter(container -> container.state.isRunning())
.map(container -> container.pid)
.orElseThrow(() -> new RuntimeException("PID not found for container with name: " +
containerName.asString()));
- Path procPath = environment.getPathResolver().getPathToRootOfHost().resolve("proc");
-
final String[] wrappedCommand = Stream.concat(
- Stream.of("nsenter", String.format("--net=%s/%d/ns/net", procPath, containerPid), "--"),
+ Stream.of("nsenter", String.format("--net=/proc/%d/ns/net", containerPid), "--"),
Stream.of(command))
.toArray(String[]::new);
try {
Pair<Integer, String> result = processExecuter.exec(wrappedCommand);
if (result.getFirst() != 0) {
- String msg = String.format(
+ throw new RuntimeException(String.format(
"Failed to execute %s in network namespace for %s (PID = %d), exit code: %d, output: %s",
- Arrays.toString(wrappedCommand), containerName.asString(), containerPid, result.getFirst(), result.getSecond());
- logger.error(msg);
- throw new RuntimeException(msg);
+ Arrays.toString(wrappedCommand), containerName.asString(), containerPid, result.getFirst(), result.getSecond()));
}
return new ProcessResult(0, result.getSecond(), "");
} catch (IOException e) {
- logger.warning(String.format("IOException while executing %s in network namespace for %s (PID = %d)",
+ throw new RuntimeException(String.format("IOException while executing %s in network namespace for %s (PID = %d)",
Arrays.toString(wrappedCommand), containerName.asString(), containerPid), e);
- throw new RuntimeException(e);
}
}
@Override
- public void resumeNode(ContainerName containerName) {
- executeCommandInContainer(containerName, nodeProgram, "resume");
+ public void resumeNode(NodeAgentContext context) {
+ executeNodeCtlInContainer(context, "resume");
}
@Override
- public void suspendNode(ContainerName containerName) {
- executeCommandInContainer(containerName, nodeProgram, "suspend");
+ public void suspendNode(NodeAgentContext context) {
+ executeNodeCtlInContainer(context, "suspend");
}
@Override
- public void restartVespa(ContainerName containerName) {
- executeCommandInContainer(containerName, nodeProgram, "restart-vespa");
+ public void restartVespa(NodeAgentContext context) {
+ executeNodeCtlInContainer(context, "restart-vespa");
}
@Override
- public void startServices(ContainerName containerName) {
- executeCommandInContainer(containerName, nodeProgram, "start");
+ public void startServices(NodeAgentContext context) {
+ executeNodeCtlInContainer(context, "start");
}
@Override
- public void stopServices(ContainerName containerName) {
- executeCommandInContainer(containerName, nodeProgram, "stop");
+ public void stopServices(NodeAgentContext context) {
+ executeNodeCtlInContainer(context, "stop");
+ }
+
+ ProcessResult executeNodeCtlInContainer(NodeAgentContext context, String program) {
+ String[] command = new String[] {context.pathInNodeUnderVespaHome("bin/vespa-nodectl").toString(), program};
+ ProcessResult result = executeCommandInContainerAsRoot(context, command);
+
+ if (!result.isSuccess()) {
+ throw new RuntimeException("Container " + context.containerName().asString() +
+ ": command " + Arrays.toString(command) + " failed: " + result);
+ }
+ return result;
}
@Override
- public Optional<ContainerStats> getContainerStats(ContainerName containerName) {
- return docker.getContainerStats(containerName);
+ public Optional<ContainerStats> getContainerStats(NodeAgentContext context) {
+ return docker.getContainerStats(context.containerName());
}
@Override
@@ -292,56 +259,71 @@ public class DockerOperationsImpl implements DockerOperations {
/**
* Returns map of directories to mount and whether they should be writable by everyone
*/
- private static Map<Path, Boolean> getDirectoriesToMount(Environment environment) {
- final Map<Path, Boolean> directoriesToMount = new HashMap<>();
- directoriesToMount.put(Paths.get("/etc/yamas-agent"), true);
- directoriesToMount.put(Paths.get("/etc/filebeat"), true);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/daemontools_y"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/jdisc_core"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/langdetect/"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/nginx"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/vespa"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/yca"), true);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/yck"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/yell"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/ykeykey"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/ykeykeyd"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/yms_agent"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/ysar"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/ystatus"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/zpu"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/cache"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/crash"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/db/jdisc"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/db/vespa"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/jdisc_container"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/jdisc_core"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/maven"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/mediasearch"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/run"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/scoreboards"), true);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/service"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/share"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/spool"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/vespa"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/yca"), true);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/ycore++"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/zookeeper"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("tmp"), false);
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/container-data"), false);
- if (environment.getNodeType() == NodeType.proxyhost)
- directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/vespa-hosted/routing"), true);
- if (environment.getNodeType() == NodeType.host)
- directoriesToMount.put(Paths.get("/var/lib/sia"), true);
-
- return Collections.unmodifiableMap(directoriesToMount);
+ private static void addMounts(NodeAgentContext context, Docker.CreateContainerCommand command) {
+ final Path varLibSia = Paths.get("/var/lib/sia");
+
+ // Paths unique to each container
+ List<Path> paths = new ArrayList<>(Arrays.asList(
+ Paths.get("/etc/yamas-agent"),
+ Paths.get("/etc/filebeat"),
+ context.pathInNodeUnderVespaHome("logs/daemontools_y"),
+ context.pathInNodeUnderVespaHome("logs/jdisc_core"),
+ context.pathInNodeUnderVespaHome("logs/langdetect/"),
+ context.pathInNodeUnderVespaHome("logs/nginx"),
+ context.pathInNodeUnderVespaHome("logs/vespa"),
+ context.pathInNodeUnderVespaHome("logs/yca"),
+ context.pathInNodeUnderVespaHome("logs/yck"),
+ context.pathInNodeUnderVespaHome("logs/yell"),
+ context.pathInNodeUnderVespaHome("logs/ykeykey"),
+ context.pathInNodeUnderVespaHome("logs/ykeykeyd"),
+ context.pathInNodeUnderVespaHome("logs/yms_agent"),
+ context.pathInNodeUnderVespaHome("logs/ysar"),
+ context.pathInNodeUnderVespaHome("logs/ystatus"),
+ context.pathInNodeUnderVespaHome("logs/zpu"),
+ context.pathInNodeUnderVespaHome("var/cache"),
+ context.pathInNodeUnderVespaHome("var/crash"),
+ context.pathInNodeUnderVespaHome("var/db/jdisc"),
+ context.pathInNodeUnderVespaHome("var/db/vespa"),
+ context.pathInNodeUnderVespaHome("var/jdisc_container"),
+ context.pathInNodeUnderVespaHome("var/jdisc_core"),
+ context.pathInNodeUnderVespaHome("var/maven"),
+ context.pathInNodeUnderVespaHome("var/mediasearch"),
+ context.pathInNodeUnderVespaHome("var/run"),
+ context.pathInNodeUnderVespaHome("var/scoreboards"),
+ context.pathInNodeUnderVespaHome("var/service"),
+ context.pathInNodeUnderVespaHome("var/share"),
+ context.pathInNodeUnderVespaHome("var/spool"),
+ context.pathInNodeUnderVespaHome("var/vespa"),
+ context.pathInNodeUnderVespaHome("var/yca"),
+ context.pathInNodeUnderVespaHome("var/ycore++"),
+ context.pathInNodeUnderVespaHome("var/zookeeper"),
+ context.pathInNodeUnderVespaHome("tmp"),
+ context.pathInNodeUnderVespaHome("var/container-data")));
+
+ if (context.nodeType() == NodeType.proxyhost)
+ paths.add(context.pathInNodeUnderVespaHome("var/vespa-hosted/routing"));
+ if (context.nodeType() == NodeType.host)
+ paths.add(varLibSia);
+
+ paths.forEach(path -> command.withVolume(context.pathOnHostFromPathInNode(path), path));
+
+
+ // Shared paths
+ if (isInfrastructureHost(context.nodeType()))
+ command.withSharedVolume(varLibSia, varLibSia);
+
+ if (context.nodeType() == NodeType.proxyhost)
+ command.withSharedVolume(Paths.get("/opt/yahoo/share/ssl/certs"), Paths.get("/opt/yahoo/share/ssl/certs"));
+
+ if (context.nodeType() == NodeType.host)
+ command.withSharedVolume(Paths.get("/var/zpe"), context.pathInNodeUnderVespaHome("var/zpe"));
}
/** Returns whether given nodeType is a Docker host for infrastructure nodes */
private static boolean isInfrastructureHost(NodeType nodeType) {
return nodeType == NodeType.confighost ||
- nodeType == NodeType.proxyhost ||
- nodeType == NodeType.controllerhost;
+ nodeType == NodeType.proxyhost ||
+ nodeType == NodeType.controllerhost;
}
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index 3e43ed63aea..85ec3712126 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -138,7 +138,7 @@ public class StorageMaintainer {
// Write config and restart yamas-agent
Path yamasAgentFolder = context.pathOnHostFromPathInNode("/etc/yamas-agent");
configs.forEach(s -> uncheck(() -> s.writeTo(yamasAgentFolder)));
- dockerOperations.executeCommandInContainerAsRoot(context.containerName(), "service", "yamas-agent", "restart");
+ dockerOperations.executeCommandInContainerAsRoot(context, "service", "yamas-agent", "restart");
}
private SecretAgentCheckConfig annotatedCheck(NodeSpec node, SecretAgentCheckConfig check) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
index d1c44a55c1b..2cddee6aa2a 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
@@ -39,7 +39,7 @@ public class CoreCollector {
Path readBinPathFallback(NodeAgentContext context, Path coredumpPath) {
String command = gdb + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'";
String[] wrappedCommand = {"/bin/sh", "-c", command};
- ProcessResult result = docker.executeCommandInContainerAsRoot(context.containerName(), wrappedCommand);
+ ProcessResult result = docker.executeCommandInContainerAsRoot(context, wrappedCommand);
Matcher matcher = CORE_GENERATOR_PATH_PATTERN.matcher(result.getOutput());
if (! matcher.find()) {
@@ -52,7 +52,7 @@ public class CoreCollector {
Path readBinPath(NodeAgentContext context, Path coredumpPath) {
String[] command = {"file", coredumpPath.toString()};
try {
- ProcessResult result = docker.executeCommandInContainerAsRoot(context.containerName(), command);
+ ProcessResult result = docker.executeCommandInContainerAsRoot(context, command);
if (result.getExitStatus() != 0) {
throw new RuntimeException("file command failed with " + result);
}
@@ -77,7 +77,7 @@ public class CoreCollector {
List<String> readBacktrace(NodeAgentContext context, Path coredumpPath, Path binPath, boolean allThreads) {
String threads = allThreads ? "thread apply all bt" : "bt";
String[] command = {gdb.toString(), "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()};
- ProcessResult result = docker.executeCommandInContainerAsRoot(context.containerName(), command);
+ ProcessResult result = docker.executeCommandInContainerAsRoot(context, command);
if (result.getExitStatus() != 0) {
throw new RuntimeException("Failed to read backtrace " + result + ", Command: " + Arrays.toString(command));
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
index 644ac587c34..d436e214266 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
@@ -2,7 +2,6 @@
package com.yahoo.vespa.hosted.node.admin.nodeadmin;
import com.yahoo.concurrent.ThreadFactoryFactory;
-import com.yahoo.vespa.hosted.dockerapi.Container;
import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper;
import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions;
import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper;
@@ -39,7 +38,6 @@ public class NodeAdminImpl implements NodeAdmin {
private final ScheduledExecutorService metricsScheduler =
Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("metricsscheduler"));
- private final DockerOperations dockerOperations;
private final Function<String, NodeAgent> nodeAgentFactory;
private final Runnable aclMaintainer;
@@ -58,7 +56,13 @@ public class NodeAdminImpl implements NodeAdmin {
Runnable aclMaintainer,
MetricReceiverWrapper metricReceiver,
Clock clock) {
- this.dockerOperations = dockerOperations;
+ this(nodeAgentFactory, aclMaintainer, metricReceiver, clock);
+ }
+
+ public NodeAdminImpl(Function<String, NodeAgent> nodeAgentFactory,
+ Runnable aclMaintainer,
+ MetricReceiverWrapper metricReceiver,
+ Clock clock) {
this.nodeAgentFactory = nodeAgentFactory;
this.aclMaintainer = aclMaintainer;
@@ -203,9 +207,6 @@ public class NodeAdminImpl implements NodeAdmin {
}
void synchronizeNodesToNodeAgents(Set<String> hostnamesToRun) {
- Map<String, Container> runningContainersByHostname = dockerOperations.getAllManagedContainers().stream()
- .collect(Collectors.toMap(c -> c.hostname, c -> c));
-
// Stop and remove NodeAgents that should no longer be running
diff(nodeAgentsByHostname.keySet(), hostnamesToRun)
.forEach(hostname -> nodeAgentsByHostname.remove(hostname).stop());
@@ -213,11 +214,6 @@ public class NodeAdminImpl implements NodeAdmin {
// Start NodeAgent for hostnames that should be running, but aren't yet
diff(hostnamesToRun, nodeAgentsByHostname.keySet())
.forEach(this::startNodeAgent);
-
- // Remove containers that are running, but have no NodeAgent managing it (and after the previous steps,
- // these containers shouldn't be running, otherwise a NodeAgent would have been created)
- diff(runningContainersByHostname.keySet(), nodeAgentsByHostname.keySet())
- .forEach(hostname -> dockerOperations.removeContainer(runningContainersByHostname.get(hostname)));
}
private void startNodeAgent(String hostname) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 3af78593a58..350ca2263ef 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -178,7 +178,7 @@ public class NodeAgentImpl implements NodeAgent {
serviceRestarter = service -> {
try {
ProcessResult processResult = dockerOperations.executeCommandInContainerAsRoot(
- context.containerName(), "service", service, "restart");
+ context, "service", service, "restart");
if (!processResult.isSuccess()) {
context.log(logger, LogLevel.ERROR, "Failed to restart service " + service + ": " + processResult);
@@ -219,7 +219,7 @@ public class NodeAgentImpl implements NodeAgent {
void startServicesIfNeeded() {
if (!hasStartedServices) {
context.log(logger, "Starting services");
- dockerOperations.startServices(context.containerName());
+ dockerOperations.startServices(context);
hasStartedServices = true;
}
}
@@ -234,7 +234,7 @@ public class NodeAgentImpl implements NodeAgent {
}
context.log(logger, LogLevel.DEBUG, "Starting optional node program resume command");
- dockerOperations.resumeNode(context.containerName());
+ dockerOperations.resumeNode(context);
hasResumedNode = true;
}
}
@@ -265,8 +265,8 @@ public class NodeAgentImpl implements NodeAgent {
private void startContainer(NodeSpec node) {
ContainerData containerData = createContainerData(context, node);
- dockerOperations.createContainer(context.containerName(), node, containerData);
- dockerOperations.startContainer(context.containerName());
+ dockerOperations.createContainer(context, node, containerData);
+ dockerOperations.startContainer(context);
lastCpuMetric = new CpuUsageReporter();
hasStartedServices = true; // Automatically started with the container
@@ -302,7 +302,7 @@ public class NodeAgentImpl implements NodeAgent {
context.log(logger, "Restarting services");
// Since we are restarting the services we need to suspend the node.
orchestratorSuspendNode();
- dockerOperations.restartVespa(context.containerName());
+ dockerOperations.restartVespa(context);
}
}
@@ -312,7 +312,7 @@ public class NodeAgentImpl implements NodeAgent {
if (containerState == ABSENT) return;
try {
hasStartedServices = hasResumedNode = false;
- dockerOperations.stopServices(context.containerName());
+ dockerOperations.stopServices(context);
} catch (ContainerNotFoundException e) {
containerState = ABSENT;
}
@@ -324,7 +324,7 @@ public class NodeAgentImpl implements NodeAgent {
if (containerState == ABSENT) return;
try {
hasResumedNode = false;
- dockerOperations.suspendNode(context.containerName());
+ dockerOperations.suspendNode(context);
} catch (ContainerNotFoundException e) {
containerState = ABSENT;
} catch (RuntimeException e) {
@@ -379,7 +379,7 @@ public class NodeAgentImpl implements NodeAgent {
}
stopFilebeatSchedulerIfNeeded();
storageMaintainer.handleCoreDumpsForContainer(context, node, Optional.of(existingContainer));
- dockerOperations.removeContainer(existingContainer);
+ dockerOperations.removeContainer(context, existingContainer);
containerState = ABSENT;
context.log(logger, "Container successfully removed, new containerState is " + containerState);
return Optional.empty();
@@ -600,7 +600,7 @@ public class NodeAgentImpl implements NodeAgent {
final NodeSpec node = lastNode;
if (node == null || containerState != UNKNOWN) return;
- Optional<ContainerStats> containerStats = dockerOperations.getContainerStats(context.containerName());
+ Optional<ContainerStats> containerStats = dockerOperations.getContainerStats(context);
if (!containerStats.isPresent()) return;
Dimensions.Builder dimensionsBuilder = new Dimensions.Builder()
@@ -675,7 +675,7 @@ public class NodeAgentImpl implements NodeAgent {
// Push metrics to the metrics proxy in each container - give it maximum 1 seconds to complete
String[] command = {"vespa-rpc-invoke", "-t", "2", "tcp/localhost:19091", "setExtraMetrics", wrappedMetrics};
- dockerOperations.executeCommandInContainerAsRoot(context.containerName(), 5L, command);
+ dockerOperations.executeCommandInContainerAsRoot(context, 5L, command);
} catch (DockerExecTimeoutException | JsonProcessingException e) {
context.log(logger, LogLevel.WARNING, "Failed to push metrics to container", e);
}
@@ -683,7 +683,7 @@ public class NodeAgentImpl implements NodeAgent {
private Optional<Container> getContainer() {
if (containerState == ABSENT) return Optional.empty();
- Optional<Container> container = dockerOperations.getContainer(context.containerName());
+ Optional<Container> container = dockerOperations.getContainer(context);
if (! container.isPresent()) containerState = ABSENT;
return container;
}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java
index 7f4dad6ee36..760326f061e 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java
@@ -12,6 +12,8 @@ import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
import com.yahoo.vespa.hosted.node.admin.component.Environment;
import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig;
import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImplTest;
import org.junit.Test;
import org.mockito.InOrder;
@@ -19,6 +21,7 @@ import java.io.IOException;
import java.net.InetAddress;
import java.nio.file.Paths;
import java.util.Optional;
+import java.util.OptionalLong;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;
@@ -48,43 +51,41 @@ public class DockerOperationsImplTest {
@Test
public void processResultFromNodeProgramWhenSuccess() {
- final ContainerName containerName = new ContainerName("container-name");
+ final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname("container-123.domain.tld");
final ProcessResult actualResult = new ProcessResult(0, "output", "errors");
- final String programPath = "/bin/command";
- final String[] command = new String[]{programPath, "arg"};
- when(docker.executeInContainerAsRoot(any(), anyVararg()))
+ when(docker.executeInContainerAsUser(any(), any(), any(), anyVararg()))
.thenReturn(actualResult); // output from node program
- ProcessResult result = dockerOperations.executeCommandInContainer(containerName, command);
+ ProcessResult result = dockerOperations.executeNodeCtlInContainer(context, "start");
final InOrder inOrder = inOrder(docker);
- inOrder.verify(docker, times(1)).executeInContainerAsRoot(
- eq(containerName),
- eq(command[0]),
- eq(command[1]));
+ inOrder.verify(docker, times(1)).executeInContainerAsUser(
+ eq(context.containerName()),
+ eq("root"),
+ eq(OptionalLong.empty()),
+ eq("/opt/vespa/bin/vespa-nodectl"),
+ eq("start"));
assertThat(result, is(actualResult));
}
@Test(expected = RuntimeException.class)
public void processResultFromNodeProgramWhenNonZeroExitCode() {
- final ContainerName containerName = new ContainerName("container-name");
+ final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname("container-123.domain.tld");
final ProcessResult actualResult = new ProcessResult(3, "output", "errors");
- final String programPath = "/bin/command";
- final String[] command = new String[]{programPath, "arg"};
- when(docker.executeInContainerAsRoot(any(), anyVararg()))
+ when(docker.executeInContainerAsUser(any(), any(), any(), anyVararg()))
.thenReturn(actualResult); // output from node program
- dockerOperations.executeCommandInContainer(containerName, command);
+ dockerOperations.executeNodeCtlInContainer(context, "start");
}
@Test
public void runsCommandInNetworkNamespace() throws IOException {
Container container = makeContainer("container-42", Container.State.RUNNING, 42);
- when(processExecuter.exec(aryEq(new String[]{"nsenter", "--net=/host/proc/42/ns/net", "--", "iptables", "-nvL"})))
+ when(processExecuter.exec(aryEq(new String[]{"nsenter", "--net=/proc/42/ns/net", "--", "iptables", "-nvL"})))
.thenReturn(new Pair<>(0, ""));
dockerOperations.executeCommandInNetworkNamespace(container.name, "iptables", "-nvL");
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
index ec29cf0fe25..d7e0fe3d8e5 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
@@ -37,14 +37,14 @@ public class DockerFailTest {
dockerTester.callOrderVerifier.assertInOrder(1200,
"createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
- "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
+ "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
dockerTester.dockerMock.deleteContainer(new ContainerName("host1"));
dockerTester.callOrderVerifier.assertInOrder(
"deleteContainer with ContainerName { name=host1 }",
"createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
- "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
+ "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
}
}
}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
index 313db1cfc47..26843b119de 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.dockerapi.DockerImage;
import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
import java.net.InetAddress;
+import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
@@ -17,6 +18,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.OptionalLong;
/**
* Mock with some simple logic
@@ -106,25 +108,9 @@ public class DockerMock implements Docker {
}
@Override
- public ProcessResult executeInContainer(ContainerName containerName, String... args) {
+ public ProcessResult executeInContainerAsUser(ContainerName containerName, String user, OptionalLong timeout, String... args) {
synchronized (monitor) {
- callOrderVerifier.add("executeInContainer with " + containerName + ", args: " + Arrays.toString(args));
- }
- return new ProcessResult(0, null, "");
- }
-
- @Override
- public ProcessResult executeInContainerAsRoot(ContainerName containerName, String... args) {
- synchronized (monitor) {
- callOrderVerifier.add("executeInContainerAsRoot with " + containerName + ", args: " + Arrays.toString(args));
- }
- return new ProcessResult(0, null, "");
- }
-
- @Override
- public ProcessResult executeInContainerAsRoot(ContainerName containerName, Long timeoutSeconds, String... args) {
- synchronized (monitor) {
- callOrderVerifier.add("executeInContainerAsRoot with " + containerName + ", args: " + Arrays.toString(args));
+ callOrderVerifier.add("executeInContainer " + containerName.asString() + " as " + user + ", args: " + Arrays.toString(args));
}
return new ProcessResult(0, null, "");
}
@@ -142,12 +128,12 @@ public class DockerMock implements Docker {
}
@Override
- public CreateContainerCommand withVolume(String path, String volumePath) {
+ public CreateContainerCommand withVolume(Path path, Path volumePath) {
return this;
}
@Override
- public CreateContainerCommand withSharedVolume(String path, String volumePath) {
+ public CreateContainerCommand withSharedVolume(Path path, Path volumePath) {
return this;
}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
index 3c316de94eb..ef369a21c37 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
@@ -100,7 +100,7 @@ public class DockerTester implements AutoCloseable {
Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(
NodeAgentContextImplTest.nodeAgentFromHostname(fileSystem, hostName), nodeRepositoryMock,
orchestratorMock, dockerOperations, storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL, athenzCredentialsMaintainer);
- nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, aclMaintainer, mr, Clock.systemUTC());
+ nodeAdmin = new NodeAdminImpl(nodeAgentFactory, aclMaintainer, mr, Clock.systemUTC());
nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock, storageMaintainer,
nodeAdmin, DOCKER_HOST_HOSTNAME, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL,
Optional.of(new ClassLocking()));
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
index 9c9b2afe956..e130cd0d475 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
@@ -38,16 +38,16 @@ public class MultiDockerTest {
dockerTester.callOrderVerifier.assertInOrder(
"createContainerCommand with DockerImage { imageId=image1 }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
- "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]",
+ "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]",
"createContainerCommand with DockerImage { imageId=image2 }, HostName: host2.test.yahoo.com, ContainerName { name=host2 }",
- "executeInContainerAsRoot with ContainerName { name=host2 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]",
+ "executeInContainer host2 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]",
"stopContainer with ContainerName { name=host2 }",
"deleteContainer with ContainerName { name=host2 }",
"createContainerCommand with DockerImage { imageId=image1 }, HostName: host3.test.yahoo.com, ContainerName { name=host3 }",
- "executeInContainerAsRoot with ContainerName { name=host3 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
+ "executeInContainer host3 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
dockerTester.callOrderVerifier.assertInOrderWithAssertMessage(
"Maintainer did not receive call to delete application storage",
@@ -87,7 +87,7 @@ public class MultiDockerTest {
ContainerName containerName = ContainerName.fromHostname(hostName);
tester.callOrderVerifier.assertInOrder(
"createContainerCommand with " + dockerImage + ", HostName: " + hostName + ", " + containerName,
- "executeInContainerAsRoot with " + containerName + ", args: [" + DockerTester.NODE_PROGRAM + ", resume]");
+ "executeInContainer " + containerName.asString() + " as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
return nodeSpec;
}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
index 3681878d728..da743c40e8b 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java
@@ -36,7 +36,7 @@ public class NodeStateTest {
tester.callOrderVerifier.assertInOrder(
"createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
- "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
+ "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
}
@@ -59,7 +59,7 @@ public class NodeStateTest {
}
dockerTester.callOrderVerifier.assertInOrder(
- "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", stop]",
+ "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", stop]",
"stopContainer with ContainerName { name=host1 }",
"deleteContainer with ContainerName { name=host1 }");
}
@@ -102,7 +102,7 @@ public class NodeStateTest {
"Node not started again after being put to active state",
"deleteContainer with ContainerName { name=host1 }",
"createContainerCommand with DockerImage { imageId=newDockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
- "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
+ "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
}
}
}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
index 0406934defc..2e290e014c2 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
@@ -38,7 +38,7 @@ public class RestartTest {
dockerTester.callOrderVerifier.assertInOrder(
"Suspend for host1.test.yahoo.com",
- "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", restart-vespa]");
+ "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", restart-vespa]");
}
}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
index b5950646da9..0fed9433432 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
@@ -152,7 +152,7 @@ public class CoreCollectorTest {
}
private void mockExec(String[] cmd, String output, String error) {
- when(docker.executeCommandInContainerAsRoot(context.containerName(), cmd))
+ when(docker.executeCommandInContainerAsRoot(context, cmd))
.thenReturn(new ProcessResult(error.isEmpty() ? 0 : 1, output, error));
}
}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
index 065b039c7fd..2431433b17f 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin;
import com.yahoo.metrics.simple.MetricReceiver;
import com.yahoo.test.ManualClock;
import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
-import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent;
import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl;
import org.junit.Test;
@@ -37,12 +36,11 @@ import static org.mockito.Mockito.when;
public class NodeAdminImplTest {
// Trick to allow mocking of typed interface without casts/warnings.
private interface NodeAgentFactory extends Function<String, NodeAgent> {}
- private final DockerOperations dockerOperations = mock(DockerOperations.class);
private final Function<String, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class);
private final Runnable aclMaintainer = mock(Runnable.class);
private final ManualClock clock = new ManualClock();
- private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, aclMaintainer,
+ private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentFactory, aclMaintainer,
new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock);
@Test
@@ -53,7 +51,6 @@ public class NodeAdminImplTest {
final NodeAgent nodeAgent2 = mock(NodeAgentImpl.class);
when(nodeAgentFactory.apply(eq(hostName1))).thenReturn(nodeAgent1);
when(nodeAgentFactory.apply(eq(hostName2))).thenReturn(nodeAgent2);
- when(dockerOperations.getAllManagedContainers()).thenReturn(Collections.emptyList());
final InOrder inOrder = inOrder(nodeAgentFactory, nodeAgent1, nodeAgent2);
@@ -91,8 +88,6 @@ public class NodeAdminImplTest {
@Test
public void testSetFrozen() {
- when(dockerOperations.getAllManagedContainers()).thenReturn(Collections.emptyList());
-
List<NodeAgent> nodeAgents = new ArrayList<>();
Set<String> existingContainerHostnames = new HashSet<>();
for (int i = 0; i < 3; i++) {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index fbb584ea1d9..b2facd3dd6d 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -124,15 +124,15 @@ public class NodeAgentImplTest {
nodeAgent.converge();
- verify(dockerOperations, never()).removeContainer(any());
+ verify(dockerOperations, never()).removeContainer(eq(context), any());
verify(orchestrator, never()).suspend(any(String.class));
verify(dockerOperations, never()).pullImageAsyncIfNeeded(any());
verify(storageMaintainer, never()).removeOldFilesFromNode(eq(context));
final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository);
// TODO: Verify this isn't run unless 1st time
- inOrder.verify(dockerOperations, never()).startServices(eq(context.containerName()));
- inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context.containerName()));
+ inOrder.verify(dockerOperations, never()).startServices(eq(context));
+ inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context));
inOrder.verify(orchestrator).resume(hostName);
}
@@ -176,24 +176,24 @@ public class NodeAgentImplTest {
when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(187500000000L));
nodeAgent.converge();
- inOrder.verify(dockerOperations, never()).startServices(eq(context.containerName()));
- inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context.containerName()));
+ inOrder.verify(dockerOperations, never()).startServices(eq(context));
+ inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context));
nodeAgent.suspend();
nodeAgent.converge();
- inOrder.verify(dockerOperations, never()).startServices(eq(context.containerName()));
- inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context.containerName())); // Expect a resume, but no start services
+ inOrder.verify(dockerOperations, never()).startServices(eq(context));
+ inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); // Expect a resume, but no start services
// No new suspends/stops, so no need to resume/start
nodeAgent.converge();
- inOrder.verify(dockerOperations, never()).startServices(eq(context.containerName()));
- inOrder.verify(dockerOperations, never()).resumeNode(eq(context.containerName()));
+ inOrder.verify(dockerOperations, never()).startServices(eq(context));
+ inOrder.verify(dockerOperations, never()).resumeNode(eq(context));
nodeAgent.suspend();
nodeAgent.stopServices();
nodeAgent.converge();
- inOrder.verify(dockerOperations, times(1)).startServices(eq(context.containerName()));
- inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context.containerName()));
+ inOrder.verify(dockerOperations, times(1)).startServices(eq(context));
+ inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context));
}
@Test
@@ -219,16 +219,16 @@ public class NodeAgentImplTest {
nodeAgent.converge();
- verify(dockerOperations, never()).removeContainer(any());
+ verify(dockerOperations, never()).removeContainer(eq(context), any());
verify(dockerOperations, never()).startServices(any());
verify(orchestrator, never()).suspend(any(String.class));
final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository, aclMaintainer);
inOrder.verify(dockerOperations, times(1)).pullImageAsyncIfNeeded(eq(dockerImage));
- inOrder.verify(dockerOperations, times(1)).createContainer(eq(context.containerName()), eq(node), any());
- inOrder.verify(dockerOperations, times(1)).startContainer(eq(context.containerName()));
+ inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any());
+ inOrder.verify(dockerOperations, times(1)).startContainer(eq(context));
inOrder.verify(aclMaintainer, times(1)).run();
- inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context.containerName()));
+ inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context));
inOrder.verify(nodeRepository).updateNodeAttributes(
hostName, new NodeAttributes()
.withRestartGeneration(restartGeneration)
@@ -262,7 +262,7 @@ public class NodeAgentImplTest {
verify(orchestrator, never()).suspend(any(String.class));
verify(orchestrator, never()).resume(any(String.class));
- verify(dockerOperations, never()).removeContainer(any());
+ verify(dockerOperations, never()).removeContainer(eq(context), any());
final InOrder inOrder = inOrder(dockerOperations);
inOrder.verify(dockerOperations, times(1)).pullImageAsyncIfNeeded(eq(newDockerImage));
@@ -302,9 +302,9 @@ public class NodeAgentImplTest {
inOrder.verify(orchestrator).resume(any(String.class));
inOrder.verify(orchestrator).resume(any(String.class));
inOrder.verify(orchestrator).suspend(any(String.class));
- inOrder.verify(dockerOperations).removeContainer(any());
- inOrder.verify(dockerOperations, times(1)).createContainer(eq(context.containerName()), eq(thirdSpec), any());
- inOrder.verify(dockerOperations).startContainer(eq(context.containerName()));
+ inOrder.verify(dockerOperations).removeContainer(eq(context), any());
+ inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), eq(thirdSpec), any());
+ inOrder.verify(dockerOperations).startContainer(eq(context));
inOrder.verify(orchestrator).resume(any(String.class));
}
@@ -329,8 +329,8 @@ public class NodeAgentImplTest {
fail("Expected to throw an exception");
} catch (Exception ignored) { }
- verify(dockerOperations, never()).createContainer(eq(context.containerName()), eq(node), any());
- verify(dockerOperations, never()).startContainer(eq(context.containerName()));
+ verify(dockerOperations, never()).createContainer(eq(context), eq(node), any());
+ verify(dockerOperations, never()).startContainer(eq(context));
verify(orchestrator, never()).resume(any(String.class));
verify(nodeRepository, never()).updateNodeAttributes(any(String.class), any(NodeAttributes.class));
}
@@ -354,7 +354,7 @@ public class NodeAgentImplTest {
nodeAgent.converge();
- verify(dockerOperations, never()).removeContainer(any());
+ verify(dockerOperations, never()).removeContainer(eq(context), any());
verify(orchestrator, never()).resume(any(String.class));
verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any());
}
@@ -379,10 +379,10 @@ public class NodeAgentImplTest {
nodeAgent.converge();
// Should only be called once, when we initialize
- verify(dockerOperations, times(1)).getContainer(eq(context.containerName()));
- verify(dockerOperations, never()).removeContainer(any());
- verify(dockerOperations, never()).createContainer(eq(context.containerName()), eq(node), any());
- verify(dockerOperations, never()).startContainer(eq(context.containerName()));
+ verify(dockerOperations, times(1)).getContainer(eq(context));
+ verify(dockerOperations, never()).removeContainer(eq(context), any());
+ verify(dockerOperations, never()).createContainer(eq(context), eq(node), any());
+ verify(dockerOperations, never()).startContainer(eq(context));
verify(orchestrator, never()).resume(any(String.class));
verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any());
}
@@ -410,7 +410,7 @@ public class NodeAgentImplTest {
nodeAgent.converge();
final InOrder inOrder = inOrder(storageMaintainer, dockerOperations);
- inOrder.verify(dockerOperations, never()).removeContainer(any());
+ inOrder.verify(dockerOperations, never()).removeContainer(eq(context), any());
verify(orchestrator, never()).resume(any(String.class));
verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any());
@@ -457,16 +457,16 @@ public class NodeAgentImplTest {
nodeAgent.converge();
final InOrder inOrder = inOrder(storageMaintainer, dockerOperations, nodeRepository);
- inOrder.verify(dockerOperations, times(1)).stopServices(eq(context.containerName()));
+ inOrder.verify(dockerOperations, times(1)).stopServices(eq(context));
inOrder.verify(storageMaintainer, times(1)).handleCoreDumpsForContainer(eq(context), eq(node), any());
- inOrder.verify(dockerOperations, times(1)).removeContainer(any());
+ inOrder.verify(dockerOperations, times(1)).removeContainer(eq(context), any());
inOrder.verify(storageMaintainer, times(1)).archiveNodeStorage(eq(context));
inOrder.verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(Node.State.ready));
- verify(dockerOperations, never()).createContainer(eq(context.containerName()), any(), any());
- verify(dockerOperations, never()).startContainer(eq(context.containerName()));
- verify(dockerOperations, never()).suspendNode(eq(context.containerName()));
- verify(dockerOperations, times(1)).stopServices(eq(context.containerName()));
+ verify(dockerOperations, never()).createContainer(eq(context), any(), any());
+ verify(dockerOperations, never()).startContainer(eq(context));
+ verify(dockerOperations, never()).suspendNode(eq(context));
+ verify(dockerOperations, times(1)).stopServices(eq(context));
verify(orchestrator, never()).resume(any(String.class));
verify(orchestrator, never()).suspend(any(String.class));
// current Docker image and vespa version should be cleared
@@ -519,9 +519,9 @@ public class NodeAgentImplTest {
nodeAgent.tick();
- verify(dockerOperations, times(1)).removeContainer(any());
- verify(dockerOperations, times(1)).createContainer(eq(context.containerName()), eq(node), any());
- verify(dockerOperations, times(1)).startContainer(eq(context.containerName()));
+ verify(dockerOperations, times(1)).removeContainer(eq(context), any());
+ verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any());
+ verify(dockerOperations, times(1)).startContainer(eq(context));
}
@Test
@@ -544,7 +544,7 @@ public class NodeAgentImplTest {
final InOrder inOrder = inOrder(orchestrator, dockerOperations, nodeRepository);
doThrow(new RuntimeException("Failed 1st time"))
.doNothing()
- .when(dockerOperations).resumeNode(eq(context.containerName()));
+ .when(dockerOperations).resumeNode(eq(context));
// 1st try
try {
@@ -610,16 +610,16 @@ public class NodeAgentImplTest {
when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar"));
when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false);
when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L));
- doThrow(new DockerException("Failed to set up network")).doNothing().when(dockerOperations).startContainer(eq(context.containerName()));
+ doThrow(new DockerException("Failed to set up network")).doNothing().when(dockerOperations).startContainer(eq(context));
try {
nodeAgent.converge();
fail("Expected to get DockerException");
} catch (DockerException ignored) { }
- verify(dockerOperations, never()).removeContainer(any());
- verify(dockerOperations, times(1)).createContainer(eq(context.containerName()), eq(node), any());
- verify(dockerOperations, times(1)).startContainer(eq(context.containerName()));
+ verify(dockerOperations, never()).removeContainer(eq(context), any());
+ verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any());
+ verify(dockerOperations, times(1)).startContainer(eq(context));
verify(nodeAgent, never()).resumeNodeIfNeeded(any());
// The docker container was actually started and is running, but subsequent exec calls to set up
@@ -627,9 +627,9 @@ public class NodeAgentImplTest {
mockGetContainer(dockerImage, true);
nodeAgent.converge();
- verify(dockerOperations, times(1)).removeContainer(any());
- verify(dockerOperations, times(2)).createContainer(eq(context.containerName()), eq(node), any());
- verify(dockerOperations, times(2)).startContainer(eq(context.containerName()));
+ verify(dockerOperations, times(1)).removeContainer(eq(context), any());
+ verify(dockerOperations, times(2)).createContainer(eq(context), eq(node), any());
+ verify(dockerOperations, times(2)).startContainer(eq(context));
verify(nodeAgent, times(1)).resumeNodeIfNeeded(any());
}
@@ -666,7 +666,7 @@ public class NodeAgentImplTest {
when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node));
when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(39625000000L));
- when(dockerOperations.getContainerStats(eq(context.containerName())))
+ when(dockerOperations.getContainerStats(eq(context)))
.thenReturn(Optional.of(stats1))
.thenReturn(Optional.of(stats2));
when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar"));
@@ -683,13 +683,13 @@ public class NodeAgentImplTest {
String[] expectedCommand = {"vespa-rpc-invoke", "-t", "2", "tcp/localhost:19091", "setExtraMetrics", expectedMetrics};
doAnswer(invocation -> {
- ContainerName calledContainerName = (ContainerName) invocation.getArguments()[0];
+ NodeAgentContext calledContainerName = (NodeAgentContext) invocation.getArguments()[0];
long calledTimeout = (long) invocation.getArguments()[1];
String[] calledCommand = new String[invocation.getArguments().length - 2];
System.arraycopy(invocation.getArguments(), 2, calledCommand, 0, calledCommand.length);
calledCommand[calledCommand.length - 1] = calledCommand[calledCommand.length - 1].replaceAll("\"timestamp\":\\d+", "\"timestamp\":0");
- assertEquals(context.containerName(), calledContainerName);
+ assertEquals(context, calledContainerName);
assertEquals(5L, calledTimeout);
assertArrayEquals(expectedCommand, calledCommand);
return null;
@@ -707,7 +707,7 @@ public class NodeAgentImplTest {
NodeAgentImpl nodeAgent = makeNodeAgent(null, false);
when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node));
- when(dockerOperations.getContainerStats(eq(context.containerName()))).thenReturn(Optional.empty());
+ when(dockerOperations.getContainerStats(eq(context))).thenReturn(Optional.empty());
nodeAgent.converge(); // Run the converge loop once to initialize lastNode
@@ -737,15 +737,15 @@ public class NodeAgentImplTest {
nodeAgent.converge();
- verify(dockerOperations, never()).removeContainer(any());
+ verify(dockerOperations, never()).removeContainer(eq(context), any());
verify(orchestrator, never()).suspend(any(String.class));
final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository, aclMaintainer);
inOrder.verify(dockerOperations, times(1)).pullImageAsyncIfNeeded(eq(dockerImage));
- inOrder.verify(dockerOperations, times(1)).createContainer(eq(context.containerName()), eq(node), any());
- inOrder.verify(dockerOperations, times(1)).startContainer(eq(context.containerName()));
+ inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any());
+ inOrder.verify(dockerOperations, times(1)).startContainer(eq(context));
inOrder.verify(aclMaintainer, times(1)).run();
- inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context.containerName()));
+ inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context));
inOrder.verify(nodeRepository).updateNodeAttributes(
hostName, new NodeAttributes()
.withRebootGeneration(rebootGeneration)
@@ -775,6 +775,6 @@ public class NodeAgentImplTest {
isRunning ? 1 : 0)) :
Optional.empty();
- when(dockerOperations.getContainer(eq(context.containerName()))).thenReturn(container);
+ when(dockerOperations.getContainer(eq(context))).thenReturn(container);
}
}