1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
|
// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.dockerapi;
import com.github.dockerjava.api.command.ExecCreateCmdResponse;
import com.github.dockerjava.api.command.ExecStartCmd;
import com.github.dockerjava.api.command.InspectExecResponse;
import com.github.dockerjava.core.command.ExecStartResultCallback;
import com.yahoo.system.ProcessExecuter;
import java.io.IOException;
import java.net.InetAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.logging.Logger;
import static org.junit.Assert.assertEquals;
/**
* <pre>
* Requires docker daemon, see node-admin/README.md for how to install and configure.
*
* To get started:
* 1. Add system test host hostnames to /etc/hosts:
* $ sudo ./vespa/node-admin/scripts/etc-hosts.sh
*
*
* Example usage:
DockerImage vespaDockerBase = new DockerImage("docker-registry.ops.yahoo.com:4443/vespa/ci:6.52.35");
Path pathToSystemtestsInHost = Paths.get("/home/valerijf/dev/systemtests");
RunSystemTests runSystemTests = new RunSystemTests(vespaDockerBase, pathToSystemtestsInHost);
ContainerName systemtestsHost = new ContainerName("stest-1");
// Update maven local repository and /home/y/lib/jars with the current version of these modules inside container
runSystemTests.updateContainerMavenLocalRepository(systemtestsHost);
Path systemTestToRun = Paths.get("tests/search/basicsearch/basic_search.rb");
runSystemTests.runSystemTest(systemtestsHost, systemTestToRun);
* </pre>
*
* @author freva
*/
public class RunSystemTests {
private static final DockerImage SYSTEMTESTS_DOCKER_IMAGE = new DockerImage("vespa-systest:latest");
private final DockerImpl docker;
private final DockerImage vespaBaseImage;
private final Path pathToSystemtestsInHost;
private final Path pathToSystemtestsInContainer = Paths.get("/systemtests");
private final Path pathToVespaRepoInHost = Paths.get("").toAbsolutePath();
private final Path pathToVespaRepoInContainer = Paths.get("/vespa");
private final Path pathToTestRunner = pathToSystemtestsInContainer.resolve("bin/run_test.rb");
private final Path pathToLibJars = Paths.get("/home/y/lib/jars");
private final String username = System.getProperty("user.name");
private final Logger logger = Logger.getLogger("systemtest");
public RunSystemTests(DockerImage vespaBaseImage, Path pathToSystemtestsInHost) {
this.docker = DockerTestUtils.getDocker();
this.vespaBaseImage = vespaBaseImage;
this.pathToSystemtestsInHost = pathToSystemtestsInHost;
}
/**
* @param systemtestHost name of the container that will execute the test, if it does not exist, a new
* one will be started.
* @param systemtestToRun relative path from the root of systemtests to the test to run, f.ex.
* tests/search/basicsearch/basic_search.rb
*/
void runSystemTest(ContainerName systemtestHost, Path systemtestToRun, String... arguments) throws InterruptedException, ExecutionException, IOException {
runSystemTest(Collections.singletonList(systemtestHost), systemtestToRun, arguments);
}
/**
* @param systemtestHosts name of the containers that will be used in the test, if some of them doe not exist, new
* ones will be started. First in list will be used as system test controller
* @param systemtestToRun relative path from the root of systemtests to the test to run, f.ex.
* tests/search/basicsearch/basic_search.rb
*/
void runSystemTest(List<ContainerName> systemtestHosts, Path systemtestToRun, String... arguments) throws InterruptedException, ExecutionException, IOException {
for (ContainerName systemtestHost : systemtestHosts) {
startSystemTestNodeIfNeeded(systemtestHost);
}
Path pathToSystestToRun = pathToSystemtestsInContainer.resolve(systemtestToRun);
logger.info("Running test " + pathToSystestToRun);
Integer testExitCode = runTest(systemtestHosts.get(0), pathToSystestToRun, arguments);
assertEquals("Test did not finish with exit code 0", Integer.valueOf(0), testExitCode);
}
/**
* This method updates container's local repository with all artifacts that are built on host machine, then
* copies any existing and updated file from target to /home/y/lib/jars.
*
* @param containerName name of the container to install modules in, if it does not exist, a new
* one will be started.
*/
void updateContainerMavenLocalRepository(ContainerName containerName) throws InterruptedException, IOException, ExecutionException {
startSystemTestNodeIfNeeded(containerName);
String sources = pathToVespaRepoInContainer.toString() + "/*/target/";
String destination = pathToLibJars.toString() + "/";
executeInContainer(containerName, "root","/bin/sh", "-c",
"rsync --existing --update --recursive --times " + sources + " " + destination);
executeInContainer(containerName, username, "/bin/sh", "-c", "cd " + pathToVespaRepoInContainer + ";" +
"mvn jar:jar install:install");
}
private void startSystemTestNodeIfNeeded(ContainerName containerName) throws IOException, InterruptedException, ExecutionException {
buildVespaSystestDockerImage(docker, vespaBaseImage);
Optional<Container> container = docker.getContainer(containerName);
if (container.isPresent()) {
if (container.get().state.isRunning()) return;
else docker.deleteContainer(containerName);
}
logger.info("Starting systemtests container " + containerName.asString());
InetAddress nodeInetAddress = InetAddress.getByName(containerName.asString());
docker.createContainerCommand(
SYSTEMTESTS_DOCKER_IMAGE,
containerName,
containerName.asString())
.withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME)
.withIpAddress(nodeInetAddress)
.withEnvironment("USER", "root")
.withEnvironment("VESPA_SYSTEM_TEST_USE_TLS", "false")
.withUlimit("nofile", 262_144, 262_144)
.withUlimit("nproc", 32_768, 409_600)
.withUlimit("core", -1, -1)
.withVolume(Paths.get(System.getProperty("user.home")).resolve(".m2").toString(),
Paths.get("/home/").resolve(username).resolve(".m2").toString())
.withVolume(pathToSystemtestsInHost.toString(), pathToSystemtestsInContainer.toString())
.withVolume(pathToVespaRepoInHost.toString(), pathToVespaRepoInContainer.toString())
.create();
docker.startContainer(containerName);
String uid = new ProcessExecuter().exec(new String[]{"/bin/sh", "-c", "id -u " + username}).getSecond();
docker.executeInContainerAsRoot(containerName, "useradd", "-u", uid.trim(), username);
// TODO: Should check something to see if node_server.rb is ready
Thread.sleep(1000);
}
private void buildVespaSystestDockerImage(Docker docker, DockerImage vespaBaseImage) throws IOException, ExecutionException, InterruptedException {
if (!docker.imageIsDownloaded(vespaBaseImage)) {
logger.info("Pulling " + vespaBaseImage.asString() + " (This may take a while)");
docker.pullImageAsync(vespaBaseImage).get();
}
Path systestBuildDirectory = pathToVespaRepoInHost.resolve("docker-api/src/test/resources/systest/");
Path systestDockerfile = systestBuildDirectory.resolve("Dockerfile");
String dockerfileTemplate = new String(Files.readAllBytes(systestBuildDirectory.resolve("Dockerfile.template")))
.replaceAll("\\$VESPA_BASE_IMAGE", vespaBaseImage.asString());
Files.write(systestDockerfile, dockerfileTemplate.getBytes());
logger.info("Building " + SYSTEMTESTS_DOCKER_IMAGE.asString());
docker.buildImage(systestDockerfile.toFile(), SYSTEMTESTS_DOCKER_IMAGE);
}
private Integer executeInContainer(ContainerName containerName, String runAsUser, String... args) throws InterruptedException {
logger.info("Executing as '" + runAsUser + "' in '" + containerName.asString() + "': " + String.join(" ", args));
ExecCreateCmdResponse response = docker.dockerClient.execCreateCmd(containerName.asString())
.withCmd(args)
.withAttachStdout(true)
.withAttachStderr(true)
.withUser(runAsUser)
.exec();
ExecStartCmd execStartCmd = docker.dockerClient.execStartCmd(response.getId());
execStartCmd.exec(new ExecStartResultCallback(System.out, System.err)).awaitCompletion();
InspectExecResponse state = docker.dockerClient.inspectExecCmd(execStartCmd.getExecId()).exec();
return state.getExitCode();
}
private Integer runTest(ContainerName containerName, Path testToRun, String... args) throws InterruptedException {
String[] combinedArgs = new String[args.length + 2];
combinedArgs[0] = pathToTestRunner.toString();
combinedArgs[1] = testToRun.toString();
System.arraycopy(args, 0, combinedArgs, 2, args.length);
return executeInContainer(containerName, "root", combinedArgs);
}
}
|
