diff options
author | Valerij Fredriksen <valerijf@vespa.ai> | 2023-11-03 19:15:48 +0100 |
---|---|---|
committer | Valerij Fredriksen <valerijf@vespa.ai> | 2023-11-03 19:20:57 +0100 |
commit | 9154e4490d95b1d55c4e35e16e7b469c8d79fb2e (patch) | |
tree | 17a7b4b025aa015fba1095177ae3d700ade9fdbd /node-admin/src/test | |
parent | 59107fa8733a7061b41d0122df7ca8ab524383a4 (diff) |
Move node-admin
Diffstat (limited to 'node-admin/src/test')
86 files changed, 0 insertions, 10059 deletions
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java deleted file mode 100644 index d3982af14e4..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.cgroup; - -import com.yahoo.vespa.hosted.node.admin.container.ContainerId; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.util.Map; -import java.util.Optional; - -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.sharesToWeight; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.weightToShares; -import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Device; -import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Max; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author freva - */ -public class CgroupTest { - - private static final ContainerId containerId = new ContainerId("4aec78cc"); - - private final FileSystem fileSystem = TestFileSystem.create(); - private final Cgroup containerCgroup = Cgroup.root(fileSystem).resolveContainer(containerId); - private final CpuController containerCpu = containerCgroup.cpu(); - private final NodeAgentContext context = NodeAgentContextImpl.builder("node123.yahoo.com").fileSystem(fileSystem).build(); - private final UnixPath cgroupRoot = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-4aec78cc.scope/container")).createDirectories(); - - @Test - public void updates_cpu_quota_and_period() { - assertEquals(Optional.empty(), containerCgroup.cpu().readMax()); - - cgroupRoot.resolve("cpu.max").writeUtf8File("max 100000\n"); - assertEquals(Optional.of(new CpuController.Max(Size.max(), 100000)), containerCpu.readMax()); - - cgroupRoot.resolve("cpu.max").writeUtf8File("456 123456\n"); - assertEquals(Optional.of(new CpuController.Max(Size.from(456), 123456)), containerCpu.readMax()); - - containerCgroup.cpu().updateMax(context, 456, 123456); - - assertTrue(containerCgroup.cpu().updateMax(context, 654, 123456)); - assertEquals(Optional.of(new CpuController.Max(Size.from(654), 123456)), containerCpu.readMax()); - assertEquals("654 123456\n", cgroupRoot.resolve("cpu.max").readUtf8File()); - - assertTrue(containerCgroup.cpu().updateMax(context, -1, 123456)); - assertEquals(Optional.of(new CpuController.Max(Size.max(), 123456)), containerCpu.readMax()); - assertEquals("max 123456\n", cgroupRoot.resolve("cpu.max").readUtf8File()); - } - - @Test - public void updates_cpu_shares() { - assertEquals(Optional.empty(), containerCgroup.cpu().readShares()); - - cgroupRoot.resolve("cpu.weight").writeUtf8File("1\n"); - assertEquals(Optional.of(2), containerCgroup.cpu().readShares()); - - assertFalse(containerCgroup.cpu().updateShares(context, 2)); - - assertTrue(containerCgroup.cpu().updateShares(context, 12345)); - assertEquals(Optional.of(12323), containerCgroup.cpu().readShares()); - } - - @Test - public void reads_cpu_stats() { - cgroupRoot.resolve("cpu.stat").writeUtf8File(""" - usage_usec 17794243 - user_usec 16099205 - system_usec 1695038 - nr_periods 12465 - nr_throttled 25 - throttled_usec 14256 - """); - - assertEquals(Map.of(TOTAL_USAGE_USEC, 17794243L, USER_USAGE_USEC, 16099205L, SYSTEM_USAGE_USEC, 1695038L, - TOTAL_PERIODS, 12465L, THROTTLED_PERIODS, 25L, THROTTLED_TIME_USEC, 14256L), containerCgroup.cpu().readStats()); - } - - @Test - public void reads_memory_metrics() { - cgroupRoot.resolve("memory.current").writeUtf8File("2525093888\n"); - assertEquals(2525093888L, containerCgroup.memory().readCurrent().value()); - - cgroupRoot.resolve("memory.max").writeUtf8File("4322885632\n"); - assertEquals(4322885632L, containerCgroup.memory().readMax().value()); - - cgroupRoot.resolve("memory.stat").writeUtf8File(""" - anon 3481600 - file 69206016 - kernel_stack 73728 - slab 3552304 - percpu 262336 - sock 73728 - shmem 8380416 - file_mapped 1081344 - file_dirty 135168 - slab_reclaimable 1424320 - """); - var stats = containerCgroup.memory().readStat(); - assertEquals(69206016L, stats.file().value()); - assertEquals(3481600L, stats.anon().value()); - assertEquals(3552304L, stats.slab().value()); - assertEquals(73728L, stats.sock().value()); - assertEquals(1424320L, stats.slabReclaimable().value()); - } - - @Test - public void shares_to_weight_and_back_is_stable() { - for (int i = 2; i <= 262144; i++) { - int originalShares = i; // Must be effectively final to use in lambda :( - int roundTripShares = weightToShares(sharesToWeight(i)); - int diff = i - roundTripShares; - assertTrue(diff >= 0 && diff <= 27, // ~26.2 shares / weight - () -> "Original shares: " + originalShares + ", round trip shares: " + roundTripShares + ", diff: " + diff); - } - } - - @Test - void reads_io_max() { - assertEquals(Optional.empty(), containerCgroup.io().readMax()); - - cgroupRoot.resolve("io.max").writeUtf8File(""); - assertEquals(Optional.of(Map.of()), containerCgroup.io().readMax()); - - cgroupRoot.resolve("io.max").writeUtf8File(""" - 253:1 rbps=11 wbps=max riops=22 wiops=33 - 253:0 rbps=max wbps=44 riops=max wiops=55 - """); - assertEquals(Map.of(new Device(253, 1), new Max(Size.from(11), Size.max(), Size.from(22), Size.from(33)), - new Device(253, 0), new Max(Size.max(), Size.from(44), Size.max(), Size.from(55))), - containerCgroup.io().readMax().orElseThrow()); - } - - @Test - void writes_io_max() { - Device device = new Device(253, 0); - Max initial = new Max(Size.max(), Size.from(44), Size.max(), Size.from(55)); - assertTrue(containerCgroup.io().updateMax(context, device, initial)); - assertEquals("253:0 rbps=max wbps=44 riops=max wiops=55\n", cgroupRoot.resolve("io.max").readUtf8File()); - - cgroupRoot.resolve("io.max").writeUtf8File(""" - 253:1 rbps=11 wbps=max riops=22 wiops=33 - 253:0 rbps=max wbps=44 riops=max wiops=55 - """); - assertFalse(containerCgroup.io().updateMax(context, device, initial)); - - cgroupRoot.resolve("io.max").writeUtf8File(""); - assertFalse(containerCgroup.io().updateMax(context, device, Max.UNLIMITED)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java deleted file mode 100644 index cb828394249..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.cgroup; - -import org.junit.jupiter.api.Test; - -import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Max; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -class IoControllerTest { - - @Test - void parse_io_max() { - assertEquals(Max.UNLIMITED, Max.fromString("")); - assertEquals(new Max(Size.from(1), Size.max(), Size.max(), Size.max()), Max.fromString("rbps=1 wiops=max")); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java deleted file mode 100644 index 910fd8e670a..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.http.HttpVersion; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.entity.BasicHttpEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicStatusLine; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.SocketTimeoutException; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Basic testing of retry logic. - * - * @author dybis - */ -public class ConfigServerApiImplTest { - - private static final int FAIL_RETURN_CODE = 100000; - private static final int TIMEOUT_RETURN_CODE = 100001; - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class TestPojo { - @JsonProperty("foo") - String foo; - @JsonProperty("error-code") - Integer errorCode; - } - - private final String uri1 = "http://host1:666"; - private final String uri2 = "http://host2:666"; - private final List<URI> configServers = List.of(URI.create(uri1), URI.create(uri2)); - private final StringBuilder mockLog = new StringBuilder(); - - private ConfigServerApiImpl configServerApi; - private int mockReturnCode = 200; - - @BeforeEach - public void initExecutor() throws IOException { - CloseableHttpClient httpMock = mock(CloseableHttpClient.class); - when(httpMock.execute(any())).thenAnswer(invocationOnMock -> { - HttpGet get = (HttpGet) invocationOnMock.getArguments()[0]; - mockLog.append(get.getMethod()).append(" ").append(get.getURI()).append(" "); - - switch (mockReturnCode) { - case FAIL_RETURN_CODE -> throw new RuntimeException("FAIL"); - case TIMEOUT_RETURN_CODE -> throw new SocketTimeoutException("read timed out"); - } - - BasicStatusLine statusLine = new BasicStatusLine(HttpVersion.HTTP_1_1, mockReturnCode, null); - BasicHttpEntity entity = new BasicHttpEntity(); - String returnMessage = "{\"foo\":\"bar\", \"no\":3, \"error-code\": " + mockReturnCode + "}"; - InputStream stream = new ByteArrayInputStream(returnMessage.getBytes(StandardCharsets.UTF_8)); - entity.setContent(stream); - - CloseableHttpResponse response = mock(CloseableHttpResponse.class); - when(response.getEntity()).thenReturn(entity); - when(response.getStatusLine()).thenReturn(statusLine); - - return response; - }); - configServerApi = ConfigServerApiImpl.createForTestingWithClient(configServers, httpMock); - } - - @Test - void testBasicParsingSingleServer() { - TestPojo answer = configServerApi.get("/path", TestPojo.class); - assertEquals(answer.foo, "bar"); - assertLogStringContainsGETForAHost(); - } - - @Test - void testBasicFailure() { - assertThrows(HttpException.class, () -> { - // Server is returning 400, no retries. - mockReturnCode = 400; - - TestPojo testPojo = configServerApi.get("/path", TestPojo.class); - assertEquals(testPojo.errorCode.intValue(), mockReturnCode); - assertLogStringContainsGETForAHost(); - }); - } - - @Test - void testBasicSuccessWithNoRetries() { - // Server is returning 201, no retries. - mockReturnCode = 201; - - TestPojo testPojo = configServerApi.get("/path", TestPojo.class); - assertEquals(testPojo.errorCode.intValue(), mockReturnCode); - assertLogStringContainsGETForAHost(); - } - - @Test - void testBasicSuccessWithCustomTimeouts() { - mockReturnCode = TIMEOUT_RETURN_CODE; - - var params = new ConfigServerApi.Params<TestPojo>(); - params.setConnectionTimeout(Duration.ofSeconds(3)); - - try { - configServerApi.get("/path", TestPojo.class, params); - fail(); - } catch (ConnectionException e) { - assertNotNull(e.getCause()); - assertEquals("read timed out", e.getCause().getMessage()); - } - } - - @Test - void testRetries() { - // Client is throwing exception, should be retries. - mockReturnCode = FAIL_RETURN_CODE; - try { - configServerApi.get("/path", TestPojo.class); - fail("Expected failure"); - } catch (Exception e) { - // ignore - } - - List<String> log = List.of(mockLog.toString().split(" ")); - assertTrue(log.containsAll(List.of("GET http://host1:666/path", "GET http://host2:666/path"))); - } - - @Test - void testNoRetriesOnBadHttpResponseCode() { - // Client is throwing exception, should be retries. - mockReturnCode = 503; - try { - configServerApi.get("/path", TestPojo.class); - fail("Expected failure"); - } catch (Exception e) { - // ignore - } - - assertLogStringContainsGETForAHost(); - } - - @Test - void testForbidden() { - mockReturnCode = 403; - try { - configServerApi.get("/path", TestPojo.class); - fail("Expected exception"); - } catch (HttpException.ForbiddenException e) { - // ignore - } - assertLogStringContainsGETForAHost(); - } - - @Test - void testNotFound() { - // Server is returning 404, special exception is thrown. - mockReturnCode = 404; - try { - configServerApi.get("/path", TestPojo.class); - fail("Expected exception"); - } catch (HttpException.NotFoundException e) { - // ignore - } - assertLogStringContainsGETForAHost(); - } - - @Test - void testConflict() { - // Server is returning 409, no exception is thrown. - mockReturnCode = 409; - configServerApi.get("/path", TestPojo.class); - assertLogStringContainsGETForAHost(); - } - - private void assertLogStringContainsGETForAHost() { - String logString = mockLog.toString(); - assertTrue((logString.equals("GET http://host1:666/path ") || logString.equals("GET http://host2:666/path ")), - "log does not contain expected entries:" + logString); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java deleted file mode 100644 index 430da856cfa..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.cores; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.HostName; -import com.yahoo.test.json.JsonTestHelper; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.node.admin.configserver.StandardConfigServerResponse; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.time.Instant; -import java.util.List; -import java.util.Optional; - -import static com.yahoo.yolean.Exceptions.uncheck; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author hakonhall - */ -class CoresTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final ObjectMapper mapper = new ObjectMapper(); - private final ConfigServerApi configServerApi = mock(ConfigServerApi.class); - private final Cores cores = new CoresImpl(configServerApi); - private final HostName hostname = HostName.of("foo.com"); - private final String id = "5c987afb-347a-49ee-a0c5-bef56bbddeb0"; - private final CoreDumpMetadata metadata = new CoreDumpMetadata() - .setType(CoreDumpMetadata.Type.OOM) - .setCreated(Instant.ofEpochMilli(12345678)) - .setKernelVersion("4.18.0-372.26.1.el8_6.x86_64") - .setCpuMicrocodeVersion("0x1000065") - .setCoreDumpPath(fileSystem.getPath("/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813")) - .setDecryptionToken("987def") - .setDockerImage(DockerImage.fromString("us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8")) - .setBinPath("/usr/bin/java") - .setVespaVersion("8.68.8") - .setBacktraceAllThreads(List.of("Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...", - "Debugger attached successfully.", - " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)", - " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)")) - .setBacktrace(List.of("Example", "of", "backtrace")); - - @Test - void reportOK() { - var oKResponse = new StandardConfigServerResponse(); - oKResponse.message = "OK"; - when(configServerApi.post(any(), any(), any())).thenReturn(oKResponse); - - cores.report(hostname, id, metadata); - - var pathCaptor = ArgumentCaptor.forClass(String.class); - var bodyJsonPojoCaptor = ArgumentCaptor.forClass(Object.class); - verify(configServerApi, times(1)).post(pathCaptor.capture(), bodyJsonPojoCaptor.capture(), any()); - - assertEquals("/cores/v1/report/" + hostname + "/" + id, pathCaptor.getValue()); - - assertEquals(""" - { - "backtrace": [ - "Example", - "of", - "backtrace" - ], - "backtrace_all_threads": [ - "Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...", - "Debugger attached successfully.", - " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)", - " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)" - ], - "bin_path": "/usr/bin/java", - "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813", - "cpu_microcode_version": "0x1000065", - "created": 12345678, - "decryption_token": "987def", - "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8", - "kernel_version": "4.18.0-372.26.1.el8_6.x86_64", - "type": "OOM", - "vespa_version": "8.68.8" - }""", - JsonTestHelper.normalize(uncheck(() -> mapper.writeValueAsString(bodyJsonPojoCaptor.getValue())))); - } - - @Test - void reportFails() { - var response = new StandardConfigServerResponse(); - response.errorCode = "503"; - response.message = "error detail"; - when(configServerApi.post(any(), any(), any())).thenReturn(response); - - assertThrows(ConfigServerException.class, - () -> cores.report(hostname, "abcde-1234", metadata), - "Failed to report core dump at Optional[/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813]: error detail 503"); - - var pathCaptor = ArgumentCaptor.forClass(String.class); - var bodyJsonPojoCaptor = ArgumentCaptor.forClass(Object.class); - verify(configServerApi).post(pathCaptor.capture(), bodyJsonPojoCaptor.capture(), any()); - } - - @Test - void serialization() { - Path path = fileSystem.getPath("/foo.json"); - ReportCoreDumpRequest request = new ReportCoreDumpRequest().fillFrom(metadata); - request.save(path); - assertEquals(""" - { - "backtrace": [ - "Example", - "of", - "backtrace" - ], - "backtrace_all_threads": [ - "Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...", - "Debugger attached successfully.", - " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)", - " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)" - ], - "bin_path": "/usr/bin/java", - "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813", - "cpu_microcode_version": "0x1000065", - "created": 12345678, - "decryption_token": "987def", - "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8", - "kernel_version": "4.18.0-372.26.1.el8_6.x86_64", - "type": "OOM", - "vespa_version": "8.68.8" - }""", - JsonTestHelper.normalize(new UnixPath(path).readUtf8File())); - - Optional<ReportCoreDumpRequest> loaded = ReportCoreDumpRequest.load(path); - assertTrue(loaded.isPresent()); - var meta = new CoreDumpMetadata(); - loaded.get().populateMetadata(meta, fileSystem); - assertEquals(metadata, meta); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java deleted file mode 100644 index 664e25bc744..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.flags; - -import com.yahoo.vespa.flags.FlagId; -import com.yahoo.vespa.flags.json.FlagData; -import com.yahoo.vespa.flags.json.wire.WireFlagData; -import com.yahoo.vespa.flags.json.wire.WireFlagDataList; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author hakonhall - */ -public class RealFlagRepositoryTest { - private final ConfigServerApi configServerApi = mock(ConfigServerApi.class); - private final RealFlagRepository repository = new RealFlagRepository(configServerApi); - - @Test - void test() { - WireFlagDataList list = new WireFlagDataList(); - list.flags = new ArrayList<>(); - list.flags.add(new WireFlagData()); - list.flags.get(0).id = "id1"; - - when(configServerApi.get(any(), eq(WireFlagDataList.class))).thenReturn(list); - Map<FlagId, FlagData> allFlagData = repository.getAllFlagData(); - assertEquals(1, allFlagData.size()); - assertTrue(allFlagData.containsKey(new FlagId("id1"))); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java deleted file mode 100644 index d91e9befab9..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * - * @author smorgrav - */ -public class AclTest { - - private static final Acl aclCommon = new Acl( - Set.of(1234, 453), Set.of(4321), - testNodes(Set.of(), "192.1.2.2", "fb00::1", "fe80::2", "fe80::3"), - Set.of()); - - private static final Acl aclWithoutPorts = new Acl( - Set.of(), Set.of(), - testNodes(Set.of(), "192.1.2.2", "fb00::1", "fe80::2"), - Set.of()); - - @Test - void no_trusted_ports() { - String listRulesIpv4 = String.join("\n", aclWithoutPorts.toRules(IPVersion.IPv4)); - assertEquals( - """ - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p icmp -j ACCEPT - -A INPUT -s 192.1.2.2/32 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp-port-unreachable""", - listRulesIpv4); - } - - @Test - void ipv4_rules() { - String listRulesIpv4 = String.join("\n", aclCommon.toRules(IPVersion.IPv4)); - assertEquals( - """ - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 453,1234 -j ACCEPT - -A INPUT -p udp -m multiport --dports 4321 -j ACCEPT - -A INPUT -s 192.1.2.2/32 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp-port-unreachable""", - listRulesIpv4); - } - - @Test - void ipv6_rules() { - String listRulesIpv6 = String.join("\n", aclCommon.toRules(IPVersion.IPv6)); - assertEquals( - """ - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p ipv6-icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 453,1234 -j ACCEPT - -A INPUT -p udp -m multiport --dports 4321 -j ACCEPT - -A INPUT -s fb00::1/128 -j ACCEPT - -A INPUT -s fe80::2/128 -j ACCEPT - -A INPUT -s fe80::3/128 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp6-port-unreachable""", listRulesIpv6); - } - - @Test - void ipv6_rules_stable_order() { - Acl aclCommonDifferentOrder = new Acl( - Set.of(453, 1234), Set.of(4321), - testNodes(Set.of(), "fe80::2", "192.1.2.2", "fb00::1", "fe80::3"), - Set.of()); - - for (IPVersion ipVersion : IPVersion.values()) { - assertEquals(aclCommon.toRules(ipVersion), aclCommonDifferentOrder.toRules(ipVersion)); - } - } - - @Test - void trusted_networks() { - Acl acl = new Acl(Set.of(4080), Set.of(), testNodes(Set.of(), "127.0.0.1"), Set.of("10.0.0.0/24", "2001:db8::/32")); - - assertEquals(""" - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 4080 -j ACCEPT - -A INPUT -s 127.0.0.1/32 -j ACCEPT - -A INPUT -s 10.0.0.0/24 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp-port-unreachable""", - String.join("\n", acl.toRules(IPVersion.IPv4))); - - assertEquals(""" - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p ipv6-icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 4080 -j ACCEPT - -A INPUT -s 2001:db8::/32 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp6-port-unreachable""", - String.join("\n", acl.toRules(IPVersion.IPv6))); - } - - @Test - void config_server_acl() { - Set<Acl.Node> testNodes = Stream.concat(testNodes(NodeType.config, Set.of(), "172.17.0.41", "172.17.0.42", "172.17.0.43").stream(), - testNodes(NodeType.tenant, Set.of(19070), "172.17.0.81", "172.17.0.82", "172.17.0.83").stream()) - .collect(Collectors.toSet()); - Acl acl = new Acl(Set.of(22, 4443), Set.of(), testNodes, Set.of()); - assertEquals(""" - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT - -A INPUT -s 172.17.0.41/32 -j ACCEPT - -A INPUT -s 172.17.0.42/32 -j ACCEPT - -A INPUT -s 172.17.0.43/32 -j ACCEPT - -A INPUT -s 172.17.0.81/32 -p tcp -m multiport --dports 19070 -j ACCEPT - -A INPUT -s 172.17.0.82/32 -p tcp -m multiport --dports 19070 -j ACCEPT - -A INPUT -s 172.17.0.83/32 -p tcp -m multiport --dports 19070 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp-port-unreachable""", - String.join("\n", acl.toRules(IPVersion.IPv4))); - - Set<Acl.Node> testNodes2 = Stream.concat(testNodes(NodeType.config, Set.of(), "2001:db8::41", "2001:db8::42", "2001:db8::43").stream(), - testNodes(NodeType.tenant, Set.of(19070), "2001:db8::81", "2001:db8::82", "2001:db8::83").stream()) - .collect(Collectors.toSet()); - Acl acl2 = new Acl(Set.of(22, 4443), Set.of(), testNodes2, Set.of()); - - assertEquals(""" - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p ipv6-icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT - -A INPUT -s 2001:db8::41/128 -j ACCEPT - -A INPUT -s 2001:db8::42/128 -j ACCEPT - -A INPUT -s 2001:db8::43/128 -j ACCEPT - -A INPUT -s 2001:db8::81/128 -p tcp -m multiport --dports 19070 -j ACCEPT - -A INPUT -s 2001:db8::82/128 -p tcp -m multiport --dports 19070 -j ACCEPT - -A INPUT -s 2001:db8::83/128 -p tcp -m multiport --dports 19070 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp6-port-unreachable""", - String.join("\n", acl2.toRules(IPVersion.IPv6))); - } - - private static Set<Acl.Node> testNodes(Set<Integer> ports, String... address) { - return testNodes(NodeType.tenant, ports, address); - } - - private static Set<Acl.Node> testNodes(NodeType nodeType, Set<Integer> ports, String... address) { - return Arrays.stream(address) - .map(addr -> new Acl.Node("hostname", addr, ports)) - .collect(Collectors.toUnmodifiableSet()); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java deleted file mode 100644 index b236c223078..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.yahoo.vespa.hosted.provision.Node; -import org.junit.jupiter.api.Test; - -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class NodeStateTest { - - @Test - void is_equal_to_node_repository_states() { - Set<String> nodeRepositoryStates = Stream.of(Node.State.values()).map(Enum::name).collect(Collectors.toSet()); - Set<String> nodeAdminStates = Stream.of(NodeState.values()).map(Enum::name).collect(Collectors.toSet()); - - assertEquals(nodeAdminStates, nodeRepositoryStates); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java deleted file mode 100644 index 4100b3cf102..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.yahoo.application.Networking; -import com.yahoo.application.container.JDisc; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.WireguardKey; -import com.yahoo.config.provision.WireguardKeyWithTimestamp; -import com.yahoo.config.provision.host.FlavorOverrides; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; -import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer; -import com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler; -import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.ServerSocket; -import java.net.URI; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * Tests the NodeRepository class used for talking to the node repository. It uses a mock from the node repository - * which already contains some data. - * - * @author dybdahl - */ -public class RealNodeRepositoryTest { - - private static final double delta = 0.00000001; - private JDisc container; - private NodeRepository nodeRepositoryApi; - - private int findRandomOpenPort() throws IOException { - try (ServerSocket socket = new ServerSocket(0)) { - socket.setReuseAddress(true); - return socket.getLocalPort(); - } - } - - /** - * Starts NodeRepository with - * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors} - * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository} - * {@link NodesV2ApiHandler} - * These classes define some test data that is used in these tests. - */ - @BeforeEach - public void startContainer() throws Exception { - Exception lastException = null; - - // This tries to bind a random open port for the node-repo mock, which is a race condition, so try - // a few times before giving up - for (int i = 0; i < 3; i++) { - try { - int port = findRandomOpenPort(); - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port, SystemName.main, CloudAccount.from("123456789012")), Networking.enable); - ConfigServerApi configServerApi = ConfigServerApiImpl.createForTesting( - List.of(URI.create("http://127.0.0.1:" + port))); - waitForJdiscContainerToServe(configServerApi); - return; - } catch (RuntimeException e) { - lastException = e; - } - } - throw new RuntimeException("Failed to bind a port in three attempts, giving up", lastException); - } - - private void waitForJdiscContainerToServe(ConfigServerApi configServerApi) throws InterruptedException { - Instant start = Instant.now(); - nodeRepositoryApi = new RealNodeRepository(configServerApi); - while (Instant.now().minusSeconds(120).isBefore(start)) { - try { - nodeRepositoryApi.getNodes("foobar"); - return; - } catch (Exception e) { - Thread.sleep(100); - } - } - throw new RuntimeException("Could not get answer from container."); - } - - @AfterEach - public void stopContainer() { - if (container != null) { - container.close(); - } - } - - @Test - void testGetContainersToRunApi() { - String dockerHostHostname = "dockerhost1.yahoo.com"; - - List<NodeSpec> containersToRun = nodeRepositoryApi.getNodes(dockerHostHostname); - assertEquals(1, containersToRun.size()); - NodeSpec node = containersToRun.get(0); - assertEquals("host4.yahoo.com", node.hostname()); - assertEquals(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.42.0"), node.wantedDockerImage().get()); - assertEquals(NodeState.active, node.state()); - assertEquals(Long.valueOf(0), node.wantedRestartGeneration().get()); - assertEquals(Long.valueOf(0), node.currentRestartGeneration().get()); - assertEquals(1, node.vcpu(), delta); - assertEquals(4, node.memoryGb(), delta); - assertEquals(100, node.diskGb(), delta); - } - - @Test - void testGetContainer() { - String hostname = "host4.yahoo.com"; - Optional<NodeSpec> node = nodeRepositoryApi.getOptionalNode(hostname); - assertTrue(node.isPresent()); - assertEquals(hostname, node.get().hostname()); - assertEquals(CloudAccount.from("123456789012"), node.get().cloudAccount()); - } - - @Test - void testGetContainerForNonExistingNode() { - String hostname = "host-that-does-not-exist"; - Optional<NodeSpec> node = nodeRepositoryApi.getOptionalNode(hostname); - assertFalse(node.isPresent()); - } - - @Test - void testUpdateNodeAttributes() { - var hostname = "host4.yahoo.com"; - var dockerImage = "registry.example.com/repo/image-1:6.2.3"; - var wireguardKey = WireguardKey.from("111122223333444455556666777788889999000042c="); - var wireguardKeyTimestamp = Instant.ofEpochMilli(123L); // Instant from clock in MockNodeRepository - var keyWithTimestamp = new WireguardKeyWithTimestamp(wireguardKey, wireguardKeyTimestamp); - - nodeRepositoryApi.updateNodeAttributes( - hostname, - new NodeAttributes() - .withRestartGeneration(1) - .withDockerImage(DockerImage.fromString(dockerImage)) - .withWireguardPubkey(wireguardKey)); - - NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode(hostname).orElseThrow(); - assertEquals(1, hostSpec.currentRestartGeneration().orElseThrow()); - assertEquals(dockerImage, hostSpec.currentDockerImage().orElseThrow().asString()); - assertEquals(keyWithTimestamp, hostSpec.wireguardKeyWithTimestamp().orElseThrow()); - } - - @Test - void testMarkAsReady() { - nodeRepositoryApi.setNodeState("host5.yahoo.com", NodeState.dirty); - nodeRepositoryApi.setNodeState("host5.yahoo.com", NodeState.ready); - - try { - nodeRepositoryApi.setNodeState("host4.yahoo.com", NodeState.ready); - fail("Should not be allowed to be marked ready as it is not registered as provisioned, dirty, failed or parked"); - } catch (RuntimeException ignored) { - // expected - } - - try { - nodeRepositoryApi.setNodeState("host101.yahoo.com", NodeState.ready); - fail("Expected failure because host101 does not exist"); - } catch (RuntimeException ignored) { - // expected - } - } - - @Test - void testAddNodes() { - AddNode host = AddNode.forHost("host123.domain.tld", - "id1", - "default", - Optional.of(FlavorOverrides.ofDisk(123)), - NodeType.confighost, - Set.of("::1"), Set.of("::2", "::3")); - - NodeResources nodeResources = new NodeResources(1, 2, 3, 4, NodeResources.DiskSpeed.slow, NodeResources.StorageType.local); - AddNode node = AddNode.forNode("host123-1.domain.tld", "id1", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3")); - - assertFalse(nodeRepositoryApi.getOptionalNode("host123.domain.tld").isPresent()); - nodeRepositoryApi.addNodes(List.of(host, node)); - - NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode("host123.domain.tld").orElseThrow(); - assertEquals("id1", hostSpec.id()); - assertEquals("default", hostSpec.flavor()); - assertEquals(123, hostSpec.diskGb(), 0); - assertEquals(NodeType.confighost, hostSpec.type()); - assertEquals(NodeResources.Architecture.x86_64, hostSpec.resources().architecture()); - - NodeSpec nodeSpec = nodeRepositoryApi.getOptionalNode("host123-1.domain.tld").orElseThrow(); - assertEquals(nodeResources, nodeSpec.resources()); - assertEquals(NodeType.config, nodeSpec.type()); - } - - @Test - void wireguard_peer_config_can_be_retrieved_for_configservers_and_exclave_nodes() { - - //// Configservers //// - - List<WireguardPeer> cfgPeers = nodeRepositoryApi.getConfigserverPeers(); - - // cfg2 does not have a wg public key, so should not be included - assertEquals(1, cfgPeers.size()); - - assertWireguardPeer(cfgPeers.get(0), "cfg1.yahoo.com", - "::201:1", - "lololololololololololololololololololololoo=", - 456L); - - //// Exclave nodes //// - - List<WireguardPeer> exclavePeers = nodeRepositoryApi.getExclavePeers(); - - // host3 does not have a wg public key, so should not be included - assertEquals(1, exclavePeers.size()); - - assertWireguardPeer(exclavePeers.get(0), "dockerhost2.yahoo.com", - "::101:1", - "000011112222333344445555666677778888999900c=", - 123L); - } - - private void assertWireguardPeer(WireguardPeer peer, String hostname, String ipv6, - String publicKey, long keyTimestamp) { - assertEquals(hostname, peer.hostname().value()); - assertEquals(1, peer.ipAddresses().size()); - assertIp(peer.ipAddresses().get(0), ipv6, 6); - var expectedKeyWithTimestamp = new WireguardKeyWithTimestamp(WireguardKey.from(publicKey), - Instant.ofEpochMilli(keyTimestamp)); - assertEquals(expectedKeyWithTimestamp, peer.keyWithTimestamp()); - } - - private void assertIp(VersionedIpAddress ip, String expectedIp, int expectedVersion) { - assertEquals(expectedIp, ip.asString()); - assertEquals(expectedVersion, ip.version().version()); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java deleted file mode 100644 index 8de5986739e..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.yahoo.test.json.JsonTestHelper; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.RealNodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport; -import org.junit.jupiter.api.Test; - -import java.util.HashMap; - -import static com.yahoo.yolean.Exceptions.uncheck; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author hakonhall - */ -public class NodeRepositoryNodeTest { - private static final ObjectMapper mapper = new ObjectMapper(); - private final NodeRepositoryNode node = new NodeRepositoryNode(); - private final NodeAttributes attributes = new NodeAttributes(); - - - /** - * Test both how NodeRepositoryNode serialize, and the serialization of an empty NodeRepositoryNode - * patched with a NodeAttributes, as they work in tandem: - * NodeAttributes -> NodeRepositoryNode -> JSON. - */ - @Test - void testReportsSerialization() { - // Make sure we don't accidentally patch with "reports": null, as that actually means removing all reports. - assertEquals(JsonInclude.Include.NON_NULL, NodeRepositoryNode.class.getAnnotation(JsonInclude.class).value()); - - // Absent report and unmodified attributes => nothing about reports in JSON - node.reports = null; - assertNodeAndAttributes("{}"); - - // Make sure we're able to patch with a null report value ("reportId": null), as that means removing the report. - node.reports = new HashMap<>(); - node.reports.put("rid", null); - attributes.withReportRemoved("rid"); - assertNodeAndAttributes("{\"reports\": {\"rid\": null}}"); - - // Add ridTwo report to node - ObjectNode reportJson = mapper.createObjectNode(); - reportJson.set(BaseReport.CREATED_FIELD, mapper.valueToTree(3)); - reportJson.set(BaseReport.DESCRIPTION_FIELD, mapper.valueToTree("desc")); - node.reports.put("ridTwo", reportJson); - - // Add ridTwo report to attributes - BaseReport reportTwo = new BaseReport(3L, "desc", null); - attributes.withReport("ridTwo", reportTwo.toJsonNode()); - - // Verify node serializes to expected, as well as attributes patched on node. - assertNodeAndAttributes("{\"reports\": {\"rid\": null, \"ridTwo\": {\"createdMillis\": 3, \"description\": \"desc\"}}}"); - } - - private void assertNodeAndAttributes(String expectedJson) { - assertNodeJson(node, expectedJson); - assertNodeJson(RealNodeRepository.nodeRepositoryNodeFromNodeAttributes(attributes), expectedJson); - } - - private void assertNodeJson(NodeRepositoryNode node, String json) { - JsonNode expected = uncheck(() -> mapper.readTree(json)); - JsonNode actual = uncheck(() -> mapper.valueToTree(node)); - JsonTestHelper.assertJsonEquals(actual, expected); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java deleted file mode 100644 index 69e79ec8720..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports; - -import com.yahoo.test.json.JsonTestHelper; -import org.junit.jupiter.api.Test; - -import static com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport.Type.SOFT_FAIL; -import static com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport.Type.UNSPECIFIED; -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author hakonhall - */ -public class BaseReportTest { - private static final String JSON_1 = "{\"createdMillis\": 1, \"description\": \"desc\"}"; - private static final String JSON_2 = "{\"createdMillis\": 1, \"description\": \"desc\", \"type\": \"SOFT_FAIL\"}"; - - @Test - void testSerialization1() { - JsonTestHelper.assertJsonEquals(new BaseReport(1L, "desc", SOFT_FAIL).toJsonNode(), - JSON_2); - JsonTestHelper.assertJsonEquals(new BaseReport(null, "desc", SOFT_FAIL).toJsonNode(), - "{\"description\": \"desc\", \"type\": \"SOFT_FAIL\"}"); - JsonTestHelper.assertJsonEquals(new BaseReport(1L, null, SOFT_FAIL).toJsonNode(), - "{\"createdMillis\": 1, \"type\": \"SOFT_FAIL\"}"); - JsonTestHelper.assertJsonEquals(new BaseReport(null, null, SOFT_FAIL).toJsonNode(), - "{\"type\": \"SOFT_FAIL\"}"); - - JsonTestHelper.assertJsonEquals(new BaseReport(1L, "desc", null).toJsonNode(), - JSON_1); - JsonTestHelper.assertJsonEquals(new BaseReport(null, "desc", null).toJsonNode(), - "{\"description\": \"desc\"}"); - JsonTestHelper.assertJsonEquals(new BaseReport(1L, null, null).toJsonNode(), - "{\"createdMillis\": 1}"); - JsonTestHelper.assertJsonEquals(new BaseReport(null, null, null).toJsonNode(), - "{}"); - } - - @Test - void testShouldUpdate() { - BaseReport report = new BaseReport(1L, "desc", SOFT_FAIL); - assertFalse(report.updates(report)); - - // createdMillis is ignored - assertFalse(new BaseReport(1L, "desc", SOFT_FAIL).updates(report)); - assertFalse(new BaseReport(2L, "desc", SOFT_FAIL).updates(report)); - assertFalse(new BaseReport(null, "desc", SOFT_FAIL).updates(report)); - - // description is not ignored - assertTrue(new BaseReport(1L, "desc 2", SOFT_FAIL).updates(report)); - assertTrue(new BaseReport(1L, null, SOFT_FAIL).updates(report)); - - // type is not ignored - assertTrue(new BaseReport(1L, "desc", null).updates(report)); - assertTrue(new BaseReport(1L, "desc", BaseReport.Type.HARD_FAIL).updates(report)); - } - - @Test - void testJsonSerialization() { - BaseReport report = BaseReport.fromJson(JSON_2); - assertEquals(1L, (long) report.getCreatedMillisOrNull()); - assertEquals("desc", report.getDescriptionOrNull()); - assertEquals(SOFT_FAIL, report.getTypeOrNull()); - JsonTestHelper.assertJsonEquals(report.toJson(), JSON_2); - } - - @Test - void testUnspecifiedType() { - BaseReport report = new BaseReport(1L, "desc", null); - assertNull(report.getTypeOrNull()); - assertEquals(UNSPECIFIED, report.getType()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java deleted file mode 100644 index bb9c075ad74..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; - -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; -import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; -import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; -import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason; -import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class OrchestratorImplTest { - - private static final String hostName = "host123.yahoo.com"; - - private final ConfigServerApiImpl configServerApi = mock(ConfigServerApiImpl.class); - private final OrchestratorImpl orchestrator = new OrchestratorImpl(configServerApi); - - @Test - void testSuspendCall() { - when(configServerApi.put( - eq(OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended"), - eq(Optional.empty()), - eq(UpdateHostResponse.class), - any() - )).thenReturn(new UpdateHostResponse(hostName, null)); - - orchestrator.suspend(hostName); - } - - @Test - void testSuspendCallWithFailureReason() { - assertThrows(OrchestratorException.class, () -> { - when(configServerApi.put( - eq(OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended"), - eq(Optional.empty()), - eq(UpdateHostResponse.class), - any() - )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail"))); - - orchestrator.suspend(hostName); - }); - } - - @Test - void testSuspendCallWithNotFound() { - assertThrows(OrchestratorNotFoundException.class, () -> { - when(configServerApi.put(any(String.class), any(), any(), any())) - .thenThrow(new HttpException.NotFoundException("Not Found")); - - orchestrator.suspend(hostName); - }); - } - - @Test - void testSuspendCallWithSomeOtherException() { - assertThrows(RuntimeException.class, () -> { - when(configServerApi.put(any(String.class), any(), any(), any())) - .thenThrow(new RuntimeException("Some parameter was wrong")); - - orchestrator.suspend(hostName); - }); - } - - - @Test - void testResumeCall() { - when(configServerApi.delete( - OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended", - UpdateHostResponse.class - )).thenReturn(new UpdateHostResponse(hostName, null)); - - orchestrator.resume(hostName); - } - - @Test - void testResumeCallWithFailureReason() { - assertThrows(OrchestratorException.class, () -> { - when(configServerApi.delete( - OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended", - UpdateHostResponse.class - )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail"))); - - orchestrator.resume(hostName); - }); - } - - @Test - void testResumeCallWithNotFound() { - assertThrows(OrchestratorNotFoundException.class, () -> { - when(configServerApi.delete( - any(String.class), - any() - )).thenThrow(new HttpException.NotFoundException("Not Found")); - - orchestrator.resume(hostName); - }); - } - - @Test - void testResumeCallWithSomeOtherException() { - assertThrows(RuntimeException.class, () -> { - when(configServerApi.put(any(String.class), any(), any(), any())) - .thenThrow(new RuntimeException("Some parameter was wrong")); - - orchestrator.suspend(hostName); - }); - } - - @Test - void testBatchSuspendCall() { - String parentHostName = "host1.test.yahoo.com"; - List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); - - when(configServerApi.put( - eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"), - eq(Optional.empty()), - eq(BatchOperationResult.class), - any() - )).thenReturn(BatchOperationResult.successResult()); - - orchestrator.suspend(parentHostName, hostNames); - } - - @Test - void testBatchSuspendCallWithFailureReason() { - assertThrows(OrchestratorException.class, () -> { - String parentHostName = "host1.test.yahoo.com"; - List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); - String failureReason = "Failed to suspend"; - - when(configServerApi.put( - eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"), - eq(Optional.empty()), - eq(BatchOperationResult.class), - any() - )).thenReturn(new BatchOperationResult(failureReason)); - - orchestrator.suspend(parentHostName, hostNames); - }); - } - - @Test - void testBatchSuspendCallWithSomeException() { - assertThrows(RuntimeException.class, () -> { - String parentHostName = "host1.test.yahoo.com"; - List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); - String exceptionMessage = "Exception: Something crashed!"; - - when(configServerApi.put( - eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"), - eq(Optional.empty()), - eq(BatchOperationResult.class), - any() - )).thenThrow(new RuntimeException(exceptionMessage)); - - orchestrator.suspend(parentHostName, hostNames); - }); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java deleted file mode 100644 index 478e89cde34..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.state; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class HealthResponseTest { - @Test - void deserializationOfNormalResponse() throws Exception { - String jsonResponse = "{\n" + - " \"metrics\": {\n" + - " \"snapshot\": {\n" + - " \"from\": 1.523614569023E9,\n" + - " \"to\": 1.523614629023E9\n" + - " },\n" + - " \"values\": [\n" + - " {\n" + - " \"name\": \"requestsPerSecond\",\n" + - " \"values\": {\n" + - " \"count\": 121,\n" + - " \"rate\": 2.0166666666666666\n" + - " }\n" + - " },\n" + - " {\n" + - " \"name\": \"latencySeconds\",\n" + - " \"values\": {\n" + - " \"average\": 5.537190082644628E-4,\n" + - " \"count\": 121,\n" + - " \"last\": 0.001,\n" + - " \"max\": 0.001,\n" + - " \"min\": 0,\n" + - " \"rate\": 2.0166666666666666\n" + - " }\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"status\": {\"code\": \"up\"},\n" + - " \"time\": 1523614629451\n" + - "}"; - - HealthResponse response = deserialize(jsonResponse); - - assertEquals(response.status.code, "up"); - } - - private static HealthResponse deserialize(String json) throws Exception { - ObjectMapper mapper = new ObjectMapper(); - - return mapper.readValue(json, HealthResponse.class); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java deleted file mode 100644 index 733a105f047..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.state; - -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException; -import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; -import org.junit.jupiter.api.Test; - -import java.net.ConnectException; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class StateImplTest { - private final ConfigServerApi api = mock(ConfigServerApi.class); - private final StateImpl state = new StateImpl(api); - - @Test - void testWhenUp() { - HealthResponse response = new HealthResponse(); - response.status.code = "up"; - when(api.get(any(), any())).thenReturn(response); - - HealthCode code = state.getHealth(); - assertEquals(HealthCode.UP, code); - } - - @Test - void connectException() { - RuntimeException exception = - ConnectionException.handleException("Error: ", new ConnectException("connection refused")); - when(api.get(any(), any())).thenThrow(exception); - - HealthCode code = state.getHealth(); - assertEquals(HealthCode.DOWN, code); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java deleted file mode 100644 index 45b74368fd8..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.container.image.Image; -import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; - -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; - -/** - * @author mpolden - */ -public class ContainerEngineMock implements ContainerEngine { - - private final Map<ContainerName, Container> containers = new ConcurrentHashMap<>(); - private final Map<String, ImageDownload> images = new ConcurrentHashMap<>(); - private boolean asyncImageDownload = false; - - private final TestTerminal terminal; - - public ContainerEngineMock() { - this(null); - } - - public ContainerEngineMock(TestTerminal terminal) { - this.terminal = terminal; - } - - public ContainerEngineMock asyncImageDownload(boolean enabled) { - this.asyncImageDownload = enabled; - return this; - } - - public ContainerEngineMock completeDownloadOf(DockerImage image) { - String imageId = image.asString(); - ImageDownload download; - while ((download = images.get(imageId)) == null); - download.complete(); - return this; - } - - public ContainerEngineMock setImages(List<Image> images) { - this.images.clear(); - for (var image : images) { - ImageDownload imageDownload = new ImageDownload(image); - imageDownload.complete(); - this.images.put(image.id(), imageDownload); - } - return this; - } - - public ContainerEngineMock addContainers(List<Container> containers) { - for (var container : containers) { - if (this.containers.containsKey(container.name())) { - throw new IllegalArgumentException("Container " + container.name() + " already exists"); - } - this.containers.put(container.name(), container); - } - return this; - } - - public ContainerEngineMock addContainer(Container container) { - return addContainers(List.of(container)); - } - - @Override - public ContainerData createContainer(NodeAgentContext context, ContainerResources containerResources) { - addContainer(createContainer(context, PartialContainer.State.created, containerResources)); - return new ContainerData() { - @Override - public void addFile(ContainerPath path, String data) { - throw new UnsupportedOperationException("addFile not implemented"); - } - - @Override - public void addFile(ContainerPath path, String data, String permissions) { - throw new UnsupportedOperationException("addFile not implemented"); - } - - @Override - public void addDirectory(ContainerPath path, String... permissions) { - throw new UnsupportedOperationException("addDirectory not implemented"); - } - - @Override - public void addSymlink(ContainerPath symlink, Path target) { - throw new UnsupportedOperationException("addSymlink not implemented"); - } - - @Override - public void converge(NodeAgentContext context) { - throw new UnsupportedOperationException("converge not implemented"); - } - }; - } - - @Override - public void startContainer(NodeAgentContext context) { - Container container = requireContainer(context.containerName(), PartialContainer.State.created); - Container newContainer = createContainer(context, PartialContainer.State.running, container.resources()); - containers.put(newContainer.name(), newContainer); - } - - @Override - public void removeContainer(TaskContext context, PartialContainer container) { - requireContainer(container.name()); - containers.remove(container.name()); - } - - @Override - public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) { - Container container = requireContainer(context.containerName()); - containers.put(container.name(), new Container(containerId, container.name(), container.createdAt(), container.state(), - container.imageId(), container.image(), - container.labels(), container.pid(), - container.conmonPid(), container.hostname(), - containerResources, container.networks(), - container.managed())); - } - - @Override - public Optional<Container> getContainer(NodeAgentContext context) { - return Optional.ofNullable(containers.get(context.containerName())); - } - - @Override - public List<PartialContainer> listContainers(TaskContext context) { - return List.copyOf(containers.values()); - } - - @Override - public String networkInterface(NodeAgentContext context) { - return "eth0"; - } - - @Override - public CommandResult execute(NodeAgentContext context, UnixUser user, Duration timeout, String... command) { - if (terminal == null) { - return new CommandResult(null, 0, ""); - } - return terminal.newCommandLine(context) - .add(command) - .executeSilently(); - } - - @Override - public CommandResult executeInNetworkNamespace(NodeAgentContext context, CommandLine.Options options, String... command) { - if (terminal == null) { - return new CommandResult(null, 0, ""); - } - return terminal.newCommandLine(context).add(command).execute(options); - } - - @Override - public void pullImage(TaskContext context, DockerImage image, RegistryCredentials registryCredentials) { - String imageId = image.asString(); - ImageDownload imageDownload = images.computeIfAbsent(imageId, (ignored) -> new ImageDownload(new Image(imageId, List.of(imageId)))); - if (!asyncImageDownload) { - imageDownload.complete(); - } - imageDownload.awaitCompletion(); - } - - @Override - public boolean hasImage(TaskContext context, DockerImage image) { - ImageDownload download = images.get(image.asString()); - return download != null && download.isComplete(); - } - - @Override - public void removeImage(TaskContext context, String id) { - images.remove(id); - } - - @Override - public List<Image> listImages(TaskContext context) { - return images.values().stream() - .filter(ImageDownload::isComplete) - .map(ImageDownload::image) - .toList(); - } - - private Container requireContainer(ContainerName name) { - return requireContainer(name, null); - } - - private Container requireContainer(ContainerName name, PartialContainer.State wantedState) { - Container container = containers.get(name); - if (container == null) throw new IllegalArgumentException("No such container: " + name); - if (wantedState != null && container.state() != wantedState) throw new IllegalArgumentException("Container is " + container.state() + ", wanted " + wantedState); - return container; - } - - public Container createContainer(NodeAgentContext context, PartialContainer.State state, ContainerResources containerResources) { - return new Container(new ContainerId("id-of-" + context.containerName()), - context.containerName(), - Instant.EPOCH, - state, - "image-id", - context.node().wantedDockerImage().get(), - Map.of(), - 41, - 42, - context.hostname().value(), - containerResources, - List.of(), - true); - } - - private static class ImageDownload { - - private final Image image; - private final CountDownLatch done = new CountDownLatch(1); - - ImageDownload(Image image) { - this.image = Objects.requireNonNull(image); - } - - Image image() { - return image; - } - - boolean isComplete() { - return done.getCount() == 0; - } - - void complete() { - done.countDown(); - } - - void awaitCompletion() { - try { - done.await(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java deleted file mode 100644 index f9f7a18597c..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author freva - */ -public class ContainerNameTest { - @Test - void testAlphanumericalContainerName() { - String name = "container123"; - ContainerName containerName = new ContainerName(name); - assertEquals(containerName.asString(), name); - } - - @Test - void testAlphanumericalWithDashContainerName() { - String name = "container-123"; - ContainerName containerName = new ContainerName(name); - assertEquals(containerName.asString(), name); - } - - @Test - void testContainerNameFromHostname() { - assertEquals(new ContainerName("container-123"), ContainerName.fromHostname("container-123.sub.domain.tld")); - } - - @Test - void testAlphanumericalWithSlashContainerName() { - assertThrows(IllegalArgumentException.class, () -> { - new ContainerName("container/123"); - }); - } - - @Test - void testEmptyContainerName() { - assertThrows(IllegalArgumentException.class, () -> { - new ContainerName(""); - }); - } - - @Test - void testNullContainerName() { - assertThrows(NullPointerException.class, () -> { - new ContainerName(null); - }); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java deleted file mode 100644 index a72e926a471..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -/** - * @author mpolden - */ -public class ContainerOperationsTest { - - private final TestTaskContext context = new TestTaskContext(); - private final ContainerEngineMock containerEngine = new ContainerEngineMock(); - private final FileSystem fileSystem = TestFileSystem.create(); - private final TestTimer timer = new TestTimer(); - private final ContainerOperations containerOperations = new ContainerOperations(containerEngine, mock(Cgroup.class), fileSystem, timer); - - @Test - void no_managed_containers_running() { - Container c1 = createContainer("c1", true); - Container c2 = createContainer("c2", false); - - containerEngine.addContainer(c1); - assertFalse(containerOperations.noManagedContainersRunning(context)); - - containerEngine.removeContainer(context, c1); - assertTrue(containerOperations.noManagedContainersRunning(context)); - - containerEngine.addContainer(c2); - assertTrue(containerOperations.noManagedContainersRunning(context)); - } - - @Test - void retain_managed_containers() { - Container c1 = createContainer("c1", true); - Container c2 = createContainer("c2", true); - Container c3 = createContainer("c3", false); - containerEngine.addContainers(List.of(c1, c2, c3)); - - assertEquals(3, containerEngine.listContainers(context).size()); - containerOperations.retainManagedContainers(context, Set.of(c1.name())); - - assertEquals(List.of(c1.name(), c3.name()), containerEngine.listContainers(context).stream() - .map(PartialContainer::name) - .sorted() - .toList()); - } - - private Container createContainer(String name, boolean managed) { - return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, PartialContainer.State.running, - "image-id", DockerImage.EMPTY, Map.of(), 42, 43, name, - ContainerResources.UNLIMITED, List.of(), managed); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java deleted file mode 100644 index cbc803b6105..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * @author freva - */ -public class ContainerResourcesTest { - - @Test - void verify_unlimited() { - assertEquals(-1, ContainerResources.UNLIMITED.cpuQuota()); - assertEquals(100_000, ContainerResources.UNLIMITED.cpuPeriod()); - assertEquals(0, ContainerResources.UNLIMITED.cpuShares()); - } - - @Test - void validate_shares() { - new ContainerResources(0, 0, 0); - new ContainerResources(0, 2, 0); - new ContainerResources(0, 2048, 0); - new ContainerResources(0, 262_144, 0); - - assertThrows(IllegalArgumentException.class, () -> new ContainerResources(0, -1, 0)); // Negative shares not allowed - assertThrows(IllegalArgumentException.class, () -> new ContainerResources(0, 1, 0)); // 1 share not allowed - assertThrows(IllegalArgumentException.class, () -> new ContainerResources(0, 262_145, 0)); - } - - @Test - void cpu_shares_scaling() { - ContainerResources resources = ContainerResources.from(5.3, 2.5, 0); - assertEquals(530_000, resources.cpuQuota()); - assertEquals(100_000, resources.cpuPeriod()); - assertEquals(80, resources.cpuShares()); - } - - private static void assertThrows(Class<? extends Throwable> clazz, Runnable runnable) { - try { - runnable.run(); - fail("Expected " + clazz); - } catch (Throwable e) { - if (!clazz.isInstance(e)) throw e; - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java deleted file mode 100644 index 8cd3d6529c5..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; -import com.yahoo.vespa.hosted.node.admin.cgroup.MemoryController; -import com.yahoo.vespa.hosted.node.admin.cgroup.Size; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; -import org.mockito.Answers; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author mpolden - */ -public class ContainerStatsCollectorTest { - - private final TestTerminal testTerminal = new TestTerminal(); - private final ContainerEngineMock containerEngine = new ContainerEngineMock(testTerminal); - private final FileSystem fileSystem = TestFileSystem.create(); - private final Cgroup cgroup = mock(Cgroup.class, Answers.RETURNS_DEEP_STUBS); - private final NodeAgentContext context = NodeAgentContextImpl.builder(NodeSpec.Builder.testSpec("c1").build()) - .fileSystem(TestFileSystem.create()) - .build(); - @Test - void collect() throws Exception { - ContainerStatsCollector collector = new ContainerStatsCollector(containerEngine, cgroup, fileSystem, 24); - ContainerId containerId = new ContainerId("id1"); - int containerPid = 42; - assertTrue(collector.collect(context, containerId, containerPid, "eth0").isEmpty(), "No stats found"); - - mockMemoryStats(containerId); - mockCpuStats(containerId); - mockNetworkStats(containerPid); - - Optional<ContainerStats> stats = collector.collect(context, containerId, containerPid, "eth0"); - assertTrue(stats.isPresent()); - assertEquals(new ContainerStats.CpuStats(24, 6049374780000L, 691675615472L, - 262190000000L, 3L, 1L, 2L), - stats.get().cpuStats()); - assertEquals(new ContainerStats.MemoryStats(470790144L, 1228017664L, 2147483648L), - stats.get().memoryStats()); - assertEquals(Map.of("eth0", new ContainerStats.NetworkStats(22280813L, 4L, 3L, - 19859383L, 6L, 5L)), - stats.get().networks()); - assertEquals(List.of(), stats.get().gpuStats()); - - mockGpuStats(); - stats = collector.collect(context, containerId, containerPid, "eth0"); - assertTrue(stats.isPresent()); - assertEquals(List.of(new ContainerStats.GpuStats(0, 35, 16106127360L, 6144655360L), - new ContainerStats.GpuStats(1, 67, 32212254720L, 19314769920L)), - stats.get().gpuStats()); - } - - private void mockGpuStats() throws IOException { - Path devPath = fileSystem.getPath("/dev"); - Files.createDirectories(devPath); - Files.createFile(devPath.resolve("nvidia0")); - testTerminal.expectCommand("nvidia-smi --query-gpu=index,utilization.gpu,memory.total,memory.free --format=csv,noheader,nounits 2>&1", 0, - """ - 0, 35, 15360, 9500 - 1, 67, 30720, 12300 - """); - } - - private void mockNetworkStats(int pid) { - UnixPath dev = new UnixPath(fileSystem.getPath("/proc/" + pid + "/net/dev")); - dev.createParents().writeUtf8File("Inter-| Receive | Transmit\n" + - " face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n" + - " lo: 36289258 149700 0 0 0 0 0 0 36289258 149700 0 0 0 0 0 0\n" + - " eth0: 22280813 118083 3 4 0 0 0 0 19859383 115415 5 6 0 0 0 0\n"); - } - - private void mockMemoryStats(ContainerId containerId) { - when(cgroup.resolveContainer(eq(containerId)).memory().readCurrent()).thenReturn(Size.from(1228017664L)); - when(cgroup.resolveContainer(eq(containerId)).memory().readMax()).thenReturn(Size.from(2147483648L)); - when(cgroup.resolveContainer(eq(containerId)).memory().readStat()).thenReturn( - new MemoryController.Stats(Size.from(470790144L), Size.from(0), Size.from(0), Size.from(0), Size.from(0))); - } - - private void mockCpuStats(ContainerId containerId) throws IOException { - UnixPath proc = new UnixPath(fileSystem.getPath("/proc")); - proc.createDirectories(); - - when(cgroup.resolveContainer(eq(containerId)).cpu().readStats()).thenReturn(Map.of( - TOTAL_USAGE_USEC, 691675615472L, SYSTEM_USAGE_USEC, 262190000000L, USER_USAGE_USEC, 40900L, - TOTAL_PERIODS, 1L, THROTTLED_PERIODS, 2L, THROTTLED_TIME_USEC, 3L)); - - proc.resolve("stat").writeUtf8File("cpu 7991366 978222 2346238 565556517 1935450 25514479 615206 0 0 0\n" + - "cpu0 387906 61529 99088 23516506 42258 1063359 29882 0 0 0\n" + - "cpu1 271253 49383 86149 23655234 41703 1061416 31885 0 0 0\n" + - "cpu2 349420 50987 93560 23571695 59437 1051977 24461 0 0 0\n" + - "cpu3 328107 50628 93406 23605135 44378 1048549 30199 0 0 0\n" + - "cpu4 267474 50404 99253 23606041 113094 1038572 26494 0 0 0\n" + - "cpu5 309584 50677 94284 23550372 132616 1033661 29436 0 0 0\n" + - "cpu6 477926 56888 121251 23367023 83121 1074930 28818 0 0 0\n" + - "cpu7 335335 29350 106130 23551107 95606 1066394 26156 0 0 0\n" + - "cpu8 323678 28629 99171 23586501 82183 1064708 25403 0 0 0\n" + - "cpu9 329805 27516 98538 23579458 89235 1061561 25140 0 0 0\n" + - "cpu10 291536 26455 93934 23642345 81282 1049736 25228 0 0 0\n" + - "cpu11 271103 25302 90630 23663641 85711 1048781 24291 0 0 0\n" + - "cpu12 323634 63392 100406 23465340 132684 1089157 28319 0 0 0\n" + - "cpu13 348085 49568 100772 23490388 114190 1079474 20948 0 0 0\n" + - "cpu14 310712 51208 90461 23547980 101601 1071940 26712 0 0 0\n" + - "cpu15 360405 52754 94620 23524878 79851 1062050 26836 0 0 0\n" + - "cpu16 367893 52141 98074 23541314 57500 1058968 25242 0 0 0\n" + - "cpu17 412756 51486 101592 23515056 47653 1044874 27467 0 0 0\n" + - "cpu18 287307 25478 106011 23599505 79848 1089812 23160 0 0 0\n" + - "cpu19 275001 24421 98338 23628694 79675 1084074 22083 0 0 0\n" + - "cpu20 288038 24805 94432 23629908 74735 1078501 21915 0 0 0\n" + - "cpu21 295373 25017 91344 23628585 75282 1071019 22026 0 0 0\n" + - "cpu22 326739 25588 90385 23608217 69186 1068494 21108 0 0 0\n" + - "cpu23 452284 24602 104397 23481583 72612 1052462 21985 0 0 0\n" + - "intr 6645352968 64 0 0 0 1481 0 0 0 1 0 0 0 0 0 0 0 39 0 0 0 0 0 0 37 0 0 0 0 0 0 0 0 4334106 1 6949071 5814662 5415344 6939471 6961483 6358810 5271953 6718644 0 126114 126114 126114 126114 126114 126114 126114 126114 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n" + - "ctxt 2495530303\n" + - "btime 1611928223\n" + - "processes 4839481\n" + - "procs_running 4\n" + - "procs_blocked 0\n" + - "softirq 2202631388 4 20504999 46734 54405637 4330276 0 6951 1664780312 10130 458546345\n"); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java deleted file mode 100644 index 37db6895040..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.image; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; -import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author mpolden - */ -public class ContainerImageDownloaderTest { - - @Test - @Timeout(5_000) - void test_download() { - ContainerEngineMock podman = new ContainerEngineMock().asyncImageDownload(true); - ContainerImageDownloader downloader = new ContainerImageDownloader(podman, new TestTimer()); - TaskContext context = new TestTaskContext(); - DockerImage image = DockerImage.fromString("registry.example.com/repo/vespa:7.42"); - - assertFalse(downloader.get(context, image, () -> RegistryCredentials.none), "Download started"); - assertFalse(downloader.get(context, image, () -> RegistryCredentials.none), "Download pending"); - podman.completeDownloadOf(image); - boolean downloadCompleted; - while (!(downloadCompleted = downloader.get(context, image, () -> RegistryCredentials.none))) ; - assertTrue(downloadCompleted, "Download completed"); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java deleted file mode 100644 index 71312125cbc..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.image; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.hosted.node.admin.container.Container; -import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; -import com.yahoo.vespa.hosted.node.admin.container.ContainerId; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.container.ContainerResources; -import org.junit.jupiter.api.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author freva - * @author mpolden - */ -public class ContainerImagePrunerTest { - - private final Tester tester = new Tester(); - - @Test - void noImagesMeansNoUnusedImages() { - tester.withExistingImages() - .expectDeletedImages(); - } - - @Test - void singleImageWithoutContainersIsUnused() { - tester.withExistingImages(image("image-1")) - // Even though nothing is using the image, we will keep it for at least 1h - .expectDeletedImagesAfterMinutes(0) - .expectDeletedImagesAfterMinutes(30) - .expectDeletedImagesAfterMinutes(30, "image-1"); - } - - @Test - void singleImageWithContainerIsUsed() { - tester.withExistingImages(image("image-1")) - .withExistingContainers(container("container-1", "image-1")) - .expectDeletedImages(); - } - - @Test - void multipleUnusedImagesAreIdentified() { - tester.withExistingImages(image("image-1"), image("image-2")) - .expectDeletedImages("image-1", "image-2"); - } - - @Test - void unusedImagesWithMultipleTags() { - tester.withExistingImages(image("image-1", "vespa-6", "vespa-6.28", "vespa:latest")) - .expectDeletedImages("vespa-6", "vespa-6.28", "vespa:latest"); - } - - @Test - void unusedImagesWithMultipleUntagged() { - tester.withExistingImages(image("image1", "<none>:<none>"), - image("image2", "<none>:<none>")) - .expectDeletedImages("image1", "image2"); - } - - @Test - void taggedImageWithNoContainersIsUnused() { - tester.withExistingImages(image("image-1", "vespa-6")) - .expectDeletedImages("vespa-6"); - } - - @Test - void reDownloadingImageIsNotImmediatelyDeleted() { - tester.withExistingImages(image("image")) - .expectDeletedImages("image") // After 1h we delete image - .expectDeletedImagesAfterMinutes(0) // image is immediately re-downloaded, but is not deleted - .expectDeletedImagesAfterMinutes(10) - .expectDeletedImages("image"); // 1h after re-download it is deleted again - } - - @Test - void reDownloadingImageIsNotImmediatelyDeletedWhenDeletingByTag() { - tester.withExistingImages(image("image", "my-tag")) - .expectDeletedImages("my-tag") // After 1h we delete image - .expectDeletedImagesAfterMinutes(0) // image is immediately re-downloaded, but is not deleted - .expectDeletedImagesAfterMinutes(10) - .expectDeletedImages("my-tag"); // 1h after re-download it is deleted again - } - - /** Same scenario as in {@link #multipleUnusedImagesAreIdentified()} */ - @Test - void doesNotDeleteExcludedByIdImages() { - tester.withExistingImages(image("image-1"), image("image-2")) - // Normally, image-1 should also be deleted, but because we exclude image-1 only image-2 is deleted - .expectDeletedImages(List.of("image-1"), "image-2"); - } - - /** Same as in {@link #doesNotDeleteExcludedByIdImages()} but with tags */ - @Test - void doesNotDeleteExcludedByTagImages() { - tester.withExistingImages(image("image-1", "vespa:6.288.16"), image("image-2", "vespa:6.289.94")) - .expectDeletedImages(List.of("vespa:6.288.16"), "vespa:6.289.94"); - } - - @Test - void excludingNotDownloadedImageIsNoop() { - tester.withExistingImages(image("image-1", "vespa:6.288.16"), - image("image-2", "vespa:6.289.94")) - .expectDeletedImages(List.of("vespa:6.300.1"), "vespa:6.288.16", "vespa:6.289.94", "rhel-6"); - } - - private static Image image(String id, String... tags) { - return new Image(id, List.of(tags)); - } - - private static Container container(String name, String imageId) { - return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, - Container.State.running, imageId, DockerImage.EMPTY, Map.of(), - 42, 43, name + ".example.com", ContainerResources.UNLIMITED, - List.of(), true); - } - - private static class Tester { - - private final ContainerEngineMock containerEngine = new ContainerEngineMock(); - private final TaskContext context = new TestTaskContext(); - private final TestTimer timer = new TestTimer(); - private final ContainerImagePruner pruner = new ContainerImagePruner(containerEngine, timer); - private final Map<String, Integer> removalCountByImageId = new HashMap<>(); - - private boolean initialized = false; - - private Tester withExistingImages(Image... images) { - containerEngine.setImages(List.of(images)); - return this; - } - - private Tester withExistingContainers(Container... containers) { - containerEngine.addContainers(List.of(containers)); - return this; - } - - private Tester expectDeletedImages(String... imageIds) { - return expectDeletedImagesAfterMinutes(60, imageIds); - } - - private Tester expectDeletedImages(List<String> excludedRefs, String... imageIds) { - return expectDeletedImagesAfterMinutes(60, excludedRefs, imageIds); - } - - private Tester expectDeletedImagesAfterMinutes(int minutesAfter, String... imageIds) { - return expectDeletedImagesAfterMinutes(minutesAfter, List.of(), imageIds); - } - - private Tester expectDeletedImagesAfterMinutes(int minutesAfter, List<String> excludedRefs, String... imageIds) { - if (!initialized) { - // Run once with a very long expiry to initialize internal state of existing images - pruner.removeUnusedImages(context, List.of(), Duration.ofDays(999)); - initialized = true; - } - - timer.advance(Duration.ofMinutes(minutesAfter)); - - pruner.removeUnusedImages(context, excludedRefs, Duration.ofHours(1).minusSeconds(1)); - - List.of(imageIds) - .forEach(imageId -> { - int newValue = removalCountByImageId.getOrDefault(imageId, 0) + 1; - removalCountByImageId.put(imageId, newValue); - - assertTrue(containerEngine.listImages(context).stream().noneMatch(image -> image.id().equals(imageId)), - "Image " + imageId + " removed"); - }); - return this; - } - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java deleted file mode 100644 index 8e23c7e54b6..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.metrics; - -import org.junit.jupiter.api.Test; - -import java.util.Map; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics.APPLICATION_HOST; -import static com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics.DimensionType.DEFAULT; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class MetricsTest { - private static final Dimensions hostDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); - private final Metrics metrics = new Metrics(); - - @Test - void testDefaultValue() { - metrics.declareCounter("some.name", hostDimension); - - assertEquals(getMetricsForDimension(hostDimension).get("some.name"), 0L); - } - - @Test - void testSimpleIncrementMetric() { - Counter counter = metrics.declareCounter("a_counter.value", hostDimension); - - counter.add(5); - counter.add(8); - - Map<String, Number> latestMetrics = getMetricsForDimension(hostDimension); - assertEquals(1, latestMetrics.size(), "Expected only 1 metric value to be set"); - assertEquals(latestMetrics.get("a_counter.value"), 13L); // 5 + 8 - } - - @Test - void testSimpleGauge() { - Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); - - gauge.sample(42); - gauge.sample(-342.23); - - Map<String, Number> latestMetrics = getMetricsForDimension(hostDimension); - assertEquals(1, latestMetrics.size(), "Expected only 1 metric value to be set"); - assertEquals(latestMetrics.get("test.gauge"), -342.23); - } - - @Test - void testRedeclaringSameGauge() { - Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); - gauge.sample(42); - - // Same as hostDimension, but new instance. - Dimensions newDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); - Gauge newGauge = metrics.declareGauge("test.gauge", newDimension); - newGauge.sample(56); - - assertEquals(getMetricsForDimension(hostDimension).get("test.gauge"), 56.); - } - - @Test - void testSameMetricNameButDifferentDimensions() { - Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); - gauge.sample(42); - - // Not the same as hostDimension. - Dimensions newDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); - Gauge newGauge = metrics.declareGauge("test.gauge", newDimension); - newGauge.sample(56); - - assertEquals(getMetricsForDimension(hostDimension).get("test.gauge"), 42.); - assertEquals(getMetricsForDimension(newDimension).get("test.gauge"), 56.); - } - - @Test - void testDeletingMetric() { - metrics.declareGauge("test.gauge", hostDimension); - - Dimensions differentDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); - metrics.declareGauge("test.gauge", differentDimension); - - assertEquals(2, metrics.getMetricsByType(DEFAULT).size()); - metrics.deleteMetricByDimension(APPLICATION_HOST, differentDimension, DEFAULT); - assertEquals(1, metrics.getMetricsByType(DEFAULT).size()); - assertEquals(getMetricsForDimension(hostDimension).size(), 1); - assertEquals(getMetricsForDimension(differentDimension).size(), 0); - } - - private Map<String, Number> getMetricsForDimension(Dimensions dimensions) { - return metrics.getOrCreateApplicationMetrics(APPLICATION_HOST, DEFAULT) - .getOrDefault(dimensions, Map.of()) - .entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue())); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java deleted file mode 100644 index de41da7329b..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integration; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; - -/** - * @author freva - */ -public class ContainerFailTest { - - @Test - void test() { - DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image"); - try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { - ContainerName containerName = new ContainerName("host1"); - String hostname = "host1.test.yahoo.com"; - NodeSpec nodeSpec = NodeSpec.Builder - .testSpec(hostname) - .wantedDockerImage(dockerImage) - .currentDockerImage(dockerImage) - .build(); - tester.addChildNodeRepositoryNode(nodeSpec); - - NodeAgentContext context = NodeAgentContextImpl.builder(nodeSpec).fileSystem(TestFileSystem.create()).build(); - - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any()); - tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); - - tester.containerOperations.removeContainer(context, tester.containerOperations.getContainer(context).get()); - - tester.inOrder(tester.containerOperations).removeContainer(containerMatcher(containerName), any()); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any()); - tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); - - verify(tester.nodeRepository, never()).updateNodeAttributes(any(), any()); - } - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java deleted file mode 100644 index b4d85a5e974..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integration; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials; -import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ProcMeminfo; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ProcMeminfoReader; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentFactory; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.mockito.InOrder; -import org.mockito.Mockito; - -import java.nio.file.FileSystem; -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Phaser; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.logging.Logger; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -/** - * @author musum - */ -// Need to deconstruct nodeAdminStateUpdater -public class ContainerTester implements AutoCloseable { - - private static final Logger log = Logger.getLogger(ContainerTester.class.getName()); - static final HostName HOST_HOSTNAME = HostName.of("host.test.yahoo.com"); - - private final Thread loopThread; - private final Phaser phaser = new Phaser(1); - - private final ContainerEngineMock containerEngine = new ContainerEngineMock(); - private final FileSystem fileSystem = TestFileSystem.create(); - private final TestTimer timer = new TestTimer(); - final ContainerOperations containerOperations = spy(new ContainerOperations(containerEngine, mock(Cgroup.class), fileSystem, timer)); - final NodeRepoMock nodeRepository = spy(new NodeRepoMock()); - final Orchestrator orchestrator = mock(Orchestrator.class); - final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); - final InOrder inOrder = Mockito.inOrder(containerOperations, nodeRepository, orchestrator, storageMaintainer); - final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - - final NodeAdminStateUpdater nodeAdminStateUpdater; - final NodeAdminImpl nodeAdmin; - - private volatile NodeAdminStateUpdater.State wantedState = NodeAdminStateUpdater.State.RESUMED; - - - ContainerTester(List<DockerImage> images) { - images.forEach(image -> containerEngine.pullImage(null, image, RegistryCredentials.none)); - when(storageMaintainer.diskUsageFor(any())).thenReturn(Optional.empty()); - - IPAddressesMock ipAddresses = new IPAddressesMock(); - ipAddresses.addAddress(HOST_HOSTNAME.value(), "1.1.1.1"); - ipAddresses.addAddress(HOST_HOSTNAME.value(), "f000::"); - for (int i = 1; i < 4; i++) ipAddresses.addAddress("host" + i + ".test.yahoo.com", "f000::" + i); - - NodeSpec hostSpec = NodeSpec.Builder.testSpec(HOST_HOSTNAME.value()).type(NodeType.host).build(); - nodeRepository.updateNodeSpec(hostSpec); - - Metrics metrics = new Metrics(); - FileSystem fileSystem = TestFileSystem.create(); - ProcMeminfoReader procMeminfoReader = mock(ProcMeminfoReader.class); - when(procMeminfoReader.read()).thenReturn(new ProcMeminfo(1, 2)); - - NodeAgentFactory nodeAgentFactory = (contextSupplier, nodeContext) -> - new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, containerOperations, () -> RegistryCredentials.none, - storageMaintainer, flagSource, - List.of(), Optional.empty(), Optional.empty(), timer, Duration.ofSeconds(-1), - VespaServiceDumper.DUMMY_INSTANCE, List.of()) { - @Override public void converge(NodeAgentContext context) { - super.converge(context); - phaser.arriveAndAwaitAdvance(); - } - @Override public void stopForHostSuspension(NodeAgentContext context) { - super.stopForHostSuspension(context); - phaser.arriveAndAwaitAdvance(); - } - @Override public void stopForRemoval(NodeAgentContext context) { - super.stopForRemoval(context); - phaser.arriveAndDeregister(); - } - }; - nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, timer, Duration.ofMillis(10), Duration.ZERO, procMeminfoReader); - NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) -> - NodeAgentContextImpl.builder(nodeSpec).acl(acl).fileSystem(fileSystem).build(); - nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator, - nodeAdmin, HOST_HOSTNAME); - - loopThread = new Thread(() -> { - nodeAdminStateUpdater.start(); - while ( ! phaser.isTerminated()) { - try { - nodeAdminStateUpdater.converge(wantedState); - } catch (RuntimeException e) { - log.info(e.getMessage()); - } - } - nodeAdminStateUpdater.stop(); - }); - loopThread.start(); - } - - /** Adds a node to node-repository mock that is running on this host */ - void addChildNodeRepositoryNode(NodeSpec nodeSpec) { - if (nodeSpec.wantedDockerImage().isPresent()) { - if (!containerEngine.hasImage(null, nodeSpec.wantedDockerImage().get())) { - throw new IllegalArgumentException("Want to use image " + nodeSpec.wantedDockerImage().get() + - ", but that image does not exist in the container engine"); - } - } - - if (nodeRepository.getOptionalNode(nodeSpec.hostname()).isEmpty()) - phaser.register(); - - nodeRepository.updateNodeSpec(new NodeSpec.Builder(nodeSpec) - .parentHostname(HOST_HOSTNAME.value()) - .build()); - } - - void setWantedState(NodeAdminStateUpdater.State wantedState) { - this.wantedState = wantedState; - } - - <T> T inOrder(T t) { - waitSomeTicks(); - return inOrder.verify(t); - } - - void waitSomeTicks() { - try { - // 3 is enough for everyone! (Well, maybe not for all eternity ...) - for (int i = 0; i < 3; i++) - phaser.awaitAdvanceInterruptibly(phaser.arrive(), 1000, TimeUnit.MILLISECONDS); - } - catch (InterruptedException | TimeoutException e) { - throw new RuntimeException(e); - } - } - - public static NodeAgentContext containerMatcher(ContainerName containerName) { - return argThat((ctx) -> ctx.containerName().equals(containerName)); - } - - @Override - public void close() { - phaser.forceTermination(); - try { - loopThread.join(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java deleted file mode 100644 index 7e874bcd5a7..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integration; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; - -/** - * @author freva - */ -public class MultiContainerTest { - - @Test - void test() { - DockerImage image1 = DockerImage.fromString("registry.example.com/repo/image1"); - DockerImage image2 = DockerImage.fromString("registry.example.com/repo/image2"); - try (ContainerTester tester = new ContainerTester(List.of(image1, image2))) { - addAndWaitForNode(tester, "host1.test.yahoo.com", image1); - NodeSpec nodeSpec2 = addAndWaitForNode(tester, "host2.test.yahoo.com", image2); - - tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(nodeSpec2.hostname(), NodeState.dirty).build()); - - ContainerName host2 = new ContainerName("host2"); - tester.inOrder(tester.containerOperations).removeContainer(containerMatcher(host2), any()); - tester.inOrder(tester.storageMaintainer).archiveNodeStorage( - argThat(context -> context.containerName().equals(host2))); - tester.inOrder(tester.nodeRepository).setNodeState(eq(nodeSpec2.hostname()), eq(NodeState.ready)); - - addAndWaitForNode(tester, "host3.test.yahoo.com", image1); - } - } - - private NodeAgentContext containerMatcher(ContainerName containerName) { - return argThat((ctx) -> ctx.containerName().equals(containerName)); - } - - private NodeSpec addAndWaitForNode(ContainerTester tester, String hostName, DockerImage dockerImage) { - NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostName).wantedDockerImage(dockerImage).build(); - tester.addChildNodeRepositoryNode(nodeSpec); - - ContainerName containerName = ContainerName.fromHostname(hostName); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any()); - tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); - tester.inOrder(tester.nodeRepository).updateNodeAttributes(eq(hostName), any()); - - return nodeSpec; - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java deleted file mode 100644 index da14c5aa47b..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integration; - -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.AddNode; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NoSuchNodeException; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; - -/** - * Mock with some simple logic - * - * @author dybis - */ -public class NodeRepoMock implements NodeRepository { - - private final Map<String, NodeSpec> nodeSpecByHostname = new ConcurrentHashMap<>(); - private volatile Map<String, Acl> aclByHostname = Map.of(); - - @Override - public void addNodes(List<AddNode> nodes) { } - - @Override - public List<NodeSpec> getNodes(String baseHostName) { - return nodeSpecByHostname.values().stream() - .filter(node -> baseHostName.equals(node.parentHostname().orElse(null))) - .toList(); - } - - @Override - public Optional<NodeSpec> getOptionalNode(String hostName) { - return Optional.ofNullable(nodeSpecByHostname.get(hostName)); - } - - @Override - public Map<String, Acl> getAcls(String hostname) { - return aclByHostname; - } - - @Override - public List<WireguardPeer> getExclavePeers() { - throw new UnsupportedOperationException(); - } - - @Override - public List<WireguardPeer> getConfigserverPeers() { - throw new UnsupportedOperationException(); - } - - @Override - public void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes) { - updateNodeSpec(new NodeSpec.Builder(getNode(hostName)) - .updateFromNodeAttributes(nodeAttributes) - .build()); - } - - @Override - public void setNodeState(String hostName, NodeState nodeState) { - updateNodeSpec(new NodeSpec.Builder(getNode(hostName)) - .state(nodeState) - .build()); - } - - public void updateNodeSpec(NodeSpec nodeSpec) { - nodeSpecByHostname.put(nodeSpec.hostname(), nodeSpec); - } - - public void updateNodeSpec(String hostname, Function<NodeSpec.Builder, NodeSpec.Builder> mapper) { - nodeSpecByHostname.compute(hostname, (__, nodeSpec) -> { - if (nodeSpec == null) throw new NoSuchNodeException(hostname); - return mapper.apply(new NodeSpec.Builder(nodeSpec)).build(); - }); - } - - public void resetNodeSpecs() { - nodeSpecByHostname.clear(); - } - - public void setAcl(Map<String, Acl> aclByHostname) { - this.aclByHostname = Map.copyOf(aclByHostname); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java deleted file mode 100644 index a1440ba8669..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integration; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.HOST_HOSTNAME; -import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; - -/** - * Tests rebooting of Docker host - * - * @author musum - */ -public class RebootTest { - - private final String hostname = "host1.test.yahoo.com"; - private final DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image"); - - @Test - void test() { - try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { - tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build()); - - ContainerName host1 = new ContainerName("host1"); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any()); - - tester.setWantedState(NodeAdminStateUpdater.State.SUSPENDED); - - tester.inOrder(tester.orchestrator).suspend(eq(HOST_HOSTNAME.value()), eq(List.of(hostname, HOST_HOSTNAME.value()))); - tester.inOrder(tester.containerOperations).stopServices(containerMatcher(host1)); - assertTrue(tester.nodeAdmin.setFrozen(true)); - } - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java deleted file mode 100644 index 1445546097a..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integration; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; - -/** - * Tests that different wanted and current restart generation leads to execution of restart command - * - * @author musum - */ -public class RestartTest { - - @Test - void test() { - DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image:1.2.3"); - try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { - String hostname = "host1.test.yahoo.com"; - NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostname) - .wantedDockerImage(dockerImage) - .wantedVespaVersion(dockerImage.tagAsVersion()) - .build(); - tester.addChildNodeRepositoryNode(nodeSpec); - - ContainerName host1 = new ContainerName("host1"); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any()); - tester.inOrder(tester.nodeRepository).updateNodeAttributes( - eq(hostname), eq(new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()))); - - // Increment wantedRestartGeneration to 2 in node-repo - tester.addChildNodeRepositoryNode(new NodeSpec.Builder(tester.nodeRepository.getNode(hostname)) - .wantedRestartGeneration(2).build()); - - tester.inOrder(tester.orchestrator).suspend(eq(hostname)); - tester.inOrder(tester.containerOperations).restartVespa(containerMatcher(host1)); - tester.inOrder(tester.nodeRepository).updateNodeAttributes( - eq(hostname), eq(new NodeAttributes().withRestartGeneration(2))); - } - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java deleted file mode 100644 index 51b3bb5e6c4..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import com.yahoo.config.provision.NodeResources; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; -import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup; -import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; - -/** - * @author dybis - */ -public class StorageMaintainerTest { - - private final TestTerminal terminal = new TestTerminal(); - private final CoredumpHandler coredumpHandler = mock(CoredumpHandler.class); - private final DiskCleanup diskCleanup = mock(DiskCleanup.class); - private final SyncClient syncClient = mock(SyncClient.class); - private final TestTimer timer = new TestTimer(Instant.ofEpochSecond(1234567890)); - private final FileSystem fileSystem = TestFileSystem.create(); - private final StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, coredumpHandler, diskCleanup, syncClient, timer, - fileSystem.getPath("/data/vespa/storage/container-archive")); - - @Test - void testDiskUsed() { - NodeAgentContext context = NodeAgentContextImpl.builder("host-1.domain.tld").fileSystem(fileSystem).build(); - - terminal.expectCommand("du -xsk /data/vespa/storage/host-1 2>&1", 0, "321\t/data/vespa/storage/host-1/"); - assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); - - // Value should still be cached, no new execution against the terminal - assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); - } - - @Test - void testNonExistingDiskUsed() { - DiskSize size = storageMaintainer.getDiskUsed(null, Path.of("/fake/path")); - assertEquals(DiskSize.ZERO, size); - } - - @Test - void archive_container_data_test() throws IOException { - // Create some files in containers - NodeAgentContext context1 = createNodeAgentContextAndContainerStorage(fileSystem, "container-1"); - createNodeAgentContextAndContainerStorage(fileSystem, "container-2"); - - Path pathToArchiveDir = fileSystem.getPath("/data/vespa/storage/container-archive"); - Files.createDirectories(pathToArchiveDir); - - Path containerStorageRoot = context1.paths().of("/").pathOnHost().getParent(); - Set<String> containerStorageRootContentsBeforeArchive = FileFinder.from(containerStorageRoot) - .maxDepth(1) - .stream() - .map(FileFinder.FileAttributes::filename) - .collect(Collectors.toSet()); - assertEquals(Set.of("container-archive", "container-1", "container-2"), containerStorageRootContentsBeforeArchive); - - - // Archive container-1 - storageMaintainer.archiveNodeStorage(context1); - - timer.advance(Duration.ofSeconds(3)); - storageMaintainer.archiveNodeStorage(context1); - - // container-1 should be gone from container-storage - Set<String> containerStorageRootContentsAfterArchive = FileFinder.from(containerStorageRoot) - .maxDepth(1) - .stream() - .map(FileFinder.FileAttributes::filename) - .collect(Collectors.toSet()); - assertEquals(Set.of("container-archive", "container-2"), containerStorageRootContentsAfterArchive); - - // container archive directory should contain exactly 1 directory - the one we just archived - List<FileFinder.FileAttributes> containerArchiveContentsAfterArchive = FileFinder.from(pathToArchiveDir).maxDepth(1).list(); - assertEquals(1, containerArchiveContentsAfterArchive.size()); - Path archivedContainerStoragePath = containerArchiveContentsAfterArchive.get(0).path(); - assertEquals("container-1_20090213233130", archivedContainerStoragePath.getFileName().toString()); - Set<String> archivedContainerStorageContents = FileFinder.files(archivedContainerStoragePath) - .stream() - .map(fileAttributes -> archivedContainerStoragePath.relativize(fileAttributes.path()).toString()) - .collect(Collectors.toSet()); - assertEquals(Set.of("opt/vespa/logs/vespa/vespa.log", "opt/vespa/logs/vespa/zookeeper.log"), archivedContainerStorageContents); - } - - private static NodeAgentContext createNodeAgentContextAndContainerStorage(FileSystem fileSystem, String containerName) throws IOException { - NodeAgentContext context = NodeAgentContextImpl.builder(containerName + ".domain.tld") - .fileSystem(fileSystem).build(); - - ContainerPath containerVespaHome = context.paths().underVespaHome(""); - Files.createDirectories(context.paths().of("/etc/something")); - Files.createFile(context.paths().of("/etc/something/conf")); - - Files.createDirectories(containerVespaHome.resolve("logs/vespa")); - Files.createFile(containerVespaHome.resolve("logs/vespa/vespa.log")); - Files.createFile(containerVespaHome.resolve("logs/vespa/zookeeper.log")); - - Files.createDirectories(containerVespaHome.resolve("var/db")); - Files.createFile(containerVespaHome.resolve("var/db/some-file")); - - Files.createDirectories(containerVespaHome.resolve("var/tmp")); - Files.createFile(containerVespaHome.resolve("var/tmp/some-file")); - - ContainerPath containerRoot = context.paths().of("/"); - Set<String> actualContents = FileFinder.files(containerRoot) - .stream() - .map(fileAttributes -> containerRoot.relativize(fileAttributes.path()).toString()) - .collect(Collectors.toSet()); - Set<String> expectedContents = Set.of( - "etc/something/conf", - "opt/vespa/logs/vespa/vespa.log", - "opt/vespa/logs/vespa/zookeeper.log", - "opt/vespa/var/tmp/some-file", - "opt/vespa/var/db/some-file"); - assertEquals(expectedContents, actualContents); - return context; - } - - @Test - void not_run_if_not_enough_used() { - NodeAgentContext context = NodeAgentContextImpl.builder( - NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build()) - .fileSystem(fileSystem).build(); - mockDiskUsage(500L); - - storageMaintainer.cleanDiskIfFull(context); - verifyNoInteractions(diskCleanup); - } - - @Test - void deletes_correct_amount() { - NodeAgentContext context = NodeAgentContextImpl.builder( - NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build()) - .fileSystem(fileSystem).build(); - - mockDiskUsage(950_000L); - - storageMaintainer.cleanDiskIfFull(context); - // Allocated size: 1 GB, usage: 950_000 kiB (972.8 MB). Wanted usage: 70% => 700 MB - verify(diskCleanup).cleanup(eq(context), any(), eq(272_800_000L)); - } - - @AfterEach - public void after() { - terminal.verifyAllCommandsExecuted(); - } - - private void mockDiskUsage(long kBytes) { - terminal.expectCommand("du -xsk /data/vespa/storage/h123a 2>&1", 0, kBytes + "\t/path"); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java deleted file mode 100644 index 063e8cb3f77..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ /dev/null @@ -1,351 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.acl; - -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Function; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.endsWith; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -public class AclMaintainerTest { - - private static final String EMPTY_FILTER_TABLE = "-P INPUT ACCEPT\n-P FORWARD ACCEPT\n-P OUTPUT ACCEPT\n"; - private static final String EMPTY_NAT_TABLE = "-P PREROUTING ACCEPT\n-P INPUT ACCEPT\n-P OUTPUT ACCEPT\n-P POSTROUTING ACCEPT\n"; - - private final ContainerOperations containerOperations = mock(ContainerOperations.class); - private final IPAddressesMock ipAddresses = new IPAddressesMock(); - private final AclMaintainer aclMaintainer = new AclMaintainer(containerOperations, ipAddresses); - - private final FileSystem fileSystem = TestFileSystem.create(); - private final Function<Acl, NodeAgentContext> contextGenerator = - acl -> NodeAgentContextImpl.builder("container1.host.com").fileSystem(fileSystem).acl(acl).build(); - private final List<String> writtenFileContents = new ArrayList<>(); - - @Test - void configures_full_container_acl_from_empty() { - Acl acl = new Acl.Builder().withTrustedPorts(22, 4443) - .withTrustedNode("hostname1", "3001::abcd") - .withTrustedNode("hostname2", "3001::1234") - .withTrustedNode("hostname1", "192.168.0.5") - .withTrustedNode("hostname4", "172.16.5.234").build(); - NodeAgentContext context = contextGenerator.apply(acl); - - ipAddresses.addAddress(context.hostname().value(), "2001::1"); - ipAddresses.addAddress(context.hostname().value(), "10.0.0.1"); - - whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); - whenListRules(context, "filter", IPVersion.IPv6, EMPTY_FILTER_TABLE); - whenListRules(context, "nat", IPVersion.IPv4, EMPTY_NAT_TABLE); - whenListRules(context, "nat", IPVersion.IPv6, EMPTY_NAT_TABLE); - - aclMaintainer.converge(context); - - verify(containerOperations, times(4)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); - verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); - verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); - verifyNoMoreInteractions(containerOperations); - - List<String> expected = List.of( - // IPv4 filter table restore - "*filter\n" + - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + - "-A INPUT -s 172.16.5.234/32 -j ACCEPT\n" + - "-A INPUT -s 192.168.0.5/32 -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp-port-unreachable\n" + - "COMMIT\n", - - // IPv6 filter table restore - "*filter\n" + - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + - "-A INPUT -s 3001::1234/128 -j ACCEPT\n" + - "-A INPUT -s 3001::abcd/128 -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n" + - "COMMIT\n", - - // IPv4 nat table restore - "*nat\n" + - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 10.0.0.1/32 -j REDIRECT\n" + - "COMMIT\n", - - // IPv6 nat table restore - "*nat\n" + - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 2001::1/128 -j REDIRECT\n" + - "COMMIT\n"); - assertEquals(expected, writtenFileContents); - } - - @Test - void configures_minimal_container_acl_from_empty() { - // The ACL spec is empty and our this node's addresses do not resolve - Acl acl = new Acl.Builder().withTrustedPorts().build(); - NodeAgentContext context = contextGenerator.apply(acl); - - whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); - whenListRules(context, "filter", IPVersion.IPv6, EMPTY_FILTER_TABLE); - whenListRules(context, "nat", IPVersion.IPv4, EMPTY_NAT_TABLE); - whenListRules(context, "nat", IPVersion.IPv6, EMPTY_NAT_TABLE); - - aclMaintainer.converge(context); - - verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); - verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); - verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); - verifyNoMoreInteractions(containerOperations); - - List<String> expected = List.of( - // IPv4 filter table restore - "*filter\n" + - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p icmp -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp-port-unreachable\n" + - "COMMIT\n", - - // IPv6 filter table restore - "*filter\n" + - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n" + - "COMMIT\n"); - assertEquals(expected, writtenFileContents); - } - - @Test - void only_configure_iptables_for_ipversion_that_differs() { - Acl acl = new Acl.Builder().withTrustedPorts(22, 4443).withTrustedNode("hostname1", "3001::abcd").build(); - NodeAgentContext context = contextGenerator.apply(acl); - - ipAddresses.addAddress(context.hostname().value(), "2001::1"); - - whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); - whenListRules(context, "filter", IPVersion.IPv6, - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + - "-A INPUT -s 3001::abcd/128 -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n"); - whenListRules(context, "nat", IPVersion.IPv6, - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 2001::1/128 -j REDIRECT\n"); - - aclMaintainer.converge(context); - - verify(containerOperations, times(3)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); - verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); - verify(containerOperations, never()).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); //we don't have a ip4 address for the container so no redirect - verifyNoMoreInteractions(containerOperations); - - List<String> expected = List.of( - "*filter\n" + - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp-port-unreachable\n" + - "COMMIT\n"); - assertEquals(expected, writtenFileContents); - } - - @Test - void rollback_is_attempted_when_applying_acl_fail() { - Acl acl = new Acl.Builder().withTrustedPorts(22, 4443).withTrustedNode("hostname1", "3001::abcd").build(); - NodeAgentContext context = contextGenerator.apply(acl); - - ipAddresses.addAddress(context.hostname().value(), "2001::1"); - - whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); - whenListRules(context, "filter", IPVersion.IPv6, - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + - "-A INPUT -s 3001::abcd/128 -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n"); - whenListRules(context, "nat", IPVersion.IPv6, - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 2001::1/128 -j REDIRECT\n"); - - when(containerOperations.executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any())) - .thenThrow(new RuntimeException("iptables restore failed")); - - aclMaintainer.converge(context); - - verify(containerOperations, times(3)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); - verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); - verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables"), eq("-F"), eq("-t"), eq("filter")); - verifyNoMoreInteractions(containerOperations); - - aclMaintainer.converge(context); - } - - @Test - public void config_server_acl() { - Acl acl = new Acl.Builder().withTrustedPorts(22, 4443) - .withTrustedNode("cfg1", "2001:db8::1") - .withTrustedNode("cfg2", "2001:db8::2") - .withTrustedNode("cfg3", "2001:db8::3") - .withTrustedNode("cfg1", "172.17.0.41") - .withTrustedNode("cfg2", "172.17.0.42") - .withTrustedNode("cfg3", "172.17.0.43") - .build(); - NodeAgentContext context = NodeAgentContextImpl.builder("cfg3.example.com") - .fileSystem(fileSystem) - .acl(acl) - .nodeSpecBuilder(builder -> builder.type(NodeType.config)) - .build(); - - ipAddresses.addAddress(context.hostname().value(), "2001:db8::3"); - ipAddresses.addAddress(context.hostname().value(), "172.17.0.43"); - - whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); - whenListRules(context, "filter", IPVersion.IPv6, EMPTY_FILTER_TABLE); - whenListRules(context, "nat", IPVersion.IPv4, EMPTY_NAT_TABLE); - whenListRules(context, "nat", IPVersion.IPv6, EMPTY_NAT_TABLE); - - aclMaintainer.converge(context); - - verify(containerOperations, times(4)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); - verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); - verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); - verifyNoMoreInteractions(containerOperations); - - List<String> expected = List.of( - // IPv4 filter table restore - """ - *filter - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT - -A INPUT -s 172.17.0.41/32 -j ACCEPT - -A INPUT -s 172.17.0.42/32 -j ACCEPT - -A INPUT -s 172.17.0.43/32 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp-port-unreachable - COMMIT - """, - // IPv6 filter table restore - """ - *filter - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p ipv6-icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT - -A INPUT -s 2001:db8::1/128 -j ACCEPT - -A INPUT -s 2001:db8::2/128 -j ACCEPT - -A INPUT -s 2001:db8::3/128 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp6-port-unreachable - COMMIT - """, - // IPv4 nat table restore - """ - *nat - -P PREROUTING ACCEPT - -P INPUT ACCEPT - -P OUTPUT ACCEPT - -P POSTROUTING ACCEPT - -A OUTPUT -d 172.17.0.43/32 -j REDIRECT - COMMIT - """, - // IPv6 nat table restore - """ - *nat - -P PREROUTING ACCEPT - -P INPUT ACCEPT - -P OUTPUT ACCEPT - -P POSTROUTING ACCEPT - -A OUTPUT -d 2001:db8::3/128 -j REDIRECT - COMMIT - """); - assertEquals(expected, writtenFileContents); - } - - @BeforeEach - public void setup() { - doAnswer(invoc -> { - String path = invoc.getArgument(2); - writtenFileContents.add(new UnixPath(path).readUtf8File()); - return new CommandResult(null, 0, ""); - }).when(containerOperations).executeCommandInNetworkNamespace(any(), endsWith("-restore"), any()); - } - - private void whenListRules(NodeAgentContext context, String table, IPVersion ipVersion, String output) { - when(containerOperations.executeCommandInNetworkNamespace( - eq(context), any(CommandLine.Options.class), eq(ipVersion.iptablesCmd()), eq("-S"), eq("-t"), eq(table))) - .thenReturn(new CommandResult(null, 0, output)); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java deleted file mode 100644 index 52eac44fbc3..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.acl; - -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class FilterTableLineEditorTest { - - @Test - void filter_set_wanted_rules() { - Acl acl = new Acl.Builder().withTrustedPorts(22).withTrustedNode("hostname", "3001::1").build(); - - assertFilterTableLineEditorResult( - acl, IPVersion.IPv6, - - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n", - - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22 -j ACCEPT\n" + - "-A INPUT -s 3001::1/128 -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp6-port-unreachable"); - } - - @Test - void produces_minimal_diff_simple() { - assertFilterTableDiff(List.of(2, 5, 3, 6, 1, 4), List.of(2, 5, 6, 1, 4), - "Patching file table:\n" + - "--A INPUT -s 2001::3/128 -j ACCEPT\n"); - } - - @Test - void produces_minimal_diff_complex() { - assertFilterTableDiff(List.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), List.of(5, 11, 6, 3, 10, 4, 8, 12), - "Patching file table:\n" + - "--A INPUT -s 2001::1/128 -j ACCEPT\n" + - "--A INPUT -s 2001::2/128 -j ACCEPT\n" + - "+-A INPUT -s 2001::11/128 -j ACCEPT\n" + - "+-A INPUT -s 2001::12/128 -j ACCEPT\n" + - "--A INPUT -s 2001::7/128 -j ACCEPT\n" + - "--A INPUT -s 2001::9/128 -j ACCEPT\n"); - } - - private static void assertFilterTableLineEditorResult( - Acl acl, IPVersion ipVersion, String currentFilterTable, String expectedRestoreFileContent) { - FilterTableLineEditor filterLineEditor = FilterTableLineEditor.from(acl, ipVersion); - Editor editor = new Editor( - "nat-table", - () -> List.of(currentFilterTable.split("\n")), - result -> assertEquals(expectedRestoreFileContent, String.join("\n", result)), - filterLineEditor); - editor.edit(m -> {}); - } - - private static void assertFilterTableDiff(List<Integer> currentIpSuffix, List<Integer> wantedIpSuffix, String diff) { - Acl.Builder currentAcl = new Acl.Builder(); - NodeType nodeType = NodeType.tenant; - currentIpSuffix.forEach(i -> currentAcl.withTrustedNode("host" + i, "2001::" + i)); - List<String> currentTable = new ArrayList<>(); - - Acl.Builder wantedAcl = new Acl.Builder(); - wantedIpSuffix.forEach(i -> wantedAcl.withTrustedNode("host" + i, "2001::" + i)); - - new Editor("table", List::of, currentTable::addAll, FilterTableLineEditor.from(currentAcl.build(), IPVersion.IPv6)) - .edit(log -> {}); - - new Editor("table", () -> currentTable, result -> {}, FilterTableLineEditor.from(wantedAcl.build(), IPVersion.IPv6)) - .edit(log -> assertEquals(diff, log)); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java deleted file mode 100644 index d8d526050d7..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.acl; - -import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class NatTableLineEditorTest { - - @Test - void nat_set_redirect_rule_without_touching_docker_rules() { - assertNatTableLineEditorResult( - "-A OUTPUT -d 3001::1/128 -j REDIRECT", - - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-N DOCKER_OUTPUT\n" + - "-N DOCKER_POSTROUTING\n" + - "-A OUTPUT -d 127.0.0.11/32 -j DOCKER_OUTPUT\n" + - "-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING\n" + - "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p tcp -m tcp --dport 53 -j DNAT --to-destination 127.0.0.11:43500\n" + - "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp -m udp --dport 53 -j DNAT --to-destination 127.0.0.11:57392\n" + - "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p tcp -m tcp --sport 43500 -j SNAT --to-source :53\n" + - "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p udp -m udp --sport 57392 -j SNAT --to-source :53\n", - - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-N DOCKER_OUTPUT\n" + - "-N DOCKER_POSTROUTING\n" + - "-A OUTPUT -d 127.0.0.11/32 -j DOCKER_OUTPUT\n" + - "-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING\n" + - "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p tcp -m tcp --dport 53 -j DNAT --to-destination 127.0.0.11:43500\n" + - "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp -m udp --dport 53 -j DNAT --to-destination 127.0.0.11:57392\n" + - "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p tcp -m tcp --sport 43500 -j SNAT --to-source :53\n" + - "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p udp -m udp --sport 57392 -j SNAT --to-source :53\n" + - "-A OUTPUT -d 3001::1/128 -j REDIRECT"); - } - - @Test - void nat_cleanup_wrong_redirect_rules() { - assertNatTableLineEditorResult( - "-A OUTPUT -d 3001::1/128 -j REDIRECT", - - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 3001::2/128 -j REDIRECT\n", - - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 3001::1/128 -j REDIRECT"); - } - - @Test - void nat_delete_duplicate_rules() { - assertNatTableLineEditorResult( - "-A OUTPUT -d 3001::1/128 -j REDIRECT", - - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 3001::2/128 -j REDIRECT\n" + - "-A OUTPUT -d 3001::1/128 -j REDIRECT\n" + - "-A OUTPUT -d 3001::4/128 -j REDIRECT\n", - - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 3001::1/128 -j REDIRECT"); - } - - private static void assertNatTableLineEditorResult(String redirectRule, String currentNatTable, String expectedNatTable) { - NatTableLineEditor natLineEditor = NatTableLineEditor.from(redirectRule); - Editor editor = new Editor( - "nat-table", - () -> List.of(currentNatTable.split("\n")), - result -> assertEquals(expectedNatTable, String.join("\n", result)), - natLineEditor); - editor.edit(m -> {}); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java deleted file mode 100644 index 9487affd376..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ /dev/null @@ -1,234 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; - -import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL8; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class CoreCollectorTest { - private static final Instant CORE_CREATED = Instant.ofEpochMilli(2233445566L); - - private final ContainerOperations docker = mock(ContainerOperations.class); - private final CoreCollector coreCollector = new CoreCollector(docker); - private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld") - .fileSystem(TestFileSystem.create()).build(); - - private final ContainerPath TEST_CORE_PATH = (ContainerPath) new UnixPath(context.paths().of("/tmp/core.1234")) - .createParents() - .createNewFile() - .setLastModifiedTime(CORE_CREATED) - .toPath(); - private final String TEST_BIN_PATH = "/usr/bin/program"; - private final List<String> GDB_BACKTRACE = List.of("[New Thread 2703]", - "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.", - "#0 0x00000000004004d8 in main (argv=...) at main.c:4", "4\t printf(argv[3]);", - "#0 0x00000000004004d8 in main (argv=...) at main.c:4"); - - @Test - void extractsBinaryPathTest() { - final String[] cmd = {"file", TEST_CORE_PATH.pathInContainer()}; - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program'"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH)); - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program --foo --bar baz'"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH)); - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, " + - "from 'program', real uid: 0, effective uid: 0, real gid: 0, effective gid: 0, " + - "execfn: '/usr/bin/program', platform: 'x86_64"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH)); - - String fallbackResponse = "/response/from/fallback"; - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-batch", "-core", "/tmp/core.1234"}, - """ - GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.2) 7.7.1 - Type “apropos word” to search for commands related to “word”… - Reading symbols from abc…(no debugging symbols found)…done. - [New LWP 23678] - Core was generated by `/response/from/fallback'. \s - Program terminated with signal SIGSEGV, Segmentation fault. \s - #0 0x0000000000400541 in main () - #0 0x0000000000400541 in main () - (gdb) bt - #0 0x0000000000400541 in main () - (gdb) - """); - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style"); - assertEquals(fallbackResponse, coreCollector.readBinPath(context, TEST_CORE_PATH)); - - mockExec(cmd, "", "Error code 1234"); - assertEquals(fallbackResponse, coreCollector.readBinPath(context, TEST_CORE_PATH)); - } - - @Test - void extractsBinaryPathUsingGdbTest() { - String[] cmd = new String[]{GDB_PATH_RHEL8, "-n", "-batch", "-core", "/tmp/core.1234"}; - - mockExec(cmd, "Core was generated by `/usr/bin/program-from-gdb --identity foo/search/cluster.content_'."); - assertEquals("/usr/bin/program-from-gdb", coreCollector.readBinPathFallback(context, TEST_CORE_PATH)); - - mockExec(cmd, "", "Error 123"); - try { - coreCollector.readBinPathFallback(context, TEST_CORE_PATH); - fail("Expected not to be able to get bin path"); - } catch (RuntimeException e) { - assertEquals("Failed to extract binary path from GDB, result: exit status 1, output 'Error 123', command: " + - "[/opt/rh/gcc-toolset-12/root/bin/gdb, -n, -batch, -core, /tmp/core.1234]", e.getMessage()); - } - } - - @Test - void extractsBacktraceUsingGdb() { - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false)); - - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, - "", "Failure"); - try { - coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false); - fail("Expected not to be able to read backtrace"); - } catch (RuntimeException e) { - assertEquals("Failed to read backtrace exit status 1, output 'Failure', Command: " + - "[" + GDB_PATH_RHEL8 + ", -n, -ex, set print frame-arguments none, -ex, bt, -batch, " + - "/usr/bin/program, /tmp/core.1234]", e.getMessage()); - } - } - - @Test - void extractsBacktraceFromAllThreadsUsingGdb() { - mockExec(new String[]{GDB_PATH_RHEL8, "-n", - "-ex", "set print frame-arguments none", - "-ex", "thread apply all bt", "-batch", - "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, true)); - } - - @Test - void collectsDataTest() { - mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program'"); - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", "thread apply all bt", "-batch", - "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - - var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH) - .setCreated(CORE_CREATED) - .setType(CoreDumpMetadata.Type.CORE_DUMP) - .setBacktrace(GDB_BACKTRACE) - .setBacktraceAllThreads(GDB_BACKTRACE); - assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); - } - - @Test - void collectsDataRelativePath() { - mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from 'sbin/distributord-bin'"); - String absolutePath = "/opt/vespa/sbin/distributord-bin"; - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", "bt", "-batch", absolutePath, "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", "thread apply all bt", "-batch", absolutePath, "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - - var expected = new CoreDumpMetadata() - .setBinPath(absolutePath) - .setCreated(CORE_CREATED) - .setType(CoreDumpMetadata.Type.CORE_DUMP) - .setBacktrace(GDB_BACKTRACE) - .setBacktraceAllThreads(GDB_BACKTRACE); - assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); - } - - @Test - void collectsPartialIfBacktraceFailsTest() { - mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program'"); - mockExec(new String[]{GDB_PATH_RHEL8 + " -n -ex set print frame-arguments none -ex bt -batch /usr/bin/program /tmp/core.1234"}, - "", "Failure"); - - var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH).setCreated(CORE_CREATED).setType(CoreDumpMetadata.Type.CORE_DUMP); - assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); - } - - @Test - void reportsJstackInsteadOfGdbForJdkCores() { - mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, - "dump.core.5954: ELF 64-bit LSB core file x86-64, version 1 (SYSV), too many program header sections (33172)"); - - String jdkPath = "/path/to/jdk/java"; - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-batch", "-core", "/tmp/core.1234"}, - "Core was generated by `" + jdkPath + " -Dconfig.id=default/container.11 -XX:+Pre'."); - - String jstack = "jstack11"; - mockExec(new String[]{"jhsdb", "jstack", "--exe", jdkPath, "--core", "/tmp/core.1234"}, - jstack); - - var expected = new CoreDumpMetadata().setBinPath(jdkPath) - .setCreated(CORE_CREATED) - .setType(CoreDumpMetadata.Type.CORE_DUMP) - .setBacktraceAllThreads(List.of(jstack)); - assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); - } - - @Test - void metadata_for_java_heap_dump() { - var expected = new CoreDumpMetadata().setBinPath("java") - .setType(CoreDumpMetadata.Type.JVM_HEAP) - .setCreated(CORE_CREATED) - .setBacktrace(List.of("Heap dump, no backtrace available")); - - assertEquals(expected, coreCollector.collect(context, (ContainerPath) new UnixPath(context.paths().of("/dump_java_pid123.hprof")) - .createNewFile() - .setLastModifiedTime(CORE_CREATED) - .toPath())); - } - - private void mockExec(String[] cmd, String output) { - mockExec(cmd, output, ""); - } - - private void mockExec(String[] cmd, String output, String error) { - mockExec(context, cmd, output, error); - } - - private void mockExec(NodeAgentContext context, String[] cmd, String output, String error) { - when(docker.executeCommandInContainer(context, context.users().root(), cmd)) - .thenReturn(new CommandResult(null, error.isEmpty() ? 0 : 1, error.isEmpty() ? output : error)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java deleted file mode 100644 index e65a226b789..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.security.KeyId; -import com.yahoo.security.SealedSharedKey; -import com.yahoo.security.SecretSharedKey; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores; -import com.yahoo.vespa.hosted.node.admin.container.metrics.DimensionMetrics; -import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import javax.crypto.spec.SecretKeySpec; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.FileTime; -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.yolean.Exceptions.uncheck; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class CoredumpHandlerTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld") - .fileSystem(fileSystem).build(); - private final ContainerPath containerCrashPath = context.paths().of("/var/crash"); - private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps"); - - private final CoreCollector coreCollector = mock(CoreCollector.class); - private final Cores cores = mock(Cores.class); - private final Metrics metrics = new Metrics(); - private final TestTimer timer = new TestTimer(); - @SuppressWarnings("unchecked") - private final Supplier<String> coredumpIdSupplier = mock(Supplier.class); - private final SecretSharedKeySupplier secretSharedKeySupplier = mock(SecretSharedKeySupplier.class); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - private final CoredumpHandler coredumpHandler = - new CoredumpHandler(coreCollector, cores, containerCrashPath.pathInContainer(), - doneCoredumpsPath, metrics, timer, coredumpIdSupplier, secretSharedKeySupplier, - flagSource); - - @Test - void coredump_enqueue_test() throws IOException { - ContainerPath crashPath = context.paths().of("/some/crash/path"); - ContainerPath processingDir = context.paths().of("/some/other/processing"); - - Files.createDirectories(crashPath); - createFileAged(crashPath.resolve("bash.core.431"), Duration.ZERO); - - assertFolderContents(crashPath, "bash.core.431"); - Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); - assertEquals(Optional.empty(), enqueuedPath); - - // bash.core.431 finished writing... and 2 more have since been written - timer.advance(Duration.ofMinutes(3)); - createFileAged(crashPath.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); - createFileAged(crashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); - - when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321"); - enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); - assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); - assertFolderContents(crashPath, "bash.core.431", "vespa-slobrok.core.673"); - assertFolderContents(processingDir, "id-123"); - assertFolderContents(processingDir.resolve("id-123"), "dump_vespa-proton.core.119"); - verify(coredumpIdSupplier, times(1)).get(); - - // Enqueue another - enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); - assertEquals(Optional.of(processingDir.resolve("id-321")), enqueuedPath); - assertFolderContents(crashPath, "bash.core.431"); - assertFolderContents(processingDir, "id-123", "id-321"); - assertFolderContents(processingDir.resolve("id-321"), "dump_vespa-slobrok.core.673"); - verify(coredumpIdSupplier, times(2)).get(); - } - - @Test - void enqueue_with_hs_err_files() throws IOException { - ContainerPath crashPath = context.paths().of("/some/crash/path"); - ContainerPath processingDir = context.paths().of("/some/other/processing"); - Files.createDirectories(crashPath); - - createFileAged(crashPath.resolve("java.core.69"), Duration.ofSeconds(515)); - createFileAged(crashPath.resolve("hs_err_pid69.log"), Duration.ofSeconds(520)); - - createFileAged(crashPath.resolve("java.core.2420"), Duration.ofSeconds(540)); - createFileAged(crashPath.resolve("hs_err_pid2420.log"), Duration.ofSeconds(549)); - createFileAged(crashPath.resolve("hs_err_pid2421.log"), Duration.ofSeconds(550)); - - when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321"); - Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); - assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); - assertFolderContents(crashPath, "hs_err_pid69.log", "java.core.69"); - assertFolderContents(processingDir, "id-123"); - assertFolderContents(processingDir.resolve("id-123"), "hs_err_pid2420.log", "hs_err_pid2421.log", "dump_java.core.2420"); - } - - @Test - void coredump_to_process_test() throws IOException { - ContainerPath processingDir = context.paths().of("/some/other/processing"); - - // Initially there are no core dumps - Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, containerCrashPath, processingDir); - assertEquals(Optional.empty(), enqueuedPath); - - // 3 core dumps occur - Files.createDirectories(containerCrashPath); - createFileAged(containerCrashPath.resolve("bash.core.431"), Duration.ZERO); - createFileAged(containerCrashPath.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); - createFileAged(containerCrashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); - - when(coredumpIdSupplier.get()).thenReturn("id-123"); - enqueuedPath = coredumpHandler.getCoredumpToProcess(context, containerCrashPath, processingDir); - assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); - - // Running this again wont enqueue new core dumps as we are still processing the one enqueued previously - enqueuedPath = coredumpHandler.getCoredumpToProcess(context, containerCrashPath, processingDir); - assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); - verify(coredumpIdSupplier, times(1)).get(); - } - - @Test - void gather_metadata_test() throws IOException { - var metadata = new CoreDumpMetadata().setKernelVersion("3.10.0-862.9.1.el7.x86_64") - .setBacktrace(List.of("call 1", "function 2", "something something")) - .setVespaVersion("6.48.4") - .setBinPath("/bin/bash") - .setCoreDumpPath(context.paths().of("/home/docker/dumps/container-123/id-123/dump_core.456")) - .setDockerImage(DockerImage.fromString("example.com/vespa/ci:6.48.4")); - - new UnixPath(fileSystem.getPath("/proc/cpuinfo")).createParents().writeUtf8File("microcode\t: 0xf0"); - - ContainerPath coredumpDirectory = context.paths().of("/var/crash/id-123"); - Files.createDirectories(coredumpDirectory.pathOnHost()); - Files.createFile(coredumpDirectory.resolve("dump_core.456")); - when(coreCollector.collect(eq(context), eq(coredumpDirectory.resolve("dump_core.456")))) - .thenReturn(metadata); - - assertEquals(metadata, coredumpHandler.gatherMetadata(context, coredumpDirectory)); - verify(coreCollector, times(1)).collect(any(), any()); - - // On second invocation the test already exist, so times(1) is not incremented - assertEquals(metadata, coredumpHandler.gatherMetadata(context, coredumpDirectory)); - doThrow(new IllegalStateException("Should not be invoked")) - .when(coreCollector).collect(any(), any()); - verify(coreCollector, times(1)).collect(any(), any()); - } - - @Test - void cant_get_metadata_if_no_core_file() { - assertThrows(IllegalStateException.class, () -> { - coredumpHandler.gatherMetadata(context, context.paths().of("/fake/path")); - }); - } - - @Test - void fails_to_get_core_file_if_only_compressed_or_encrypted() { - assertThrows(IllegalStateException.class, () -> { - ContainerPath coredumpDirectory = context.paths().of("/path/to/coredump/proccessing/id-123"); - Files.createDirectories(coredumpDirectory); - Files.createFile(coredumpDirectory.resolve("dump_bash.core.431.zst")); - Files.createFile(coredumpDirectory.resolve("dump_bash.core.543.zst.enc")); - coredumpHandler.findCoredumpFileInProcessingDirectory(coredumpDirectory); - }); - } - - void do_process_single_coredump_test(String expectedCoreFileName) throws IOException { - ContainerPath coredumpDirectory = context.paths().of("/path/to/coredump/proccessing/id-123"); - Files.createDirectories(coredumpDirectory); - Files.write(coredumpDirectory.resolve("metadata2.json"), "{\"test-metadata\":{}}".getBytes()); - Files.createFile(coredumpDirectory.resolve("dump_bash.core.431")); - assertFolderContents(coredumpDirectory, "metadata2.json", "dump_bash.core.431"); - CoreDumpMetadata expectedMetadata = new CoreDumpMetadata(); - expectedMetadata.setDecryptionToken("131Q0MMF3hBuMVnXg1WnSFexZGrcwa9ZhfHlegLNwPIN6hQJnBxq5srLf3aZbYdlRVE"); - - coredumpHandler.processAndReportSingleCoreDump(context, coredumpDirectory, Optional.empty()); - verify(coreCollector, never()).collect(any(), any()); - verify(cores, times(1)).report(eq(context.hostname()), eq("id-123"), eq(expectedMetadata)); - assertFalse(Files.exists(coredumpDirectory)); - assertFolderContents(doneCoredumpsPath.resolve("container-123"), "id-123"); - assertFolderContents(doneCoredumpsPath.resolve("container-123").resolve("id-123"), "metadata2.json", expectedCoreFileName); - } - - @Test - void processing_single_coredump_test_without_encryption_throws() throws IOException { - assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst")); - } - - @Test - void process_single_coredump_test_with_encryption() throws IOException { - flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), "bar-key"); - when(secretSharedKeySupplier.create(KeyId.ofString("bar-key"))).thenReturn(Optional.of(makeFixedSecretSharedKey())); - do_process_single_coredump_test("dump_bash.core.431.zst.enc"); - } - - @Test - void processing_throws_when_no_public_key_set_in_feature_flag() throws IOException { - flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), ""); // empty -> not set - verify(secretSharedKeySupplier, never()).create(any()); - assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst")); - } - - @Test - void processing_throws_when_no_key_returned_for_key_id_specified_by_feature_flag() throws IOException { - flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), "baz-key"); - when(secretSharedKeySupplier.create(KeyId.ofString("baz-key"))).thenReturn(Optional.empty()); - assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst")); - } - - @Test - void report_enqueued_and_processed_metrics() throws IOException { - Path processingPath = containerCrashPath.resolve("processing"); - Files.createFile(containerCrashPath.resolve("dump-1")); - Files.createFile(containerCrashPath.resolve("dump-2")); - Files.createFile(containerCrashPath.resolve("hs_err_pid2.log")); - Files.createDirectory(processingPath); - Files.createFile(processingPath.resolve("metadata2.json")); - Files.createFile(processingPath.resolve("dump-3")); - - new UnixPath(doneCoredumpsPath.resolve("container-123").resolve("dump-3-folder").resolve("dump-3")) - .createParents() - .createNewFile(); - - coredumpHandler.updateMetrics(context, containerCrashPath); - List<DimensionMetrics> updatedMetrics = metrics.getMetricsByType(Metrics.DimensionType.PRETAGGED); - assertEquals(1, updatedMetrics.size()); - Map<String, Number> values = updatedMetrics.get(0).getMetrics(); - assertEquals(3, values.get("coredumps.enqueued").intValue()); - assertEquals(1, values.get("coredumps.processed").intValue()); - } - - @BeforeEach - public void setup() throws IOException { - Files.createDirectories(containerCrashPath.pathOnHost()); - } - - private static void assertFolderContents(Path pathToFolder, String... filenames) { - Set<String> expectedContentsOfFolder = Set.of(filenames); - Set<String> actualContentsOfFolder; - try (Stream<UnixPath> paths = new UnixPath(pathToFolder).listContentsOfDirectory()) { - actualContentsOfFolder = paths.map(unixPath -> unixPath.toPath().getFileName().toString()) - .collect(Collectors.toSet()); - } - assertEquals(expectedContentsOfFolder, actualContentsOfFolder); - } - - private Path createFileAged(Path path, Duration age) { - return uncheck(() -> Files.setLastModifiedTime( - Files.createFile(path), - FileTime.from(timer.currentTime().minus(age)))); - } - - private static byte[] bytesOf(String str) { - return str.getBytes(StandardCharsets.UTF_8); - } - - private static SecretSharedKey makeFixedSecretSharedKey() { - byte[] keyBytes = bytesOf("very secret yes!"); // 128 bits - var secretKey = new SecretKeySpec(keyBytes, "AES"); - var keyId = KeyId.ofString("the shiniest key"); - // We don't parse any of these fields in the test, so just use dummy contents. - byte[] enc = bytesOf("hello world"); - byte[] ciphertext = bytesOf("imaginary ciphertext"); - return new SecretSharedKey(secretKey, new SealedSharedKey(SealedSharedKey.CURRENT_TOKEN_VERSION, keyId, enc, ciphertext)); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java deleted file mode 100644 index 0e20d3965a0..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.disk; - -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.FileTime; -import java.time.Instant; -import java.util.Map; -import java.util.TreeMap; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class CoredumpCleanupRuleTest { - - private final FileSystem fileSystem = TestFileSystem.create(); - - @Test - void for_container_test() throws IOException { - Path path = fileSystem.getPath("/test/path"); - DiskCleanupRule rule = CoredumpCleanupRule.forContainer(path); - - assertPriorities(rule, Map.of()); - - createFile(path.resolve("core1"), Instant.ofEpochSecond(232)); - assertPriorities(rule, Map.of("/test/path/core1", Priority.MEDIUM)); - - createFile(path.resolve("core2"), Instant.ofEpochSecond(123)); - assertPriorities(rule, Map.of( - "/test/path/core2", Priority.MEDIUM, - "/test/path/core1", Priority.HIGHEST)); - - createFile(path.resolve("vespa-proton-bin.core.325"), Instant.ofEpochSecond(456)); - createFile(path.resolve("vespa-distributor.core.764"), Instant.ofEpochSecond(256)); - var expected = Map.of( - "/test/path/core2", Priority.HIGHEST, - "/test/path/core1", Priority.HIGHEST, - "/test/path/vespa-proton-bin.core.325", Priority.HIGHEST, - "/test/path/vespa-distributor.core.764", Priority.MEDIUM); - assertPriorities(rule, expected); - - // processing core has no effect on this - Files.createDirectories(path.resolve("processing/abcd-1234")); - createFile(path.resolve("processing/abcd-1234/core5"), Instant.ofEpochSecond(67)); - assertPriorities(rule, expected); - } - - @Test - void for_host_test() throws IOException { - Path path = fileSystem.getPath("/test/path"); - DiskCleanupRule rule = CoredumpCleanupRule.forHost(path); - - assertPriorities(rule, Map.of()); - - createFile(path.resolve("h123a/abcd-1234/dump_core1"), Instant.parse("2020-04-21T19:21:00Z")); - createFile(path.resolve("h123a/abcd-1234/metadata.json"), Instant.parse("2020-04-21T19:26:00Z")); - assertPriorities(rule, Map.of("/test/path/h123a/abcd-1234/dump_core1", Priority.MEDIUM)); - - createFile(path.resolve("h123a/abcd-efgh/dump_core1"), Instant.parse("2020-04-21T07:13:00Z")); - createFile(path.resolve("h123a/56ad-af42/dump_vespa-distributor.321"), Instant.parse("2020-04-21T23:37:00Z")); - createFile(path.resolve("h123a/4324-a23d/dump_core2"), Instant.parse("2020-04-22T04:56:00Z")); - createFile(path.resolve("h123a/8534-7da3/dump_vespa-proton-bin.123"), Instant.parse("2020-04-19T15:35:00Z")); - - // Also create a core for a second container: h123b - createFile(path.resolve("h123b/db1a-ab34/dump_core1"), Instant.parse("2020-04-21T07:01:00Z")); - createFile(path.resolve("h123b/7392-59ad/dump_vespa-proton-bin.342"), Instant.parse("2020-04-22T12:05:00Z")); - - assertPriorities(rule, Map.of( - "/test/path/h123a/abcd-1234/dump_core1", Priority.HIGH, - "/test/path/h123a/abcd-efgh/dump_core1", Priority.HIGH, - - // Although it is the oldest core of the day for h123a, it is the first one that starts with vespa- - "/test/path/h123a/56ad-af42/dump_vespa-distributor.321", Priority.MEDIUM, - "/test/path/h123a/4324-a23d/dump_core2", Priority.MEDIUM, - "/test/path/h123a/8534-7da3/dump_vespa-proton-bin.123", Priority.MEDIUM, - "/test/path/h123b/db1a-ab34/dump_core1", Priority.MEDIUM, - "/test/path/h123b/7392-59ad/dump_vespa-proton-bin.342", Priority.MEDIUM - )); - } - - private static void createFile(Path path, Instant instant) throws IOException { - Files.createDirectories(path.getParent()); - Files.createFile(path); - Files.setLastModifiedTime(path, FileTime.from(instant)); - } - - private static void assertPriorities(DiskCleanupRule rule, Map<String, Priority> expected) { - Map<String, Priority> actual = rule.prioritize().stream() - .collect(Collectors.toMap(pfa -> pfa.fileAttributes().path().toString(), PrioritizedFileAttributes::priority)); - - assertEquals(new TreeMap<>(expected), new TreeMap<>(actual)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java deleted file mode 100644 index 390501a4530..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.disk; - -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.PosixFileAttributeView; -import java.nio.file.attribute.PosixFileAttributes; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author freva - */ -public class DiskCleanupTest { - - private final TestTaskContext context = new TestTaskContext(); - private final DiskCleanupTester tester = new DiskCleanupTester(); - private final DiskCleanup diskCleanup = new DiskCleanup(); - - @Test - void nothing_deleted() throws IOException { - assertFalse(diskCleanup.cleanup(context, List.of(), 0)); - assertFalse(diskCleanup.cleanup(context, List.of(), 10)); - - DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock(); - DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock(); - assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 0)); - assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); - - tester.createFile("/path/that-should-not-be-deleted", 5); - assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); - tester.assertAllFilesExistExcept(); - - // Create a file and let rule return it, but before cleanup is run, the file is deleted - rule1.addFile(tester.createFile("/path/file-does-not-exist", 1), Priority.HIGHEST); - Files.delete(tester.path("/path/file-does-not-exist")); - assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); - } - - @Test - void delete_test() throws IOException { - tester.createFile("/opt/vespa/var/db/do-not-delete-1.db", 1); - tester.createFile("/opt/vespa/var/db/do-not-delete-2.db", 1); - tester.createFile("/opt/vespa/var/zookeeper/do-not-delete-3", 1); - tester.createFile("/opt/vespa/var/index/something-important", 1); - - DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock() - .addFile(tester.createFile("/opt/vespa/logs/vespa-1.log", 10), Priority.MEDIUM) - .addFile(tester.createFile("/opt/vespa/logs/vespa-2.log", 8), Priority.HIGH) - .addFile(tester.createFile("/opt/vespa/logs/vespa-3.log", 13), Priority.HIGHEST) - .addFile(tester.createFile("/opt/vespa/logs/vespa-4.log", 10), Priority.HIGHEST); - DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock() - .addFile(tester.createFile("/opt/vespa/var/crash/core1", 105), Priority.LOW) - .addFile(tester.createFile("/opt/vespa/var/crash/vespa-proton-bin.core-232", 190), Priority.HIGH) - .addFile(tester.createFile("/opt/vespa/var/crash/core3", 54), Priority.MEDIUM) - .addFile(tester.createFile("/opt/vespa/var/crash/core4", 300), Priority.HIGH); - - // 2 files with HIGHEST priority, tie broken by the largest size which is won by "vespa-3.log", since - // it is >= 10 bytes, no more files are deleted - assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); - tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log"); - - // Called with the same arguments, but vespa-3.log is still missing... - assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); - tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log"); - - assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 500)); - tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log", // from before - // 300 + 190 + 8 + 54 - "/opt/vespa/var/crash/core4", "/opt/vespa/var/crash/vespa-proton-bin.core-232", "/opt/vespa/logs/vespa-2.log", "/opt/vespa/var/crash/core3"); - } - - private static class DiskCleanupRuleMock implements DiskCleanupRule { - private final ArrayList<PrioritizedFileAttributes> pfa = new ArrayList<>(); - - private DiskCleanupRuleMock addFile(Path path, Priority priority) throws IOException { - PosixFileAttributes attributes = Files.getFileAttributeView(path, PosixFileAttributeView.class).readAttributes(); - pfa.add(new PrioritizedFileAttributes(new FileAttributes(path, attributes), priority)); - return this; - } - - @Override - public Collection<PrioritizedFileAttributes> prioritize() { - return Collections.unmodifiableList(pfa); - } - } - - private static class DiskCleanupTester { - private final FileSystem fileSystem = TestFileSystem.create(); - private final Set<String> files = new HashSet<>(); - - private Path path(String path) { - return fileSystem.getPath(path); - } - - private Path createFile(String pathStr, int size) throws IOException { - Path path = path(pathStr); - Files.createDirectories(path.getParent()); - Files.write(path, new byte[size]); - files.add(path.toString()); - return path; - } - - private void assertAllFilesExistExcept(String... deletedPaths) { - Set<String> actual = FileFinder.files(path("/")).stream().map(fa -> fa.path().toString()).collect(Collectors.toSet()); - Set<String> expected = new HashSet<>(files); - expected.removeAll(Set.of(deletedPaths)); - assertEquals(expected, actual); - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java deleted file mode 100644 index c85ddf41906..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.disk; - -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; -import static org.mockito.Mockito.mock; - -/** - * @author freva - */ -public class LinearCleanupRuleTest { - - @Test - void basic() { - assertRule(Map.of(), Priority.LOWEST, Priority.HIGHEST); - - assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.LOW, 1.0, Priority.LOW), Priority.LOW, Priority.LOW); - assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.MEDIUM, 1.0, Priority.MEDIUM), Priority.LOW, Priority.MEDIUM); - - assertRule(Map.of( - -5.0, Priority.LOW, - 0.0, Priority.LOW, - 0.2, Priority.LOW, - 0.35, Priority.MEDIUM, - 0.65, Priority.MEDIUM, - 0.8, Priority.HIGH, - 1.0, Priority.HIGH, - 5.0, Priority.HIGH), - Priority.LOW, Priority.HIGH); - } - - @Test - void fail_if_high_priority_lower_than_low() { - assertThrows(IllegalArgumentException.class, () -> { - assertRule(Map.of(), Priority.HIGHEST, Priority.LOWEST); - }); - } - - private static void assertRule(Map<Double, Priority> expectedPriorities, Priority low, Priority high) { - Map<FileAttributes, Double> fileAttributesByScore = expectedPriorities.keySet().stream() - .collect(Collectors.toMap(score -> mock(FileAttributes.class), score -> score)); - LinearCleanupRule rule = new LinearCleanupRule( - () -> List.copyOf(fileAttributesByScore.keySet()), fileAttributesByScore::get, low, high); - - Map<Double, Priority> actualPriorities = rule.prioritize().stream() - .collect(Collectors.toMap(pfa -> fileAttributesByScore.get(pfa.fileAttributes()), PrioritizedFileAttributes::priority)); - assertEquals(expectedPriorities, actualPriorities); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java deleted file mode 100644 index 607efa9771a..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.yolean.concurrent.Sleeper; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author bjorncs - */ -class ArtifactProducersTest { - - @Test - void generates_exception_on_unknown_artifact() { - ArtifactProducers instance = ArtifactProducers.createDefault(Sleeper.NOOP); - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, () -> instance.resolve(List.of("unknown-artifact"))); - String expectedMsg = - "Invalid artifact type 'unknown-artifact'. Valid types are " + - "['config-dump', 'jvm-heap-dump', 'jvm-jfr', 'jvm-jmap', 'jvm-jstack', 'jvm-jstat', 'perf-report', 'pmap', " + - "'vespa-log', 'zookeeper-snapshot'] " + - "and valid aliases are " + - "['jvm-dump': ['jvm-heap-dump', 'jvm-jmap', 'jvm-jstack', 'jvm-jstat', 'vespa-log']]"; - assertEquals(expectedMsg, exception.getMessage()); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java deleted file mode 100644 index db19d6b0074..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java +++ /dev/null @@ -1,319 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.integration.NodeRepoMock; -import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient; -import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; -import com.yahoo.vespa.test.file.TestFileSystem; -import com.yahoo.yolean.concurrent.Sleeper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -import java.io.IOException; -import java.net.URI; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.ServiceDumpReport.DumpOptions; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author bjorncs - */ -class VespaServiceDumperImplTest { - - private static final String HOSTNAME = "host-1.domain.tld"; - - private final FileSystem fileSystem = TestFileSystem.create(); - private final Path tmpDirectory = fileSystem.getPath("/data/vespa/storage/host-1/opt/vespa/var/tmp"); - - @BeforeEach - void create_tmp_directory() throws IOException { - // Create temporary directory in container - Files.createDirectories(tmpDirectory); - } - - @Test - void creates_valid_dump_id_from_dump_request() { - long nowMillis = Instant.now().toEpochMilli(); - ServiceDumpReport request = new ServiceDumpReport( - nowMillis, null, null, null, null, "default/container.3", null, null, List.of("perf-report"), null); - String dumpId = VespaServiceDumperImpl.createDumpId(request); - assertEquals("default-container-3-" + nowMillis, dumpId); - } - - @Test - void invokes_perf_commands_when_generating_perf_report() { - // Setup mocks - ContainerOperations operations = mock(ContainerOperations.class); - when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) - .thenReturn(new CommandResult(null, 0, "12345")) - .thenReturn(new CommandResult(null, 0, "")) - .thenReturn(new CommandResult(null, 0, "")); - SyncClient syncClient = createSyncClientMock(); - NodeRepoMock nodeRepository = new NodeRepoMock(); - TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); - NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("perf-report"), new ServiceDumpReport.DumpOptions(true, 45.0, null)); - - VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); - NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) - .fileSystem(fileSystem) - .build(); - reporter.processServiceDumpRequest(context); - - verify(operations).executeCommandInContainer( - context, context.users().vespa(), "/opt/vespa/libexec/vespa/find-pid", "default/container.1"); - verify(operations).executeCommandInContainer( - context, context.users().vespa(), "perf", "record", "-g", "--output=/opt/vespa/var/tmp/vespa-service-dump-1600000000000/perf-record.bin", - "--pid=12345", "sleep", "45"); - verify(operations).executeCommandInContainer( - context, context.users().vespa(), "bash", "-c", "perf report --input=/opt/vespa/var/tmp/vespa-service-dump-1600000000000/perf-record.bin" + - " > /opt/vespa/var/tmp/vespa-service-dump-1600000000000/perf-report.txt"); - - String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," + - "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + - "\"configId\":\"default/container.1\",\"artifacts\":[\"perf-report\"]," + - "\"dumpOptions\":{\"callGraphRecording\":true,\"duration\":45.0}}"; - assertReportEquals(nodeRepository, expectedJson); - - List<URI> expectedUris = List.of( - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-record.bin.zst"), - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-report.txt")); - assertSyncedFiles(context, syncClient, expectedUris); - } - - @Test - void invokes_jcmd_commands_when_creating_jfr_recording() { - // Setup mocks - ContainerOperations operations = mock(ContainerOperations.class); - when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) - .thenReturn(new CommandResult(null, 0, "12345")) - .thenReturn(new CommandResult(null, 0, "ok")) - .thenReturn(new CommandResult(null, 0, "name=host-admin success")); - SyncClient syncClient = createSyncClientMock(); - NodeRepoMock nodeRepository = new NodeRepoMock(); - TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); - NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("jvm-jfr")); - - VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); - NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) - .fileSystem(fileSystem) - .build(); - reporter.processServiceDumpRequest(context); - - verify(operations).executeCommandInContainer( - context, context.users().vespa(), "/opt/vespa/libexec/vespa/find-pid", "default/container.1"); - verify(operations).executeCommandInContainer( - context, context.users().vespa(), "jcmd", "12345", "JFR.start", "name=host-admin", "path-to-gc-roots=true", "settings=profile", - "filename=/opt/vespa/var/tmp/vespa-service-dump-1600000000000/recording.jfr", "duration=30s"); - verify(operations).executeCommandInContainer(context, context.users().vespa(), "jcmd", "12345", "JFR.check", "name=host-admin"); - - String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000," + - "\"completedAt\":1600001000000," + - "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + - "\"configId\":\"default/container.1\",\"artifacts\":[\"jvm-jfr\"],\"dumpOptions\":{}}"; - assertReportEquals(nodeRepository, expectedJson); - - List<URI> expectedUris = List.of( - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/recording.jfr.zst")); - assertSyncedFiles(context, syncClient, expectedUris); - } - - @Test - void invokes_zookeeper_backup_command_when_generating_snapshot() { - // Setup mocks - ContainerOperations operations = mock(ContainerOperations.class); - when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) - .thenReturn(new CommandResult(null, 0, "12345")); - SyncClient syncClient = createSyncClientMock(); - NodeRepoMock nodeRepository = new NodeRepoMock(); - TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); - NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("zookeeper-snapshot")); - - VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); - NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) - .fileSystem(fileSystem) - .build(); - reporter.processServiceDumpRequest(context); - - verify(operations).executeCommandInContainer( - context, - context.users().vespa(), - "bash", - "-c", - "/opt/vespa/bin/vespa-backup-zk-data.sh -o /opt/vespa/var/tmp/vespa-service-dump-1600000000000/zookeeper-snapshot.tgz -k -f"); - - String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," + - "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + - "\"configId\":\"default/container.1\",\"artifacts\":[\"zookeeper-snapshot\"],\"dumpOptions\":{}}"; - assertReportEquals(nodeRepository, expectedJson); - - List<URI> expectedUris = List.of( - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/zookeeper-snapshot.tgz")); - assertSyncedFiles(context, syncClient, expectedUris); - } - - @Test - void invokes_config_proxy_command_whn_invoking_config_dump() { - // Setup mocks - ContainerOperations operations = mock(ContainerOperations.class); - when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) - .thenReturn(new CommandResult(null, 0, "12345")); - SyncClient syncClient = createSyncClientMock(); - NodeRepoMock nodeRepository = new NodeRepoMock(); - TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); - NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("config-dump")); - - VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); - NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) - .fileSystem(fileSystem) - .build(); - reporter.processServiceDumpRequest(context); - - verify(operations).executeCommandInContainer( - context, - context.users().vespa(), - "bash", - "-c", - "mkdir -p /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config;" + - " /opt/vespa/bin/vespa-configproxy-cmd -m dumpcache /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config;" + - " tar cvf /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config.tar /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config;" + - " zstd /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config.tar -o /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config-dump.tar.zst"); - - String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," + - "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + - "\"configId\":\"default/container.1\",\"artifacts\":[\"config-dump\"],\"dumpOptions\":{}}"; - assertReportEquals(nodeRepository, expectedJson); - - List<URI> expectedUris = List.of( - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/config-dump.tar.zst")); - assertSyncedFiles(context, syncClient, expectedUris); - } - - @Test - void handles_multiple_artifact_types() { - // Setup mocks - ContainerOperations operations = mock(ContainerOperations.class); - when(operations.executeCommandInContainer( - any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) - // For perf report: - .thenReturn(new CommandResult(null, 0, "12345")) - .thenReturn(new CommandResult(null, 0, "")) - .thenReturn(new CommandResult(null, 0, "")) - // For jfr recording: - .thenReturn(new CommandResult(null, 0, "12345")) - .thenReturn(new CommandResult(null, 0, "ok")) - .thenReturn(new CommandResult(null, 0, "name=host-admin success")); - SyncClient syncClient = createSyncClientMock(); - NodeRepoMock nodeRepository = new NodeRepoMock(); - TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); - NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("perf-report", "jvm-jfr"), - new ServiceDumpReport.DumpOptions(true, 20.0, null)); - VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); - NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) - .fileSystem(fileSystem) - .build(); - reporter.processServiceDumpRequest(context); - - List<URI> expectedUris = List.of( - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-record.bin.zst"), - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-report.txt"), - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/recording.jfr.zst")); - assertSyncedFiles(context, syncClient, expectedUris); - } - - @Test - void fails_gracefully_on_invalid_request_json() { - // Setup mocks - ContainerOperations operations = mock(ContainerOperations.class); - SyncClient syncClient = createSyncClientMock(); - NodeRepoMock nodeRepository = new NodeRepoMock(); - TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); - JsonNodeFactory fac = new ObjectMapper().getNodeFactory(); - ObjectNode invalidRequest = new ObjectNode(fac) - .set("dumpOptions", new ObjectNode(fac).put("duration", "invalidDurationDataType")); - NodeSpec spec = NodeSpec.Builder - .testSpec(HOSTNAME, NodeState.active) - .report(ServiceDumpReport.REPORT_ID, invalidRequest) - .build(); - nodeRepository.updateNodeSpec(spec); - VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); - NodeAgentContextImpl context = NodeAgentContextImpl.builder(spec) - .fileSystem(fileSystem) - .build(); - reporter.processServiceDumpRequest(context); - String expectedJson = "{\"createdMillis\":1600001000000,\"startedAt\":1600001000000,\"failedAt\":1600001000000," + - "\"configId\":\"unknown\",\"error\":\"Invalid JSON in service dump request\",\"artifacts\":[]}"; - assertReportEquals(nodeRepository, expectedJson); - } - - private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, List<String> artifacts) { - return createNodeSpecWithDumpRequest(repository, artifacts, new DumpOptions(null, null, null)); - } - - private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, List<String> artifacts, DumpOptions options) { - ServiceDumpReport request = ServiceDumpReport.createRequestReport( - Instant.ofEpochMilli(1600000000000L), null, "default/container.1", artifacts, options); - NodeSpec spec = NodeSpec.Builder - .testSpec(HOSTNAME, NodeState.active) - .report(ServiceDumpReport.REPORT_ID, request.toJsonNode()) - .archiveUri(URI.create("s3://uri-1/tenant1/")) - .build(); - repository.updateNodeSpec(spec); - return spec; - } - - private static void assertReportEquals(NodeRepoMock nodeRepository, String expectedJson) { - ServiceDumpReport report = nodeRepository.getNode(HOSTNAME).reports() - .getReport(ServiceDumpReport.REPORT_ID, ServiceDumpReport.class).get(); - String actualJson = report.toJson(); - assertEquals(expectedJson, actualJson); - } - - @SuppressWarnings("unchecked") - private static void assertSyncedFiles(NodeAgentContextImpl context, SyncClient client, List<URI> expectedDestinations) { - ArgumentCaptor<List<SyncFileInfo>> filesCaptor = ArgumentCaptor.forClass(List.class); - verify(client).sync(eq(context), filesCaptor.capture(), eq(Integer.MAX_VALUE)); - List<SyncFileInfo> actualFiles = filesCaptor.getValue(); - List<URI> actualFilenames = actualFiles.stream() - .map(SyncFileInfo::destination) - .sorted() - .toList(); - assertEquals(expectedDestinations, actualFilenames); - } - - private SyncClient createSyncClientMock() { - SyncClient client = mock(SyncClient.class); - when(client.sync(any(TaskContext.class), anyList(), anyInt())) - .thenReturn(true); - return client; - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java deleted file mode 100644 index 8e56741274e..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.sync; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.net.URI; -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.NONE; -import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.ZSTD; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class SyncFileInfoTest { - - private static final FileSystem fileSystem = TestFileSystem.create(); - - private static final URI nodeArchiveUri = URI.create("s3://vespa-data-bucket/vespa/music/main/h432a/"); - private static final Path accessLogPath1 = fileSystem.getPath("/opt/vespa/logs/access/access.log.20210211"); - private static final Path accessLogPath2 = fileSystem.getPath("/opt/vespa/logs/access/access.log.20210212.zst"); - private static final Path accessLogPath3 = fileSystem.getPath("/opt/vespa/logs/access/access-json.log.20210213.zst"); - private static final Path accessLogPath4 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.20210214.zst"); - private static final Path accessLogPath5 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.container.20210214.zst"); - private static final Path accessLogPath6 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.metrics-proxy.20210214.zst"); - private static final Path connectionLogPath1 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.20210210"); - private static final Path connectionLogPath2 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.20210212.zst"); - private static final Path connectionLogPath3 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.metrics-proxy.20210210"); - private static final Path vespaLogPath1 = fileSystem.getPath("/opt/vespa/logs/vespa.log"); - private static final Path vespaLogPath2 = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12"); - private static final Path zkLogPath0 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.0.log"); - private static final Path zkLogPath1 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.1.log"); - private static final Path startServicesPath1 = fileSystem.getPath("/opt/vespa/logs/start-services.out"); - private static final Path startServicesPath2 = fileSystem.getPath("/opt/vespa/logs/start-services.out-20230808100143"); - private static final Path rotatedNginxErrorLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-error.log.20231019-1234555"); - private static final Path currentNginxErrorLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-error.log"); - private static final Path nginxAccessLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-access.log.20231019-1234"); - - @Test - void access_logs() { - assertForLogFile(accessLogPath1, null, null, true); - assertForLogFile(accessLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210211.zst", ZSTD, false); - - assertForLogFile(accessLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210212.zst", NONE, true); - assertForLogFile(accessLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210212.zst", NONE, false); - - assertForLogFile(accessLogPath3, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access-json.log.20210213.zst", NONE, true); - assertForLogFile(accessLogPath3, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access-json.log.20210213.zst", NONE, false); - - assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.20210214.zst", NONE, true); - assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.20210214.zst", NONE, false); - - assertForLogFile(accessLogPath5, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.container.20210214.zst", NONE, true); - assertForLogFile(accessLogPath5, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.container.20210214.zst", NONE, false); - - assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, accessLogPath6, true, ApplicationId.defaultId())); - assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, accessLogPath6, false, ApplicationId.defaultId())); - } - - @Test - void connection_logs() { - assertForLogFile(connectionLogPath1, null, null, true); - assertForLogFile(connectionLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210210.zst", ZSTD, false); - - assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210212.zst", NONE, true); - assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210212.zst", NONE, false); - - assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, connectionLogPath3, true, ApplicationId.defaultId())); - assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, connectionLogPath3, false, ApplicationId.defaultId())); - } - - @Test - void vespa_logs() { - new UnixPath(vespaLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); - assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ofHours(1), true); - assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2022-05-09.14-22-11.zst", ZSTD, Duration.ZERO, false); - - assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, true); - assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, false); - } - - @Test - void zookeeper_logs() { - new UnixPath(zkLogPath0).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-13T13:13:45Z")); - assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ofHours(1), true); - assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-13.13-13-45.zst", ZSTD, Duration.ZERO, false); - - new UnixPath(zkLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); - assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, true); - assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, false); - } - - @Test - void nginx_error_logs() { - new UnixPath(currentNginxErrorLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); - assertForLogFile(currentNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.zst", ZSTD, Duration.ofHours(1),true); - assertForLogFile(currentNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.zst", ZSTD, Duration.ZERO,false); - - new UnixPath(rotatedNginxErrorLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); - assertForLogFile(rotatedNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.20231019-1234555.zst", ZSTD, true); - assertForLogFile(rotatedNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.20231019-1234555.zst", ZSTD, false); - - // Does not sync access logs - new UnixPath(nginxAccessLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); - Optional<SyncFileInfo> sfi = SyncFileInfo.forLogFile(nodeArchiveUri, nginxAccessLog, false, ApplicationId.defaultId()); - assertEquals(Optional.empty(), sfi); - } - - @Test - void start_services() { - assertForLogFile(startServicesPath1, null, null, true); - assertForLogFile(startServicesPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/start-services/start-services.out-20230808100143.zst", ZSTD, true); - } - - private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, boolean rotatedOnly) { - assertForLogFile(srcPath, destination, compression, null, rotatedOnly); - } - - private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, Duration minDurationBetweenSync, boolean rotatedOnly) { - Optional<SyncFileInfo> sfi = SyncFileInfo.forLogFile(nodeArchiveUri, srcPath, rotatedOnly, ApplicationId.defaultId()); - assertEquals(destination, sfi.map(SyncFileInfo::destination).map(URI::toString).orElse(null)); - assertEquals(compression, sfi.map(SyncFileInfo::uploadCompression).orElse(null)); - assertEquals(minDurationBetweenSync, sfi.flatMap(SyncFileInfo::minDurationBetweenSync).orElse(null)); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java deleted file mode 100644 index 616100363e9..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.sync; - -import com.yahoo.compress.ZstdCompressor; -import org.junit.jupiter.api.Test; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Random; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class ZstdCompressingInputStreamTest { - - @Test - void compression_test() { - Random rnd = new Random(); - byte[] data = new byte[(int) (100_000 * (10 + rnd.nextDouble()))]; - rnd.nextBytes(data); - assertCompression(data, 1 << 14); - } - - @Test - void compress_empty_file_test() { - byte[] compressedData = compress(new byte[0], 1 << 10); - assertEquals(13, compressedData.length, "zstd compressing an empty file results in a 13 bytes file"); - } - - private static void assertCompression(byte[] data, int bufferSize) { - byte[] compressedData = compress(data, bufferSize); - byte[] decompressedData = new byte[data.length]; - var compressor = new ZstdCompressor(); - compressor.decompress(compressedData, 0, compressedData.length, decompressedData, 0, decompressedData.length); - - assertArrayEquals(data, decompressedData); - } - - private static byte[] compress(byte[] data, int bufferSize) { - ByteArrayInputStream bais = new ByteArrayInputStream(data); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(bais, bufferSize)) { - byte[] buffer = new byte[bufferSize]; - for (int nRead; (nRead = zcis.read(buffer, 0, buffer.length)) != -1; ) - baos.write(buffer, 0, nRead); - baos.flush(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - - return baos.toByteArray(); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java deleted file mode 100644 index 355c997a3e0..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; -import org.mockito.InOrder; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithScheduler; -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithSchedulerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -/** - * @author bakksjo - */ -public class NodeAdminImplTest { - - private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory = mock(NodeAgentWithSchedulerFactory.class); - private final TestTimer timer = new TestTimer(); - private final ProcMeminfoReader procMeminfoReader = mock(ProcMeminfoReader.class); - private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentWithSchedulerFactory, - new Metrics(), timer, Duration.ZERO, Duration.ZERO, procMeminfoReader); - - @Test - void nodeAgentsAreProperlyLifeCycleManaged() { - final NodeAgentContext context1 = createNodeAgentContext("host1.test.yahoo.com"); - final NodeAgentContext context2 = createNodeAgentContext("host2.test.yahoo.com"); - final NodeAgentWithScheduler nodeAgent1 = mockNodeAgentWithSchedulerFactory(context1); - final NodeAgentWithScheduler nodeAgent2 = mockNodeAgentWithSchedulerFactory(context2); - - final InOrder inOrder = inOrder(nodeAgentWithSchedulerFactory, nodeAgent1, nodeAgent2); - nodeAdmin.refreshContainersToRun(Set.of()); - verifyNoMoreInteractions(nodeAgentWithSchedulerFactory); - - nodeAdmin.refreshContainersToRun(Set.of(context1)); - inOrder.verify(nodeAgent1).start(); - inOrder.verify(nodeAgent2, never()).start(); - inOrder.verify(nodeAgent1, never()).stopForRemoval(); - - nodeAdmin.refreshContainersToRun(Set.of(context1)); - inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any()); - inOrder.verify(nodeAgent1, never()).start(); - inOrder.verify(nodeAgent1, never()).stopForRemoval(); - - nodeAdmin.refreshContainersToRun(Set.of()); - inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any()); - verify(nodeAgent1).stopForRemoval(); - - nodeAdmin.refreshContainersToRun(Set.of(context2)); - inOrder.verify(nodeAgent2).start(); - inOrder.verify(nodeAgent2, never()).stopForRemoval(); - inOrder.verify(nodeAgent1, never()).stopForRemoval(); - - nodeAdmin.refreshContainersToRun(Set.of()); - inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any()); - inOrder.verify(nodeAgent2, never()).start(); - inOrder.verify(nodeAgent2).stopForRemoval(); - inOrder.verify(nodeAgent1, never()).start(); - inOrder.verify(nodeAgent1, never()).stopForRemoval(); - } - - @Test - void testSetFrozen() { - Set<NodeAgentContext> contexts = new HashSet<>(); - List<NodeAgentWithScheduler> nodeAgents = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - NodeAgentContext context = createNodeAgentContext("host" + i + ".test.yahoo.com"); - NodeAgentWithScheduler nodeAgent = mockNodeAgentWithSchedulerFactory(context); - - contexts.add(context); - nodeAgents.add(nodeAgent); - } - - nodeAdmin.refreshContainersToRun(contexts); - - assertTrue(nodeAdmin.isFrozen()); // Initially everything is frozen to force convergence - mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); - assertTrue(nodeAdmin.setFrozen(false)); // Unfreeze everything - - - mockNodeAgentSetFrozenResponse(nodeAgents, false, false, false); - assertFalse(nodeAdmin.setFrozen(true)); // NodeAdmin freezes only when all the NodeAgents are frozen - - mockNodeAgentSetFrozenResponse(nodeAgents, false, true, true); - assertFalse(nodeAdmin.setFrozen(true)); - assertFalse(nodeAdmin.isFrozen()); - - mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); - assertTrue(nodeAdmin.setFrozen(true)); - assertTrue(nodeAdmin.isFrozen()); - - mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); - assertTrue(nodeAdmin.setFrozen(true)); - assertTrue(nodeAdmin.isFrozen()); - - mockNodeAgentSetFrozenResponse(nodeAgents, false, false, false); - assertFalse(nodeAdmin.setFrozen(false)); - assertFalse(nodeAdmin.isFrozen()); // NodeAdmin unfreezes instantly - - mockNodeAgentSetFrozenResponse(nodeAgents, false, false, true); - assertFalse(nodeAdmin.setFrozen(false)); - assertFalse(nodeAdmin.isFrozen()); - - mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); - assertTrue(nodeAdmin.setFrozen(false)); - assertFalse(nodeAdmin.isFrozen()); - } - - @Test - void testSubsystemFreezeDuration() { - // Initially everything is frozen to force convergence - assertTrue(nodeAdmin.isFrozen()); - assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); - timer.advance(Duration.ofSeconds(1)); - assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration()); - - // Unfreezing floors freeze duration - assertTrue(nodeAdmin.setFrozen(false)); // Unfreeze everything - assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); - timer.advance(Duration.ofSeconds(1)); - assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); - - // Advancing time now will make freeze duration proceed according to clock - assertTrue(nodeAdmin.setFrozen(true)); - assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); - timer.advance(Duration.ofSeconds(1)); - assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration()); - } - - private void mockNodeAgentSetFrozenResponse(List<NodeAgentWithScheduler> nodeAgents, boolean... responses) { - for (int i = 0; i < nodeAgents.size(); i++) { - NodeAgentWithScheduler nodeAgent = nodeAgents.get(i); - when(nodeAgent.setFrozen(anyBoolean(), any())).thenReturn(responses[i]); - } - } - - private NodeAgentContext createNodeAgentContext(String hostname) { - return NodeAgentContextImpl.builder(hostname).fileSystem(TestFileSystem.create()).build(); - } - - private NodeAgentWithScheduler mockNodeAgentWithSchedulerFactory(NodeAgentContext context) { - NodeAgentWithScheduler nodeAgentWithScheduler = mock(NodeAgentWithScheduler.class); - when(nodeAgentWithSchedulerFactory.create(eq(context))).thenReturn(nodeAgentWithScheduler); - return nodeAgentWithScheduler; - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java deleted file mode 100644 index 420146b52f0..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.OrchestratorStatus; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.integration.NodeRepoMock; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; -import org.junit.jupiter.api.Test; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.RESUMED; -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED; -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * Basic test of NodeAdminStateUpdater - * - * @author freva - */ -public class NodeAdminStateUpdaterTest { - private final NodeAgentContextFactory nodeAgentContextFactory = mock(NodeAgentContextFactory.class); - private final NodeRepoMock nodeRepository = spy(new NodeRepoMock()); - private final Orchestrator orchestrator = mock(Orchestrator.class); - private final NodeAdmin nodeAdmin = mock(NodeAdmin.class); - private final HostName hostHostname = HostName.of("basehost1.test.yahoo.com"); - - private final NodeAdminStateUpdater updater = spy(new NodeAdminStateUpdater( - nodeAgentContextFactory, nodeRepository, orchestrator, nodeAdmin, hostHostname)); - - - @Test - void state_convergence() { - mockNodeRepo(NodeState.active, 4); - List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream() - .map(NodeSpec::hostname) - .toList(); - List<String> suspendHostnames = new ArrayList<>(activeHostnames); - suspendHostnames.add(hostHostname.value()); - when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); - - { - // Initially everything is frozen to force convergence - assertConvergeError(RESUMED, "NodeAdmin is not yet unfrozen"); - when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); - updater.converge(RESUMED); - verify(orchestrator, times(1)).resume(hostHostname.value()); - - // We are already resumed, so this should return without resuming again - updater.converge(RESUMED); - verify(orchestrator, times(1)).resume(hostHostname.value()); - verify(nodeAdmin, times(2)).setFrozen(eq(false)); - - // Host is externally suspended in orchestrator, should be resumed by node-admin - setHostOrchestratorStatus(hostHostname, OrchestratorStatus.ALLOWED_TO_BE_DOWN); - updater.converge(RESUMED); - verify(orchestrator, times(2)).resume(hostHostname.value()); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - setHostOrchestratorStatus(hostHostname, OrchestratorStatus.NO_REMARKS); - - // Lets try to suspend node admin only - when(nodeAdmin.setFrozen(eq(true))).thenReturn(false); - assertConvergeError(SUSPENDED_NODE_ADMIN, "NodeAdmin is not yet frozen"); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - } - - { - // First orchestration failure happens within the freeze convergence timeout, - // and so should not call setFrozen(false) - final String exceptionMessage = "Cannot allow to suspend because some reason"; - when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); - doThrow(new RuntimeException(exceptionMessage)).doNothing() - .when(orchestrator).suspend(eq(hostHostname.value())); - assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMessage); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - - updater.converge(SUSPENDED_NODE_ADMIN); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - verify(orchestrator, times(2)).suspend(hostHostname.value()); - setHostOrchestratorStatus(hostHostname, OrchestratorStatus.ALLOWED_TO_BE_DOWN); - - // Already suspended, no changes - updater.converge(SUSPENDED_NODE_ADMIN); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - verify(orchestrator, times(2)).suspend(hostHostname.value()); - - // Host is externally resumed - setHostOrchestratorStatus(hostHostname, OrchestratorStatus.NO_REMARKS); - updater.converge(SUSPENDED_NODE_ADMIN); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - verify(orchestrator, times(3)).suspend(hostHostname.value()); - setHostOrchestratorStatus(hostHostname, OrchestratorStatus.ALLOWED_TO_BE_DOWN); - } - - { - // At this point orchestrator will say its OK to suspend, but something goes wrong when we try to stop services - final String exceptionMessage = "Failed to stop services"; - verify(orchestrator, times(0)).suspend(eq(hostHostname.value()), eq(suspendHostnames)); - doThrow(new RuntimeException(exceptionMessage)).doNothing().when(nodeAdmin).stopNodeAgentServices(); - assertConvergeError(SUSPENDED, exceptionMessage); - verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(suspendHostnames)); - // Make sure we dont roll back if we fail to stop services - we will try to stop again next tick - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - - // Finally we are successful in transitioning to frozen - updater.converge(SUSPENDED); - } - } - - @Test - void half_transition_revert() { - final String exceptionMsg = "Cannot allow to suspend because some reason"; - mockNodeRepo(NodeState.active, 3); - - // Initially everything is frozen to force convergence - when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); - updater.converge(RESUMED); - verify(nodeAdmin, times(1)).setFrozen(eq(false)); - verify(nodeAdmin, times(1)).refreshContainersToRun(any()); - - // Let's start suspending, we are able to freeze the nodes, but orchestrator denies suspension - when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); - when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); - doThrow(new RuntimeException(exceptionMsg)).when(orchestrator).suspend(eq(hostHostname.value())); - - assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg); - verify(nodeAdmin, times(1)).setFrozen(eq(true)); - verify(orchestrator, times(1)).suspend(eq(hostHostname.value())); - assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg); - verify(nodeAdmin, times(2)).setFrozen(eq(true)); - verify(orchestrator, times(2)).suspend(eq(hostHostname.value())); - assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg); - verify(nodeAdmin, times(3)).setFrozen(eq(true)); - verify(orchestrator, times(3)).suspend(eq(hostHostname.value())); - - // No new unfreezes nor refresh while trying to freeze - verify(nodeAdmin, times(1)).setFrozen(eq(false)); - verify(nodeAdmin, times(1)).refreshContainersToRun(any()); - - // Only resume and fetch containers when subsystem freeze duration expires - when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofHours(1)); - assertConvergeError(SUSPENDED_NODE_ADMIN, "Timed out trying to freeze all nodes: will force an unfrozen tick"); - verify(nodeAdmin, times(2)).setFrozen(eq(false)); - verify(orchestrator, times(3)).suspend(eq(hostHostname.value())); // no new suspend calls - verify(nodeAdmin, times(2)).refreshContainersToRun(any()); - - // We change our mind, want to remain resumed - updater.converge(RESUMED); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); // Make sure that we unfreeze! - } - - @Test - void do_not_orchestrate_host_when_not_active() { - when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofHours(1)); - when(nodeAdmin.setFrozen(anyBoolean())).thenReturn(true); - mockNodeRepo(NodeState.ready, 3); - - // Resume and suspend only require that node-agents are frozen and permission from - // orchestrator to resume/suspend host. Therefore, if host is not active, we only need to freeze. - updater.converge(RESUMED); - verify(orchestrator, never()).resume(eq(hostHostname.value())); - - updater.converge(SUSPENDED_NODE_ADMIN); - verify(orchestrator, never()).suspend(eq(hostHostname.value())); - - // When doing batch suspend, only suspend the containers if the host is not active - List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream() - .map(NodeSpec::hostname) - .toList(); - updater.converge(SUSPENDED); - verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(activeHostnames)); - } - - @Test - void node_spec_and_acl_aligned() { - Acl acl = new Acl.Builder().withTrustedPorts(22).build(); - mockNodeRepo(NodeState.active, 3); - mockAcl(acl, 1, 2, 3); - - updater.adjustNodeAgentsToRunFromNodeRepository(); - updater.adjustNodeAgentsToRunFromNodeRepository(); - updater.adjustNodeAgentsToRunFromNodeRepository(); - - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host3.yahoo.com")), eq(acl)); - verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value())); - verify(nodeRepository, times(3)).getAcls(eq(hostHostname.value())); - } - - @Test - void node_spec_and_acl_mismatch_missing_one_acl() { - Acl acl = new Acl.Builder().withTrustedPorts(22).build(); - mockNodeRepo(NodeState.active, 3); - mockAcl(acl, 1, 2); // Acl for 3 is missing - - updater.adjustNodeAgentsToRunFromNodeRepository(); - mockNodeRepo(NodeState.active, 2); // Next tick, the spec for 3 is no longer returned - updater.adjustNodeAgentsToRunFromNodeRepository(); - updater.adjustNodeAgentsToRunFromNodeRepository(); - - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(1)).create(argThat(spec -> spec.hostname().equals("host3.yahoo.com")), eq(Acl.EMPTY)); - verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value())); - verify(nodeRepository, times(3)).getAcls(eq(hostHostname.value())); - } - - @Test - void node_spec_and_acl_mismatch_additional_acl() { - Acl acl = new Acl.Builder().withTrustedPorts(22).build(); - mockNodeRepo(NodeState.active, 2); - mockAcl(acl, 1, 2, 3); // Acl for 3 is extra - - updater.adjustNodeAgentsToRunFromNodeRepository(); - updater.adjustNodeAgentsToRunFromNodeRepository(); - updater.adjustNodeAgentsToRunFromNodeRepository(); - - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl)); - verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value())); - verify(nodeRepository, times(3)).getAcls(eq(hostHostname.value())); - } - - private void assertConvergeError(NodeAdminStateUpdater.State targetState, String reason) { - try { - updater.converge(targetState); - fail("Expected converging to " + targetState + " to fail with \"" + reason + "\", but it succeeded without error"); - } catch (RuntimeException e) { - assertEquals(reason, e.getMessage()); - } - } - - private void mockNodeRepo(NodeState hostState, int numberOfNodes) { - nodeRepository.resetNodeSpecs(); - - IntStream.rangeClosed(1, numberOfNodes) - .mapToObj(i -> NodeSpec.Builder.testSpec("host" + i + ".yahoo.com").parentHostname(hostHostname.value()).build()) - .forEach(nodeRepository::updateNodeSpec); - - nodeRepository.updateNodeSpec(NodeSpec.Builder.testSpec(hostHostname.value(), hostState).type(NodeType.host).build()); - } - - private void mockAcl(Acl acl, int... nodeIds) { - nodeRepository.setAcl(Arrays.stream(nodeIds) - .mapToObj(i -> "host" + i + ".yahoo.com") - .collect(Collectors.toMap(Function.identity(), h -> acl))); - } - - private void setHostOrchestratorStatus(HostName hostname, OrchestratorStatus orchestratorStatus) { - nodeRepository.updateNodeSpec(hostname.value(), node -> node.orchestratorStatus(orchestratorStatus)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java deleted file mode 100644 index 589eceebb74..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author freva - */ -public class NodeAgentContextImplTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final NodeAgentContext context = NodeAgentContextImpl.builder("container-1.domain.tld") - .fileSystem(fileSystem).build(); - - @Test - void path_on_host_from_path_in_node_test() { - assertEquals( - "/data/vespa/storage/container-1", - context.paths().of("/").pathOnHost().toString()); - - assertEquals( - "/data/vespa/storage/container-1/dev/null", - context.paths().of("/dev/null").pathOnHost().toString()); - } - - @Test - void path_in_container_must_be_absolute() { - assertThrows(IllegalArgumentException.class, () -> { - context.paths().of("some/relative/path"); - }); - } - - @Test - void path_in_node_from_path_on_host_test() { - assertEquals( - "/dev/null", - context.paths().fromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-1/dev/null")).pathInContainer()); - } - - @Test - void path_on_host_must_be_absolute() { - assertThrows(IllegalArgumentException.class, () -> { - context.paths().fromPathOnHost(Path.of("some/relative/path")); - }); - } - - @Test - void path_on_host_must_be_inside_container_storage_of_context() { - assertThrows(IllegalArgumentException.class, () -> { - context.paths().fromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-2/dev/null")); - }); - } - - @Test - void path_on_host_must_be_inside_container_storage() { - assertThrows(IllegalArgumentException.class, () -> { - context.paths().fromPathOnHost(fileSystem.getPath("/home")); - }); - } - - @Test - void path_under_vespa_host_in_container_test() { - assertEquals( - "/opt/vespa", - context.paths().underVespaHome("").pathInContainer()); - - assertEquals( - "/opt/vespa/logs/vespa/vespa.log", - context.paths().underVespaHome("logs/vespa/vespa.log").pathInContainer()); - } - - @Test - void path_under_vespa_home_must_be_relative() { - assertThrows(IllegalArgumentException.class, () -> { - context.paths().underVespaHome("/home"); - }); - } - - @Test - void disabledTasksTest() { - NodeAgentContext context1 = createContextWithDisabledTasks(); - assertFalse(context1.isDisabled(NodeAgentTask.DiskCleanup)); - assertFalse(context1.isDisabled(NodeAgentTask.CoreDumps)); - - NodeAgentContext context2 = createContextWithDisabledTasks("root>UpgradeTask", "DiskCleanup", "node>CoreDumps"); - assertFalse(context2.isDisabled(NodeAgentTask.DiskCleanup)); - assertTrue(context2.isDisabled(NodeAgentTask.CoreDumps)); - } - - private NodeAgentContext createContextWithDisabledTasks(String... tasks) { - InMemoryFlagSource flagSource = new InMemoryFlagSource(); - flagSource.withListFlag(PermanentFlags.DISABLED_HOST_ADMIN_TASKS.id(), List.of(tasks), String.class); - return NodeAgentContextImpl.builder("node123").fileSystem(fileSystem).flagSource(flagSource).build(); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java deleted file mode 100644 index 5e09c45d217..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.jdisc.core.SystemTimer; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; - -import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextSupplier.ContextSupplierInterruptedException; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author freva - */ -public class NodeAgentContextManagerTest { - - private static final int TIMEOUT = 10_000; - - private final SystemTimer timer = new SystemTimer(); - private final NodeAgentContext initialContext = generateContext(); - private final NodeAgentContextManager manager = new NodeAgentContextManager(timer, initialContext); - - @Test - @Timeout(TIMEOUT) - void context_is_ignored_unless_scheduled_while_waiting() { - NodeAgentContext context1 = generateContext(); - manager.scheduleTickWith(context1, timer.currentTime()); - assertSame(initialContext, manager.currentContext()); - - AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - manager.waitUntilWaitingForNextContext(); - assertFalse(async.isCompleted()); - - NodeAgentContext context2 = generateContext(); - manager.scheduleTickWith(context2, timer.currentTime()); - - assertSame(context2, async.awaitResult().response.get()); - assertSame(context2, manager.currentContext()); - } - - @Test - @Timeout(TIMEOUT) - void returns_no_earlier_than_at_given_time() { - AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - manager.waitUntilWaitingForNextContext(); - - NodeAgentContext context1 = generateContext(); - Instant returnAt = timer.currentTime().plusMillis(500); - manager.scheduleTickWith(context1, returnAt); - - assertSame(context1, async.awaitResult().response.get()); - assertSame(context1, manager.currentContext()); - // Is accurate to a millisecond - assertFalse(timer.currentTime().plusMillis(1).isBefore(returnAt)); - } - - @Test - @Timeout(TIMEOUT) - void blocks_in_nextContext_until_one_is_scheduled() { - AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - manager.waitUntilWaitingForNextContext(); - assertFalse(async.isCompleted()); - - NodeAgentContext context1 = generateContext(); - manager.scheduleTickWith(context1, timer.currentTime()); - - async.awaitResult(); - assertEquals(Optional.of(context1), async.response); - assertFalse(async.exception.isPresent()); - } - - @Test - @Timeout(TIMEOUT) - void blocks_in_nextContext_until_interrupt() { - AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - manager.waitUntilWaitingForNextContext(); - assertFalse(async.isCompleted()); - - manager.interrupt(); - - async.awaitResult(); - assertEquals(Optional.of(ContextSupplierInterruptedException.class), async.exception.map(Exception::getClass)); - assertFalse(async.response.isPresent()); - } - - @Test - @Timeout(TIMEOUT) - void setFrozen_does_not_block_with_no_timeout() { - assertFalse(manager.setFrozen(false, Duration.ZERO)); - - // Generate new context and get it from the supplier, this completes the unfreeze - NodeAgentContext context1 = generateContext(); - AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - manager.waitUntilWaitingForNextContext(); - manager.scheduleTickWith(context1, timer.currentTime()); - assertSame(context1, async.awaitResult().response.get()); - - assertTrue(manager.setFrozen(false, Duration.ZERO)); - } - - @Test - @Timeout(TIMEOUT) - void setFrozen_blocks_at_least_for_duration_of_timeout() { - long wantedDurationMillis = 100; - long start = timer.currentTimeMillis(); - assertFalse(manager.setFrozen(false, Duration.ofMillis(wantedDurationMillis))); - long actualDurationMillis = timer.currentTimeMillis() - start; - - assertTrue(actualDurationMillis >= wantedDurationMillis); - } - - private static NodeAgentContext generateContext() { - return NodeAgentContextImpl.builder("container-123.domain.tld").fileSystem(TestFileSystem.create()).build(); - } - - private static class AsyncExecutor<T> { - private final CountDownLatch latch = new CountDownLatch(1); - private volatile Optional<T> response = Optional.empty(); - private volatile Optional<Exception> exception = Optional.empty(); - - private AsyncExecutor(Callable<T> supplier) { - new Thread(() -> { - try { - response = Optional.of(supplier.call()); - } catch (Exception e) { - exception = Optional.of(e); - } - latch.countDown(); - }).start(); - } - - private AsyncExecutor<T> awaitResult() { - try { - latch.await(); - } catch (InterruptedException ignored) { } - return this; - } - - private boolean isCompleted() { - return latch.getCount() == 0; - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java deleted file mode 100644 index 709326cc3b8..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ /dev/null @@ -1,889 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.OrchestratorStatus; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; -import com.yahoo.vespa.hosted.node.admin.container.Container; -import com.yahoo.vespa.hosted.node.admin.container.ContainerId; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.container.ContainerResources; -import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.InOrder; - -import java.nio.file.FileSystem; -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.BiFunction; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author Øyvind Bakksjø - */ -public class NodeAgentImplTest { - private static final NodeResources resources = new NodeResources(2, 16, 250, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local); - private static final Version vespaVersion = Version.fromString("1.2.3"); - private static final ContainerId containerId = new ContainerId("af23"); - private static final String hostName = "host1.test.yahoo.com"; - - private final NodeAgentContextSupplier contextSupplier = mock(NodeAgentContextSupplier.class); - private final DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image"); - private final ContainerOperations containerOperations = mock(ContainerOperations.class); - private final NodeRepository nodeRepository = mock(NodeRepository.class); - private final Orchestrator orchestrator = mock(Orchestrator.class); - private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); - private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); - private final HealthChecker healthChecker = mock(HealthChecker.class); - private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - private final TestTimer timer = new TestTimer(Instant.now()); - private final FileSystem fileSystem = TestFileSystem.create(); - - @BeforeEach - public void setUp() { - when(containerOperations.suspendNode(any())).thenReturn(""); - when(containerOperations.resumeNode(any())).thenReturn(""); - when(containerOperations.restartVespa(any())).thenReturn(""); - when(containerOperations.startServices(any())).thenReturn(""); - when(containerOperations.stopServices(any())).thenReturn(""); - } - - @Test - void upToDateContainerIsUntouched() { - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .orchestratorStatus(OrchestratorStatus.NO_REMARKS) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(orchestrator, never()).suspend(any(String.class)); - verify(containerOperations, never()).pullImageAsyncIfNeeded(any(), any(), any()); - - final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository); - // TODO: Verify this isn't run unless 1st time - inOrder.verify(containerOperations, never()).startServices(eq(context)); - inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); - inOrder.verify(orchestrator, never()).resume(hostName); - } - - @Test - void verifyRemoveOldFilesIfDiskFull() { - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - verify(storageMaintainer, times(1)).cleanDiskIfFull(eq(context)); - } - - @Test - void startsAfterStoppingServices() { - final InOrder inOrder = inOrder(containerOperations); - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - inOrder.verify(containerOperations, never()).startServices(eq(context)); - inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); - - nodeAgent.stopForHostSuspension(context); - nodeAgent.doConverge(context); - inOrder.verify(containerOperations, never()).startServices(eq(context)); - inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); // Expect a resume, but no start services - - // No new suspends/stops, so no need to resume/start - nodeAgent.doConverge(context); - inOrder.verify(containerOperations, never()).startServices(eq(context)); - inOrder.verify(containerOperations, never()).resumeNode(eq(context)); - - nodeAgent.stopForHostSuspension(context); - nodeAgent.doConverge(context); - inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any()); - inOrder.verify(containerOperations, times(1)).startContainer(eq(context)); - inOrder.verify(containerOperations, times(0)).startServices(eq(context)); // done as part of startContainer - inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); - } - - @Test - void absentContainerCausesStart() { - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(null, false); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false); - - nodeAgent.doConverge(context); - - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(containerOperations, never()).startServices(any()); - verify(orchestrator, never()).suspend(any(String.class)); - - final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository, aclMaintainer, healthChecker); - inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(dockerImage), any()); - inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any()); - inOrder.verify(containerOperations, times(1)).startContainer(eq(context)); - inOrder.verify(aclMaintainer, times(1)).converge(eq(context)); - inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); - inOrder.verify(healthChecker, times(1)).verifyHealth(eq(context)); - inOrder.verify(nodeRepository).updateNodeAttributes( - hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(vespaVersion).withRebootGeneration(0)); - inOrder.verify(orchestrator, never()).resume(hostName); - } - - @Test - void containerIsNotStoppedIfNewImageMustBePulled() { - final DockerImage newDockerImage = DockerImage.fromString("registry.example.com/repo/new-image"); - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(newDockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(containerOperations.pullImageAsyncIfNeeded(any(), any(), any())).thenReturn(true); - - nodeAgent.doConverge(context); - - verify(orchestrator, never()).suspend(any(String.class)); - verify(orchestrator, never()).resume(any(String.class)); - verify(containerOperations, never()).removeContainer(eq(context), any()); - - final InOrder inOrder = inOrder(containerOperations); - inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(newDockerImage), any()); - } - - @Test - void containerIsUpdatedIfCpuChanged() { - NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .orchestratorStatus(OrchestratorStatus.NO_REMARKS); - - NodeAgentContext firstContext = createContext(specBuilder.build()); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(containerOperations.pullImageAsyncIfNeeded(any(), any(), any())).thenReturn(true); - - InOrder inOrder = inOrder(orchestrator, containerOperations); - - nodeAgent.doConverge(firstContext); - inOrder.verify(orchestrator, never()).resume(any(String.class)); - - NodeAgentContext secondContext = createContext(specBuilder.diskGb(200).build()); - nodeAgent.doConverge(secondContext); - inOrder.verify(orchestrator, never()).resume(any(String.class)); - - NodeAgentContext thirdContext = NodeAgentContextImpl.builder(specBuilder.vcpu(5).build()).fileSystem(fileSystem).cpuSpeedUp(1.25).build(); - nodeAgent.doConverge(thirdContext); - ContainerResources resourcesAfterThird = ContainerResources.from(0, 4, 16); - mockGetContainer(dockerImage, resourcesAfterThird, true); - - inOrder.verify(orchestrator, never()).suspend(any()); - inOrder.verify(containerOperations).updateContainer(eq(thirdContext), eq(containerId), eq(resourcesAfterThird)); - inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - inOrder.verify(containerOperations, never()).startContainer(any()); - inOrder.verify(orchestrator, never()).resume(any()); - - // No changes - nodeAgent.converge(thirdContext); - inOrder.verify(orchestrator, never()).suspend(any()); - inOrder.verify(containerOperations, never()).updateContainer(eq(thirdContext), eq(containerId), any()); - inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - inOrder.verify(orchestrator, never()).resume(any()); - - // Set the feature flag - flagSource.withDoubleFlag(PermanentFlags.CONTAINER_CPU_CAP.id(), 2.3); - - nodeAgent.doConverge(thirdContext); - inOrder.verify(containerOperations).updateContainer(eq(thirdContext), eq(containerId), eq(ContainerResources.from(9.2, 4, 16))); - inOrder.verify(orchestrator, never()).resume(any()); - } - - @Test - void containerIsRecreatedIfMemoryChanged() { - NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .wantedRestartGeneration(2).currentRestartGeneration(1); - - NodeAgentContext firstContext = createContext(specBuilder.build()); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(containerOperations.pullImageAsyncIfNeeded(any(), any(), any())).thenReturn(true); - - nodeAgent.doConverge(firstContext); - NodeAgentContext secondContext = createContext(specBuilder.memoryGb(20).build()); - nodeAgent.doConverge(secondContext); - ContainerResources resourcesAfterThird = ContainerResources.from(0, 2, 20); - mockGetContainer(dockerImage, resourcesAfterThird, true); - - InOrder inOrder = inOrder(orchestrator, containerOperations, nodeRepository); - inOrder.verify(orchestrator).resume(any(String.class)); - inOrder.verify(containerOperations).removeContainer(eq(secondContext), any()); - inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); - inOrder.verify(containerOperations, never()).restartVespa(any()); - inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withRestartGeneration(2).withRebootGeneration(0))); - - nodeAgent.doConverge(secondContext); - inOrder.verify(orchestrator).resume(any(String.class)); - inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); - inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - } - - @Test - void noRestartIfOrchestratorSuspendFails() { - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .wantedRestartGeneration(2).currentRestartGeneration(1) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - doThrow(new OrchestratorException("Denied")).when(orchestrator).suspend(eq(hostName)); - try { - nodeAgent.doConverge(context); - fail("Expected to throw an exception"); - } catch (OrchestratorException ignored) { - } - - verify(containerOperations, never()).createContainer(eq(context), any()); - verify(containerOperations, never()).startContainer(eq(context)); - verify(orchestrator, never()).resume(any(String.class)); - verify(nodeRepository, never()).updateNodeAttributes(any(String.class), any(NodeAttributes.class)); - - // Verify aclMaintainer is called even if suspension fails - verify(aclMaintainer, times(1)).converge(eq(context)); - } - - @Test - void recreatesContainerIfRebootWanted() { - final long wantedRebootGeneration = 2; - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .wantedRebootGeneration(wantedRebootGeneration).currentRebootGeneration(1) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false); - doThrow(ConvergenceException.ofTransient("Connection refused")).doNothing() - .when(healthChecker).verifyHealth(eq(context)); - - try { - nodeAgent.doConverge(context); - } catch (ConvergenceException ignored) { - } - - // First time we fail to resume because health verification fails - verify(orchestrator, times(1)).suspend(eq(hostName)); - verify(containerOperations, times(1)).removeContainer(eq(context), any()); - verify(containerOperations, times(1)).createContainer(eq(context), any()); - verify(containerOperations, times(1)).startContainer(eq(context)); - verify(orchestrator, never()).resume(eq(hostName)); - verify(nodeRepository, never()).updateNodeAttributes(any(), any()); - - nodeAgent.doConverge(context); - - // Do not reboot the container again - verify(containerOperations, times(1)).removeContainer(eq(context), any()); - verify(containerOperations, times(1)).createContainer(eq(context), any()); - verify(orchestrator, times(1)).resume(eq(hostName)); - verify(nodeRepository, times(1)).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() - .withRebootGeneration(wantedRebootGeneration))); - } - - @Test - void failedNodeRunningContainerShouldStillBeRunning() { - final NodeSpec node = nodeBuilder(NodeState.failed) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(orchestrator, never()).resume(any(String.class)); - verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); - } - - @Test - void readyNodeLeadsToNoAction() { - final NodeSpec node = nodeBuilder(NodeState.ready).build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(null, false); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - nodeAgent.doConverge(context); - nodeAgent.doConverge(context); - - // Should only be called once, when we initialize - verify(containerOperations, times(1)).getContainer(eq(context)); - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(containerOperations, never()).createContainer(eq(context), any()); - verify(containerOperations, never()).startContainer(eq(context)); - verify(orchestrator, never()).resume(any(String.class)); - verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); - } - - @Test - void inactiveNodeRunningContainerShouldStillBeRunning() { - final NodeSpec node = nodeBuilder(NodeState.inactive) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - final InOrder inOrder = inOrder(storageMaintainer, containerOperations); - inOrder.verify(containerOperations, never()).removeContainer(eq(context), any()); - - verify(orchestrator, never()).resume(any(String.class)); - verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); - } - - @Test - void reservedNodeDoesNotUpdateNodeRepoWithVersion() { - final NodeSpec node = nodeBuilder(NodeState.reserved) - .wantedDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(null, false); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); - } - - private void nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState nodeState, Optional<Long> wantedRestartGeneration) { - NodeSpec.Builder builder = nodeBuilder(nodeState) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage); - wantedRestartGeneration.ifPresent(restartGeneration -> builder - .wantedRestartGeneration(restartGeneration).currentRestartGeneration(restartGeneration)); - - NodeSpec node = builder.build(); - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - final InOrder inOrder = inOrder(storageMaintainer, containerOperations, nodeRepository); - inOrder.verify(containerOperations, times(1)).stopServices(eq(context)); - inOrder.verify(storageMaintainer, times(1)).handleCoreDumpsForContainer(eq(context), any(), eq(true)); - inOrder.verify(containerOperations, times(1)).removeContainer(eq(context), any()); - inOrder.verify(storageMaintainer, times(1)).archiveNodeStorage(eq(context)); - inOrder.verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(NodeState.ready)); - - verify(containerOperations, never()).createContainer(eq(context), any()); - verify(containerOperations, never()).startContainer(eq(context)); - verify(containerOperations, never()).suspendNode(eq(context)); - verify(containerOperations, times(1)).stopServices(eq(context)); - verify(orchestrator, never()).resume(any(String.class)); - verify(orchestrator, never()).suspend(any(String.class)); - // current Docker image and vespa version should be cleared - verify(nodeRepository, times(1)).updateNodeAttributes( - eq(hostName), eq(new NodeAttributes().withDockerImage(DockerImage.EMPTY).withVespaVersion(Version.emptyVersion))); - } - - @Test - void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycled() { - nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState.dirty, Optional.of(1L)); - } - - @Test - void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycledNoRestartGeneration() { - nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState.dirty, Optional.empty()); - } - - @Test - void testRestartDeadContainerAfterNodeAdminRestart() { - final NodeSpec node = nodeBuilder(NodeState.active) - .currentDockerImage(dockerImage).wantedDockerImage(dockerImage) - .currentVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, false); - - when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - verify(containerOperations, times(1)).removeContainer(eq(context), any()); - verify(containerOperations, times(1)).createContainer(eq(context), any()); - verify(containerOperations, times(1)).startContainer(eq(context)); - } - - @Test - void resumeProgramRunsUntilSuccess() { - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .currentVespaVersion(vespaVersion) - .wantedRestartGeneration(1).currentRestartGeneration(1) - .orchestratorStatus(OrchestratorStatus.ALLOWED_TO_BE_DOWN) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); - - final InOrder inOrder = inOrder(orchestrator, containerOperations, nodeRepository); - doThrow(new RuntimeException("Failed 1st time")) - .doReturn("") - .when(containerOperations).resumeNode(eq(context)); - - // 1st try - try { - nodeAgent.doConverge(context); - fail("Expected to throw an exception"); - } catch (RuntimeException ignored) { - } - - inOrder.verify(containerOperations, times(1)).resumeNode(any()); - inOrder.verifyNoMoreInteractions(); - - // 2nd try - nodeAgent.doConverge(context); - - inOrder.verify(containerOperations).resumeNode(any()); - inOrder.verify(orchestrator).resume(hostName); - inOrder.verifyNoMoreInteractions(); - } - - @Test - void start_container_subtask_failure_leads_to_container_restart() { - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion) - .wantedRestartGeneration(1).currentRestartGeneration(1) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = spy(makeNodeAgent(null, false)); - - when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false); - doThrow(new RuntimeException("Failed to set up network")).doNothing().when(containerOperations).startContainer(eq(context)); - - try { - nodeAgent.doConverge(context); - fail("Expected to get RuntimeException"); - } catch (RuntimeException ignored) { - } - - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(containerOperations, times(1)).createContainer(eq(context), any()); - verify(containerOperations, times(1)).startContainer(eq(context)); - verify(nodeAgent, never()).resumeNodeIfNeeded(any()); - - // The docker container was actually started and is running, but subsequent exec calls to set up - // networking failed - mockGetContainer(dockerImage, true); - nodeAgent.doConverge(context); - - verify(containerOperations, times(1)).removeContainer(eq(context), any()); - verify(containerOperations, times(2)).createContainer(eq(context), any()); - verify(containerOperations, times(2)).startContainer(eq(context)); - verify(nodeAgent, times(1)).resumeNodeIfNeeded(any()); - } - - @Test - void testRunningConfigServer() { - final NodeSpec node = nodeBuilder(NodeState.active) - .type(NodeType.config) - .wantedDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion) - .orchestratorStatus(OrchestratorStatus.ALLOWED_TO_BE_DOWN) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(null, false); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false); - - nodeAgent.doConverge(context); - - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(orchestrator, never()).suspend(any(String.class)); - - final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository, aclMaintainer); - inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(dockerImage), any()); - inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any()); - inOrder.verify(containerOperations, times(1)).startContainer(eq(context)); - inOrder.verify(aclMaintainer, times(1)).converge(eq(context)); - inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); - inOrder.verify(nodeRepository).updateNodeAttributes( - hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(vespaVersion).withRebootGeneration(0)); - inOrder.verify(orchestrator).resume(hostName); - } - - - // Tests that only containers without owners are stopped - @Test - void testThatStopContainerDependsOnOwnerPresent() { - verifyThatContainerIsStopped(NodeState.parked, Optional.empty()); - verifyThatContainerIsStopped(NodeState.parked, Optional.of(ApplicationId.defaultId())); - verifyThatContainerIsStopped(NodeState.failed, Optional.empty()); - verifyThatContainerIsStopped(NodeState.failed, Optional.of(ApplicationId.defaultId())); - verifyThatContainerIsStopped(NodeState.inactive, Optional.of(ApplicationId.defaultId())); - } - - @Test - void initial_cpu_cap_test() { - NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion); - - NodeAgentContext context = createContext(specBuilder.build()); - NodeAgentImpl nodeAgent = makeNodeAgent(null, false, Duration.ofSeconds(30)); - - InOrder inOrder = inOrder(orchestrator, containerOperations); - - ConvergenceException healthCheckException = ConvergenceException.ofTransient("Not yet up"); - doThrow(healthCheckException).when(healthChecker).verifyHealth(any()); - for (int i = 0; i < 3; i++) { - try { - nodeAgent.doConverge(context); - fail("Expected to fail with health check exception"); - } catch (ConvergenceException e) { - assertEquals(healthCheckException, e); - } - timer.advance(Duration.ofSeconds(30)); - } - - doNothing().when(healthChecker).verifyHealth(any()); - try { - nodeAgent.doConverge(context); - fail("Expected to fail due to warm up period not yet done"); - } catch (ConvergenceException e) { - assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage()); - } - inOrder.verify(orchestrator, never()).resume(any()); - inOrder.verify(orchestrator, never()).suspend(any()); - inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); - - - timer.advance(Duration.ofSeconds(31)); - nodeAgent.doConverge(context); - - inOrder.verify(orchestrator, never()).suspend(any()); - inOrder.verify(containerOperations).updateContainer(eq(context), eq(containerId), eq(ContainerResources.from(0, 2, 16))); - inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - inOrder.verify(containerOperations, never()).startContainer(any()); - inOrder.verify(orchestrator, never()).resume(any()); - - // No changes - nodeAgent.converge(context); - inOrder.verify(orchestrator, never()).suspend(any()); - inOrder.verify(containerOperations, never()).updateContainer(eq(context), eq(containerId), any()); - inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - inOrder.verify(orchestrator, never()).resume(any()); - } - - @Test - void resumes_normally_if_container_is_already_capped_on_start() { - NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .wantedRestartGeneration(1).currentRestartGeneration(1); - - NodeAgentContext context = createContext(specBuilder.build()); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); - mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); - - InOrder inOrder = inOrder(orchestrator, containerOperations); - - nodeAgent.doConverge(context); - - nodeAgent.converge(context); - inOrder.verify(orchestrator, never()).suspend(any(String.class)); - inOrder.verify(containerOperations, never()).updateContainer(eq(context), eq(containerId), any()); - inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - inOrder.verify(orchestrator, never()).resume(any(String.class)); - } - - @Test - void uncaps_and_caps_cpu_for_services_restart() { - NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .wantedRestartGeneration(2).currentRestartGeneration(1); - - NodeAgentContext context = createContext(specBuilder.build()); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); - mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true); - - InOrder inOrder = inOrder(orchestrator, containerOperations); - - nodeAgent.converge(context); - inOrder.verify(orchestrator, times(1)).suspend(eq(hostName)); - inOrder.verify(containerOperations, times(1)).updateContainer(eq(context), eq(containerId), eq(ContainerResources.from(0, 0, 16))); - inOrder.verify(containerOperations, times(1)).restartVespa(eq(context)); - - mockGetContainer(dockerImage, ContainerResources.from(0, 0, 16), true); - doNothing().when(healthChecker).verifyHealth(any()); - try { - nodeAgent.doConverge(context); - fail("Expected to fail due to warm up period not yet done"); - } catch (ConvergenceException e) { - assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage()); - } - inOrder.verify(orchestrator, never()).resume(any()); - inOrder.verify(orchestrator, never()).suspend(any()); - inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); - - - timer.advance(Duration.ofSeconds(31)); - nodeAgent.doConverge(context); - inOrder.verify(orchestrator, times(1)).resume(eq(hostName)); - } - - @Test - void resume_during_first_warmup() { - InOrder inOrder = inOrder(orchestrator, nodeRepository); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); - mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true); - - // Warmup period prevents resume when node has a current docker image, i.e., already existed. - nodeAgent.converge(createContext(nodeBuilder(NodeState.active).wantedDockerImage(dockerImage).currentDockerImage(dockerImage).build())); - inOrder.verifyNoMoreInteractions(); - - nodeAgent.converge(createContext(nodeBuilder(NodeState.active).wantedDockerImage(dockerImage).build())); - inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withDockerImage(dockerImage) - .withRebootGeneration(0) - .withVespaVersion(Version.fromString("7.1.1")))); - inOrder.verifyNoMoreInteractions(); - } - - - @Test - void drop_all_documents() { - InOrder inOrder = inOrder(orchestrator, nodeRepository); - BiFunction<NodeState, DropDocumentsReport, NodeSpec> specBuilder = (state, report) -> (report == null ? - nodeBuilder(state) : nodeBuilder(state).report(DropDocumentsReport.reportId(), report.toJsonNode())) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .build(); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); - - NodeAgentContext context = createContext(specBuilder.apply(NodeState.active, null)); - UnixPath indexPath = new UnixPath(context.paths().underVespaHome("var/db/vespa/search/cluster.foo/0/doc")).createParents().createNewFile(); - mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true); - assertTrue(indexPath.exists()); - - // Initially no changes, index is not dropped - nodeAgent.converge(context); - assertTrue(indexPath.exists()); - inOrder.verifyNoMoreInteractions(); - - context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, null, null, null))); - nodeAgent.converge(context); - verify(containerOperations).removeContainer(eq(context), any()); - assertFalse(indexPath.exists()); - inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, timer.currentTimeMillis(), null, null).toJsonNode()))); - inOrder.verifyNoMoreInteractions(); - - // After droppedAt and before readiedAt are set, we cannot proceed - mockGetContainer(null, false); - context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, null, null))); - nodeAgent.converge(context); - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(containerOperations, never()).startContainer(eq(context)); - inOrder.verifyNoMoreInteractions(); - - context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, 3L, null))); - nodeAgent.converge(context); - verify(containerOperations).startContainer(eq(context)); - inOrder.verifyNoMoreInteractions(); - - mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); - timer.advance(Duration.ofSeconds(31)); - nodeAgent.converge(context); - verify(containerOperations, times(1)).startContainer(eq(context)); - verify(containerOperations, never()).removeContainer(eq(context), any()); - inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() - .withRebootGeneration(0) - .withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, 2L, 3L, timer.currentTimeMillis()).toJsonNode()))); - inOrder.verifyNoMoreInteractions(); - } - - private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) { - NodeSpec.Builder nodeBuilder = nodeBuilder(nodeState) - .type(NodeType.tenant) - .flavor("docker") - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage); - - owner.ifPresent(nodeBuilder::owner); - NodeSpec node = nodeBuilder.build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - verify(containerOperations, never()).removeContainer(eq(context), any()); - if (owner.isPresent()) { - verify(containerOperations, never()).stopServices(eq(context)); - } else { - verify(containerOperations, times(1)).stopServices(eq(context)); - nodeAgent.doConverge(context); - // Should not be called more than once, have already been stopped - verify(containerOperations, times(1)).stopServices(eq(context)); - } - } - - private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) { - return makeNodeAgent(dockerImage, isRunning, Duration.ofSeconds(-1)); - } - - private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning, Duration warmUpDuration) { - mockGetContainer(dockerImage, isRunning); - doAnswer(invoc -> { - NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class); - ContainerResources resources = invoc.getArgument(1, ContainerResources.class); - mockGetContainer(context.node().wantedDockerImage().get(), resources, true); - return null; - }).when(containerOperations).createContainer(any(), any()); - - doAnswer(invoc -> { - NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class); - ContainerResources resources = invoc.getArgument(2, ContainerResources.class); - mockGetContainer(context.node().wantedDockerImage().get(), resources, true); - return null; - }).when(containerOperations).updateContainer(any(), any(), any()); - - return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, containerOperations, - () -> RegistryCredentials.none, storageMaintainer, flagSource, - List.of(credentialsMaintainer), Optional.of(aclMaintainer), Optional.of(healthChecker), - timer, warmUpDuration, VespaServiceDumper.DUMMY_INSTANCE, List.of()); - } - - private void mockGetContainer(DockerImage dockerImage, boolean isRunning) { - mockGetContainer(dockerImage, ContainerResources.from(0, resources.vcpu(), resources.memoryGb()), isRunning); - } - - private void mockGetContainer(DockerImage dockerImage, ContainerResources containerResources, boolean isRunning) { - doAnswer(invoc -> { - NodeAgentContext context = invoc.getArgument(0); - if (!hostName.equals(context.hostname().value())) - throw new IllegalArgumentException(); - return dockerImage != null ? - Optional.of(new Container( - containerId, - ContainerName.fromHostname(hostName), - timer.currentTime(), - isRunning ? Container.State.running : Container.State.exited, - "image-id-1", - dockerImage, - Map.of(), - 42, - 43, - hostName, - containerResources, - List.of(), - true)) : - Optional.empty(); - }).when(containerOperations).getContainer(any()); - } - - private NodeAgentContext createContext(NodeSpec nodeSpec) { - return NodeAgentContextImpl.builder(nodeSpec).fileSystem(fileSystem).build(); - } - - private NodeSpec.Builder nodeBuilder(NodeState state) { - return NodeSpec.Builder.testSpec(hostName, state).realResources(resources); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java deleted file mode 100644 index c45d9ab4b61..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author freva - */ -class UserNamespaceTest { - - private final UserNamespace userNamespace = new UserNamespace(1000, 2000, 10000); - - @Test - public void translates_between_ids() { - assertEquals(1001, userNamespace.userIdOnHost(1)); - assertEquals(2001, userNamespace.groupIdOnHost(1)); - assertEquals(1, userNamespace.userIdInContainer(1001)); - assertEquals(1, userNamespace.groupIdInContainer(2001)); - - assertEquals(userNamespace.overflowId(), userNamespace.userIdInContainer(1)); - assertEquals(userNamespace.overflowId(), userNamespace.userIdInContainer(999999)); - - assertThrows(IllegalArgumentException.class, () -> userNamespace.userIdOnHost(-1)); - assertThrows(IllegalArgumentException.class, () -> userNamespace.userIdOnHost(70_000)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java deleted file mode 100644 index eddc7edd597..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.provider; - -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class DebugHandlerHelperTest { - @Test - void trivial() { - DebugHandlerHelper helper = new DebugHandlerHelper(); - helper.addConstant("constant-key", "constant-value"); - - NodeAdminDebugHandler handler = () -> Map.of("handler-value-key", "handler-value-value"); - helper.addHandler("handler-key", handler); - - helper.addThreadSafeSupplier("supplier-key", () -> "supplier-value"); - - assertEquals("{" + - "supplier-key=supplier-value, " + - "handler-key={handler-value-key=handler-value-value}, " + - "constant-key=constant-value" + - "}", - helper.getDebugPage().toString()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java deleted file mode 100644 index 115969c5ded..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.logging.Logger; - -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -/** - * @author bjorncs - */ -public class DefaultEnvWriterTest { - - @TempDir - public File temporaryFolder; - - private static final Path EXAMPLE_FILE = Path.of("src/test/resources/default-env-example.txt"); - private static final Path EXPECTED_RESULT_FILE = Path.of("src/test/resources/default-env-rewritten.txt"); - - private final TaskContext context = mock(TaskContext.class); - - @Test - void default_env_is_correctly_rewritten() throws IOException { - Path tempFile = File.createTempFile("junit", null, temporaryFolder).toPath(); - Files.copy(EXAMPLE_FILE, tempFile, REPLACE_EXISTING); - - DefaultEnvWriter writer = new DefaultEnvWriter(); - writer.addOverride("VESPA_HOSTNAME", "my-new-hostname"); - writer.addFallback("VESPA_CONFIGSERVER", "new-fallback-configserver"); - writer.addOverride("VESPA_TLS_CONFIG_FILE", "/override/path/to/config.file"); - - boolean modified = writer.updateFile(context, tempFile); - - assertTrue(modified); - assertEquals(Files.readString(EXPECTED_RESULT_FILE), Files.readString(tempFile)); - verify(context, times(1)).log(any(Logger.class), any(String.class)); - - modified = writer.updateFile(context, tempFile); - assertFalse(modified); - assertEquals(Files.readString(EXPECTED_RESULT_FILE), Files.readString(tempFile)); - verify(context, times(1)).log(any(Logger.class), any(String.class)); - } - - @Test - void generates_default_env_content() throws IOException { - DefaultEnvWriter writer = new DefaultEnvWriter(); - writer.addOverride("VESPA_HOSTNAME", "my-new-hostname"); - writer.addFallback("VESPA_CONFIGSERVER", "new-fallback-configserver"); - writer.addOverride("VESPA_TLS_CONFIG_FILE", "/override/path/to/config.file"); - writer.addUnset("VESPA_LEGACY_OPTION"); - String generatedContent = writer.generateContent(); - assertEquals(Files.readString(EXPECTED_RESULT_FILE), generatedContent); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java deleted file mode 100644 index 76676739613..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import org.junit.jupiter.api.Test; - -import java.util.Optional; -import java.util.regex.Pattern; - -import static org.junit.jupiter.api.Assertions.*; - -public class StringEditorTest { - private final StringEditor editor = new StringEditor(); - private final Cursor cursor = editor.cursor(); - - @Test - void testBasics() { - assertCursor(0, 0, ""); - - cursor.write("hello"); - assertCursor(0, 5, "hello"); - - cursor.write("one\ntwo"); - assertCursor(1, 3, "helloone\ntwo"); - - cursor.deleteAll(); - assertCursor(0, 0, ""); - - cursor.moveForward(); - assertCursor(0, 0, ""); - - cursor.writeLine("foo"); - assertCursor(1, 0, "foo\n"); - - cursor.writeLines("one", "two"); - assertCursor(3, 0, "foo\none\ntwo\n"); - - cursor.deleteBackward(); - assertCursor(2, 3, "foo\none\ntwo"); - - cursor.deleteBackward(2); - assertCursor(2, 1, "foo\none\nt"); - - Mark mark = cursor.createMark(); - - cursor.moveToStartOfPreviousLine().moveBackward(2); - assertCursor(0, 2, "foo\none\nt"); - - assertEquals("o\none\nt", cursor.getTextTo(mark)); - - cursor.deleteTo(mark); - assertCursor(0, 2, "fo"); - - cursor.deleteBackward(2); - assertCursor(0, 0, ""); - - cursor.writeLines("one", "two", "three").moveToStartOfBuffer(); - assertCursor(0, 0, "one\ntwo\nthree\n"); - - Pattern pattern = Pattern.compile("t(.)"); - Optional<Match> match = cursor.moveForwardToEndOfMatch(pattern); - assertCursor(1, 2, "one\ntwo\nthree\n"); - assertTrue(match.isPresent()); - assertEquals("tw", match.get().match()); - assertEquals("", match.get().prefix()); - assertEquals("o", match.get().suffix()); - assertEquals(new Position(1, 0), match.get().startOfMatch()); - assertEquals(new Position(1, 2), match.get().endOfMatch()); - assertEquals(1, match.get().groupCount()); - assertEquals("w", match.get().group(1)); - - match = cursor.moveForwardToEndOfMatch(pattern); - assertCursor(2, 2, "one\ntwo\nthree\n"); - assertTrue(match.isPresent()); - assertEquals("th", match.get().match()); - assertEquals(1, match.get().groupCount()); - assertEquals("h", match.get().group(1)); - - match = cursor.moveForwardToEndOfMatch(pattern); - assertCursor(2, 2, "one\ntwo\nthree\n"); - assertFalse(match.isPresent()); - - assertTrue(cursor.skipBackward("h")); - assertCursor(2, 1, "one\ntwo\nthree\n"); - assertFalse(cursor.skipBackward("x")); - - assertTrue(cursor.skipForward("hre")); - assertCursor(2, 4, "one\ntwo\nthree\n"); - assertFalse(cursor.skipForward("x")); - - try { - cursor.moveTo(mark); - fail(); - } catch (IllegalArgumentException e) { - // expected - } - - mark = cursor.createMark(); - cursor.moveToStartOfBuffer(); - assertEquals(new Position(0, 0), cursor.getPosition()); - cursor.moveTo(mark); - assertEquals(new Position(2, 4), cursor.getPosition()); - - cursor.moveTo(1, 2); - assertCursor(1, 2, "one\ntwo\nthree\n"); - - cursor.deleteSuffix(); - assertCursor(1, 2, "one\ntw\nthree\n"); - - cursor.deletePrefix(); - assertCursor(1, 0, "one\n\nthree\n"); - - cursor.deleteLine(); - assertCursor(1, 0, "one\nthree\n"); - - cursor.deleteLine(); - assertCursor(1, 0, "one\n"); - - cursor.deleteLine(); - assertCursor(1, 0, "one\n"); - - cursor.moveToStartOfBuffer().moveForward().writeNewlineAfter(); - assertCursor(0, 1, "o\nne\n"); - - cursor.deleteAll().writeLines("one", "two", "three", "four"); - cursor.moveToStartOfBuffer().moveToStartOfNextLine(); - assertCursor(1, 0, "one\ntwo\nthree\nfour\n"); - Pattern pattern2 = Pattern.compile("(o)(.)?"); - int count = cursor.replaceMatches(pattern2, m -> { - String prefix = m.group(2) == null ? "" : m.group(2); - return prefix + m.match() + m.group(1); - }); - assertCursor(3, 5, "one\ntwoo\nthree\nfuouor\n"); - assertEquals(2, count); - - cursor.moveToStartOfBuffer().moveToEndOfLine(); - Pattern pattern3 = Pattern.compile("o"); - count = cursor.replaceMatches(pattern3, m -> "a"); - assertEquals(4, count); - assertCursor(3, 5, "one\ntwaa\nthree\nfuauar\n"); - } - - private void assertCursor(int lineIndex, int columnIndex, String text) { - assertEquals(text, cursor.getBufferText()); - assertEquals(new Position(lineIndex, columnIndex), cursor.getPosition()); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java deleted file mode 100644 index 15fb36dc3d5..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class TextBufferImplTest { - private final TextBufferImpl textBuffer = new TextBufferImpl(); - - @Test - void testWrite() { - assertEquals("", textBuffer.getString()); - assertWrite(2, 0, "foo\nbar\n", - 0, 0, "foo\nbar\n"); - - assertWrite(1, 6, "fofirst\nsecondo\nbar\n", - 0, 2, "first\nsecond"); - - assertWrite(3, 1, "fofirst\nsecondo\nbar\na", - 3, 0, "a"); - assertWrite(4, 0, "fofirst\nsecondo\nbar\na\n", - 3, 1, "\n"); - } - - @Test - void testDelete() { - write(0, 0, "foo\nbar\nzoo\n"); - delete(0, 2, 2, 1); - assertEquals("fooo\n", textBuffer.getString()); - - delete(0, 4, 1, 0); - assertEquals("fooo", textBuffer.getString()); - - delete(0, 0, 0, 4); - assertEquals("", textBuffer.getString()); - - delete(0, 0, 0, 0); - assertEquals("", textBuffer.getString()); - } - - private void assertWrite(int expectedLineIndex, int expectedColumnIndex, String expectedString, - int lineIndex, int columnIndex, String text) { - Position position = write(lineIndex, columnIndex, text); - assertEquals(new Position(expectedLineIndex, expectedColumnIndex), position); - assertEquals(expectedString, textBuffer.getString()); - } - - private Position write(int lineIndex, int columnIndex, String text) { - return textBuffer.write(new Position(lineIndex, columnIndex), text); - } - - private void delete(int startLineIndex, int startColumnIndex, - int endLineIndex, int endColumnIndex) { - textBuffer.delete(new Position(startLineIndex, startColumnIndex), - new Position(endLineIndex, endColumnIndex)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java deleted file mode 100644 index 507bf706484..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class DiskSizeTest { - - @Test - void bytes_to_display_count_test() { - assertEquals("-1 bytes", DiskSize.of(-1).asString()); - assertEquals("123 bytes", DiskSize.of(123).asString()); - assertEquals("1 kB", DiskSize.of(1_000).asString()); - assertEquals("15 MB", DiskSize.of(15_000_000).asString()); - assertEquals("123 GB", DiskSize.of(123_456_789_012L).asString()); - assertEquals("988 TB", DiskSize.of(987_654_321_098_765L).asString()); - assertEquals("987.7 TB", DiskSize.of(987_654_321_098_765L).asString(1)); - assertEquals("987.65 TB", DiskSize.of(987_654_321_098_765L).asString(2)); - assertEquals("2 PB", DiskSize.of(2_000_000_000_000_000L).asString()); - assertEquals("9 EB", DiskSize.of(Long.MAX_VALUE).asString()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java deleted file mode 100644 index 9a651494854..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -import java.nio.file.FileSystem; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class EditorTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final UnixPath path = new UnixPath(fileSystem.getPath("/file")); - - @Test - void testEdit() { - path.writeUtf8File(joinLines("first", "second", "third")); - - LineEditor lineEditor = mock(LineEditor.class); - when(lineEditor.edit(any())).thenReturn( - LineEdit.none(), // don't edit the first line - LineEdit.remove(), // remove the second - LineEdit.replaceWith("replacement")); // replace the third - - Editor editor = new Editor(path.toPath(), lineEditor); - TaskContext context = mock(TaskContext.class); - - assertTrue(editor.converge(context)); - - verify(lineEditor, times(3)).edit(any()); - - // Verify the system modification message - ArgumentCaptor<String> modificationMessage = ArgumentCaptor.forClass(String.class); - verify(context).recordSystemModification(any(), modificationMessage.capture()); - assertEquals( - "Patching file /file:\n-second\n-third\n+replacement\n", - modificationMessage.getValue()); - - // Verify the new contents of the file: - assertEquals(joinLines("first", "replacement"), path.readUtf8File()); - } - - @Test - void testInsert() { - path.writeUtf8File(joinLines("second", "eight", "fifth", "seventh")); - - LineEditor lineEditor = mock(LineEditor.class); - when(lineEditor.edit(any())).thenReturn( - LineEdit.insertBefore("first"), // insert first, and keep the second line - LineEdit.replaceWith("third", "fourth"), // remove eight, and replace with third and fourth instead - LineEdit.none(), // Keep fifth - LineEdit.insert(List.of("sixth"), // insert sixth before seventh - List.of("eight"))); // add eight after seventh - - Editor editor = new Editor(path.toPath(), lineEditor); - TaskContext context = mock(TaskContext.class); - - assertTrue(editor.converge(context)); - - // Verify the system modification message - ArgumentCaptor<String> modificationMessage = ArgumentCaptor.forClass(String.class); - verify(context).recordSystemModification(any(), modificationMessage.capture()); - assertEquals( - "Patching file /file:\n" + - "+first\n" + - "-eight\n" + - "+third\n" + - "+fourth\n" + - "+sixth\n" + - "+eight\n", - modificationMessage.getValue()); - - // Verify the new contents of the file: - assertEquals(joinLines("first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eight"), - path.readUtf8File()); - } - - @Test - void noop() { - path.writeUtf8File("line\n"); - - LineEditor lineEditor = mock(LineEditor.class); - when(lineEditor.edit(any())).thenReturn(LineEdit.none()); - - Editor editor = new Editor(path.toPath(), lineEditor); - TaskContext context = mock(TaskContext.class); - - assertFalse(editor.converge(context)); - - verify(lineEditor, times(1)).edit(any()); - - // Verify the system modification message - verify(context, times(0)).recordSystemModification(any(), any()); - - // Verify same contents - assertEquals("line\n", path.readUtf8File()); - } - - @Test - void testMissingFile() { - LineEditor lineEditor = mock(LineEditor.class); - when(lineEditor.onComplete()).thenReturn(List.of("line")); - - TaskContext context = mock(TaskContext.class); - var editor = new Editor(path.toPath(), lineEditor); - editor.converge(context); - - assertEquals("line\n", path.readUtf8File()); - } - - private static String joinLines(String... lines) { - return String.join("\n", lines) + "\n"; - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java deleted file mode 100644 index 8559e36fe8b..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -public class FileAttributesCacheTest { - @Test - void exists() { - UnixPath unixPath = mock(UnixPath.class); - FileAttributesCache cache = new FileAttributesCache(unixPath); - - when(unixPath.getAttributesIfExists()).thenReturn(Optional.empty()); - assertFalse(cache.get().isPresent()); - verify(unixPath, times(1)).getAttributesIfExists(); - verifyNoMoreInteractions(unixPath); - - FileAttributes attributes = new FileAttributes(Instant.EPOCH, 0, 0, "", false, false, 0, 0, 0); - when(unixPath.getAttributesIfExists()).thenReturn(Optional.of(attributes)); - when(unixPath.getAttributesIfExists()).thenReturn(Optional.of(attributes)); - assertTrue(cache.get().isPresent()); - verify(unixPath, times(1 + 1)).getAttributesIfExists(); - verifyNoMoreInteractions(unixPath); - - assertEquals(attributes, cache.getOrThrow()); - verifyNoMoreInteractions(unixPath); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java deleted file mode 100644 index ed183738ef0..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import org.junit.jupiter.api.Test; - -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileAttributes.deviceMajor; -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileAttributes.deviceMinor; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -class FileAttributesTest { - - @Test - void parse_dev_t() { - assertEquals(0x12345BCD, deviceMajor(0x1234567890ABCDEFL)); - assertEquals(0x67890AEF, deviceMinor(0x1234567890ABCDEFL)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java deleted file mode 100644 index e1cea37ccbc..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.time.Instant; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -public class FileContentCacheTest { - private final UnixPath unixPath = mock(UnixPath.class); - private final FileContentCache cache = new FileContentCache(unixPath); - - private final byte[] content = "content".getBytes(StandardCharsets.UTF_8); - private final byte[] newContent = "new-content".getBytes(StandardCharsets.UTF_8); - - @Test - void get() { - when(unixPath.readBytes()).thenReturn(content); - assertArrayEquals(content, cache.get(Instant.ofEpochMilli(0))); - verify(unixPath, times(1)).readBytes(); - verifyNoMoreInteractions(unixPath); - - // cache hit - assertArrayEquals(content, cache.get(Instant.ofEpochMilli(0))); - verify(unixPath, times(1)).readBytes(); - verifyNoMoreInteractions(unixPath); - - // cache miss - when(unixPath.readBytes()).thenReturn(newContent); - assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(1))); - verify(unixPath, times(1 + 1)).readBytes(); - verifyNoMoreInteractions(unixPath); - - // cache hit both at times 0 and 1 - assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(0))); - verify(unixPath, times(1 + 1)).readBytes(); - verifyNoMoreInteractions(unixPath); - assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(1))); - verify(unixPath, times(1 + 1)).readBytes(); - verifyNoMoreInteractions(unixPath); - } - - @Test - void updateWith() { - cache.updateWith(content, Instant.ofEpochMilli(2)); - assertArrayEquals(content, cache.get(Instant.ofEpochMilli(2))); - verifyNoMoreInteractions(unixPath); - - cache.updateWith(newContent, Instant.ofEpochMilli(4)); - assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(4))); - verifyNoMoreInteractions(unixPath); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java deleted file mode 100644 index f7fb66fca94..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -public class FileDeleterTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final UnixPath path = new UnixPath(fileSystem.getPath("/tmp/foo")); - private final FileDeleter deleter = new FileDeleter(path.toPath()); - private final TaskContext context = mock(TaskContext.class); - - @Test - void deleteExisting() { - assertFalse(deleter.converge(context)); - path.createParents().writeUtf8File("bar"); - assertTrue(deleter.converge(context)); - assertFalse(deleter.converge(context)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java deleted file mode 100644 index 76941d3333b..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileTime; -import java.time.Duration; -import java.time.Instant; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Set.of; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - - -/** - * @author freva - */ - -public class FileFinderTest { - - @Nested - public class GeneralLogicTests { - - private final FileSystem fileSystem = TestFileSystem.create(); - - @Test - void all_files_non_recursive() { - assertFileHelper(FileFinder.files(testRoot()) - .maxDepth(1), - - of("file-1.json", "test.json", "test.txt"), - of("test", "test/file.txt", "test/data.json", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); - } - - @Test - void all_files_recursive() { - assertFileHelper(FileFinder.files(testRoot()), - - of("file-1.json", "test.json", "test.txt", "test/file.txt", "test/data.json", "test/subdir-1/test"), - of("test", "test/subdir-1", "test/subdir-2")); - } - - @Test - void all_files_recursive_with_prune_relative() { - assertFileHelper(FileFinder.files(testRoot()).prune(fileSystem.getPath("test")), - - of("file-1.json", "test.json", "test.txt"), - of("test", "test/file.txt", "test/data.json", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); - } - - @Test - void all_files_recursive_with_prune_absolute() { - assertFileHelper(FileFinder.files(testRoot()).prune(testRoot().resolve("test/subdir-1")), - - of("file-1.json", "test.json", "test.txt", "test/file.txt", "test/data.json"), - of("test", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); - } - - @Test - void throws_if_prune_path_not_under_base_path() { - assertThrows(IllegalArgumentException.class, () -> { - FileFinder.files(Path.of("/some/path")).prune(Path.of("/other/path")); - }); - } - - @Test - void with_file_filter_recursive() { - assertFileHelper(FileFinder.files(testRoot()) - .match(FileFinder.nameEndsWith(".json")), - - of("file-1.json", "test.json", "test/data.json"), - of("test.txt", "test", "test/file.txt", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); - } - - @Test - void all_files_limited_depth() { - assertFileHelper(FileFinder.files(testRoot()) - .maxDepth(2), - - of("test.txt", "file-1.json", "test.json", "test/file.txt", "test/data.json"), - of("test", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); - } - - @Test - void directory_with_filter() { - assertFileHelper(FileFinder.directories(testRoot()) - .match(FileFinder.nameStartsWith("subdir")) - .maxDepth(2), - - of("test/subdir-1", "test/subdir-2"), - of("file-1.json", "test.json", "test.txt", "test", "test/file.txt", "test/data.json")); - } - - @Test - void match_file_and_directory_with_same_name() { - assertFileHelper(FileFinder.from(testRoot()) - .match(FileFinder.nameEndsWith("test")), - - of("test", "test/subdir-1/test"), - of("file-1.json", "test.json", "test.txt")); - } - - @Test - void all_contents() { - assertFileHelper(FileFinder.from(testRoot()) - .maxDepth(1), - - of("file-1.json", "test.json", "test.txt", "test"), - of()); - - assertTrue(Files.exists(testRoot())); - } - - @BeforeEach - public void setup() throws IOException { - Path root = testRoot(); - Files.createDirectories(root); - - Files.createFile(root.resolve("file-1.json")); - Files.createFile(root.resolve("test.json")); - Files.createFile(root.resolve("test.txt")); - - Files.createDirectories(root.resolve("test")); - Files.createFile(root.resolve("test/file.txt")); - Files.createFile(root.resolve("test/data.json")); - - Files.createDirectories(root.resolve("test/subdir-1")); - Files.createFile(root.resolve("test/subdir-1/test")); - - Files.createDirectories(root.resolve("test/subdir-2")); - } - - private Path testRoot() { - return fileSystem.getPath("/file-finder"); - } - - private void assertFileHelper(FileFinder fileFinder, Set<String> expectedList, Set<String> expectedContentsAfterDelete) { - Set<String> actualList = fileFinder.stream() - .map(FileFinder.FileAttributes::path) - .map(testRoot()::relativize) - .map(Path::toString) - .collect(Collectors.toSet()); - assertEquals(expectedList, actualList); - - fileFinder.deleteRecursively(mock(TaskContext.class)); - Set<String> actualContentsAfterDelete = recursivelyListContents(testRoot()).stream() - .map(testRoot()::relativize) - .map(Path::toString) - .collect(Collectors.toSet()); - assertEquals(expectedContentsAfterDelete, actualContentsAfterDelete); - } - - private List<Path> recursivelyListContents(Path basePath) { - try (Stream<Path> pathStream = Files.list(basePath)) { - List<Path> paths = new LinkedList<>(); - pathStream.forEach(path -> { - paths.add(path); - if (Files.isDirectory(path)) - paths.addAll(recursivelyListContents(path)); - }); - return paths; - } catch (NoSuchFileException e) { - return List.of(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - } - - @Nested - public class FilterUnitTests { - - private final BasicFileAttributes attributes = mock(BasicFileAttributes.class); - - @Test - void age_filter_test() { - Path path = Path.of("/my/fake/path"); - when(attributes.lastModifiedTime()).thenReturn(FileTime.from(Instant.now().minus(Duration.ofHours(1)))); - FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes); - - assertFalse(FileFinder.olderThan(Duration.ofMinutes(61)).test(fileAttributes)); - assertTrue(FileFinder.olderThan(Duration.ofMinutes(59)).test(fileAttributes)); - - assertTrue(FileFinder.youngerThan(Duration.ofMinutes(61)).test(fileAttributes)); - assertFalse(FileFinder.youngerThan(Duration.ofMinutes(59)).test(fileAttributes)); - } - - @Test - void size_filters() { - Path path = Path.of("/my/fake/path"); - when(attributes.size()).thenReturn(100L); - FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes); - - assertFalse(FileFinder.largerThan(101).test(fileAttributes)); - assertTrue(FileFinder.largerThan(99).test(fileAttributes)); - - assertTrue(FileFinder.smallerThan(101).test(fileAttributes)); - assertFalse(FileFinder.smallerThan(99).test(fileAttributes)); - } - - @Test - void filename_filters() { - Path path = Path.of("/my/fake/path/some-12352-file.json"); - FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes); - - assertTrue(FileFinder.nameStartsWith("some-").test(fileAttributes)); - assertFalse(FileFinder.nameStartsWith("som-").test(fileAttributes)); - - assertTrue(FileFinder.nameEndsWith(".json").test(fileAttributes)); - assertFalse(FileFinder.nameEndsWith("file").test(fileAttributes)); - - assertTrue(FileFinder.nameMatches(Pattern.compile("some-[0-9]+-file.json")).test(fileAttributes)); - assertTrue(FileFinder.nameMatches(Pattern.compile("^some-[0-9]+-file.json$")).test(fileAttributes)); - assertFalse(FileFinder.nameMatches(Pattern.compile("some-[0-9]-file.json")).test(fileAttributes)); - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java deleted file mode 100644 index e418833ab50..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.FileSystem; -import java.nio.file.NoSuchFileException; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.mock; - -/** - * @author hakonhall - */ -class FileMoverTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final TaskContext context = mock(TaskContext.class); - private final UnixPath source = new UnixPath(fileSystem.getPath("/from/source")); - private final UnixPath destination = new UnixPath(fileSystem.getPath("/to/destination")); - private final FileMover mover = new FileMover(source.toPath(), destination.toPath()); - - @Test - void movingRegularFile() { - assertConvergeThrows(() -> mover.converge(context), NoSuchFileException.class, "/from/source"); - - source.createParents().writeUtf8File("content"); - assertConvergeThrows(() -> mover.converge(context), NoSuchFileException.class, "/to/destination"); - - destination.createParents(); - assertTrue(mover.converge(context)); - assertFalse(source.exists()); - assertTrue(destination.exists()); - assertEquals("content", destination.readUtf8File()); - - assertFalse(mover.converge(context)); - - source.writeUtf8File("content 2"); - assertConvergeThrows(() -> mover.converge(context), FileAlreadyExistsException.class, "/to/destination"); - - mover.replaceExisting(); - assertTrue(mover.converge(context)); - - source.writeUtf8File("content 3"); - destination.deleteIfExists(); - destination.createDirectory(); - assertTrue(mover.converge(context)); - } - - private void assertConvergeThrows(Runnable runnable, Class<?> expectedRootExceptionClass, String expectedMessage) { - try { - runnable.run(); - fail(); - } catch (Throwable t) { - Throwable rootCause = t; - do { - Throwable cause = rootCause.getCause(); - if (cause == null) break; - rootCause = cause; - } while (true); - - assertTrue(expectedRootExceptionClass.isInstance(rootCause), "Unexpected root cause: " + rootCause); - assertEquals(expectedMessage, rootCause.getMessage()); - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java deleted file mode 100644 index b0992e9826a..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author hakonhall - */ -public class FileSnapshotTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final UnixPath path = new UnixPath(fileSystem.getPath("/var/lib/file.txt")); - - private FileSnapshot fileSnapshot = FileSnapshot.forPath(path.toPath()); - - @Test - void fileDoesNotExist() { - assertFalse(fileSnapshot.exists()); - assertFalse(fileSnapshot.attributes().isPresent()); - assertFalse(fileSnapshot.content().isPresent()); - assertEquals(path.toPath(), fileSnapshot.path()); - } - - @Test - void directory() { - path.createParents().createDirectory(); - fileSnapshot = fileSnapshot.snapshot(); - assertTrue(fileSnapshot.exists()); - assertTrue(fileSnapshot.attributes().isPresent()); - assertTrue(fileSnapshot.attributes().get().isDirectory()); - } - - @Test - void regularFile() { - path.createParents().writeUtf8File("file content"); - fileSnapshot = fileSnapshot.snapshot(); - assertTrue(fileSnapshot.exists()); - assertTrue(fileSnapshot.attributes().isPresent()); - assertTrue(fileSnapshot.attributes().get().isRegularFile()); - assertTrue(fileSnapshot.utf8Content().isPresent()); - assertEquals("file content", fileSnapshot.utf8Content().get()); - - FileSnapshot newFileSnapshot = fileSnapshot.snapshot(); - assertSame(fileSnapshot, newFileSnapshot); - } - - @Test - void fileRemoval() { - path.createParents().writeUtf8File("file content"); - fileSnapshot = fileSnapshot.snapshot(); - assertTrue(fileSnapshot.exists()); - path.deleteIfExists(); - fileSnapshot = fileSnapshot.snapshot(); - assertFalse(fileSnapshot.exists()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java deleted file mode 100644 index c60de78bf8c..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -public class FileSyncTest { - private final TestTaskContext taskContext = new TestTaskContext(); - private final FileSystem fileSystem = TestFileSystem.create(); - - private final Path path = fileSystem.getPath("/dir/file.txt"); - private final UnixPath unixPath = new UnixPath(path); - private final FileSync fileSync = new FileSync(path); - - private String content = "content"; - private int ownerId = 123; // default is 1 - private int groupId = 456; // default is 2 - private String permissions = "rw-r-xr--"; - - @Test - void trivial() { - assertConvergence("Creating file /dir/file.txt with permissions rw-r-xr--", - "Changing user ID of /dir/file.txt from 1 to 123", - "Changing group ID of /dir/file.txt from 2 to 456"); - - content = "new-content"; - assertConvergence("Patching file /dir/file.txt"); - - ownerId = 124; - assertConvergence("Changing user ID of /dir/file.txt from 123 to 124"); - - groupId = 457; - assertConvergence("Changing group ID of /dir/file.txt from 456 to 457"); - - permissions = "rwxr--rwx"; - assertConvergence("Changing permissions of /dir/file.txt from rw-r-xr-- to " + - permissions); - } - - private void assertConvergence(String... systemModificationMessages) { - PartialFileData fileData = PartialFileData.builder() - .withContent(content) - .withOwnerId(ownerId) - .withGroupId(groupId) - .withPermissions(permissions) - .create(); - taskContext.clearSystemModificationLog(); - assertTrue(fileSync.convergeTo(taskContext, fileData)); - - assertTrue(Files.isRegularFile(path)); - fileData.getContent().ifPresent(content -> assertArrayEquals(content, unixPath.readBytes())); - fileData.getOwnerId().ifPresent(owner -> assertEquals((int) owner, unixPath.getOwnerId())); - fileData.getGroupId().ifPresent(group -> assertEquals((int) group, unixPath.getGroupId())); - fileData.getPermissions().ifPresent(permissions -> assertEquals(permissions, unixPath.getPermissions())); - - List<String> actualMods = taskContext.getSystemModificationLog(); - List<String> expectedMods = List.of(systemModificationMessages); - assertEquals(expectedMods, actualMods); - - UnixPath unixPath = new UnixPath(path); - Instant lastModifiedTime = unixPath.getLastModifiedTime(); - taskContext.clearSystemModificationLog(); - assertFalse(fileSync.convergeTo(taskContext, fileData)); - assertEquals(lastModifiedTime, unixPath.getLastModifiedTime()); - - actualMods = taskContext.getSystemModificationLog(); - assertEquals(new ArrayList<>(), actualMods); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java deleted file mode 100644 index 1264206bef3..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.test.file.TestFileSystem; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.time.Instant; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -public class FileWriterTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final TaskContext context = mock(TaskContext.class); - - @Test - void testWrite() { - final String content = "content"; - final String permissions = "rwxr-xr-x"; - final int owner = 123; - final int group = 456; - - Path path = fileSystem.getPath("/opt/vespa/tmp/file.txt"); - FileWriter writer = new FileWriter(path, () -> content) - .withPermissions(permissions) - .withOwnerId(owner) - .withGroupId(group) - .onlyIfFileDoesNotAlreadyExist(); - assertTrue(writer.converge(context)); - verify(context, times(1)).recordSystemModification(any(), eq("Creating file " + path + " with permissions rwxr-xr-x")); - - UnixPath unixPath = new UnixPath(path); - assertEquals(content, unixPath.readUtf8File()); - assertEquals(permissions, unixPath.getPermissions()); - assertEquals(owner, unixPath.getOwnerId()); - assertEquals(group, unixPath.getGroupId()); - Instant fileTime = unixPath.getLastModifiedTime(); - - // Second time is a no-op. - assertFalse(writer.converge(context)); - assertEquals(fileTime, unixPath.getLastModifiedTime()); - } - - @Test - void testAtomicWrite() { - FileWriter writer = new FileWriter(fileSystem.getPath("/foo/bar")) - .atomicWrite(true); - - assertTrue(writer.converge(context, "content")); - - verify(context).recordSystemModification(any(), eq("Creating file /foo/bar")); - assertEquals("content", new UnixPath(writer.path()).readUtf8File()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java deleted file mode 100644 index 11675bbe46f..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.UncheckedIOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author hakonhall - */ -public class MakeDirectoryTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final TestTaskContext context = new TestTaskContext(); - - private final String path = "/parent/dir"; - private String permissions = "rwxr----x"; - private int ownerId = 123; - private int groupId = 456; - - @Test - void newDirectory() { - verifySystemModifications( - "Creating directory " + path, - "Changing user ID of /parent/dir from 1 to 123", - "Changing group ID of /parent/dir from 2 to 456"); - - ownerId = 124; - verifySystemModifications("Changing user ID of /parent/dir from 123 to 124"); - - groupId = 457; - verifySystemModifications("Changing group ID of /parent/dir from 456 to 457"); - - permissions = "--x---r--"; - verifySystemModifications("Changing permissions of /parent/dir from rwxr----x to --x---r--"); - } - - private void verifySystemModifications(String... modifications) { - context.clearSystemModificationLog(); - MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path)) - .createParents() - .withPermissions(permissions) - .withOwnerId(ownerId) - .withGroupId(groupId); - assertTrue(makeDirectory.converge(context)); - - assertEquals(List.of(modifications), context.getSystemModificationLog()); - - context.clearSystemModificationLog(); - assertFalse(makeDirectory.converge(context)); - assertEquals(List.of(), context.getSystemModificationLog()); - } - - @Test - void exceptionIfMissingParent() { - String path = "/parent/dir"; - MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path)); - - try { - makeDirectory.converge(context); - } catch (UncheckedIOException e) { - if (e.getCause() instanceof NoSuchFileException) { - return; - } - throw e; - } - fail(); - } - - @Test - void okIfParentExists() { - String path = "/dir"; - MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path)); - assertTrue(makeDirectory.converge(context)); - assertTrue(Files.isDirectory(fileSystem.getPath(path))); - - MakeDirectory makeDirectory2 = new MakeDirectory(fileSystem.getPath(path)); - assertFalse(makeDirectory2.converge(context)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java deleted file mode 100644 index 79fa1cf6ea2..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -/** - * @author hakonhall - */ -public class StoredBooleanTest { - private final TaskContext context = mock(TaskContext.class); - private final FileSystem fileSystem = TestFileSystem.create(); - private final Path path = fileSystem.getPath("/foo"); - private final StoredBoolean storedBoolean = new StoredBoolean(path); - - @Test - void storedBoolean() { - assertFalse(storedBoolean.value()); - storedBoolean.set(context); - assertTrue(storedBoolean.value()); - storedBoolean.clear(context); - assertFalse(storedBoolean.value()); - } - - @Test - void testCompatibility() throws IOException { - StoredInteger storedInteger = new StoredInteger(path); - assertFalse(storedBoolean.value()); - - storedInteger.write(context, 1); - assertTrue(storedBoolean.value()); - - storedInteger.write(context, 2); - assertTrue(storedBoolean.value()); - - storedInteger.write(context, 0); - assertFalse(storedBoolean.value()); - - Files.delete(path); - assertFalse(storedBoolean.value()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java deleted file mode 100644 index d9dfcefc7e3..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -public class TemplateTest { - - @Test - void basic() { - FileSystem fileSystem = TestFileSystem.create(); - Path templatePath = fileSystem.getPath("/example.vm"); - String templateContent = "a $x, $y b"; - new UnixPath(templatePath).writeUtf8File(templateContent); - - Path toPath = fileSystem.getPath("/example"); - TaskContext taskContext = mock(TaskContext.class); - boolean converged = Template.at(templatePath) - .set("x", "foo") - .set("y", "bar") - .getFileWriterTo(toPath) - .converge(taskContext); - - assertTrue(converged); - - String actualContent = new UnixPath(toPath).readUtf8File(); - assertEquals("a foo, bar b", actualContent); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java deleted file mode 100644 index 5892a9b9f53..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * @author hakonhall - */ -public class UnixPathTest { - - private final FileSystem fs = TestFileSystem.create(); - - @Test - void createParents() { - Path parentDirectory = fs.getPath("/a/b/c"); - Path filePath = parentDirectory.resolve("bar"); - UnixPath path = new UnixPath(filePath); - - assertFalse(Files.exists(fs.getPath("/a"))); - path.createParents(); - assertTrue(Files.exists(parentDirectory)); - } - - @Test - void utf8File() { - String original = "foo\nbar\n"; - UnixPath path = new UnixPath(fs.getPath("example.txt")); - path.writeUtf8File(original); - String fromFile = path.readUtf8File(); - assertEquals(original, fromFile); - assertEquals(List.of("foo", "bar"), path.readLines()); - } - - @Test - void touch() { - UnixPath path = new UnixPath(fs.getPath("example.txt")); - assertTrue(path.create()); - assertEquals("", path.readUtf8File()); - assertFalse(path.create()); - } - - @Test - void permissions() { - String expectedPermissions = "rwxr-x---"; - UnixPath path = new UnixPath(fs.getPath("file.txt")); - path.writeUtf8File("foo"); - path.setPermissions(expectedPermissions); - assertEquals(expectedPermissions, path.getPermissions()); - } - - @Test - void badPermissionsString() { - assertThrows(IllegalArgumentException.class, () -> { - new UnixPath(fs.getPath("file.txt")).setPermissions("abcdefghi"); - }); - } - - @Test - void owner() { - Path path = fs.getPath("file.txt"); - UnixPath unixPath = new UnixPath(path); - unixPath.writeUtf8File("foo"); - - unixPath.setOwnerId(123); - assertEquals(123, unixPath.getOwnerId()); - - unixPath.setGroupId(456); - assertEquals(456, unixPath.getGroupId()); - } - - @Test - void createDirectoryWithPermissions() { - Path path = fs.getPath("dir"); - UnixPath unixPath = new UnixPath(path); - String permissions = "rwxr-xr--"; - assertTrue(unixPath.createDirectory(permissions)); - assertTrue(unixPath.isDirectory()); - assertEquals(permissions, unixPath.getPermissions()); - assertFalse(unixPath.createDirectory(permissions)); - } - - @Test - void createSymbolicLink() { - String original = "foo\nbar\n"; - UnixPath path = new UnixPath(fs.getPath("example.txt")); - path.writeUtf8File(original); - String fromFile = path.readUtf8File(); - assertEquals(original, fromFile); - - UnixPath link = path.createSymbolicLink(fs.getPath("link-to-example.txt")); - assertEquals(original, link.readUtf8File()); - } - - @Test - void readBytesIfExists() { - UnixPath path = new UnixPath(fs.getPath("example.txt")); - assertFalse(path.readBytesIfExists().isPresent()); - path.writeBytes(new byte[]{42}); - assertArrayEquals(new byte[]{42}, path.readBytesIfExists().get()); - } - - @Test - void deleteRecursively() throws Exception { - // Create the following file tree: - // - // /dir1 - // |--- dir2 - // |--- file1 - // /link1 -> /dir1/dir2 - // - var dir1 = fs.getPath("/dir1"); - var dir2 = dir1.resolve("dir2"); - var file1 = dir2.resolve("file1"); - Files.createDirectories(dir2); - Files.writeString(file1, "file1"); - var link1 = Files.createSymbolicLink(fs.getPath("/link1"), dir2); - - new UnixPath(link1).deleteRecursively(); - assertTrue(Files.exists(dir2), "Deleting " + link1 + " recursively does not remove " + dir2); - assertTrue(Files.exists(file1), "Deleting " + link1 + " recursively does not remove " + file1); - - new UnixPath(dir1).deleteRecursively(); - assertFalse(Files.exists(file1), dir1 + " deleted recursively"); - assertFalse(Files.exists(dir2), dir1 + " deleted recursively"); - assertFalse(Files.exists(dir1), dir1 + " deleted recursively"); - } - - @Test - void isEmptyDirectory() { - var path = new UnixPath((fs.getPath("/foo"))); - assertFalse(path.isEmptyDirectory()); - - path.writeUtf8File(""); - assertFalse(path.isEmptyDirectory()); - - path.deleteIfExists(); - path.createDirectory(); - assertTrue(path.isEmptyDirectory()); - - path.resolve("bar").writeUtf8File(""); - assertFalse(path.isEmptyDirectory()); - } - - @Test - void atomicWrite() { - var path = new UnixPath(fs.getPath("/dir/foo")); - path.createParents(); - path.writeUtf8File("bar"); - path.atomicWriteBytes("bar v2".getBytes(StandardCharsets.UTF_8)); - assertEquals("bar v2", path.readUtf8File()); - } - - @Test - void testParentAndFilename() { - var absolutePath = new UnixPath("/foo/bar"); - assertEquals("/foo", absolutePath.getParent().toString()); - assertEquals("bar", absolutePath.getFilename()); - - var pathWithoutSlash = new UnixPath("foo"); - assertRuntimeException(IllegalStateException.class, "Path has no parent directory: 'foo'", pathWithoutSlash::getParent); - assertEquals("foo", pathWithoutSlash.getFilename()); - - var pathWithSlash = new UnixPath("/foo"); - assertEquals("/", pathWithSlash.getParent().toString()); - assertEquals("foo", pathWithSlash.getFilename()); - - assertRuntimeException(IllegalStateException.class, "Path has no parent directory: '/'", () -> new UnixPath("/").getParent()); - assertRuntimeException(IllegalStateException.class, "Path has no filename: '/'", () -> new UnixPath("/").getFilename()); - } - - private <T extends RuntimeException> void assertRuntimeException(Class<T> baseClass, String message, Runnable runnable) { - try { - runnable.run(); - fail("No exception was thrown"); - } catch (RuntimeException e) { - if (!baseClass.isInstance(e)) { - fail("Exception class mismatch " + baseClass.getName() + " != " + e.getClass().getName()); - } - - assertEquals(message, e.getMessage()); - } - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java deleted file mode 100644 index 37fe90209ea..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.fs; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace; -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author freva - */ -class ContainerFileSystemTest { - - private final FileSystem fileSystem = TestFileSystem.create(); - private final UnixPath containerRootOnHost = new UnixPath(fileSystem.getPath("/data/storage/ctr1")); - private final UserScope userScope = UserScope.create(new UserNamespace(10_000, 11_000, 10000)); - private final ContainerFileSystem containerFs = ContainerFileSystem.create(containerRootOnHost.createDirectories().toPath(), userScope); - - @Test - public void creates_files_and_directories_with_container_root_as_owner() throws IOException { - ContainerPath containerPath = ContainerPath.fromPathInContainer(containerFs, Path.of("/opt/vespa/logs/file"), userScope.root()); - UnixPath unixPath = new UnixPath(containerPath).createParents().writeUtf8File("hello world"); - - for (ContainerPath p = containerPath; p.getParent() != null; p = p.getParent()) - assertOwnership(p, 0, 0, 10000, 11000); - - unixPath.setOwnerId(500).setGroupId(1000); - assertOwnership(containerPath, 500, 1000, 10500, 12000); - - UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); - ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/copy1"), userScope.root()); - Files.copy(hostFile.toPath(), destination); - assertOwnership(destination, 0, 0, 10000, 11000); - } - - @Test - public void file_write_and_read() throws IOException { - ContainerPath containerPath = ContainerPath.fromPathInContainer(containerFs, Path.of("/file"), userScope.root()); - UnixPath unixPath = new UnixPath(containerPath); - unixPath.writeUtf8File("hello"); - assertOwnership(containerPath, 0, 0, 10000, 11000); - - unixPath.setOwnerId(500).setGroupId(200); - assertOwnership(containerPath, 500, 200, 10500, 11200); - Files.writeString(containerPath, " world", StandardOpenOption.APPEND); - assertOwnership(containerPath, 500, 200, 10500, 11200); // Owner should not have been updated as the file already existed - - assertEquals("hello world", unixPath.readUtf8File()); - - unixPath.deleteIfExists(); - new UnixPath(containerPath.withUser(userScope.vespa())).writeUtf8File("test123"); - assertOwnership(containerPath, 1000, 1000, 11000, 12000); - } - - @Test - public void copy() throws IOException { - UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); - ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest"), userScope.root()); - - // If file is copied to JimFS path, the UID/GIDs are not fixed - Files.copy(hostFile.toPath(), destination.pathOnHost()); - assertEquals(String.valueOf(userScope.namespace().overflowId()), Files.getOwner(destination).getName()); - Files.delete(destination); - - Files.copy(hostFile.toPath(), destination); - assertOwnership(destination, 0, 0, 10000, 11000); - - // Set owner + group on both source host file and destination container file - hostFile.setOwnerId(5).setGroupId(10); - new UnixPath(destination).setOwnerId(500).setGroupId(200); - assertOwnership(destination, 500, 200, 10500, 11200); - // Copy the host file to destination again with COPY_ATTRIBUTES and REPLACE_EXISTING - Files.copy(hostFile.toPath(), destination, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - // The destination is recreated, so the owner should be root - assertOwnership(destination, 0, 0, 10000, 11000); - - // Set owner + group and copy within ContainerFS - new UnixPath(destination).setOwnerId(500).setGroupId(200); - ContainerPath destination2 = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest2"), userScope.root()); - Files.copy(destination, destination2, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - assertOwnership(destination2, 500, 200, 10500, 11200); - } - - @Test - public void move() throws IOException { - UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); - ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest"), userScope.root()); - - // If file is moved to JimFS path, the UID/GIDs are not fixed - Files.move(hostFile.toPath(), destination.pathOnHost()); - assertEquals(String.valueOf(userScope.namespace().overflowId()), Files.getOwner(destination).getName()); - Files.delete(destination); - - hostFile.createNewFile(); - Files.move(hostFile.toPath(), destination); - assertOwnership(destination, 0, 0, 10000, 11000); - - // Set owner + group on both source host file and destination container file - hostFile.createNewFile(); - hostFile.setOwnerId(5).setGroupId(10); - new UnixPath(destination).setOwnerId(500).setGroupId(200); - assertOwnership(destination, 500, 200, 10500, 11200); - // Move the host file to destination again with COPY_ATTRIBUTES and REPLACE_EXISTING - Files.move(hostFile.toPath(), destination, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - // The destination is recreated, so the owner should be root - assertOwnership(destination, 0, 0, 10000, 11000); - - // Set owner + group and move within ContainerFS - new UnixPath(destination).setOwnerId(500).setGroupId(200); - ContainerPath destination2 = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest2"), userScope.root()); - Files.move(destination, destination2, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - assertOwnership(destination2, 500, 200, 10500, 11200); - } - - @Test - public void symlink() throws IOException { - ContainerPath source = ContainerPath.fromPathInContainer(containerFs, Path.of("/src"), userScope.root()); - // Symlink from ContainerPath to some relative path (different FS provider) - Files.createSymbolicLink(source, fileSystem.getPath("../relative/target")); - assertEquals(fileSystem.getPath("../relative/target"), Files.readSymbolicLink(source)); - Files.delete(source); - - // Symlinks from ContainerPath to a ContainerPath: Target is resolved within container with base FS provider - Files.createSymbolicLink(source, ContainerPath.fromPathInContainer(containerFs, Path.of("/path/in/container"), userScope.root())); - assertEquals(fileSystem.getPath("/path/in/container"), Files.readSymbolicLink(source)); - assertOwnership(source, 0, 0, 10000, 11000); - } - - @Test - public void disallow_operations_on_symlink() throws IOException { - Path destination = fileSystem.getPath("/dir/file"); - Files.createDirectories(destination.getParent()); - - ContainerPath link = containerFs.getPath("/link"); - Files.createSymbolicLink(link, destination); - - // Cannot write file via symlink - assertThrows(IOException.class, () -> Files.writeString(link, "hello")); - - assertOwnership(link, 0, 0, 10_000, 11_000); - Files.setAttribute(link, "unix:uid", 10); // This succeeds because attribute is set on the link (destination does not exist) - assertFalse(Files.exists(destination)); - assertOwnership(link, 10, 0, 10_010, 11_000); - } - - @Test - public void disallow_operations_on_parent_symlink() throws IOException { - Path destination = fileSystem.getPath("/dir/sub/folder"); - Files.createDirectories(destination.getParent()); - - // Create symlink /some/dir/link -> /dir/sub - ContainerPath link = containerFs.getPath("/some/dir/link"); - Files.createDirectories(link.getParent()); - Files.createSymbolicLink(link, destination.getParent()); - - ContainerPath file = link.resolve("file"); - assertThrows(IOException.class, () -> Files.writeString(file, "hello")); - Files.writeString(file.pathOnHost(), "hello"); // Writing through host FS works - } - - @Test - public void permissions() throws IOException { - assertPermissions(Files.createDirectory(containerFs.getPath("/dir1")), "rwxr-x---"); - assertPermissions(Files.createDirectory(containerFs.getPath("/dir2"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-"); - - assertPermissions(Files.createDirectories(containerFs.getPath("/sub/dir/leaf"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-"); - assertPermissions(containerFs.getPath("/sub/dir"), "r-x-w-rw-"); // Non-leafs get the same permission as the leaf - - // TODO: Uncomment when JimFS forwards attributes for SecureDirectoryStream::newByteChannel -// assertPermissions(Files.createFile(containerFs.getPath("/file1")), "rw-r-----"); -// assertPermissions(Files.createFile(containerFs.getPath("/file2"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-"); - } - - private static void assertOwnership(ContainerPath path, int contUid, int contGid, int hostUid, int hostGid) throws IOException { - assertOwnership(path, contUid, contGid); - assertOwnership(path.pathOnHost(), hostUid, hostGid); - } - - private static void assertOwnership(Path path, int uid, int gid) throws IOException { - Map<String, Object> attrs = Files.readAttributes(path, "unix:*", LinkOption.NOFOLLOW_LINKS); - assertEquals(uid, attrs.get("uid")); - assertEquals(gid, attrs.get("gid")); - } - - private static void assertPermissions(Path path, String expected) throws IOException { - String actual = PosixFilePermissions.toString(Files.getPosixFilePermissions(path)); - assertEquals(expected, actual); - } - - private static FileAttribute<?> permissionsFromString(String permissions) { - return PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString(permissions)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java deleted file mode 100644 index eb7a8e13925..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.fs; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.function.Executable; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -/** - * @author freva - */ -class ContainerPathTest { - - private final FileSystem baseFs = TestFileSystem.create(); - private final ContainerFileSystem containerFs = ContainerFileSystem.create(baseFs.getPath("/data/storage/ctr1"), mock(UserScope.class)); - - @Test - public void create_new_container_path() { - ContainerPath path = fromPathInContainer(Path.of("/opt/vespa//logs/./file")); - assertPaths(path, "/data/storage/ctr1/opt/vespa/logs/file", "/opt/vespa/logs/file"); - - path = fromPathOnHost(baseFs.getPath("/data/storage/ctr1/opt/vespa/logs/file")); - assertPaths(path, "/data/storage/ctr1/opt/vespa/logs/file", "/opt/vespa/logs/file"); - - path = fromPathOnHost(baseFs.getPath("/data/storage/ctr2/..////./ctr1/./opt")); - assertPaths(path, "/data/storage/ctr1/opt", "/opt"); - - assertThrows(() -> fromPathInContainer(Path.of("relative/path")), "Path in container must be absolute: relative/path"); - assertThrows(() -> fromPathOnHost(baseFs.getPath("relative/path")), "Paths have different roots: /data/storage/ctr1, relative/path"); - assertThrows(() -> fromPathOnHost(baseFs.getPath("/data/storage/ctr2")), "Path /data/storage/ctr2 is not under container root /data/storage/ctr1"); - assertThrows(() -> fromPathOnHost(baseFs.getPath("/data/storage/ctr1/../ctr2")), "Path /data/storage/ctr2 is not under container root /data/storage/ctr1"); - } - - @Test - public void container_path_operations() { - ContainerPath path = fromPathInContainer(Path.of("/opt/vespa/logs/file")); - ContainerPath parent = path.getParent(); - assertPaths(path.getRoot(), "/data/storage/ctr1", "/"); - assertPaths(parent, "/data/storage/ctr1/opt/vespa/logs", "/opt/vespa/logs"); - assertNull(path.getRoot().getParent()); - - assertEquals(Path.of("file"), path.getFileName()); - assertEquals(Path.of("logs"), path.getName(2)); - assertEquals(4, path.getNameCount()); - assertEquals(Path.of("vespa/logs"), path.subpath(1, 3)); - - assertTrue(path.startsWith(path)); - assertTrue(path.startsWith(parent)); - assertFalse(parent.startsWith(path)); - assertFalse(path.startsWith(Path.of(path.toString()))); - - assertTrue(path.endsWith(Path.of(path.pathInContainer()))); - assertTrue(path.endsWith(Path.of("logs/file"))); - assertFalse(path.endsWith(Path.of("/logs/file"))); - } - - @Test - public void resolution() { - ContainerPath path = fromPathInContainer(Path.of("/opt/vespa/logs")); - assertPaths(path.resolve(Path.of("/root")), "/data/storage/ctr1/root", "/root"); - assertPaths(path.resolve(Path.of("relative")), "/data/storage/ctr1/opt/vespa/logs/relative", "/opt/vespa/logs/relative"); - assertPaths(path.resolve(Path.of("/../../../dir2/../../../dir2")), "/data/storage/ctr1/dir2", "/dir2"); - assertPaths(path.resolve(Path.of("/some/././///path")), "/data/storage/ctr1/some/path", "/some/path"); - - assertPaths(path.resolve(Path.of("../dir")), "/data/storage/ctr1/opt/vespa/dir", "/opt/vespa/dir"); - assertEquals(path.resolve(Path.of("../dir")), path.resolveSibling("dir")); - } - - @Test - public void resolves_real_paths() throws IOException { - ContainerPath path = fromPathInContainer(Path.of("/opt/vespa/logs")); - Files.createDirectories(path.pathOnHost().getParent()); - - Files.createFile(baseFs.getPath("/data/storage/ctr1/opt/vespa/target1")); - Files.createSymbolicLink(path.pathOnHost(), path.pathOnHost().resolveSibling("target1")); - assertPaths(path.toRealPath(LinkOption.NOFOLLOW_LINKS), "/data/storage/ctr1/opt/vespa/logs", "/opt/vespa/logs"); - assertPaths(path.toRealPath(), "/data/storage/ctr1/opt/vespa/target1", "/opt/vespa/target1"); - - Files.delete(path.pathOnHost()); - Files.createFile(baseFs.getPath("/data/storage/ctr1/opt/target2")); - Files.createSymbolicLink(path.pathOnHost(), baseFs.getPath("../target2")); - assertPaths(path.toRealPath(), "/data/storage/ctr1/opt/target2", "/opt/target2"); - - Files.delete(path.pathOnHost()); - Files.createFile(baseFs.getPath("/data/storage/ctr2")); - Files.createSymbolicLink(path.pathOnHost(), path.getRoot().pathOnHost().resolveSibling("ctr2")); - assertThrows(path::toRealPath, "Path /data/storage/ctr2 is not under container root /data/storage/ctr1"); - } - - private ContainerPath fromPathInContainer(Path pathInContainer) { - return ContainerPath.fromPathInContainer(containerFs, pathInContainer, UnixUser.ROOT); - } - private ContainerPath fromPathOnHost(Path pathOnHost) { - return ContainerPath.fromPathOnHost(containerFs, pathOnHost, UnixUser.ROOT); - } - - private static void assertPaths(ContainerPath actual, String expectedPathOnHost, String expectedPathInContainer) { - assertEquals(expectedPathOnHost, actual.pathOnHost().toString()); - assertEquals(expectedPathInContainer, actual.pathInContainer()); - } - - private static void assertThrows(Executable executable, String expectedMsg) { - String actualMsg = Assertions.assertThrows(IllegalArgumentException.class, executable).getMessage(); - assertEquals(expectedMsg, actualMsg); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java deleted file mode 100644 index 525c6d9162c..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.fs; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace; -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.attribute.UserPrincipalNotFoundException; - -import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal; -import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author freva - */ -class ContainerUserPrincipalLookupServiceTest { - - private final UserScope userScope = UserScope.create(new UserNamespace(10_000, 11_000, 10000)); - private final ContainerUserPrincipalLookupService userPrincipalLookupService = - new ContainerUserPrincipalLookupService(TestFileSystem.create().getUserPrincipalLookupService(), userScope); - - @Test - public void correctly_resolves_ids() throws IOException { - ContainerUserPrincipal user = userPrincipalLookupService.lookupPrincipalByName("1000"); - assertEquals("vespa", user.getName()); - assertEquals("11000", user.baseFsPrincipal().getName()); - assertEquals(user, userPrincipalLookupService.lookupPrincipalByName("vespa")); - - ContainerGroupPrincipal group = userPrincipalLookupService.lookupPrincipalByGroupName("1000"); - assertEquals("vespa", group.getName()); - assertEquals("12000", group.baseFsPrincipal().getName()); - assertEquals(group, userPrincipalLookupService.lookupPrincipalByGroupName("vespa")); - - assertThrows(UserPrincipalNotFoundException.class, () -> userPrincipalLookupService.lookupPrincipalByName("test")); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java deleted file mode 100644 index 299d3e4b441..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.network; - -import com.google.common.net.InetAddresses; - -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author smorgrav - */ -public class IPAddressesMock implements IPAddresses { - - private final Map<String, List<InetAddress>> otherAddresses = new HashMap<>(); - - public IPAddressesMock addAddress(String hostname, String ip) { - List<InetAddress> addresses = otherAddresses.getOrDefault(hostname, new ArrayList<>()); - addresses.add(InetAddresses.forString(ip)); - otherAddresses.put(hostname, addresses); - return this; - } - - @Override - public InetAddress[] getAddresses(String hostname) { - List<InetAddress> addresses = otherAddresses.get(hostname); - if (addresses == null) return new InetAddress[0]; - return addresses.toArray(new InetAddress[addresses.size()]); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java deleted file mode 100644 index 59ddc1f6c8d..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.network; - -import com.google.common.net.InetAddresses; -import org.junit.jupiter.api.Test; - -import java.net.Inet6Address; -import java.net.InetAddress; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author smorgrav - */ -public class IPAddressesTest { - - private final IPAddressesMock mock = new IPAddressesMock(); - - @Test - void choose_sitelocal_ipv4_over_public() { - mock.addAddress("localhost", "38.3.4.2") - .addAddress("localhost", "10.0.2.2") - .addAddress("localhost", "fe80::1") - .addAddress("localhost", "2001::1"); - - assertEquals(InetAddresses.forString("10.0.2.2"), mock.getIPv4Address("localhost").get()); - } - - @Test - void choose_ipv6_public_over_local() { - mock.addAddress("localhost", "38.3.4.2") - .addAddress("localhost", "10.0.2.2") - .addAddress("localhost", "fe80::1") - .addAddress("localhost", "2001::1"); - - assertEquals(InetAddresses.forString("2001::1"), mock.getIPv6Address("localhost").get()); - } - - @Test - void throws_when_multiple_ipv6_addresses() { - assertThrows(RuntimeException.class, () -> { - mock.addAddress("localhost", "2001::1") - .addAddress("localhost", "2001::2"); - mock.getIPv6Address("localhost"); - }); - } - - @Test - void throws_when_multiple_private_ipv4_addresses() { - assertThrows(RuntimeException.class, () -> { - mock.addAddress("localhost", "38.3.4.2") - .addAddress("localhost", "10.0.2.2") - .addAddress("localhost", "10.0.2.3"); - mock.getIPv4Address("localhost"); - }); - } - - @Test - void translator_with_valid_parameters() { - - // Test simplest possible address - Inet6Address original = (Inet6Address) InetAddresses.forString("2001:db8::1"); - Inet6Address prefix = (Inet6Address) InetAddresses.forString("fd00::"); - InetAddress translated = IPAddresses.prefixTranslate(original, prefix, 8); - assertEquals("fd00:0:0:0:0:0:0:1", translated.getHostAddress()); - - - // Test an actual aws address we use - original = (Inet6Address) InetAddresses.forString("2600:1f16:f34:5300:ccc6:1703:b7c2:369d"); - translated = IPAddresses.prefixTranslate(original, prefix, 8); - assertEquals("fd00:0:0:0:ccc6:1703:b7c2:369d", translated.getHostAddress()); - - // Test different subnet size - translated = IPAddresses.prefixTranslate(original, prefix, 6); - assertEquals("fd00:0:0:5300:ccc6:1703:b7c2:369d", translated.getHostAddress()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java deleted file mode 100644 index 69d5c6f2c31..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.network; - -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; - -/** - * @author gjoranv - */ -public class VersionedIpAddressTest { - - @Test - void ip4_address_can_be_generated_from_string() { - var ip4 = VersionedIpAddress.from("10.0.0.1"); - assertEquals(IPVersion.IPv4, ip4.version()); - assertEquals("10.0.0.1", ip4.asString()); - } - - @Test - void ip6_address_can_be_generated_from_string() { - var ip6 = VersionedIpAddress.from("::1"); - assertEquals(IPVersion.IPv6, ip6.version()); - assertEquals("::1", ip6.asString()); - } - - @Test - void they_are_sorted_by_version_then_by_address() { - var ip4 = VersionedIpAddress.from("10.0.0.1"); - var ip4_2 = VersionedIpAddress.from("127.0.0.1"); - var ip6 = VersionedIpAddress.from("::1"); - var ip6_2 = VersionedIpAddress.from("::2"); - - var sorted = Stream.of(ip4_2, ip6, ip4, ip6_2) - .sorted() - .toList(); - assertEquals(List.of(ip6, ip6_2, ip4, ip4_2), sorted); - } - - @Test - void endpoint_with_port_is_generated_correctly_for_both_versions() { - var ip4 = VersionedIpAddress.from("10.0.0.1"); - var ip6 = VersionedIpAddress.from("::1"); - - assertEquals("10.0.0.1:8080", ip4.asEndpoint(8080)); - assertEquals("[::1]:8080", ip6.asEndpoint(8080)); - } - - @Test - void equals_and_hashCode_are_implemented() { - var one = VersionedIpAddress.from("::1"); - var two = VersionedIpAddress.from("::2"); - var local = VersionedIpAddress.from("127.0.0.1"); - var ten = VersionedIpAddress.from("10.0.0.1"); - assertEquals(one, VersionedIpAddress.from("::1")); - assertNotEquals(one, two); - assertNotEquals(one, local); - assertNotEquals(one, ten); - - assertEquals(local, VersionedIpAddress.from("127.0.0.1")); - assertNotEquals(local, two); - assertNotEquals(local, 10); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java deleted file mode 100644 index 19bc2d59bb2..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.jdisc.Timer; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author hakonhall - */ -public class ChildProcess2ImplTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final Timer timer = mock(Timer.class); - private final CommandLine commandLine = mock(CommandLine.class); - private final ProcessApi2 processApi = mock(ProcessApi2.class); - private Path temporaryFile; - - @BeforeEach - public void setUp() throws IOException { - temporaryFile = Files.createTempFile(fileSystem.getPath("/"), "", ""); - } - - @Test - void testSuccess() throws Exception { - when(commandLine.getTimeout()).thenReturn(Duration.ofHours(1)); - when(commandLine.getMaxOutputBytes()).thenReturn(10L); - when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); - when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); - when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); - when(commandLine.toString()).thenReturn("program arg"); - - when(timer.currentTime()).thenReturn( - Instant.ofEpochMilli(1), - Instant.ofEpochMilli(2)); - - when(processApi.waitFor(anyLong(), any())).thenReturn(true); - - try (ChildProcess2Impl child = - new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { - child.waitForTermination(); - } - } - - @Test - void testTimeout() throws Exception { - when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1)); - when(commandLine.getMaxOutputBytes()).thenReturn(10L); - when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); - when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); - when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); - when(commandLine.toString()).thenReturn("program arg"); - - when(timer.currentTime()).thenReturn( - Instant.ofEpochSecond(0), - Instant.ofEpochSecond(2)); - - when(processApi.waitFor(anyLong(), any())).thenReturn(true); - - try (ChildProcess2Impl child = - new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { - try { - child.waitForTermination(); - fail(); - } catch (TimeoutChildProcessException e) { - assertEquals( - "Command 'program arg' timed out after PT1S: stdout/stderr: ''", - e.getMessage()); - } - } - } - - @Test - void testMaxOutputBytes() throws Exception { - when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1)); - when(commandLine.getMaxOutputBytes()).thenReturn(10L); - when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); - when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); - when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); - when(commandLine.toString()).thenReturn("program arg"); - - when(timer.currentTime()).thenReturn( - Instant.ofEpochMilli(0), - Instant.ofEpochMilli(1)); - - when(processApi.waitFor(anyLong(), any())).thenReturn(true); - - Files.writeString(temporaryFile, "1234567890123"); - - try (ChildProcess2Impl child = - new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { - try { - child.waitForTermination(); - fail(); - } catch (LargeOutputChildProcessException e) { - assertEquals( - "Command 'program arg' output more than 13 bytes: stdout/stderr: '1234567890123'", - e.getMessage()); - } - } - } - - @Test - void testUnkillable() throws Exception { - when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1)); - when(commandLine.getMaxOutputBytes()).thenReturn(10L); - when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); - when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); - when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); - when(commandLine.toString()).thenReturn("program arg"); - - when(timer.currentTime()).thenReturn( - Instant.ofEpochMilli(0), - Instant.ofEpochMilli(1)); - - when(processApi.waitFor(anyLong(), any())).thenReturn(false); - - Files.writeString(temporaryFile, "1234567890123"); - - try (ChildProcess2Impl child = - new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { - try { - child.waitForTermination(); - fail(); - } catch (UnkillableChildProcessException e) { - assertEquals( - "Command 'program arg' did not terminate even after SIGTERM, +PT2M, SIGKILL, and +PT3M: stdout/stderr: '1234567890123'", - e.getMessage()); - } - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java deleted file mode 100644 index fead96404a5..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Optional; -import java.util.function.Predicate; - -import static org.junit.jupiter.api.Assertions.*; - -public class CommandLineTest { - private final TestTerminal terminal = new TestTerminal(); - private final TestTaskContext context = new TestTaskContext(); - private final CommandLine commandLine = terminal.newCommandLine(context); - - @AfterEach - public void tearDown() { - terminal.verifyAllCommandsExecuted(); - } - - @Test - void testStrings() { - terminal.expectCommand( - "/bin/bash \"with space\" \"speci&l\" \"\" \"double\\\"quote\" 2>&1", - 0, - ""); - commandLine.add("/bin/bash", "with space", "speci&l", "", "double\"quote").execute(); - assertEquals("bash", commandLine.programName()); - } - - @Test - void testBasicExecute() { - terminal.expectCommand("foo bar 2>&1", 0, "line1\nline2\n\n"); - CommandResult result = commandLine.add("foo", "bar").execute(); - assertEquals(0, result.getExitCode()); - assertEquals("line1\nline2", result.getOutput()); - assertEquals("line1\nline2\n\n", result.getUntrimmedOutput()); - assertEquals(List.of("line1", "line2"), result.getOutputLines()); - assertEquals(1, context.getSystemModificationLog().size()); - assertEquals("Executing command: foo bar 2>&1", context.getSystemModificationLog().get(0)); - - List<CommandLine> commandLines = terminal.getTestProcessFactory().getMutableCommandLines(); - assertEquals(1, commandLines.size()); - assertEquals(commandLine, commandLines.get(0)); - - int lines = result.map(r -> r.getOutputLines().size()); - assertEquals(2, lines); - } - - @Test - void verifyDefaults() { - assertEquals(CommandLine.DEFAULT_TIMEOUT, commandLine.getTimeout()); - assertEquals(CommandLine.DEFAULT_MAX_OUTPUT_BYTES, commandLine.getMaxOutputBytes()); - assertEquals(CommandLine.DEFAULT_SIGTERM_GRACE_PERIOD, commandLine.getSigTermGracePeriod()); - assertEquals(CommandLine.DEFAULT_SIGKILL_GRACE_PERIOD, commandLine.getSigKillGracePeriod()); - assertEquals(0, commandLine.getArguments().size()); - assertEquals(Optional.empty(), commandLine.getOutputFile()); - assertEquals(StandardCharsets.UTF_8, commandLine.getOutputEncoding()); - assertTrue(commandLine.getRedirectStderrToStdoutInsteadOfDiscard()); - Predicate<Integer> defaultExitCodePredicate = commandLine.getSuccessfulExitCodePredicate(); - assertTrue(defaultExitCodePredicate.test(0)); - assertFalse(defaultExitCodePredicate.test(1)); - } - - @Test - void executeSilently() { - terminal.ignoreCommand(""); - commandLine.add("foo", "bar").executeSilently(); - assertEquals(0, context.getSystemModificationLog().size()); - commandLine.recordSilentExecutionAsSystemModification(); - assertEquals(1, context.getSystemModificationLog().size()); - assertEquals("Executed command: foo bar 2>&1", context.getSystemModificationLog().get(0)); - } - - @Test - void processFactorySpawnFails() { - assertThrows(NegativeArraySizeException.class, () -> { - terminal.interceptCommand( - commandLine.toString(), - command -> { - throw new NegativeArraySizeException(); - }); - commandLine.add("foo").execute(); - }); - } - - @Test - void waitingForTerminationExceptionStillClosesChild() { - TestChildProcess2 child = new TestChildProcess2(0, ""); - child.throwInWaitForTermination(new NegativeArraySizeException()); - terminal.interceptCommand(commandLine.toString(), command -> child); - assertFalse(child.closeCalled()); - try { - commandLine.add("foo").execute(); - fail(); - } catch (NegativeArraySizeException e) { - // OK - } - - assertTrue(child.closeCalled()); - } - - @Test - void programFails() { - terminal.expectCommand("foo 2>&1", 1, ""); - try { - commandLine.add("foo").execute(); - fail(); - } catch (ChildProcessFailureException e) { - assertEquals( - "Command 'foo 2>&1' terminated with exit code 1: stdout/stderr: ''", - e.getMessage()); - } - } - - @Test - void mapException() { - terminal.ignoreCommand("output"); - CommandResult result = terminal.newCommandLine(context).add("program").execute(); - IllegalArgumentException exception = new IllegalArgumentException("foo"); - try { - result.mapOutput(output -> { - throw exception; - }); - fail(); - } catch (UnexpectedOutputException e) { - assertEquals("Command 'program 2>&1' output was not of the expected format: " + - "Failed to map output: stdout/stderr: 'output'", e.getMessage()); - assertEquals(e.getCause(), exception); - } - } - - @Test - void testMapEachLine() { - assertEquals( - 1 + 2 + 3, - terminal.ignoreCommand("1\n2\n3\n") - .newCommandLine(context) - .add("foo") - .execute() - .mapEachLine(Integer::valueOf) - .stream() - .mapToInt(i -> i) - .sum()); - } - - @Test - void addTokensWithMultipleWhiteSpaces() { - terminal.expectCommand("iptables -L 2>&1"); - commandLine.addTokens("iptables -L").execute(); - - terminal.verifyAllCommandsExecuted(); - } - - @Test - void addTokensWithSpecialCharacters() { - terminal.expectCommand("find . ! -name hei 2>&1"); - commandLine.addTokens("find . ! -name hei").execute(); - - terminal.verifyAllCommandsExecuted(); - } - - @Test - void testEnvironment() { - terminal.expectCommand("env k1=v1 -u k2 \"key 3=value 3\" programname 2>&1"); - commandLine.add("programname") - .setEnvironmentVariable("key 3", "value 3") - .removeEnvironmentVariable("k2") - .setEnvironmentVariable("k1", "v1") - .execute(); - terminal.verifyAllCommandsExecuted(); - } - - @Test - public void testToString() { - commandLine.add("bash", "-c", "echo", "$MY_SECRET"); - assertEquals("bash -c echo \"$MY_SECRET\" 2>&1", commandLine.toString()); - commandLine.censorArgument(); - assertEquals("bash -c echo <censored> 2>&1", commandLine.toString()); - - terminal.expectCommand("bash -c echo \"$MY_SECRET\" 2>&1"); - commandLine.execute(); - terminal.verifyAllCommandsExecuted(); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java deleted file mode 100644 index 58429f9f084..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.jdisc.test.TestTimer; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import static com.yahoo.yolean.Exceptions.uncheck; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class ProcessFactoryImplTest { - private final ProcessStarter starter = mock(ProcessStarter.class); - private final TestTimer timer = new TestTimer(); - private final ProcessFactoryImpl processFactory = new ProcessFactoryImpl(starter, timer); - - @Test - void testSpawn() { - CommandLine commandLine = mock(CommandLine.class); - when(commandLine.getArguments()).thenReturn(List.of("program")); - when(commandLine.getRedirectStderrToStdoutInsteadOfDiscard()).thenReturn(true); - when(commandLine.programName()).thenReturn("program"); - Path outputPath; - try (ChildProcess2Impl child = processFactory.spawn(commandLine)) { - outputPath = child.getOutputPath(); - assertTrue(Files.exists(outputPath)); - assertEquals("rw-------", new UnixPath(outputPath).getPermissions()); - ArgumentCaptor<ProcessBuilder> processBuilderCaptor = - ArgumentCaptor.forClass(ProcessBuilder.class); - verify(starter).start(processBuilderCaptor.capture()); - ProcessBuilder processBuilder = processBuilderCaptor.getValue(); - assertTrue(processBuilder.redirectErrorStream()); - ProcessBuilder.Redirect redirect = processBuilder.redirectOutput(); - assertEquals(ProcessBuilder.Redirect.Type.WRITE, redirect.type()); - assertEquals(outputPath.toFile(), redirect.file()); - } - - assertFalse(Files.exists(outputPath)); - } - - @Test - void testSpawnWithPersistentOutputFile() { - - class TemporaryFile implements AutoCloseable { - private final Path path; - - private TemporaryFile() { - String outputFileName = ProcessFactoryImplTest.class.getSimpleName() + "-temporary-test-file.out"; - FileAttribute<Set<PosixFilePermission>> fileAttribute = PosixFilePermissions.asFileAttribute( - PosixFilePermissions.fromString("rw-------")); - path = uncheck(() -> Files.createTempFile(outputFileName, ".out", fileAttribute)); - } - - @Override - public void close() { - uncheck(() -> Files.deleteIfExists(path)); - } - } - - try (TemporaryFile outputPath = new TemporaryFile()) { - CommandLine commandLine = mock(CommandLine.class); - when(commandLine.getArguments()).thenReturn(List.of("program")); - when(commandLine.programName()).thenReturn("program"); - when(commandLine.getOutputFile()).thenReturn(Optional.of(outputPath.path)); - try (ChildProcess2Impl child = processFactory.spawn(commandLine)) { - assertEquals(outputPath.path, child.getOutputPath()); - assertTrue(Files.exists(outputPath.path)); - assertEquals("rw-------", new UnixPath(outputPath.path).getPermissions()); - } - - assertTrue(Files.exists(outputPath.path)); - } - - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java deleted file mode 100644 index f6a695ea003..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.systemd; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcessFailureException; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.mock; - -/** - * @author hakonhall - */ -public class SystemCtlTest { - - private final TaskContext taskContext = mock(TaskContext.class); - private final TestTerminal terminal = new TestTerminal(); - - @Test - void enable() { - terminal.expectCommand("systemctl --quiet is-enabled docker 2>&1", 1, "") - .expectCommand("systemctl enable docker 2>&1") - .expectCommand("systemctl --quiet is-enabled docker 2>&1"); - - SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(terminal).enable("docker"); - assertTrue(enableDockerService.converge(taskContext)); - assertFalse(enableDockerService.converge(taskContext), "Already converged"); - } - - @Test - void enableCommandFailure() { - terminal.expectCommand("systemctl --quiet is-enabled docker 2>&1", 1, "") - .expectCommand("systemctl enable docker 2>&1", 1, "error enabling service"); - SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(terminal).enable("docker"); - try { - enableDockerService.converge(taskContext); - fail(); - } catch (ChildProcessFailureException e) { - // success - } - } - - - @Test - void start() { - terminal.expectCommand( - "systemctl show docker 2>&1", - 0, - "a=b\n" + - "ActiveState=failed\n" + - "bar=zoo\n") - .expectCommand("systemctl start docker 2>&1", 0, ""); - - SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker"); - assertTrue(startDockerService.converge(taskContext)); - } - - @Test - void startIsNoop() { - terminal.expectCommand( - "systemctl show docker 2>&1", - 0, - "a=b\n" + - "ActiveState=active\n" + - "bar=zoo\n") - .expectCommand("systemctl start docker 2>&1", 0, ""); - - SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker"); - assertFalse(startDockerService.converge(taskContext)); - } - - - @Test - void startCommandFailre() { - terminal.expectCommand("systemctl show docker 2>&1", 1, "error"); - SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker"); - try { - startDockerService.converge(taskContext); - fail(); - } catch (ChildProcessFailureException e) { - // success - } - } - - - @Test - void disable() { - terminal.expectCommand("systemctl --quiet is-enabled docker 2>&1") - .expectCommand("systemctl disable docker 2>&1") - .expectCommand("systemctl --quiet is-enabled docker 2>&1", 1, ""); - - assertTrue(new SystemCtl(terminal).disable("docker").converge(taskContext)); - assertFalse(new SystemCtl(terminal).disable("docker").converge(taskContext), "Already converged"); - } - - @Test - void stop() { - terminal.expectCommand( - "systemctl show docker 2>&1", - 0, - "a=b\n" + - "ActiveState=active\n" + - "bar=zoo\n") - .expectCommand("systemctl stop docker 2>&1", 0, ""); - - assertTrue(new SystemCtl(terminal).stop("docker").converge(taskContext)); - } - - @Test - void restart() { - terminal.expectCommand("systemctl restart docker 2>&1", 0, ""); - assertTrue(new SystemCtl(terminal).restart("docker").converge(taskContext)); - } - - @Test - void testUnitExists() { - SystemCtl systemCtl = new SystemCtl(terminal); - - terminal.expectCommand("systemctl list-unit-files foo.service 2>&1", 0, - "UNIT FILE STATE\n" + - "\n" + - "0 unit files listed.\n"); - assertFalse(systemCtl.serviceExists(taskContext, "foo")); - - terminal.expectCommand("systemctl list-unit-files foo.service 2>&1", 0, - "UNIT FILE STATE \n" + - "foo.service enabled\n" + - "\n" + - "1 unit files listed.\n"); - assertTrue(systemCtl.serviceExists(taskContext, "foo")); - - terminal.expectCommand("systemctl list-unit-files foo.service 2>&1", 0, "garbage"); - try { - systemCtl.serviceExists(taskContext, "foo"); - fail(); - } catch (Exception e) { - assertTrue(e.getMessage().contains("garbage")); - } - } - - @Test - void withSudo() { - SystemCtl systemCtl = new SystemCtl(terminal).withSudo(); - terminal.expectCommand("sudo systemctl restart docker 2>&1", 0, ""); - assertTrue(systemCtl.restart("docker").converge(taskContext)); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java deleted file mode 100644 index 3fc10a38a99..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.systemd; - -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.function.Function; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class SystemCtlTesterTest { - - private static final String unit = "my-unit"; - private final TestTerminal terminal = new TestTerminal(); - private final SystemCtlTester systemCtl = new SystemCtlTester(terminal); - private final TestTaskContext context = new TestTaskContext(); - - @Test - void return_expectations() { - assertSystemCtlMethod(sct -> sct.expectEnable(unit), sc -> sc.enable(unit).converge(context)); - assertSystemCtlMethod(sct -> sct.expectDisable(unit), sc -> sc.disable(unit).converge(context)); - assertSystemCtlMethod(sct -> sct.expectStart(unit), sc -> sc.start(unit).converge(context)); - assertSystemCtlMethod(sct -> sct.expectStop(unit), sc -> sc.stop(unit).converge(context)); - assertSystemCtlMethod(sct -> sct.expectServiceExists(unit), sc -> sc.serviceExists(context, unit)); - assertSystemCtlMethod(sct -> sct.expectIsActive(unit), sc -> sc.isActive(context, unit)); - } - - @Test - void void_tests() { - systemCtl.expectRestart(unit); - systemCtl.restart(unit).converge(context); - terminal.verifyAllCommandsExecuted(); - - systemCtl.expectDaemonReload(); - systemCtl.daemonReload(context); - terminal.verifyAllCommandsExecuted(); - } - - private void assertSystemCtlMethod(Function<SystemCtlTester, SystemCtlTester.Expectation> systemCtlTesterExpectationFunction, - Function<SystemCtl, Boolean> systemCtlFunction) { - List.of(true, false).forEach(wantedReturnValue -> { - systemCtlTesterExpectationFunction.apply(systemCtl).andReturn(wantedReturnValue); - assertEquals(wantedReturnValue, systemCtlFunction.apply(systemCtl)); - terminal.verifyAllCommandsExecuted(); - }); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java deleted file mode 100644 index 1e2f69d7bc8..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import org.junit.jupiter.api.Test; - -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author hakonhall - */ -public class TemplateTest { - @Test - void verifyNewlineRemoval() { - Template template = Template.from("a%{list a}\n" + - "b%{end}\n" + - "c%{list c-}\n" + - "d%{end-}\n" + - "e\n", - new TemplateDescriptor().setRemoveNewline(false)); - template.add("a"); - template.add("c"); - - assertEquals("a\n" + - "b\n" + - "cde\n", - template.render()); - } - - @Test - void verifyIfSection() { - Template template = Template.from("Hello%{if cond} world%{end}!"); - assertEquals("Hello world!", template.snapshot().set("cond", true).render()); - assertEquals("Hello!", template.snapshot().set("cond", false).render()); - } - - @Test - void verifyComplexIfSection() { - Template template = Template.from("%{if cond}\n" + - "var: %{=varname}\n" + - "if: %{if !inner}inner is false%{end-}\n" + - "list: %{list formname}element%{end-}\n" + - "%{end}\n"); - - assertEquals("", template.snapshot().set("cond", false).render()); - - assertEquals("var: varvalue\n" + - "if: \n" + - "list: \n", - template.snapshot() - .set("cond", true) - .set("varname", "varvalue") - .set("inner", true) - .render()); - - Template template2 = template.snapshot() - .set("cond", true) - .set("varname", "varvalue") - .set("inner", false); - template2.add("formname"); - - assertEquals("var: varvalue\n" + - "if: inner is false\n" + - "list: element\n", template2.render()); - } - - @Test - void verifyElse() { - var template = Template.from("%{if cond}\n" + - "if body\n" + - "%{else}\n" + - "else body\n" + - "%{end}\n"); - assertEquals("if body\n", template.snapshot().set("cond", true).render()); - assertEquals("else body\n", template.snapshot().set("cond", false).render()); - } - - @Test - void verifySnapshotPreservesList() { - var template = Template.from("%{list foo}hello %{=area}%{end}"); - template.add("foo") - .set("area", "world"); - - assertEquals("hello world", template.render()); - assertEquals("hello world", template.snapshot().render()); - - Template snapshot = template.snapshot(); - snapshot.add("foo") - .set("area", "Norway"); - assertEquals("hello worldhello Norway", snapshot.render()); - } - - @Test - void verifyVariableSection() { - Template template = getTemplate("template1.tmp"); - template.set("varname", "varvalue"); - assertEquals("variable section 'varvalue'\n" + - "end of text\n", template.render()); - } - - @Test - void verifySimpleListSection() { - Template template = getTemplate("template1.tmp"); - template.set("varname", "varvalue") - .add("listname") - .set("varname", "different varvalue") - .set("varname2", "varvalue2"); - assertEquals("variable section 'varvalue'\n" + - "same variable section 'different varvalue'\n" + - "different variable section 'varvalue2'\n" + - "between ends\n" + - "end of text\n", template.render()); - } - - @Test - void verifyNestedListSection() { - Template template = getTemplate("template2.tmp"); - ListElement A0 = template.add("listA"); - ListElement A0B0 = A0.add("listB"); - ListElement A0B1 = A0.add("listB"); - - ListElement A1 = template.add("listA"); - ListElement A1B0 = A1.add("listB"); - assertEquals("body A\n" + - "body B\n" + - "body B\n" + - "body A\n" + - "body B\n", - template.render()); - } - - @Test - void verifyVariableReferences() { - Template template = getTemplate("template3.tmp"); - template.set("varname", "varvalue") - .set("innerVarSetAtTop", "val2"); - template.add("l"); - template.add("l") - .set("varname", "varvalue2"); - assertEquals("varvalue\n" + - "varvalue\n" + - "inner varvalue\n" + - "val2\n" + - "inner varvalue2\n" + - "val2\n", - template.render()); - } - - @Test - void badTemplates() { - assertException(BadTemplateException.class, "Unknown section 'zoo' at line 2 and column 6", - () -> Template.from("foo\nbar%{zoo}")); - - assertException(BadTemplateException.class, "Expected identifier at line 1 and column 4", - () -> Template.from("%{=")); - - assertException(BadTemplateException.class, "Expected identifier at line 1 and column 4", - () -> Template.from("%{=¬atoken}")); - - assertException(BadTemplateException.class, "Expected identifier at line 1 and column 8", - () -> Template.from("%{list ¬atoken}")); - - assertException(BadTemplateException.class, "Missing end directive for section started at line 1 and column 12", - () -> Template.from("%{list foo}missing end")); - - assertException(BadTemplateException.class, "Stray 'end' at line 1 and column 3", - () -> Template.from("%{end}stray end")); - - assertException(TemplateNameNotSetException.class, "Variable at line 1 and column 4 has not been set: notset", - () -> Template.from("%{=notset}").render()); - - assertException(TemplateNameNotSetException.class, "Variable at line 1 and column 6 has not been set: cond", - () -> Template.from("%{if cond}%{end}").render()); - - assertException(NotBooleanValueTemplateException.class, "cond was set to a non-boolean value: must be true or false", - () -> Template.from("%{if cond}%{end}").set("cond", 1).render()); - - assertException(NoSuchNameTemplateException.class, "No such element 'listname' in the template section starting at " + - "line 1 and column 1, and ending at line 1 and column 4", - () -> Template.from("foo").add("listname")); - - assertException(NameAlreadyExistsTemplateException.class, - "The name 'a' of the list section at line 1 and column 16 is in conflict with the identically " + - "named list section at line 1 and column 1", - () -> Template.from("%{list a}%{end}%{list a}%{end}")); - - assertException(NameAlreadyExistsTemplateException.class, - "The name 'a' of the list section at line 1 and column 6 is in conflict with the identically " + - "named variable section at line 1 and column 1", - () -> Template.from("%{=a}%{list a}%{end}")); - - assertException(NameAlreadyExistsTemplateException.class, - "The name 'a' of the variable section at line 1 and column 16 is in conflict with the identically " + - "named list section at line 1 and column 1", - () -> Template.from("%{list a}%{end}%{=a}")); - - assertException(NameAlreadyExistsTemplateException.class, - "The name 'a' of the list section at line 1 and column 14 is in conflict with the identically " + - "named if section at line 1 and column 1", - () -> Template.from("%{if a}%{end}%{list a}%{end}")); - - assertException(NameAlreadyExistsTemplateException.class, - "The name 'a' of the if section at line 1 and column 16 is in conflict with the identically " + - "named list section at line 1 and column 1", - () -> Template.from("%{list a}%{end}%{if a}%{end}")); - } - - private <T extends Throwable> void assertException(Class<T> class_, String message, Runnable runnable) { - T exception = assertThrows(class_, runnable::run); - assertEquals(message, exception.getMessage()); - } - - private Template getTemplate(String filename) { - return Template.at(Path.of("src/test/resources/" + filename)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java deleted file mode 100644 index 505cf807116..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import org.junit.jupiter.api.Test; - -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author hakonhall - */ -public class YumPackageNameTest { - - @Test - void testBuilder() { - YumPackageName yumPackage = new YumPackageName.Builder("docker") - .setEpoch("2") - .setVersion("1.12.6") - .setRelease("71.git3e8e77d.el7.centos.1") - .setArchitecture("x86_64") - .build(); - assertEquals("docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", yumPackage.toName()); - } - - @Test - void testAllValidFormats() { - // name - verifyPackageName( - "docker-engine-selinux", - null, - "docker-engine-selinux", - null, - null, - null, - "docker-engine-selinux", - null); - - // name with parenthesis - verifyPackageName( - "dnf-command(versionlock)", - null, - "dnf-command(versionlock)", - null, - null, - null, - "dnf-command(versionlock)", - null); - - // name.arch - verifyPackageName( - "docker-engine-selinux.x86_64", - null, - "docker-engine-selinux", - null, - null, - "x86_64", - "docker-engine-selinux.x86_64", - null); - - // name-ver - verifyPackageName("docker-engine-selinux-1.12.6", - null, - "docker-engine-selinux", - "1.12.6", - null, - null, - "docker-engine-selinux-0:1.12.6", - null); - - // name-ver-rel - verifyPackageName("docker-engine-selinux-1.12.6-1.el7", - null, - "docker-engine-selinux", - "1.12.6", - "1.el7", - null, - "docker-engine-selinux-0:1.12.6-1.el7", - "docker-engine-selinux-0:1.12.6-1.el7.*"); - - // name-ver-rel.arch - verifyPackageName("docker-engine-selinux-1.12.6-1.el7.x86_64", - null, - "docker-engine-selinux", - "1.12.6", - "1.el7", - "x86_64", - "docker-engine-selinux-0:1.12.6-1.el7.x86_64", - "docker-engine-selinux-0:1.12.6-1.el7.*"); - - // name-epoch:ver-rel.arch - verifyPackageName( - "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", - "2", - "docker", - "1.12.6", - "71.git3e8e77d.el7.centos.1", - "x86_64", - "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", - "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.*"); - - // epoch:name-ver-rel.arch - verifyPackageName( - "2:docker-1.12.6-71.git3e8e77d.el7.centos.1.x86_64", - "2", - "docker", - "1.12.6", - "71.git3e8e77d.el7.centos.1", - "x86_64", - "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", - "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.*"); - } - - private void verifyPackageName(String input, - String epoch, - String name, - String version, - String release, - String architecture, - String toName, - String toVersionName) { - YumPackageName yumPackageName = YumPackageName.fromString(input); - assertPackageField("epoch", epoch, yumPackageName.getEpoch()); - assertPackageField("name", name, Optional.of(yumPackageName.getName())); - assertPackageField("version", version, yumPackageName.getVersion()); - assertPackageField("release", release, yumPackageName.getRelease()); - assertPackageField("architecture", architecture, yumPackageName.getArchitecture()); - assertPackageField("toName()", toName, Optional.of(yumPackageName.toName())); - - if (toVersionName == null) { - try { - yumPackageName.toVersionLockName(); - fail(); - } catch (IllegalStateException e) { - assertTrue(e.getMessage().contains("Version is missing ") || - e.getMessage().contains("Release is missing "), - "Exception message contains expected substring: " + e.getMessage()); - } - } else { - assertEquals(toVersionName, yumPackageName.toVersionLockName()); - } - } - - private void assertPackageField(String field, String expected, Optional<String> actual) { - if (expected == null) { - assertFalse(actual.isPresent(), field + " is not present"); - } else { - assertEquals(expected, actual.get(), field + " has expected value"); - } - } - - @Test - void testArchitectures() { - assertEquals("x86_64", YumPackageName.fromString("docker.x86_64").getArchitecture().get()); - assertEquals("i686", YumPackageName.fromString("docker.i686").getArchitecture().get()); - assertEquals("noarch", YumPackageName.fromString("docker.noarch").getArchitecture().get()); - } - - @Test - void unrecognizedArchitectureGetsGobbledUp() { - YumPackageName packageName = YumPackageName.fromString("docker-engine-selinux-1.12.6-1.el7.i486"); - // This is not a great feature - please use YumPackageName.Builder instead. - assertEquals("1.el7.i486", packageName.getRelease().get()); - } - - @Test - void failParsingOfPackageNameWithEpochAndArchitecture() { - try { - YumPackageName.fromString("epoch:docker-engine-selinux-1.12.6-1.el7.x86_64"); - fail(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().toLowerCase().contains("epoch")); - } - } - - @Test - void testSubset() { - YumPackageName yumPackage = new YumPackageName.Builder("docker") - .setVersion("1.12.6") - .build(); - - assertTrue(yumPackage.isSubsetOf(yumPackage)); - assertTrue(yumPackage.isSubsetOf(new YumPackageName.Builder("docker") - .setVersion("1.12.6") - .setEpoch("2") - .setRelease("71.git3e8e77d.el7.centos.1") - .setArchitecture("x86_64") - .build())); - assertFalse(yumPackage.isSubsetOf(new YumPackageName.Builder("docker") - .setVersion("1.13.1") - .build())); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java deleted file mode 100644 index 27b23d26b24..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcessFailureException; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.mock; - -/** - * @author hakonhall - */ -public class YumTest { - - private final TaskContext taskContext = mock(TaskContext.class); - private final TestTerminal terminal = new TestTerminal(); - private final Yum yum = new Yum(terminal); - - @AfterEach - public void after() { - terminal.verifyAllCommandsExecuted(); - } - - @Test - void testQueryInstalled() { - terminal.expectCommand( - "rpm -q docker --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", - 0, - "docker\n2\n1.13.1\n74.git6e3bb8e.el7.centos\nx86_64"); - - Optional<YumPackageName> installed = yum.queryInstalled(taskContext, "docker"); - - assertTrue(installed.isPresent()); - assertEquals("docker", installed.get().getName()); - assertEquals("2", installed.get().getEpoch().get()); - assertEquals("1.13.1", installed.get().getVersion().get()); - assertEquals("74.git6e3bb8e.el7.centos", installed.get().getRelease().get()); - assertEquals("x86_64", installed.get().getArchitecture().get()); - } - - @Test - void testQueryInstalledPartial() { - terminal.expectCommand( - "rpm -q vespa-node-admin --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", - 0, - "vespa-node-admin\n(none)\n6.283.62\n1.el7\nnoarch"); - - Optional<YumPackageName> installed = yum.queryInstalled(taskContext, "vespa-node-admin"); - - assertTrue(installed.isPresent()); - assertEquals("vespa-node-admin", installed.get().getName()); - assertEquals("0", installed.get().getEpoch().get()); - assertEquals("6.283.62", installed.get().getVersion().get()); - assertEquals("1.el7", installed.get().getRelease().get()); - assertEquals("noarch", installed.get().getArchitecture().get()); - } - - @Test - void testQueryNotInstalled() { - terminal.expectCommand( - "rpm -q fake-package --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", - 1, - "package fake-package is not installed"); - - Optional<YumPackageName> installed = yum.queryInstalled(taskContext, "fake-package"); - - assertFalse(installed.isPresent()); - } - - @Test - void testQueryInstalledMultiplePackages() { - terminal.expectCommand( - "rpm -q kernel-devel --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", - 0, - "kernel-devel\n" + - "(none)\n" + - "4.18.0\n" + - "305.7.1.el8_4\n" + - "x86_64\n" + - "kernel-devel\n" + - "(none)\n" + - "4.18.0\n" + - "240.15.1.el8_3\n" + - "x86_64\n"); - try { - yum.queryInstalled(taskContext, "kernel-devel"); - fail("Expected exception"); - } catch (IllegalArgumentException e) { - assertEquals("Found multiple installed packages for 'kernel-devel'. Version is required to match package exactly", e.getMessage()); - } - } - - @Test - void testAlreadyInstalled() { - mockRpmQuery("package-1", null); - terminal.expectCommand( - "yum install --assumeyes --enablerepo=repo1 --enablerepo=repo2 --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", - 0, - "foobar\nNothing to do.\n"); // Note trailing dot - assertFalse(yum.install("package-1", "package-2") - .enableRepo("repo1", "repo2") - .converge(taskContext)); - } - - @Test - void testAlreadyUpgraded() { - terminal.expectCommand( - "yum upgrade --assumeyes --setopt skip_missing_names_on_update=False package-1 package-2 2>&1", - 0, - "foobar\nNothing to do.\n"); // Same message as yum install no-op - - assertFalse(yum.upgrade("package-1", "package-2") - .converge(taskContext)); - } - - @Test - void testAlreadyRemoved() { - mockRpmQuery("package-1", YumPackageName.fromString("package-1-1.2.3-1")); - terminal.expectCommand( - "yum remove --assumeyes package-1 package-2 2>&1", - 0, - "foobar\nNo packages marked for removal.\n"); // Different output - - assertFalse(yum.remove("package-1", "package-2") - .converge(taskContext)); - } - - @Test - void skipsYumRemoveNotInRpm() { - mockRpmQuery("package-1", null); - mockRpmQuery("package-2", null); - assertFalse(yum.remove("package-1", "package-2").converge(taskContext)); - } - - @Test - void testInstall() { - mockRpmQuery("package-1", null); - terminal.expectCommand( - "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", - 0, - "installing, installing"); - - assertTrue(yum - .install("package-1", "package-2") - .converge(taskContext)); - } - - @Test - void skipsYumInstallIfInRpm() { - mockRpmQuery("package-1-0:1.2.3-1", YumPackageName.fromString("package-1-1.2.3-1")); - mockRpmQuery("package-2", YumPackageName.fromString("1:package-2-1.2.3-1.el7.x86_64")); - assertFalse(yum.install("package-1-1.2.3-1", "package-2").converge(taskContext)); - } - - @Test - void testInstallWithEnablerepo() { - mockRpmQuery("package-1", null); - terminal.expectCommand( - "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", - 0, - "installing, installing"); - - assertTrue(yum - .install("package-1", "package-2") - .enableRepo("repo-name") - .converge(taskContext)); - } - - @Test - void testInstallWithEnablerepoDisablerepo() { - mockRpmQuery("package-1", null); - terminal.expectCommand( - "yum install --assumeyes \"--disablerepo=*\" --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", - 0, - "installing, installing"); - - assertTrue(yum - .install("package-1", "package-2") - .enableRepo("repo-name") - .disableRepo("*") - .converge(taskContext)); - } - - @Test - void testWithVersionLock() { - terminal.expectCommand("yum versionlock list 2>&1", - 0, - "Last metadata expiration check: 0:51:26 ago on Thu 14 Jan 2021 09:39:24 AM UTC.\n"); - terminal.expectCommand("yum versionlock add --assumeyes \"openssh-0:8.0p1-4.el8_1.*\" 2>&1"); - terminal.expectCommand( - "yum install --assumeyes openssh-0:8.0p1-4.el8_1.x86_64 2>&1", - 0, - "installing"); - - YumPackageName pkg = new YumPackageName - .Builder("openssh") - .setVersion("8.0p1") - .setRelease("4.el8_1") - .setArchitecture("x86_64") - .build(); - assertTrue(yum.installFixedVersion(pkg).converge(taskContext)); - } - - @Test - void testWithDifferentVersionLock() { - terminal.expectCommand("yum versionlock list 2>&1", - 0, - "Repository chef_rpms-release is listed more than once in the configuration\n" + - "chef-0:12.21.1-1.el7.*\n" + - "package-0:0.1-8.el7.*\n"); - - terminal.expectCommand("yum versionlock delete \"package-0:0.1-8.el7.*\" 2>&1"); - - terminal.expectCommand("yum versionlock add --assumeyes --enablerepo=somerepo \"package-0:0.10-654.el7.*\" 2>&1"); - - terminal.expectCommand( - "yum install --assumeyes --enablerepo=somerepo package-0:0.10-654.el7 2>&1", - 0, - "Nothing to do\n"); - - - assertTrue(yum - .installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")) - .enableRepo("somerepo") - .converge(taskContext)); - } - - @Test - void testWithExistingVersionLock() { - terminal.expectCommand("yum versionlock list 2>&1", - 0, - "Repository chef_rpms-release is listed more than once in the configuration\n" + - "chef-0:12.21.1-1.el7.*\n" + - "package-0:0.10-654.el7.*\n"); - terminal.expectCommand( - "yum install --assumeyes package-0:0.10-654.el7 2>&1", - 0, - "Nothing to do\n"); - - assertFalse(yum.installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")).converge(taskContext)); - } - - @Test - void testWithDowngrade() { - terminal.expectCommand("yum versionlock list 2>&1", - 0, - "Repository chef_rpms-release is listed more than once in the configuration\n" + - "chef-0:12.21.1-1.el7.*\n" + - "package-0:0.10-654.el7.*\n"); - - terminal.expectCommand( - "yum install --assumeyes package-0:0.10-654.el7 2>&1", - 0, - "Package matching package-=.0.10-654.el7 already installed. Checking for update.\n" + - "Nothing to do\n"); - - terminal.expectCommand("yum downgrade --assumeyes package-0:0.10-654.el7 2>&1"); - - assertTrue(yum.installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")).converge(taskContext)); - } - - @Test - void testFailedInstall() { - assertThrows(ChildProcessFailureException.class, () -> { - mockRpmQuery("package-1", null); - terminal.expectCommand( - "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", - 1, - "error"); - - yum - .install("package-1", "package-2") - .enableRepo("repo-name") - .converge(taskContext); - fail(); - }); - } - - @Test - void testUnknownPackages() { - mockRpmQuery("package-1", null); - terminal.expectCommand( - "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 package-3 2>&1", - 0, - "Loaded plugins: fastestmirror, langpacks\n" + - "Loading mirror speeds from cached hostfile\n" + - "No package package-1 available.\n" + - "No package package-2 available.\n" + - "Nothing to do\n"); - - var command = yum.install("package-1", "package-2", "package-3"); - try { - command.converge(taskContext); - fail(); - } catch (Exception e) { - assertNotNull(e.getCause()); - assertEquals("Unknown package: package-1", e.getCause().getMessage()); - } - } - - @Test - void throwIfNoPackagesSpecified() { - assertThrows(IllegalArgumentException.class, () -> { - yum.install(); - }); - } - - @Test - void allowToCallUpgradeWithNoPackages() { - terminal.expectCommand("yum upgrade --assumeyes 2>&1", 0, "OK"); - yum.upgrade().converge(taskContext); - } - - @Test - void testDeleteVersionLock() { - terminal.expectCommand("yum versionlock delete openssh-0:8.0p1-4.el8_1.x86_64 2>&1"); - - YumPackageName pkg = new YumPackageName - .Builder("openssh") - .setVersion("8.0p1") - .setRelease("4.el8_1") - .setArchitecture("x86_64") - .build(); - assertTrue(yum.deleteVersionLock(pkg).converge(taskContext)); - } - - private void mockRpmQuery(String packageName, YumPackageName installedOrNull) { - new YumTester(terminal).expectQueryInstalled(packageName).andReturn(installedOrNull); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java deleted file mode 100644 index aafa0fcfd72..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Optional; -import java.util.function.Function; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class YumTesterTest { - - private static final String[] packages = {"pkg1", "pkg2"}; - private static final String[] repos = {"repo1", "repo2"}; - private static final String[] disablerepos = {"disablerepo1", "disablerepo2"}; - private static final YumPackageName minimalPackage = YumPackageName.fromString("pkg-1.13.1-0.el7"); - private static final YumPackageName fullPackage = YumPackageName.fromString("2:pkg-1.13.1-0.el7.x86_64"); - - private final TestTerminal terminal = new TestTerminal(); - private final YumTester yum = new YumTester(terminal); - private final TestTaskContext context = new TestTaskContext(); - - @Test - void generic_yum_methods() { - assertYumMethod(yum -> yum.expectInstall(packages).withDisableRepo(disablerepos).withEnableRepo(repos), - yum -> yum.install(List.of(packages)).disableRepo(disablerepos).enableRepo(repos).converge(context)); - - assertYumMethod(yum -> yum.expectUpdate(packages).withDisableRepo(disablerepos).withEnableRepo(repos), - yum -> yum.upgrade(List.of(packages)).disableRepo(disablerepos).enableRepo(repos).converge(context)); - - assertYumMethod(yum -> yum.expectRemove(packages).withDisableRepo(disablerepos).withEnableRepo(repos), - yum -> yum.remove(List.of(packages)).disableRepo(disablerepos).enableRepo(repos).converge(context)); - - assertYumMethod(yum -> yum.expectInstallFixedVersion(minimalPackage.toName()).withDisableRepo(disablerepos).withEnableRepo(repos), - yum -> yum.installFixedVersion(minimalPackage).disableRepo(disablerepos).enableRepo(repos).converge(context)); - - // versionlock always returns success - assertYumMethodAlwaysSuccess(yum -> yum.expectDeleteVersionLock(minimalPackage.toName()), - yum -> yum.deleteVersionLock(minimalPackage).converge(context)); - - } - - @Test - void disable_other_repos() { - assertYumMethod(yum -> yum.expectInstall(packages).withDisableRepo("*").withEnableRepo(repos), - yum -> yum.install(List.of(packages)).disableRepo("*").enableRepo(repos).converge(context)); - } - - @Test - void expect_query_installed() { - yum.expectQueryInstalled(packages[0]).andReturn(fullPackage); - assertEquals(Optional.of(fullPackage), yum.queryInstalled(context, packages[0])); - terminal.verifyAllCommandsExecuted(); - } - - private void assertYumMethod(Function<YumTester, YumTester.GenericYumCommandExpectation> yumTesterExpectationFunction, - Function<Yum, Boolean> yumFunction) { - List.of(true, false).forEach(wantedReturnValue -> { - yumTesterExpectationFunction.apply(yum).andReturn(wantedReturnValue); - assertEquals(wantedReturnValue, yumFunction.apply(yum)); - terminal.verifyAllCommandsExecuted(); - }); - } - - private void assertYumMethodAlwaysSuccess(Function<YumTester, YumTester.GenericYumCommandExpectation> yumTesterExpectationFunction, - Function<Yum, Boolean> yumFunction) { - List.of(true, false).forEach(wantedReturnValue -> { - yumTesterExpectationFunction.apply(yum).andReturn(wantedReturnValue); - assertEquals(true, yumFunction.apply(yum)); - terminal.verifyAllCommandsExecuted(); - }); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java deleted file mode 100644 index 7ac47aad1fa..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.wireguard; - -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.WireguardKey; -import com.yahoo.config.provision.WireguardKeyWithTimestamp; -import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.util.List; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author gjoranv - */ -public class WireguardPeerTest { - - @Test - void peers_are_sorted_by_hostname_ascending() { - List<WireguardPeer> peers = Stream.of( - peer("b"), - peer("a"), - peer("c") - ).sorted().toList(); - - assertEquals("a", peers.get(0).hostname().value()); - assertEquals("b", peers.get(1).hostname().value()); - assertEquals("c", peers.get(2).hostname().value()); - } - - private static WireguardPeer peer(String hostname) { - return new WireguardPeer(HostName.of(hostname), List.of(VersionedIpAddress.from("::1:1")), - new WireguardKeyWithTimestamp(WireguardKey.generateRandomForTesting(), Instant.EPOCH)); - } - -} diff --git a/node-admin/src/test/resources/default-env-example.txt b/node-admin/src/test/resources/default-env-example.txt deleted file mode 100644 index debae073271..00000000000 --- a/node-admin/src/test/resources/default-env-example.txt +++ /dev/null @@ -1,5 +0,0 @@ -override VESPA_HOSTNAME myhostname -fallback VESPA_CONFIGSERVER fallback-configserver -fallback VESPA_TLS_CONFIG_FILE /fallback/path/to/config.file -unset VESPA_LEGACY_OPTION -fallback VESPA_LEGACY_OPTION duplicated-variable
\ No newline at end of file diff --git a/node-admin/src/test/resources/default-env-rewritten.txt b/node-admin/src/test/resources/default-env-rewritten.txt deleted file mode 100644 index 94a91f4e793..00000000000 --- a/node-admin/src/test/resources/default-env-rewritten.txt +++ /dev/null @@ -1,4 +0,0 @@ -override VESPA_HOSTNAME my-new-hostname -fallback VESPA_CONFIGSERVER new-fallback-configserver -override VESPA_TLS_CONFIG_FILE /override/path/to/config.file -unset VESPA_LEGACY_OPTION diff --git a/node-admin/src/test/resources/template1.tmp b/node-admin/src/test/resources/template1.tmp deleted file mode 100644 index d53c875a0f3..00000000000 --- a/node-admin/src/test/resources/template1.tmp +++ /dev/null @@ -1,10 +0,0 @@ -variable section '%{=varname}' -%{list listname} -same variable section '%{=varname}' -different variable section '%{=varname2}' -%{list innerlistname} -inner form text -%{end} -between ends -%{end} -end of text diff --git a/node-admin/src/test/resources/template2.tmp b/node-admin/src/test/resources/template2.tmp deleted file mode 100644 index d36cb4a4a48..00000000000 --- a/node-admin/src/test/resources/template2.tmp +++ /dev/null @@ -1,4 +0,0 @@ -%{list listA}body A -%{list listB}body B -%{end} -%{end} diff --git a/node-admin/src/test/resources/template3.tmp b/node-admin/src/test/resources/template3.tmp deleted file mode 100644 index 27566e72a9d..00000000000 --- a/node-admin/src/test/resources/template3.tmp +++ /dev/null @@ -1,6 +0,0 @@ -%{=varname} -%{=varname} -%{list l} -inner %{=varname} -%{=innerVarSetAtTop} -%{end} |