diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2023-11-04 09:25:36 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-11-04 09:25:36 +0100 |
commit | bd42627870b4066b4b8085e17cf67cc7656468f0 (patch) | |
tree | 3ba27ec3212201516afe43745950c978c115999a /node-admin/src/test/java/com | |
parent | f547fce384fb465dc04bbbb95063be69b6b89430 (diff) |
Revert "Move node-admin"
Diffstat (limited to 'node-admin/src/test/java/com')
81 files changed, 10030 insertions, 0 deletions
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java new file mode 100644 index 00000000000..d3982af14e4 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java @@ -0,0 +1,162 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import com.yahoo.vespa.hosted.node.admin.container.ContainerId; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.nio.file.FileSystem; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.sharesToWeight; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.weightToShares; +import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Device; +import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Max; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author freva + */ +public class CgroupTest { + + private static final ContainerId containerId = new ContainerId("4aec78cc"); + + private final FileSystem fileSystem = TestFileSystem.create(); + private final Cgroup containerCgroup = Cgroup.root(fileSystem).resolveContainer(containerId); + private final CpuController containerCpu = containerCgroup.cpu(); + private final NodeAgentContext context = NodeAgentContextImpl.builder("node123.yahoo.com").fileSystem(fileSystem).build(); + private final UnixPath cgroupRoot = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-4aec78cc.scope/container")).createDirectories(); + + @Test + public void updates_cpu_quota_and_period() { + assertEquals(Optional.empty(), containerCgroup.cpu().readMax()); + + cgroupRoot.resolve("cpu.max").writeUtf8File("max 100000\n"); + assertEquals(Optional.of(new CpuController.Max(Size.max(), 100000)), containerCpu.readMax()); + + cgroupRoot.resolve("cpu.max").writeUtf8File("456 123456\n"); + assertEquals(Optional.of(new CpuController.Max(Size.from(456), 123456)), containerCpu.readMax()); + + containerCgroup.cpu().updateMax(context, 456, 123456); + + assertTrue(containerCgroup.cpu().updateMax(context, 654, 123456)); + assertEquals(Optional.of(new CpuController.Max(Size.from(654), 123456)), containerCpu.readMax()); + assertEquals("654 123456\n", cgroupRoot.resolve("cpu.max").readUtf8File()); + + assertTrue(containerCgroup.cpu().updateMax(context, -1, 123456)); + assertEquals(Optional.of(new CpuController.Max(Size.max(), 123456)), containerCpu.readMax()); + assertEquals("max 123456\n", cgroupRoot.resolve("cpu.max").readUtf8File()); + } + + @Test + public void updates_cpu_shares() { + assertEquals(Optional.empty(), containerCgroup.cpu().readShares()); + + cgroupRoot.resolve("cpu.weight").writeUtf8File("1\n"); + assertEquals(Optional.of(2), containerCgroup.cpu().readShares()); + + assertFalse(containerCgroup.cpu().updateShares(context, 2)); + + assertTrue(containerCgroup.cpu().updateShares(context, 12345)); + assertEquals(Optional.of(12323), containerCgroup.cpu().readShares()); + } + + @Test + public void reads_cpu_stats() { + cgroupRoot.resolve("cpu.stat").writeUtf8File(""" + usage_usec 17794243 + user_usec 16099205 + system_usec 1695038 + nr_periods 12465 + nr_throttled 25 + throttled_usec 14256 + """); + + assertEquals(Map.of(TOTAL_USAGE_USEC, 17794243L, USER_USAGE_USEC, 16099205L, SYSTEM_USAGE_USEC, 1695038L, + TOTAL_PERIODS, 12465L, THROTTLED_PERIODS, 25L, THROTTLED_TIME_USEC, 14256L), containerCgroup.cpu().readStats()); + } + + @Test + public void reads_memory_metrics() { + cgroupRoot.resolve("memory.current").writeUtf8File("2525093888\n"); + assertEquals(2525093888L, containerCgroup.memory().readCurrent().value()); + + cgroupRoot.resolve("memory.max").writeUtf8File("4322885632\n"); + assertEquals(4322885632L, containerCgroup.memory().readMax().value()); + + cgroupRoot.resolve("memory.stat").writeUtf8File(""" + anon 3481600 + file 69206016 + kernel_stack 73728 + slab 3552304 + percpu 262336 + sock 73728 + shmem 8380416 + file_mapped 1081344 + file_dirty 135168 + slab_reclaimable 1424320 + """); + var stats = containerCgroup.memory().readStat(); + assertEquals(69206016L, stats.file().value()); + assertEquals(3481600L, stats.anon().value()); + assertEquals(3552304L, stats.slab().value()); + assertEquals(73728L, stats.sock().value()); + assertEquals(1424320L, stats.slabReclaimable().value()); + } + + @Test + public void shares_to_weight_and_back_is_stable() { + for (int i = 2; i <= 262144; i++) { + int originalShares = i; // Must be effectively final to use in lambda :( + int roundTripShares = weightToShares(sharesToWeight(i)); + int diff = i - roundTripShares; + assertTrue(diff >= 0 && diff <= 27, // ~26.2 shares / weight + () -> "Original shares: " + originalShares + ", round trip shares: " + roundTripShares + ", diff: " + diff); + } + } + + @Test + void reads_io_max() { + assertEquals(Optional.empty(), containerCgroup.io().readMax()); + + cgroupRoot.resolve("io.max").writeUtf8File(""); + assertEquals(Optional.of(Map.of()), containerCgroup.io().readMax()); + + cgroupRoot.resolve("io.max").writeUtf8File(""" + 253:1 rbps=11 wbps=max riops=22 wiops=33 + 253:0 rbps=max wbps=44 riops=max wiops=55 + """); + assertEquals(Map.of(new Device(253, 1), new Max(Size.from(11), Size.max(), Size.from(22), Size.from(33)), + new Device(253, 0), new Max(Size.max(), Size.from(44), Size.max(), Size.from(55))), + containerCgroup.io().readMax().orElseThrow()); + } + + @Test + void writes_io_max() { + Device device = new Device(253, 0); + Max initial = new Max(Size.max(), Size.from(44), Size.max(), Size.from(55)); + assertTrue(containerCgroup.io().updateMax(context, device, initial)); + assertEquals("253:0 rbps=max wbps=44 riops=max wiops=55\n", cgroupRoot.resolve("io.max").readUtf8File()); + + cgroupRoot.resolve("io.max").writeUtf8File(""" + 253:1 rbps=11 wbps=max riops=22 wiops=33 + 253:0 rbps=max wbps=44 riops=max wiops=55 + """); + assertFalse(containerCgroup.io().updateMax(context, device, initial)); + + cgroupRoot.resolve("io.max").writeUtf8File(""); + assertFalse(containerCgroup.io().updateMax(context, device, Max.UNLIMITED)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java new file mode 100644 index 00000000000..cb828394249 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java @@ -0,0 +1,19 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import org.junit.jupiter.api.Test; + +import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Max; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +class IoControllerTest { + + @Test + void parse_io_max() { + assertEquals(Max.UNLIMITED, Max.fromString("")); + assertEquals(new Max(Size.from(1), Size.max(), Size.max(), Size.max()), Max.fromString("rbps=1 wiops=max")); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java new file mode 100644 index 00000000000..910fd8e670a --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java @@ -0,0 +1,194 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.http.HttpVersion; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.entity.BasicHttpEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicStatusLine; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.SocketTimeoutException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Basic testing of retry logic. + * + * @author dybis + */ +public class ConfigServerApiImplTest { + + private static final int FAIL_RETURN_CODE = 100000; + private static final int TIMEOUT_RETURN_CODE = 100001; + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class TestPojo { + @JsonProperty("foo") + String foo; + @JsonProperty("error-code") + Integer errorCode; + } + + private final String uri1 = "http://host1:666"; + private final String uri2 = "http://host2:666"; + private final List<URI> configServers = List.of(URI.create(uri1), URI.create(uri2)); + private final StringBuilder mockLog = new StringBuilder(); + + private ConfigServerApiImpl configServerApi; + private int mockReturnCode = 200; + + @BeforeEach + public void initExecutor() throws IOException { + CloseableHttpClient httpMock = mock(CloseableHttpClient.class); + when(httpMock.execute(any())).thenAnswer(invocationOnMock -> { + HttpGet get = (HttpGet) invocationOnMock.getArguments()[0]; + mockLog.append(get.getMethod()).append(" ").append(get.getURI()).append(" "); + + switch (mockReturnCode) { + case FAIL_RETURN_CODE -> throw new RuntimeException("FAIL"); + case TIMEOUT_RETURN_CODE -> throw new SocketTimeoutException("read timed out"); + } + + BasicStatusLine statusLine = new BasicStatusLine(HttpVersion.HTTP_1_1, mockReturnCode, null); + BasicHttpEntity entity = new BasicHttpEntity(); + String returnMessage = "{\"foo\":\"bar\", \"no\":3, \"error-code\": " + mockReturnCode + "}"; + InputStream stream = new ByteArrayInputStream(returnMessage.getBytes(StandardCharsets.UTF_8)); + entity.setContent(stream); + + CloseableHttpResponse response = mock(CloseableHttpResponse.class); + when(response.getEntity()).thenReturn(entity); + when(response.getStatusLine()).thenReturn(statusLine); + + return response; + }); + configServerApi = ConfigServerApiImpl.createForTestingWithClient(configServers, httpMock); + } + + @Test + void testBasicParsingSingleServer() { + TestPojo answer = configServerApi.get("/path", TestPojo.class); + assertEquals(answer.foo, "bar"); + assertLogStringContainsGETForAHost(); + } + + @Test + void testBasicFailure() { + assertThrows(HttpException.class, () -> { + // Server is returning 400, no retries. + mockReturnCode = 400; + + TestPojo testPojo = configServerApi.get("/path", TestPojo.class); + assertEquals(testPojo.errorCode.intValue(), mockReturnCode); + assertLogStringContainsGETForAHost(); + }); + } + + @Test + void testBasicSuccessWithNoRetries() { + // Server is returning 201, no retries. + mockReturnCode = 201; + + TestPojo testPojo = configServerApi.get("/path", TestPojo.class); + assertEquals(testPojo.errorCode.intValue(), mockReturnCode); + assertLogStringContainsGETForAHost(); + } + + @Test + void testBasicSuccessWithCustomTimeouts() { + mockReturnCode = TIMEOUT_RETURN_CODE; + + var params = new ConfigServerApi.Params<TestPojo>(); + params.setConnectionTimeout(Duration.ofSeconds(3)); + + try { + configServerApi.get("/path", TestPojo.class, params); + fail(); + } catch (ConnectionException e) { + assertNotNull(e.getCause()); + assertEquals("read timed out", e.getCause().getMessage()); + } + } + + @Test + void testRetries() { + // Client is throwing exception, should be retries. + mockReturnCode = FAIL_RETURN_CODE; + try { + configServerApi.get("/path", TestPojo.class); + fail("Expected failure"); + } catch (Exception e) { + // ignore + } + + List<String> log = List.of(mockLog.toString().split(" ")); + assertTrue(log.containsAll(List.of("GET http://host1:666/path", "GET http://host2:666/path"))); + } + + @Test + void testNoRetriesOnBadHttpResponseCode() { + // Client is throwing exception, should be retries. + mockReturnCode = 503; + try { + configServerApi.get("/path", TestPojo.class); + fail("Expected failure"); + } catch (Exception e) { + // ignore + } + + assertLogStringContainsGETForAHost(); + } + + @Test + void testForbidden() { + mockReturnCode = 403; + try { + configServerApi.get("/path", TestPojo.class); + fail("Expected exception"); + } catch (HttpException.ForbiddenException e) { + // ignore + } + assertLogStringContainsGETForAHost(); + } + + @Test + void testNotFound() { + // Server is returning 404, special exception is thrown. + mockReturnCode = 404; + try { + configServerApi.get("/path", TestPojo.class); + fail("Expected exception"); + } catch (HttpException.NotFoundException e) { + // ignore + } + assertLogStringContainsGETForAHost(); + } + + @Test + void testConflict() { + // Server is returning 409, no exception is thrown. + mockReturnCode = 409; + configServerApi.get("/path", TestPojo.class); + assertLogStringContainsGETForAHost(); + } + + private void assertLogStringContainsGETForAHost() { + String logString = mockLog.toString(); + assertTrue((logString.equals("GET http://host1:666/path ") || logString.equals("GET http://host2:666/path ")), + "log does not contain expected entries:" + logString); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java new file mode 100644 index 00000000000..430da856cfa --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java @@ -0,0 +1,151 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.cores; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.HostName; +import com.yahoo.test.json.JsonTestHelper; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.node.admin.configserver.StandardConfigServerResponse; +import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +import static com.yahoo.yolean.Exceptions.uncheck; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author hakonhall + */ +class CoresTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final ObjectMapper mapper = new ObjectMapper(); + private final ConfigServerApi configServerApi = mock(ConfigServerApi.class); + private final Cores cores = new CoresImpl(configServerApi); + private final HostName hostname = HostName.of("foo.com"); + private final String id = "5c987afb-347a-49ee-a0c5-bef56bbddeb0"; + private final CoreDumpMetadata metadata = new CoreDumpMetadata() + .setType(CoreDumpMetadata.Type.OOM) + .setCreated(Instant.ofEpochMilli(12345678)) + .setKernelVersion("4.18.0-372.26.1.el8_6.x86_64") + .setCpuMicrocodeVersion("0x1000065") + .setCoreDumpPath(fileSystem.getPath("/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813")) + .setDecryptionToken("987def") + .setDockerImage(DockerImage.fromString("us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8")) + .setBinPath("/usr/bin/java") + .setVespaVersion("8.68.8") + .setBacktraceAllThreads(List.of("Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...", + "Debugger attached successfully.", + " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)", + " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)")) + .setBacktrace(List.of("Example", "of", "backtrace")); + + @Test + void reportOK() { + var oKResponse = new StandardConfigServerResponse(); + oKResponse.message = "OK"; + when(configServerApi.post(any(), any(), any())).thenReturn(oKResponse); + + cores.report(hostname, id, metadata); + + var pathCaptor = ArgumentCaptor.forClass(String.class); + var bodyJsonPojoCaptor = ArgumentCaptor.forClass(Object.class); + verify(configServerApi, times(1)).post(pathCaptor.capture(), bodyJsonPojoCaptor.capture(), any()); + + assertEquals("/cores/v1/report/" + hostname + "/" + id, pathCaptor.getValue()); + + assertEquals(""" + { + "backtrace": [ + "Example", + "of", + "backtrace" + ], + "backtrace_all_threads": [ + "Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...", + "Debugger attached successfully.", + " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)", + " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)" + ], + "bin_path": "/usr/bin/java", + "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813", + "cpu_microcode_version": "0x1000065", + "created": 12345678, + "decryption_token": "987def", + "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8", + "kernel_version": "4.18.0-372.26.1.el8_6.x86_64", + "type": "OOM", + "vespa_version": "8.68.8" + }""", + JsonTestHelper.normalize(uncheck(() -> mapper.writeValueAsString(bodyJsonPojoCaptor.getValue())))); + } + + @Test + void reportFails() { + var response = new StandardConfigServerResponse(); + response.errorCode = "503"; + response.message = "error detail"; + when(configServerApi.post(any(), any(), any())).thenReturn(response); + + assertThrows(ConfigServerException.class, + () -> cores.report(hostname, "abcde-1234", metadata), + "Failed to report core dump at Optional[/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813]: error detail 503"); + + var pathCaptor = ArgumentCaptor.forClass(String.class); + var bodyJsonPojoCaptor = ArgumentCaptor.forClass(Object.class); + verify(configServerApi).post(pathCaptor.capture(), bodyJsonPojoCaptor.capture(), any()); + } + + @Test + void serialization() { + Path path = fileSystem.getPath("/foo.json"); + ReportCoreDumpRequest request = new ReportCoreDumpRequest().fillFrom(metadata); + request.save(path); + assertEquals(""" + { + "backtrace": [ + "Example", + "of", + "backtrace" + ], + "backtrace_all_threads": [ + "Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...", + "Debugger attached successfully.", + " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)", + " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)" + ], + "bin_path": "/usr/bin/java", + "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813", + "cpu_microcode_version": "0x1000065", + "created": 12345678, + "decryption_token": "987def", + "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8", + "kernel_version": "4.18.0-372.26.1.el8_6.x86_64", + "type": "OOM", + "vespa_version": "8.68.8" + }""", + JsonTestHelper.normalize(new UnixPath(path).readUtf8File())); + + Optional<ReportCoreDumpRequest> loaded = ReportCoreDumpRequest.load(path); + assertTrue(loaded.isPresent()); + var meta = new CoreDumpMetadata(); + loaded.get().populateMetadata(meta, fileSystem); + assertEquals(metadata, meta); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java new file mode 100644 index 00000000000..664e25bc744 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java @@ -0,0 +1,40 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.flags; + +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.flags.json.wire.WireFlagData; +import com.yahoo.vespa.flags.json.wire.WireFlagDataList; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author hakonhall + */ +public class RealFlagRepositoryTest { + private final ConfigServerApi configServerApi = mock(ConfigServerApi.class); + private final RealFlagRepository repository = new RealFlagRepository(configServerApi); + + @Test + void test() { + WireFlagDataList list = new WireFlagDataList(); + list.flags = new ArrayList<>(); + list.flags.add(new WireFlagData()); + list.flags.get(0).id = "id1"; + + when(configServerApi.get(any(), eq(WireFlagDataList.class))).thenReturn(list); + Map<FlagId, FlagData> allFlagData = repository.getAllFlagData(); + assertEquals(1, allFlagData.size()); + assertTrue(allFlagData.containsKey(new FlagId("id1"))); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java new file mode 100644 index 00000000000..d91e9befab9 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java @@ -0,0 +1,182 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; + +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * + * @author smorgrav + */ +public class AclTest { + + private static final Acl aclCommon = new Acl( + Set.of(1234, 453), Set.of(4321), + testNodes(Set.of(), "192.1.2.2", "fb00::1", "fe80::2", "fe80::3"), + Set.of()); + + private static final Acl aclWithoutPorts = new Acl( + Set.of(), Set.of(), + testNodes(Set.of(), "192.1.2.2", "fb00::1", "fe80::2"), + Set.of()); + + @Test + void no_trusted_ports() { + String listRulesIpv4 = String.join("\n", aclWithoutPorts.toRules(IPVersion.IPv4)); + assertEquals( + """ + -P INPUT ACCEPT + -P FORWARD ACCEPT + -P OUTPUT ACCEPT + -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + -A INPUT -i lo -j ACCEPT + -A INPUT -p icmp -j ACCEPT + -A INPUT -s 192.1.2.2/32 -j ACCEPT + -A INPUT -j REJECT --reject-with icmp-port-unreachable""", + listRulesIpv4); + } + + @Test + void ipv4_rules() { + String listRulesIpv4 = String.join("\n", aclCommon.toRules(IPVersion.IPv4)); + assertEquals( + """ + -P INPUT ACCEPT + -P FORWARD ACCEPT + -P OUTPUT ACCEPT + -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + -A INPUT -i lo -j ACCEPT + -A INPUT -p icmp -j ACCEPT + -A INPUT -p tcp -m multiport --dports 453,1234 -j ACCEPT + -A INPUT -p udp -m multiport --dports 4321 -j ACCEPT + -A INPUT -s 192.1.2.2/32 -j ACCEPT + -A INPUT -j REJECT --reject-with icmp-port-unreachable""", + listRulesIpv4); + } + + @Test + void ipv6_rules() { + String listRulesIpv6 = String.join("\n", aclCommon.toRules(IPVersion.IPv6)); + assertEquals( + """ + -P INPUT ACCEPT + -P FORWARD ACCEPT + -P OUTPUT ACCEPT + -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + -A INPUT -i lo -j ACCEPT + -A INPUT -p ipv6-icmp -j ACCEPT + -A INPUT -p tcp -m multiport --dports 453,1234 -j ACCEPT + -A INPUT -p udp -m multiport --dports 4321 -j ACCEPT + -A INPUT -s fb00::1/128 -j ACCEPT + -A INPUT -s fe80::2/128 -j ACCEPT + -A INPUT -s fe80::3/128 -j ACCEPT + -A INPUT -j REJECT --reject-with icmp6-port-unreachable""", listRulesIpv6); + } + + @Test + void ipv6_rules_stable_order() { + Acl aclCommonDifferentOrder = new Acl( + Set.of(453, 1234), Set.of(4321), + testNodes(Set.of(), "fe80::2", "192.1.2.2", "fb00::1", "fe80::3"), + Set.of()); + + for (IPVersion ipVersion : IPVersion.values()) { + assertEquals(aclCommon.toRules(ipVersion), aclCommonDifferentOrder.toRules(ipVersion)); + } + } + + @Test + void trusted_networks() { + Acl acl = new Acl(Set.of(4080), Set.of(), testNodes(Set.of(), "127.0.0.1"), Set.of("10.0.0.0/24", "2001:db8::/32")); + + assertEquals(""" + -P INPUT ACCEPT + -P FORWARD ACCEPT + -P OUTPUT ACCEPT + -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + -A INPUT -i lo -j ACCEPT + -A INPUT -p icmp -j ACCEPT + -A INPUT -p tcp -m multiport --dports 4080 -j ACCEPT + -A INPUT -s 127.0.0.1/32 -j ACCEPT + -A INPUT -s 10.0.0.0/24 -j ACCEPT + -A INPUT -j REJECT --reject-with icmp-port-unreachable""", + String.join("\n", acl.toRules(IPVersion.IPv4))); + + assertEquals(""" + -P INPUT ACCEPT + -P FORWARD ACCEPT + -P OUTPUT ACCEPT + -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + -A INPUT -i lo -j ACCEPT + -A INPUT -p ipv6-icmp -j ACCEPT + -A INPUT -p tcp -m multiport --dports 4080 -j ACCEPT + -A INPUT -s 2001:db8::/32 -j ACCEPT + -A INPUT -j REJECT --reject-with icmp6-port-unreachable""", + String.join("\n", acl.toRules(IPVersion.IPv6))); + } + + @Test + void config_server_acl() { + Set<Acl.Node> testNodes = Stream.concat(testNodes(NodeType.config, Set.of(), "172.17.0.41", "172.17.0.42", "172.17.0.43").stream(), + testNodes(NodeType.tenant, Set.of(19070), "172.17.0.81", "172.17.0.82", "172.17.0.83").stream()) + .collect(Collectors.toSet()); + Acl acl = new Acl(Set.of(22, 4443), Set.of(), testNodes, Set.of()); + assertEquals(""" + -P INPUT ACCEPT + -P FORWARD ACCEPT + -P OUTPUT ACCEPT + -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + -A INPUT -i lo -j ACCEPT + -A INPUT -p icmp -j ACCEPT + -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT + -A INPUT -s 172.17.0.41/32 -j ACCEPT + -A INPUT -s 172.17.0.42/32 -j ACCEPT + -A INPUT -s 172.17.0.43/32 -j ACCEPT + -A INPUT -s 172.17.0.81/32 -p tcp -m multiport --dports 19070 -j ACCEPT + -A INPUT -s 172.17.0.82/32 -p tcp -m multiport --dports 19070 -j ACCEPT + -A INPUT -s 172.17.0.83/32 -p tcp -m multiport --dports 19070 -j ACCEPT + -A INPUT -j REJECT --reject-with icmp-port-unreachable""", + String.join("\n", acl.toRules(IPVersion.IPv4))); + + Set<Acl.Node> testNodes2 = Stream.concat(testNodes(NodeType.config, Set.of(), "2001:db8::41", "2001:db8::42", "2001:db8::43").stream(), + testNodes(NodeType.tenant, Set.of(19070), "2001:db8::81", "2001:db8::82", "2001:db8::83").stream()) + .collect(Collectors.toSet()); + Acl acl2 = new Acl(Set.of(22, 4443), Set.of(), testNodes2, Set.of()); + + assertEquals(""" + -P INPUT ACCEPT + -P FORWARD ACCEPT + -P OUTPUT ACCEPT + -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + -A INPUT -i lo -j ACCEPT + -A INPUT -p ipv6-icmp -j ACCEPT + -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT + -A INPUT -s 2001:db8::41/128 -j ACCEPT + -A INPUT -s 2001:db8::42/128 -j ACCEPT + -A INPUT -s 2001:db8::43/128 -j ACCEPT + -A INPUT -s 2001:db8::81/128 -p tcp -m multiport --dports 19070 -j ACCEPT + -A INPUT -s 2001:db8::82/128 -p tcp -m multiport --dports 19070 -j ACCEPT + -A INPUT -s 2001:db8::83/128 -p tcp -m multiport --dports 19070 -j ACCEPT + -A INPUT -j REJECT --reject-with icmp6-port-unreachable""", + String.join("\n", acl2.toRules(IPVersion.IPv6))); + } + + private static Set<Acl.Node> testNodes(Set<Integer> ports, String... address) { + return testNodes(NodeType.tenant, ports, address); + } + + private static Set<Acl.Node> testNodes(NodeType nodeType, Set<Integer> ports, String... address) { + return Arrays.stream(address) + .map(addr -> new Acl.Node("hostname", addr, ports)) + .collect(Collectors.toUnmodifiableSet()); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java new file mode 100644 index 00000000000..b236c223078 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java @@ -0,0 +1,26 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; + +import com.yahoo.vespa.hosted.provision.Node; +import org.junit.jupiter.api.Test; + +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +public class NodeStateTest { + + @Test + void is_equal_to_node_repository_states() { + Set<String> nodeRepositoryStates = Stream.of(Node.State.values()).map(Enum::name).collect(Collectors.toSet()); + Set<String> nodeAdminStates = Stream.of(NodeState.values()).map(Enum::name).collect(Collectors.toSet()); + + assertEquals(nodeAdminStates, nodeRepositoryStates); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java new file mode 100644 index 00000000000..4100b3cf102 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -0,0 +1,249 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; + +import com.yahoo.application.Networking; +import com.yahoo.application.container.JDisc; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; +import com.yahoo.config.provision.host.FlavorOverrides; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; +import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer; +import com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler; +import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.ServerSocket; +import java.net.URI; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Tests the NodeRepository class used for talking to the node repository. It uses a mock from the node repository + * which already contains some data. + * + * @author dybdahl + */ +public class RealNodeRepositoryTest { + + private static final double delta = 0.00000001; + private JDisc container; + private NodeRepository nodeRepositoryApi; + + private int findRandomOpenPort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + socket.setReuseAddress(true); + return socket.getLocalPort(); + } + } + + /** + * Starts NodeRepository with + * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors} + * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository} + * {@link NodesV2ApiHandler} + * These classes define some test data that is used in these tests. + */ + @BeforeEach + public void startContainer() throws Exception { + Exception lastException = null; + + // This tries to bind a random open port for the node-repo mock, which is a race condition, so try + // a few times before giving up + for (int i = 0; i < 3; i++) { + try { + int port = findRandomOpenPort(); + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port, SystemName.main, CloudAccount.from("123456789012")), Networking.enable); + ConfigServerApi configServerApi = ConfigServerApiImpl.createForTesting( + List.of(URI.create("http://127.0.0.1:" + port))); + waitForJdiscContainerToServe(configServerApi); + return; + } catch (RuntimeException e) { + lastException = e; + } + } + throw new RuntimeException("Failed to bind a port in three attempts, giving up", lastException); + } + + private void waitForJdiscContainerToServe(ConfigServerApi configServerApi) throws InterruptedException { + Instant start = Instant.now(); + nodeRepositoryApi = new RealNodeRepository(configServerApi); + while (Instant.now().minusSeconds(120).isBefore(start)) { + try { + nodeRepositoryApi.getNodes("foobar"); + return; + } catch (Exception e) { + Thread.sleep(100); + } + } + throw new RuntimeException("Could not get answer from container."); + } + + @AfterEach + public void stopContainer() { + if (container != null) { + container.close(); + } + } + + @Test + void testGetContainersToRunApi() { + String dockerHostHostname = "dockerhost1.yahoo.com"; + + List<NodeSpec> containersToRun = nodeRepositoryApi.getNodes(dockerHostHostname); + assertEquals(1, containersToRun.size()); + NodeSpec node = containersToRun.get(0); + assertEquals("host4.yahoo.com", node.hostname()); + assertEquals(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.42.0"), node.wantedDockerImage().get()); + assertEquals(NodeState.active, node.state()); + assertEquals(Long.valueOf(0), node.wantedRestartGeneration().get()); + assertEquals(Long.valueOf(0), node.currentRestartGeneration().get()); + assertEquals(1, node.vcpu(), delta); + assertEquals(4, node.memoryGb(), delta); + assertEquals(100, node.diskGb(), delta); + } + + @Test + void testGetContainer() { + String hostname = "host4.yahoo.com"; + Optional<NodeSpec> node = nodeRepositoryApi.getOptionalNode(hostname); + assertTrue(node.isPresent()); + assertEquals(hostname, node.get().hostname()); + assertEquals(CloudAccount.from("123456789012"), node.get().cloudAccount()); + } + + @Test + void testGetContainerForNonExistingNode() { + String hostname = "host-that-does-not-exist"; + Optional<NodeSpec> node = nodeRepositoryApi.getOptionalNode(hostname); + assertFalse(node.isPresent()); + } + + @Test + void testUpdateNodeAttributes() { + var hostname = "host4.yahoo.com"; + var dockerImage = "registry.example.com/repo/image-1:6.2.3"; + var wireguardKey = WireguardKey.from("111122223333444455556666777788889999000042c="); + var wireguardKeyTimestamp = Instant.ofEpochMilli(123L); // Instant from clock in MockNodeRepository + var keyWithTimestamp = new WireguardKeyWithTimestamp(wireguardKey, wireguardKeyTimestamp); + + nodeRepositoryApi.updateNodeAttributes( + hostname, + new NodeAttributes() + .withRestartGeneration(1) + .withDockerImage(DockerImage.fromString(dockerImage)) + .withWireguardPubkey(wireguardKey)); + + NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode(hostname).orElseThrow(); + assertEquals(1, hostSpec.currentRestartGeneration().orElseThrow()); + assertEquals(dockerImage, hostSpec.currentDockerImage().orElseThrow().asString()); + assertEquals(keyWithTimestamp, hostSpec.wireguardKeyWithTimestamp().orElseThrow()); + } + + @Test + void testMarkAsReady() { + nodeRepositoryApi.setNodeState("host5.yahoo.com", NodeState.dirty); + nodeRepositoryApi.setNodeState("host5.yahoo.com", NodeState.ready); + + try { + nodeRepositoryApi.setNodeState("host4.yahoo.com", NodeState.ready); + fail("Should not be allowed to be marked ready as it is not registered as provisioned, dirty, failed or parked"); + } catch (RuntimeException ignored) { + // expected + } + + try { + nodeRepositoryApi.setNodeState("host101.yahoo.com", NodeState.ready); + fail("Expected failure because host101 does not exist"); + } catch (RuntimeException ignored) { + // expected + } + } + + @Test + void testAddNodes() { + AddNode host = AddNode.forHost("host123.domain.tld", + "id1", + "default", + Optional.of(FlavorOverrides.ofDisk(123)), + NodeType.confighost, + Set.of("::1"), Set.of("::2", "::3")); + + NodeResources nodeResources = new NodeResources(1, 2, 3, 4, NodeResources.DiskSpeed.slow, NodeResources.StorageType.local); + AddNode node = AddNode.forNode("host123-1.domain.tld", "id1", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3")); + + assertFalse(nodeRepositoryApi.getOptionalNode("host123.domain.tld").isPresent()); + nodeRepositoryApi.addNodes(List.of(host, node)); + + NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode("host123.domain.tld").orElseThrow(); + assertEquals("id1", hostSpec.id()); + assertEquals("default", hostSpec.flavor()); + assertEquals(123, hostSpec.diskGb(), 0); + assertEquals(NodeType.confighost, hostSpec.type()); + assertEquals(NodeResources.Architecture.x86_64, hostSpec.resources().architecture()); + + NodeSpec nodeSpec = nodeRepositoryApi.getOptionalNode("host123-1.domain.tld").orElseThrow(); + assertEquals(nodeResources, nodeSpec.resources()); + assertEquals(NodeType.config, nodeSpec.type()); + } + + @Test + void wireguard_peer_config_can_be_retrieved_for_configservers_and_exclave_nodes() { + + //// Configservers //// + + List<WireguardPeer> cfgPeers = nodeRepositoryApi.getConfigserverPeers(); + + // cfg2 does not have a wg public key, so should not be included + assertEquals(1, cfgPeers.size()); + + assertWireguardPeer(cfgPeers.get(0), "cfg1.yahoo.com", + "::201:1", + "lololololololololololololololololololololoo=", + 456L); + + //// Exclave nodes //// + + List<WireguardPeer> exclavePeers = nodeRepositoryApi.getExclavePeers(); + + // host3 does not have a wg public key, so should not be included + assertEquals(1, exclavePeers.size()); + + assertWireguardPeer(exclavePeers.get(0), "dockerhost2.yahoo.com", + "::101:1", + "000011112222333344445555666677778888999900c=", + 123L); + } + + private void assertWireguardPeer(WireguardPeer peer, String hostname, String ipv6, + String publicKey, long keyTimestamp) { + assertEquals(hostname, peer.hostname().value()); + assertEquals(1, peer.ipAddresses().size()); + assertIp(peer.ipAddresses().get(0), ipv6, 6); + var expectedKeyWithTimestamp = new WireguardKeyWithTimestamp(WireguardKey.from(publicKey), + Instant.ofEpochMilli(keyTimestamp)); + assertEquals(expectedKeyWithTimestamp, peer.keyWithTimestamp()); + } + + private void assertIp(VersionedIpAddress ip, String expectedIp, int expectedVersion) { + assertEquals(expectedIp, ip.asString()); + assertEquals(expectedVersion, ip.version().version()); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java new file mode 100644 index 00000000000..8de5986739e --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java @@ -0,0 +1,72 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.test.json.JsonTestHelper; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.RealNodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; + +import static com.yahoo.yolean.Exceptions.uncheck; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author hakonhall + */ +public class NodeRepositoryNodeTest { + private static final ObjectMapper mapper = new ObjectMapper(); + private final NodeRepositoryNode node = new NodeRepositoryNode(); + private final NodeAttributes attributes = new NodeAttributes(); + + + /** + * Test both how NodeRepositoryNode serialize, and the serialization of an empty NodeRepositoryNode + * patched with a NodeAttributes, as they work in tandem: + * NodeAttributes -> NodeRepositoryNode -> JSON. + */ + @Test + void testReportsSerialization() { + // Make sure we don't accidentally patch with "reports": null, as that actually means removing all reports. + assertEquals(JsonInclude.Include.NON_NULL, NodeRepositoryNode.class.getAnnotation(JsonInclude.class).value()); + + // Absent report and unmodified attributes => nothing about reports in JSON + node.reports = null; + assertNodeAndAttributes("{}"); + + // Make sure we're able to patch with a null report value ("reportId": null), as that means removing the report. + node.reports = new HashMap<>(); + node.reports.put("rid", null); + attributes.withReportRemoved("rid"); + assertNodeAndAttributes("{\"reports\": {\"rid\": null}}"); + + // Add ridTwo report to node + ObjectNode reportJson = mapper.createObjectNode(); + reportJson.set(BaseReport.CREATED_FIELD, mapper.valueToTree(3)); + reportJson.set(BaseReport.DESCRIPTION_FIELD, mapper.valueToTree("desc")); + node.reports.put("ridTwo", reportJson); + + // Add ridTwo report to attributes + BaseReport reportTwo = new BaseReport(3L, "desc", null); + attributes.withReport("ridTwo", reportTwo.toJsonNode()); + + // Verify node serializes to expected, as well as attributes patched on node. + assertNodeAndAttributes("{\"reports\": {\"rid\": null, \"ridTwo\": {\"createdMillis\": 3, \"description\": \"desc\"}}}"); + } + + private void assertNodeAndAttributes(String expectedJson) { + assertNodeJson(node, expectedJson); + assertNodeJson(RealNodeRepository.nodeRepositoryNodeFromNodeAttributes(attributes), expectedJson); + } + + private void assertNodeJson(NodeRepositoryNode node, String json) { + JsonNode expected = uncheck(() -> mapper.readTree(json)); + JsonNode actual = uncheck(() -> mapper.valueToTree(node)); + JsonTestHelper.assertJsonEquals(actual, expected); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java new file mode 100644 index 00000000000..69e79ec8720 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java @@ -0,0 +1,73 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports; + +import com.yahoo.test.json.JsonTestHelper; +import org.junit.jupiter.api.Test; + +import static com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport.Type.SOFT_FAIL; +import static com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport.Type.UNSPECIFIED; +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author hakonhall + */ +public class BaseReportTest { + private static final String JSON_1 = "{\"createdMillis\": 1, \"description\": \"desc\"}"; + private static final String JSON_2 = "{\"createdMillis\": 1, \"description\": \"desc\", \"type\": \"SOFT_FAIL\"}"; + + @Test + void testSerialization1() { + JsonTestHelper.assertJsonEquals(new BaseReport(1L, "desc", SOFT_FAIL).toJsonNode(), + JSON_2); + JsonTestHelper.assertJsonEquals(new BaseReport(null, "desc", SOFT_FAIL).toJsonNode(), + "{\"description\": \"desc\", \"type\": \"SOFT_FAIL\"}"); + JsonTestHelper.assertJsonEquals(new BaseReport(1L, null, SOFT_FAIL).toJsonNode(), + "{\"createdMillis\": 1, \"type\": \"SOFT_FAIL\"}"); + JsonTestHelper.assertJsonEquals(new BaseReport(null, null, SOFT_FAIL).toJsonNode(), + "{\"type\": \"SOFT_FAIL\"}"); + + JsonTestHelper.assertJsonEquals(new BaseReport(1L, "desc", null).toJsonNode(), + JSON_1); + JsonTestHelper.assertJsonEquals(new BaseReport(null, "desc", null).toJsonNode(), + "{\"description\": \"desc\"}"); + JsonTestHelper.assertJsonEquals(new BaseReport(1L, null, null).toJsonNode(), + "{\"createdMillis\": 1}"); + JsonTestHelper.assertJsonEquals(new BaseReport(null, null, null).toJsonNode(), + "{}"); + } + + @Test + void testShouldUpdate() { + BaseReport report = new BaseReport(1L, "desc", SOFT_FAIL); + assertFalse(report.updates(report)); + + // createdMillis is ignored + assertFalse(new BaseReport(1L, "desc", SOFT_FAIL).updates(report)); + assertFalse(new BaseReport(2L, "desc", SOFT_FAIL).updates(report)); + assertFalse(new BaseReport(null, "desc", SOFT_FAIL).updates(report)); + + // description is not ignored + assertTrue(new BaseReport(1L, "desc 2", SOFT_FAIL).updates(report)); + assertTrue(new BaseReport(1L, null, SOFT_FAIL).updates(report)); + + // type is not ignored + assertTrue(new BaseReport(1L, "desc", null).updates(report)); + assertTrue(new BaseReport(1L, "desc", BaseReport.Type.HARD_FAIL).updates(report)); + } + + @Test + void testJsonSerialization() { + BaseReport report = BaseReport.fromJson(JSON_2); + assertEquals(1L, (long) report.getCreatedMillisOrNull()); + assertEquals("desc", report.getDescriptionOrNull()); + assertEquals(SOFT_FAIL, report.getTypeOrNull()); + JsonTestHelper.assertJsonEquals(report.toJson(), JSON_2); + } + + @Test + void testUnspecifiedType() { + BaseReport report = new BaseReport(1L, "desc", null); + assertNull(report.getTypeOrNull()); + assertEquals(UNSPECIFIED, report.getType()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java new file mode 100644 index 00000000000..bb9c075ad74 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java @@ -0,0 +1,172 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; + +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; +import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; +import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; +import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason; +import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author freva + */ +public class OrchestratorImplTest { + + private static final String hostName = "host123.yahoo.com"; + + private final ConfigServerApiImpl configServerApi = mock(ConfigServerApiImpl.class); + private final OrchestratorImpl orchestrator = new OrchestratorImpl(configServerApi); + + @Test + void testSuspendCall() { + when(configServerApi.put( + eq(OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended"), + eq(Optional.empty()), + eq(UpdateHostResponse.class), + any() + )).thenReturn(new UpdateHostResponse(hostName, null)); + + orchestrator.suspend(hostName); + } + + @Test + void testSuspendCallWithFailureReason() { + assertThrows(OrchestratorException.class, () -> { + when(configServerApi.put( + eq(OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended"), + eq(Optional.empty()), + eq(UpdateHostResponse.class), + any() + )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail"))); + + orchestrator.suspend(hostName); + }); + } + + @Test + void testSuspendCallWithNotFound() { + assertThrows(OrchestratorNotFoundException.class, () -> { + when(configServerApi.put(any(String.class), any(), any(), any())) + .thenThrow(new HttpException.NotFoundException("Not Found")); + + orchestrator.suspend(hostName); + }); + } + + @Test + void testSuspendCallWithSomeOtherException() { + assertThrows(RuntimeException.class, () -> { + when(configServerApi.put(any(String.class), any(), any(), any())) + .thenThrow(new RuntimeException("Some parameter was wrong")); + + orchestrator.suspend(hostName); + }); + } + + + @Test + void testResumeCall() { + when(configServerApi.delete( + OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended", + UpdateHostResponse.class + )).thenReturn(new UpdateHostResponse(hostName, null)); + + orchestrator.resume(hostName); + } + + @Test + void testResumeCallWithFailureReason() { + assertThrows(OrchestratorException.class, () -> { + when(configServerApi.delete( + OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended", + UpdateHostResponse.class + )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail"))); + + orchestrator.resume(hostName); + }); + } + + @Test + void testResumeCallWithNotFound() { + assertThrows(OrchestratorNotFoundException.class, () -> { + when(configServerApi.delete( + any(String.class), + any() + )).thenThrow(new HttpException.NotFoundException("Not Found")); + + orchestrator.resume(hostName); + }); + } + + @Test + void testResumeCallWithSomeOtherException() { + assertThrows(RuntimeException.class, () -> { + when(configServerApi.put(any(String.class), any(), any(), any())) + .thenThrow(new RuntimeException("Some parameter was wrong")); + + orchestrator.suspend(hostName); + }); + } + + @Test + void testBatchSuspendCall() { + String parentHostName = "host1.test.yahoo.com"; + List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); + + when(configServerApi.put( + eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"), + eq(Optional.empty()), + eq(BatchOperationResult.class), + any() + )).thenReturn(BatchOperationResult.successResult()); + + orchestrator.suspend(parentHostName, hostNames); + } + + @Test + void testBatchSuspendCallWithFailureReason() { + assertThrows(OrchestratorException.class, () -> { + String parentHostName = "host1.test.yahoo.com"; + List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); + String failureReason = "Failed to suspend"; + + when(configServerApi.put( + eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"), + eq(Optional.empty()), + eq(BatchOperationResult.class), + any() + )).thenReturn(new BatchOperationResult(failureReason)); + + orchestrator.suspend(parentHostName, hostNames); + }); + } + + @Test + void testBatchSuspendCallWithSomeException() { + assertThrows(RuntimeException.class, () -> { + String parentHostName = "host1.test.yahoo.com"; + List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); + String exceptionMessage = "Exception: Something crashed!"; + + when(configServerApi.put( + eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"), + eq(Optional.empty()), + eq(BatchOperationResult.class), + any() + )).thenThrow(new RuntimeException(exceptionMessage)); + + orchestrator.suspend(parentHostName, hostNames); + }); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java new file mode 100644 index 00000000000..478e89cde34 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java @@ -0,0 +1,54 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.state; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class HealthResponseTest { + @Test + void deserializationOfNormalResponse() throws Exception { + String jsonResponse = "{\n" + + " \"metrics\": {\n" + + " \"snapshot\": {\n" + + " \"from\": 1.523614569023E9,\n" + + " \"to\": 1.523614629023E9\n" + + " },\n" + + " \"values\": [\n" + + " {\n" + + " \"name\": \"requestsPerSecond\",\n" + + " \"values\": {\n" + + " \"count\": 121,\n" + + " \"rate\": 2.0166666666666666\n" + + " }\n" + + " },\n" + + " {\n" + + " \"name\": \"latencySeconds\",\n" + + " \"values\": {\n" + + " \"average\": 5.537190082644628E-4,\n" + + " \"count\": 121,\n" + + " \"last\": 0.001,\n" + + " \"max\": 0.001,\n" + + " \"min\": 0,\n" + + " \"rate\": 2.0166666666666666\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"status\": {\"code\": \"up\"},\n" + + " \"time\": 1523614629451\n" + + "}"; + + HealthResponse response = deserialize(jsonResponse); + + assertEquals(response.status.code, "up"); + } + + private static HealthResponse deserialize(String json) throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + return mapper.readValue(json, HealthResponse.class); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java new file mode 100644 index 00000000000..733a105f047 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java @@ -0,0 +1,39 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.state; + +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException; +import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; +import org.junit.jupiter.api.Test; + +import java.net.ConnectException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class StateImplTest { + private final ConfigServerApi api = mock(ConfigServerApi.class); + private final StateImpl state = new StateImpl(api); + + @Test + void testWhenUp() { + HealthResponse response = new HealthResponse(); + response.status.code = "up"; + when(api.get(any(), any())).thenReturn(response); + + HealthCode code = state.getHealth(); + assertEquals(HealthCode.UP, code); + } + + @Test + void connectException() { + RuntimeException exception = + ConnectionException.handleException("Error: ", new ConnectException("connection refused")); + when(api.get(any(), any())).thenThrow(exception); + + HealthCode code = state.getHealth(); + assertEquals(HealthCode.DOWN, code); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java new file mode 100644 index 00000000000..45b74368fd8 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java @@ -0,0 +1,256 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container; + +import com.yahoo.config.provision.DockerImage; +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.container.image.Image; +import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; +import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; +import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; +import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; +import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; + +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; + +/** + * @author mpolden + */ +public class ContainerEngineMock implements ContainerEngine { + + private final Map<ContainerName, Container> containers = new ConcurrentHashMap<>(); + private final Map<String, ImageDownload> images = new ConcurrentHashMap<>(); + private boolean asyncImageDownload = false; + + private final TestTerminal terminal; + + public ContainerEngineMock() { + this(null); + } + + public ContainerEngineMock(TestTerminal terminal) { + this.terminal = terminal; + } + + public ContainerEngineMock asyncImageDownload(boolean enabled) { + this.asyncImageDownload = enabled; + return this; + } + + public ContainerEngineMock completeDownloadOf(DockerImage image) { + String imageId = image.asString(); + ImageDownload download; + while ((download = images.get(imageId)) == null); + download.complete(); + return this; + } + + public ContainerEngineMock setImages(List<Image> images) { + this.images.clear(); + for (var image : images) { + ImageDownload imageDownload = new ImageDownload(image); + imageDownload.complete(); + this.images.put(image.id(), imageDownload); + } + return this; + } + + public ContainerEngineMock addContainers(List<Container> containers) { + for (var container : containers) { + if (this.containers.containsKey(container.name())) { + throw new IllegalArgumentException("Container " + container.name() + " already exists"); + } + this.containers.put(container.name(), container); + } + return this; + } + + public ContainerEngineMock addContainer(Container container) { + return addContainers(List.of(container)); + } + + @Override + public ContainerData createContainer(NodeAgentContext context, ContainerResources containerResources) { + addContainer(createContainer(context, PartialContainer.State.created, containerResources)); + return new ContainerData() { + @Override + public void addFile(ContainerPath path, String data) { + throw new UnsupportedOperationException("addFile not implemented"); + } + + @Override + public void addFile(ContainerPath path, String data, String permissions) { + throw new UnsupportedOperationException("addFile not implemented"); + } + + @Override + public void addDirectory(ContainerPath path, String... permissions) { + throw new UnsupportedOperationException("addDirectory not implemented"); + } + + @Override + public void addSymlink(ContainerPath symlink, Path target) { + throw new UnsupportedOperationException("addSymlink not implemented"); + } + + @Override + public void converge(NodeAgentContext context) { + throw new UnsupportedOperationException("converge not implemented"); + } + }; + } + + @Override + public void startContainer(NodeAgentContext context) { + Container container = requireContainer(context.containerName(), PartialContainer.State.created); + Container newContainer = createContainer(context, PartialContainer.State.running, container.resources()); + containers.put(newContainer.name(), newContainer); + } + + @Override + public void removeContainer(TaskContext context, PartialContainer container) { + requireContainer(container.name()); + containers.remove(container.name()); + } + + @Override + public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) { + Container container = requireContainer(context.containerName()); + containers.put(container.name(), new Container(containerId, container.name(), container.createdAt(), container.state(), + container.imageId(), container.image(), + container.labels(), container.pid(), + container.conmonPid(), container.hostname(), + containerResources, container.networks(), + container.managed())); + } + + @Override + public Optional<Container> getContainer(NodeAgentContext context) { + return Optional.ofNullable(containers.get(context.containerName())); + } + + @Override + public List<PartialContainer> listContainers(TaskContext context) { + return List.copyOf(containers.values()); + } + + @Override + public String networkInterface(NodeAgentContext context) { + return "eth0"; + } + + @Override + public CommandResult execute(NodeAgentContext context, UnixUser user, Duration timeout, String... command) { + if (terminal == null) { + return new CommandResult(null, 0, ""); + } + return terminal.newCommandLine(context) + .add(command) + .executeSilently(); + } + + @Override + public CommandResult executeInNetworkNamespace(NodeAgentContext context, CommandLine.Options options, String... command) { + if (terminal == null) { + return new CommandResult(null, 0, ""); + } + return terminal.newCommandLine(context).add(command).execute(options); + } + + @Override + public void pullImage(TaskContext context, DockerImage image, RegistryCredentials registryCredentials) { + String imageId = image.asString(); + ImageDownload imageDownload = images.computeIfAbsent(imageId, (ignored) -> new ImageDownload(new Image(imageId, List.of(imageId)))); + if (!asyncImageDownload) { + imageDownload.complete(); + } + imageDownload.awaitCompletion(); + } + + @Override + public boolean hasImage(TaskContext context, DockerImage image) { + ImageDownload download = images.get(image.asString()); + return download != null && download.isComplete(); + } + + @Override + public void removeImage(TaskContext context, String id) { + images.remove(id); + } + + @Override + public List<Image> listImages(TaskContext context) { + return images.values().stream() + .filter(ImageDownload::isComplete) + .map(ImageDownload::image) + .toList(); + } + + private Container requireContainer(ContainerName name) { + return requireContainer(name, null); + } + + private Container requireContainer(ContainerName name, PartialContainer.State wantedState) { + Container container = containers.get(name); + if (container == null) throw new IllegalArgumentException("No such container: " + name); + if (wantedState != null && container.state() != wantedState) throw new IllegalArgumentException("Container is " + container.state() + ", wanted " + wantedState); + return container; + } + + public Container createContainer(NodeAgentContext context, PartialContainer.State state, ContainerResources containerResources) { + return new Container(new ContainerId("id-of-" + context.containerName()), + context.containerName(), + Instant.EPOCH, + state, + "image-id", + context.node().wantedDockerImage().get(), + Map.of(), + 41, + 42, + context.hostname().value(), + containerResources, + List.of(), + true); + } + + private static class ImageDownload { + + private final Image image; + private final CountDownLatch done = new CountDownLatch(1); + + ImageDownload(Image image) { + this.image = Objects.requireNonNull(image); + } + + Image image() { + return image; + } + + boolean isComplete() { + return done.getCount() == 0; + } + + void complete() { + done.countDown(); + } + + void awaitCompletion() { + try { + done.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java new file mode 100644 index 00000000000..f9f7a18597c --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java @@ -0,0 +1,52 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author freva + */ +public class ContainerNameTest { + @Test + void testAlphanumericalContainerName() { + String name = "container123"; + ContainerName containerName = new ContainerName(name); + assertEquals(containerName.asString(), name); + } + + @Test + void testAlphanumericalWithDashContainerName() { + String name = "container-123"; + ContainerName containerName = new ContainerName(name); + assertEquals(containerName.asString(), name); + } + + @Test + void testContainerNameFromHostname() { + assertEquals(new ContainerName("container-123"), ContainerName.fromHostname("container-123.sub.domain.tld")); + } + + @Test + void testAlphanumericalWithSlashContainerName() { + assertThrows(IllegalArgumentException.class, () -> { + new ContainerName("container/123"); + }); + } + + @Test + void testEmptyContainerName() { + assertThrows(IllegalArgumentException.class, () -> { + new ContainerName(""); + }); + } + + @Test + void testNullContainerName() { + assertThrows(NullPointerException.class, () -> { + new ContainerName(null); + }); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java new file mode 100644 index 00000000000..a72e926a471 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java @@ -0,0 +1,70 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container; + +import com.yahoo.config.provision.DockerImage; +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.nio.file.FileSystem; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * @author mpolden + */ +public class ContainerOperationsTest { + + private final TestTaskContext context = new TestTaskContext(); + private final ContainerEngineMock containerEngine = new ContainerEngineMock(); + private final FileSystem fileSystem = TestFileSystem.create(); + private final TestTimer timer = new TestTimer(); + private final ContainerOperations containerOperations = new ContainerOperations(containerEngine, mock(Cgroup.class), fileSystem, timer); + + @Test + void no_managed_containers_running() { + Container c1 = createContainer("c1", true); + Container c2 = createContainer("c2", false); + + containerEngine.addContainer(c1); + assertFalse(containerOperations.noManagedContainersRunning(context)); + + containerEngine.removeContainer(context, c1); + assertTrue(containerOperations.noManagedContainersRunning(context)); + + containerEngine.addContainer(c2); + assertTrue(containerOperations.noManagedContainersRunning(context)); + } + + @Test + void retain_managed_containers() { + Container c1 = createContainer("c1", true); + Container c2 = createContainer("c2", true); + Container c3 = createContainer("c3", false); + containerEngine.addContainers(List.of(c1, c2, c3)); + + assertEquals(3, containerEngine.listContainers(context).size()); + containerOperations.retainManagedContainers(context, Set.of(c1.name())); + + assertEquals(List.of(c1.name(), c3.name()), containerEngine.listContainers(context).stream() + .map(PartialContainer::name) + .sorted() + .toList()); + } + + private Container createContainer(String name, boolean managed) { + return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, PartialContainer.State.running, + "image-id", DockerImage.EMPTY, Map.of(), 42, 43, name, + ContainerResources.UNLIMITED, List.of(), managed); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java new file mode 100644 index 00000000000..cbc803b6105 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java @@ -0,0 +1,49 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * @author freva + */ +public class ContainerResourcesTest { + + @Test + void verify_unlimited() { + assertEquals(-1, ContainerResources.UNLIMITED.cpuQuota()); + assertEquals(100_000, ContainerResources.UNLIMITED.cpuPeriod()); + assertEquals(0, ContainerResources.UNLIMITED.cpuShares()); + } + + @Test + void validate_shares() { + new ContainerResources(0, 0, 0); + new ContainerResources(0, 2, 0); + new ContainerResources(0, 2048, 0); + new ContainerResources(0, 262_144, 0); + + assertThrows(IllegalArgumentException.class, () -> new ContainerResources(0, -1, 0)); // Negative shares not allowed + assertThrows(IllegalArgumentException.class, () -> new ContainerResources(0, 1, 0)); // 1 share not allowed + assertThrows(IllegalArgumentException.class, () -> new ContainerResources(0, 262_145, 0)); + } + + @Test + void cpu_shares_scaling() { + ContainerResources resources = ContainerResources.from(5.3, 2.5, 0); + assertEquals(530_000, resources.cpuQuota()); + assertEquals(100_000, resources.cpuPeriod()); + assertEquals(80, resources.cpuShares()); + } + + private static void assertThrows(Class<? extends Throwable> clazz, Runnable runnable) { + try { + runnable.run(); + fail("Expected " + clazz); + } catch (Throwable e) { + if (!clazz.isInstance(e)) throw e; + } + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java new file mode 100644 index 00000000000..8cd3d6529c5 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java @@ -0,0 +1,147 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container; + +import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; +import com.yahoo.vespa.hosted.node.admin.cgroup.MemoryController; +import com.yahoo.vespa.hosted.node.admin.cgroup.Size; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; +import org.mockito.Answers; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author mpolden + */ +public class ContainerStatsCollectorTest { + + private final TestTerminal testTerminal = new TestTerminal(); + private final ContainerEngineMock containerEngine = new ContainerEngineMock(testTerminal); + private final FileSystem fileSystem = TestFileSystem.create(); + private final Cgroup cgroup = mock(Cgroup.class, Answers.RETURNS_DEEP_STUBS); + private final NodeAgentContext context = NodeAgentContextImpl.builder(NodeSpec.Builder.testSpec("c1").build()) + .fileSystem(TestFileSystem.create()) + .build(); + @Test + void collect() throws Exception { + ContainerStatsCollector collector = new ContainerStatsCollector(containerEngine, cgroup, fileSystem, 24); + ContainerId containerId = new ContainerId("id1"); + int containerPid = 42; + assertTrue(collector.collect(context, containerId, containerPid, "eth0").isEmpty(), "No stats found"); + + mockMemoryStats(containerId); + mockCpuStats(containerId); + mockNetworkStats(containerPid); + + Optional<ContainerStats> stats = collector.collect(context, containerId, containerPid, "eth0"); + assertTrue(stats.isPresent()); + assertEquals(new ContainerStats.CpuStats(24, 6049374780000L, 691675615472L, + 262190000000L, 3L, 1L, 2L), + stats.get().cpuStats()); + assertEquals(new ContainerStats.MemoryStats(470790144L, 1228017664L, 2147483648L), + stats.get().memoryStats()); + assertEquals(Map.of("eth0", new ContainerStats.NetworkStats(22280813L, 4L, 3L, + 19859383L, 6L, 5L)), + stats.get().networks()); + assertEquals(List.of(), stats.get().gpuStats()); + + mockGpuStats(); + stats = collector.collect(context, containerId, containerPid, "eth0"); + assertTrue(stats.isPresent()); + assertEquals(List.of(new ContainerStats.GpuStats(0, 35, 16106127360L, 6144655360L), + new ContainerStats.GpuStats(1, 67, 32212254720L, 19314769920L)), + stats.get().gpuStats()); + } + + private void mockGpuStats() throws IOException { + Path devPath = fileSystem.getPath("/dev"); + Files.createDirectories(devPath); + Files.createFile(devPath.resolve("nvidia0")); + testTerminal.expectCommand("nvidia-smi --query-gpu=index,utilization.gpu,memory.total,memory.free --format=csv,noheader,nounits 2>&1", 0, + """ + 0, 35, 15360, 9500 + 1, 67, 30720, 12300 + """); + } + + private void mockNetworkStats(int pid) { + UnixPath dev = new UnixPath(fileSystem.getPath("/proc/" + pid + "/net/dev")); + dev.createParents().writeUtf8File("Inter-| Receive | Transmit\n" + + " face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n" + + " lo: 36289258 149700 0 0 0 0 0 0 36289258 149700 0 0 0 0 0 0\n" + + " eth0: 22280813 118083 3 4 0 0 0 0 19859383 115415 5 6 0 0 0 0\n"); + } + + private void mockMemoryStats(ContainerId containerId) { + when(cgroup.resolveContainer(eq(containerId)).memory().readCurrent()).thenReturn(Size.from(1228017664L)); + when(cgroup.resolveContainer(eq(containerId)).memory().readMax()).thenReturn(Size.from(2147483648L)); + when(cgroup.resolveContainer(eq(containerId)).memory().readStat()).thenReturn( + new MemoryController.Stats(Size.from(470790144L), Size.from(0), Size.from(0), Size.from(0), Size.from(0))); + } + + private void mockCpuStats(ContainerId containerId) throws IOException { + UnixPath proc = new UnixPath(fileSystem.getPath("/proc")); + proc.createDirectories(); + + when(cgroup.resolveContainer(eq(containerId)).cpu().readStats()).thenReturn(Map.of( + TOTAL_USAGE_USEC, 691675615472L, SYSTEM_USAGE_USEC, 262190000000L, USER_USAGE_USEC, 40900L, + TOTAL_PERIODS, 1L, THROTTLED_PERIODS, 2L, THROTTLED_TIME_USEC, 3L)); + + proc.resolve("stat").writeUtf8File("cpu 7991366 978222 2346238 565556517 1935450 25514479 615206 0 0 0\n" + + "cpu0 387906 61529 99088 23516506 42258 1063359 29882 0 0 0\n" + + "cpu1 271253 49383 86149 23655234 41703 1061416 31885 0 0 0\n" + + "cpu2 349420 50987 93560 23571695 59437 1051977 24461 0 0 0\n" + + "cpu3 328107 50628 93406 23605135 44378 1048549 30199 0 0 0\n" + + "cpu4 267474 50404 99253 23606041 113094 1038572 26494 0 0 0\n" + + "cpu5 309584 50677 94284 23550372 132616 1033661 29436 0 0 0\n" + + "cpu6 477926 56888 121251 23367023 83121 1074930 28818 0 0 0\n" + + "cpu7 335335 29350 106130 23551107 95606 1066394 26156 0 0 0\n" + + "cpu8 323678 28629 99171 23586501 82183 1064708 25403 0 0 0\n" + + "cpu9 329805 27516 98538 23579458 89235 1061561 25140 0 0 0\n" + + "cpu10 291536 26455 93934 23642345 81282 1049736 25228 0 0 0\n" + + "cpu11 271103 25302 90630 23663641 85711 1048781 24291 0 0 0\n" + + "cpu12 323634 63392 100406 23465340 132684 1089157 28319 0 0 0\n" + + "cpu13 348085 49568 100772 23490388 114190 1079474 20948 0 0 0\n" + + "cpu14 310712 51208 90461 23547980 101601 1071940 26712 0 0 0\n" + + "cpu15 360405 52754 94620 23524878 79851 1062050 26836 0 0 0\n" + + "cpu16 367893 52141 98074 23541314 57500 1058968 25242 0 0 0\n" + + "cpu17 412756 51486 101592 23515056 47653 1044874 27467 0 0 0\n" + + "cpu18 287307 25478 106011 23599505 79848 1089812 23160 0 0 0\n" + + "cpu19 275001 24421 98338 23628694 79675 1084074 22083 0 0 0\n" + + "cpu20 288038 24805 94432 23629908 74735 1078501 21915 0 0 0\n" + + "cpu21 295373 25017 91344 23628585 75282 1071019 22026 0 0 0\n" + + "cpu22 326739 25588 90385 23608217 69186 1068494 21108 0 0 0\n" + + "cpu23 452284 24602 104397 23481583 72612 1052462 21985 0 0 0\n" + + "intr 6645352968 64 0 0 0 1481 0 0 0 1 0 0 0 0 0 0 0 39 0 0 0 0 0 0 37 0 0 0 0 0 0 0 0 4334106 1 6949071 5814662 5415344 6939471 6961483 6358810 5271953 6718644 0 126114 126114 126114 126114 126114 126114 126114 126114 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n" + + "ctxt 2495530303\n" + + "btime 1611928223\n" + + "processes 4839481\n" + + "procs_running 4\n" + + "procs_blocked 0\n" + + "softirq 2202631388 4 20504999 46734 54405637 4330276 0 6951 1664780312 10130 458546345\n"); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java new file mode 100644 index 00000000000..37db6895040 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java @@ -0,0 +1,37 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container.image; + +import com.yahoo.config.provision.DockerImage; +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; +import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; +import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author mpolden + */ +public class ContainerImageDownloaderTest { + + @Test + @Timeout(5_000) + void test_download() { + ContainerEngineMock podman = new ContainerEngineMock().asyncImageDownload(true); + ContainerImageDownloader downloader = new ContainerImageDownloader(podman, new TestTimer()); + TaskContext context = new TestTaskContext(); + DockerImage image = DockerImage.fromString("registry.example.com/repo/vespa:7.42"); + + assertFalse(downloader.get(context, image, () -> RegistryCredentials.none), "Download started"); + assertFalse(downloader.get(context, image, () -> RegistryCredentials.none), "Download pending"); + podman.completeDownloadOf(image); + boolean downloadCompleted; + while (!(downloadCompleted = downloader.get(context, image, () -> RegistryCredentials.none))) ; + assertTrue(downloadCompleted, "Download completed"); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java new file mode 100644 index 00000000000..71312125cbc --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java @@ -0,0 +1,184 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container.image; + +import com.yahoo.config.provision.DockerImage; +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; +import com.yahoo.vespa.hosted.node.admin.container.Container; +import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; +import com.yahoo.vespa.hosted.node.admin.container.ContainerId; +import com.yahoo.vespa.hosted.node.admin.container.ContainerName; +import com.yahoo.vespa.hosted.node.admin.container.ContainerResources; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author freva + * @author mpolden + */ +public class ContainerImagePrunerTest { + + private final Tester tester = new Tester(); + + @Test + void noImagesMeansNoUnusedImages() { + tester.withExistingImages() + .expectDeletedImages(); + } + + @Test + void singleImageWithoutContainersIsUnused() { + tester.withExistingImages(image("image-1")) + // Even though nothing is using the image, we will keep it for at least 1h + .expectDeletedImagesAfterMinutes(0) + .expectDeletedImagesAfterMinutes(30) + .expectDeletedImagesAfterMinutes(30, "image-1"); + } + + @Test + void singleImageWithContainerIsUsed() { + tester.withExistingImages(image("image-1")) + .withExistingContainers(container("container-1", "image-1")) + .expectDeletedImages(); + } + + @Test + void multipleUnusedImagesAreIdentified() { + tester.withExistingImages(image("image-1"), image("image-2")) + .expectDeletedImages("image-1", "image-2"); + } + + @Test + void unusedImagesWithMultipleTags() { + tester.withExistingImages(image("image-1", "vespa-6", "vespa-6.28", "vespa:latest")) + .expectDeletedImages("vespa-6", "vespa-6.28", "vespa:latest"); + } + + @Test + void unusedImagesWithMultipleUntagged() { + tester.withExistingImages(image("image1", "<none>:<none>"), + image("image2", "<none>:<none>")) + .expectDeletedImages("image1", "image2"); + } + + @Test + void taggedImageWithNoContainersIsUnused() { + tester.withExistingImages(image("image-1", "vespa-6")) + .expectDeletedImages("vespa-6"); + } + + @Test + void reDownloadingImageIsNotImmediatelyDeleted() { + tester.withExistingImages(image("image")) + .expectDeletedImages("image") // After 1h we delete image + .expectDeletedImagesAfterMinutes(0) // image is immediately re-downloaded, but is not deleted + .expectDeletedImagesAfterMinutes(10) + .expectDeletedImages("image"); // 1h after re-download it is deleted again + } + + @Test + void reDownloadingImageIsNotImmediatelyDeletedWhenDeletingByTag() { + tester.withExistingImages(image("image", "my-tag")) + .expectDeletedImages("my-tag") // After 1h we delete image + .expectDeletedImagesAfterMinutes(0) // image is immediately re-downloaded, but is not deleted + .expectDeletedImagesAfterMinutes(10) + .expectDeletedImages("my-tag"); // 1h after re-download it is deleted again + } + + /** Same scenario as in {@link #multipleUnusedImagesAreIdentified()} */ + @Test + void doesNotDeleteExcludedByIdImages() { + tester.withExistingImages(image("image-1"), image("image-2")) + // Normally, image-1 should also be deleted, but because we exclude image-1 only image-2 is deleted + .expectDeletedImages(List.of("image-1"), "image-2"); + } + + /** Same as in {@link #doesNotDeleteExcludedByIdImages()} but with tags */ + @Test + void doesNotDeleteExcludedByTagImages() { + tester.withExistingImages(image("image-1", "vespa:6.288.16"), image("image-2", "vespa:6.289.94")) + .expectDeletedImages(List.of("vespa:6.288.16"), "vespa:6.289.94"); + } + + @Test + void excludingNotDownloadedImageIsNoop() { + tester.withExistingImages(image("image-1", "vespa:6.288.16"), + image("image-2", "vespa:6.289.94")) + .expectDeletedImages(List.of("vespa:6.300.1"), "vespa:6.288.16", "vespa:6.289.94", "rhel-6"); + } + + private static Image image(String id, String... tags) { + return new Image(id, List.of(tags)); + } + + private static Container container(String name, String imageId) { + return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, + Container.State.running, imageId, DockerImage.EMPTY, Map.of(), + 42, 43, name + ".example.com", ContainerResources.UNLIMITED, + List.of(), true); + } + + private static class Tester { + + private final ContainerEngineMock containerEngine = new ContainerEngineMock(); + private final TaskContext context = new TestTaskContext(); + private final TestTimer timer = new TestTimer(); + private final ContainerImagePruner pruner = new ContainerImagePruner(containerEngine, timer); + private final Map<String, Integer> removalCountByImageId = new HashMap<>(); + + private boolean initialized = false; + + private Tester withExistingImages(Image... images) { + containerEngine.setImages(List.of(images)); + return this; + } + + private Tester withExistingContainers(Container... containers) { + containerEngine.addContainers(List.of(containers)); + return this; + } + + private Tester expectDeletedImages(String... imageIds) { + return expectDeletedImagesAfterMinutes(60, imageIds); + } + + private Tester expectDeletedImages(List<String> excludedRefs, String... imageIds) { + return expectDeletedImagesAfterMinutes(60, excludedRefs, imageIds); + } + + private Tester expectDeletedImagesAfterMinutes(int minutesAfter, String... imageIds) { + return expectDeletedImagesAfterMinutes(minutesAfter, List.of(), imageIds); + } + + private Tester expectDeletedImagesAfterMinutes(int minutesAfter, List<String> excludedRefs, String... imageIds) { + if (!initialized) { + // Run once with a very long expiry to initialize internal state of existing images + pruner.removeUnusedImages(context, List.of(), Duration.ofDays(999)); + initialized = true; + } + + timer.advance(Duration.ofMinutes(minutesAfter)); + + pruner.removeUnusedImages(context, excludedRefs, Duration.ofHours(1).minusSeconds(1)); + + List.of(imageIds) + .forEach(imageId -> { + int newValue = removalCountByImageId.getOrDefault(imageId, 0) + 1; + removalCountByImageId.put(imageId, newValue); + + assertTrue(containerEngine.listImages(context).stream().noneMatch(image -> image.id().equals(imageId)), + "Image " + imageId + " removed"); + }); + return this; + } + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java new file mode 100644 index 00000000000..8e23c7e54b6 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java @@ -0,0 +1,99 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.container.metrics; + +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics.APPLICATION_HOST; +import static com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics.DimensionType.DEFAULT; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +public class MetricsTest { + private static final Dimensions hostDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); + private final Metrics metrics = new Metrics(); + + @Test + void testDefaultValue() { + metrics.declareCounter("some.name", hostDimension); + + assertEquals(getMetricsForDimension(hostDimension).get("some.name"), 0L); + } + + @Test + void testSimpleIncrementMetric() { + Counter counter = metrics.declareCounter("a_counter.value", hostDimension); + + counter.add(5); + counter.add(8); + + Map<String, Number> latestMetrics = getMetricsForDimension(hostDimension); + assertEquals(1, latestMetrics.size(), "Expected only 1 metric value to be set"); + assertEquals(latestMetrics.get("a_counter.value"), 13L); // 5 + 8 + } + + @Test + void testSimpleGauge() { + Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); + + gauge.sample(42); + gauge.sample(-342.23); + + Map<String, Number> latestMetrics = getMetricsForDimension(hostDimension); + assertEquals(1, latestMetrics.size(), "Expected only 1 metric value to be set"); + assertEquals(latestMetrics.get("test.gauge"), -342.23); + } + + @Test + void testRedeclaringSameGauge() { + Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); + gauge.sample(42); + + // Same as hostDimension, but new instance. + Dimensions newDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); + Gauge newGauge = metrics.declareGauge("test.gauge", newDimension); + newGauge.sample(56); + + assertEquals(getMetricsForDimension(hostDimension).get("test.gauge"), 56.); + } + + @Test + void testSameMetricNameButDifferentDimensions() { + Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); + gauge.sample(42); + + // Not the same as hostDimension. + Dimensions newDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); + Gauge newGauge = metrics.declareGauge("test.gauge", newDimension); + newGauge.sample(56); + + assertEquals(getMetricsForDimension(hostDimension).get("test.gauge"), 42.); + assertEquals(getMetricsForDimension(newDimension).get("test.gauge"), 56.); + } + + @Test + void testDeletingMetric() { + metrics.declareGauge("test.gauge", hostDimension); + + Dimensions differentDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); + metrics.declareGauge("test.gauge", differentDimension); + + assertEquals(2, metrics.getMetricsByType(DEFAULT).size()); + metrics.deleteMetricByDimension(APPLICATION_HOST, differentDimension, DEFAULT); + assertEquals(1, metrics.getMetricsByType(DEFAULT).size()); + assertEquals(getMetricsForDimension(hostDimension).size(), 1); + assertEquals(getMetricsForDimension(differentDimension).size(), 0); + } + + private Map<String, Number> getMetricsForDimension(Dimensions dimensions) { + return metrics.getOrCreateApplicationMetrics(APPLICATION_HOST, DEFAULT) + .getOrDefault(dimensions, Map.of()) + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue())); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java new file mode 100644 index 00000000000..de41da7329b --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java @@ -0,0 +1,52 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.integration; + +import com.yahoo.config.provision.DockerImage; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.container.ContainerName; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +/** + * @author freva + */ +public class ContainerFailTest { + + @Test + void test() { + DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image"); + try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { + ContainerName containerName = new ContainerName("host1"); + String hostname = "host1.test.yahoo.com"; + NodeSpec nodeSpec = NodeSpec.Builder + .testSpec(hostname) + .wantedDockerImage(dockerImage) + .currentDockerImage(dockerImage) + .build(); + tester.addChildNodeRepositoryNode(nodeSpec); + + NodeAgentContext context = NodeAgentContextImpl.builder(nodeSpec).fileSystem(TestFileSystem.create()).build(); + + tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any()); + tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); + + tester.containerOperations.removeContainer(context, tester.containerOperations.getContainer(context).get()); + + tester.inOrder(tester.containerOperations).removeContainer(containerMatcher(containerName), any()); + tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any()); + tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); + + verify(tester.nodeRepository, never()).updateNodeAttributes(any(), any()); + } + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java new file mode 100644 index 00000000000..b4d85a5e974 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java @@ -0,0 +1,182 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.integration; + +import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeType; +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; +import com.yahoo.vespa.hosted.node.admin.container.ContainerName; +import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; +import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials; +import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.ProcMeminfo; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.ProcMeminfoReader; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentFactory; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.mockito.InOrder; +import org.mockito.Mockito; + +import java.nio.file.FileSystem; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Phaser; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Logger; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +/** + * @author musum + */ +// Need to deconstruct nodeAdminStateUpdater +public class ContainerTester implements AutoCloseable { + + private static final Logger log = Logger.getLogger(ContainerTester.class.getName()); + static final HostName HOST_HOSTNAME = HostName.of("host.test.yahoo.com"); + + private final Thread loopThread; + private final Phaser phaser = new Phaser(1); + + private final ContainerEngineMock containerEngine = new ContainerEngineMock(); + private final FileSystem fileSystem = TestFileSystem.create(); + private final TestTimer timer = new TestTimer(); + final ContainerOperations containerOperations = spy(new ContainerOperations(containerEngine, mock(Cgroup.class), fileSystem, timer)); + final NodeRepoMock nodeRepository = spy(new NodeRepoMock()); + final Orchestrator orchestrator = mock(Orchestrator.class); + final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); + final InOrder inOrder = Mockito.inOrder(containerOperations, nodeRepository, orchestrator, storageMaintainer); + final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + + final NodeAdminStateUpdater nodeAdminStateUpdater; + final NodeAdminImpl nodeAdmin; + + private volatile NodeAdminStateUpdater.State wantedState = NodeAdminStateUpdater.State.RESUMED; + + + ContainerTester(List<DockerImage> images) { + images.forEach(image -> containerEngine.pullImage(null, image, RegistryCredentials.none)); + when(storageMaintainer.diskUsageFor(any())).thenReturn(Optional.empty()); + + IPAddressesMock ipAddresses = new IPAddressesMock(); + ipAddresses.addAddress(HOST_HOSTNAME.value(), "1.1.1.1"); + ipAddresses.addAddress(HOST_HOSTNAME.value(), "f000::"); + for (int i = 1; i < 4; i++) ipAddresses.addAddress("host" + i + ".test.yahoo.com", "f000::" + i); + + NodeSpec hostSpec = NodeSpec.Builder.testSpec(HOST_HOSTNAME.value()).type(NodeType.host).build(); + nodeRepository.updateNodeSpec(hostSpec); + + Metrics metrics = new Metrics(); + FileSystem fileSystem = TestFileSystem.create(); + ProcMeminfoReader procMeminfoReader = mock(ProcMeminfoReader.class); + when(procMeminfoReader.read()).thenReturn(new ProcMeminfo(1, 2)); + + NodeAgentFactory nodeAgentFactory = (contextSupplier, nodeContext) -> + new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, containerOperations, () -> RegistryCredentials.none, + storageMaintainer, flagSource, + List.of(), Optional.empty(), Optional.empty(), timer, Duration.ofSeconds(-1), + VespaServiceDumper.DUMMY_INSTANCE, List.of()) { + @Override public void converge(NodeAgentContext context) { + super.converge(context); + phaser.arriveAndAwaitAdvance(); + } + @Override public void stopForHostSuspension(NodeAgentContext context) { + super.stopForHostSuspension(context); + phaser.arriveAndAwaitAdvance(); + } + @Override public void stopForRemoval(NodeAgentContext context) { + super.stopForRemoval(context); + phaser.arriveAndDeregister(); + } + }; + nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, timer, Duration.ofMillis(10), Duration.ZERO, procMeminfoReader); + NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) -> + NodeAgentContextImpl.builder(nodeSpec).acl(acl).fileSystem(fileSystem).build(); + nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator, + nodeAdmin, HOST_HOSTNAME); + + loopThread = new Thread(() -> { + nodeAdminStateUpdater.start(); + while ( ! phaser.isTerminated()) { + try { + nodeAdminStateUpdater.converge(wantedState); + } catch (RuntimeException e) { + log.info(e.getMessage()); + } + } + nodeAdminStateUpdater.stop(); + }); + loopThread.start(); + } + + /** Adds a node to node-repository mock that is running on this host */ + void addChildNodeRepositoryNode(NodeSpec nodeSpec) { + if (nodeSpec.wantedDockerImage().isPresent()) { + if (!containerEngine.hasImage(null, nodeSpec.wantedDockerImage().get())) { + throw new IllegalArgumentException("Want to use image " + nodeSpec.wantedDockerImage().get() + + ", but that image does not exist in the container engine"); + } + } + + if (nodeRepository.getOptionalNode(nodeSpec.hostname()).isEmpty()) + phaser.register(); + + nodeRepository.updateNodeSpec(new NodeSpec.Builder(nodeSpec) + .parentHostname(HOST_HOSTNAME.value()) + .build()); + } + + void setWantedState(NodeAdminStateUpdater.State wantedState) { + this.wantedState = wantedState; + } + + <T> T inOrder(T t) { + waitSomeTicks(); + return inOrder.verify(t); + } + + void waitSomeTicks() { + try { + // 3 is enough for everyone! (Well, maybe not for all eternity ...) + for (int i = 0; i < 3; i++) + phaser.awaitAdvanceInterruptibly(phaser.arrive(), 1000, TimeUnit.MILLISECONDS); + } + catch (InterruptedException | TimeoutException e) { + throw new RuntimeException(e); + } + } + + public static NodeAgentContext containerMatcher(ContainerName containerName) { + return argThat((ctx) -> ctx.containerName().equals(containerName)); + } + + @Override + public void close() { + phaser.forceTermination(); + try { + loopThread.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java new file mode 100644 index 00000000000..7e874bcd5a7 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java @@ -0,0 +1,58 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.integration; + +import com.yahoo.config.provision.DockerImage; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; +import com.yahoo.vespa.hosted.node.admin.container.ContainerName; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; + +/** + * @author freva + */ +public class MultiContainerTest { + + @Test + void test() { + DockerImage image1 = DockerImage.fromString("registry.example.com/repo/image1"); + DockerImage image2 = DockerImage.fromString("registry.example.com/repo/image2"); + try (ContainerTester tester = new ContainerTester(List.of(image1, image2))) { + addAndWaitForNode(tester, "host1.test.yahoo.com", image1); + NodeSpec nodeSpec2 = addAndWaitForNode(tester, "host2.test.yahoo.com", image2); + + tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(nodeSpec2.hostname(), NodeState.dirty).build()); + + ContainerName host2 = new ContainerName("host2"); + tester.inOrder(tester.containerOperations).removeContainer(containerMatcher(host2), any()); + tester.inOrder(tester.storageMaintainer).archiveNodeStorage( + argThat(context -> context.containerName().equals(host2))); + tester.inOrder(tester.nodeRepository).setNodeState(eq(nodeSpec2.hostname()), eq(NodeState.ready)); + + addAndWaitForNode(tester, "host3.test.yahoo.com", image1); + } + } + + private NodeAgentContext containerMatcher(ContainerName containerName) { + return argThat((ctx) -> ctx.containerName().equals(containerName)); + } + + private NodeSpec addAndWaitForNode(ContainerTester tester, String hostName, DockerImage dockerImage) { + NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostName).wantedDockerImage(dockerImage).build(); + tester.addChildNodeRepositoryNode(nodeSpec); + + ContainerName containerName = ContainerName.fromHostname(hostName); + tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any()); + tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); + tester.inOrder(tester.nodeRepository).updateNodeAttributes(eq(hostName), any()); + + return nodeSpec; + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java new file mode 100644 index 00000000000..da14c5aa47b --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java @@ -0,0 +1,91 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.integration; + +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.AddNode; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NoSuchNodeException; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; +import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +/** + * Mock with some simple logic + * + * @author dybis + */ +public class NodeRepoMock implements NodeRepository { + + private final Map<String, NodeSpec> nodeSpecByHostname = new ConcurrentHashMap<>(); + private volatile Map<String, Acl> aclByHostname = Map.of(); + + @Override + public void addNodes(List<AddNode> nodes) { } + + @Override + public List<NodeSpec> getNodes(String baseHostName) { + return nodeSpecByHostname.values().stream() + .filter(node -> baseHostName.equals(node.parentHostname().orElse(null))) + .toList(); + } + + @Override + public Optional<NodeSpec> getOptionalNode(String hostName) { + return Optional.ofNullable(nodeSpecByHostname.get(hostName)); + } + + @Override + public Map<String, Acl> getAcls(String hostname) { + return aclByHostname; + } + + @Override + public List<WireguardPeer> getExclavePeers() { + throw new UnsupportedOperationException(); + } + + @Override + public List<WireguardPeer> getConfigserverPeers() { + throw new UnsupportedOperationException(); + } + + @Override + public void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes) { + updateNodeSpec(new NodeSpec.Builder(getNode(hostName)) + .updateFromNodeAttributes(nodeAttributes) + .build()); + } + + @Override + public void setNodeState(String hostName, NodeState nodeState) { + updateNodeSpec(new NodeSpec.Builder(getNode(hostName)) + .state(nodeState) + .build()); + } + + public void updateNodeSpec(NodeSpec nodeSpec) { + nodeSpecByHostname.put(nodeSpec.hostname(), nodeSpec); + } + + public void updateNodeSpec(String hostname, Function<NodeSpec.Builder, NodeSpec.Builder> mapper) { + nodeSpecByHostname.compute(hostname, (__, nodeSpec) -> { + if (nodeSpec == null) throw new NoSuchNodeException(hostname); + return mapper.apply(new NodeSpec.Builder(nodeSpec)).build(); + }); + } + + public void resetNodeSpecs() { + nodeSpecByHostname.clear(); + } + + public void setAcl(Map<String, Acl> aclByHostname) { + this.aclByHostname = Map.copyOf(aclByHostname); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java new file mode 100644 index 00000000000..a1440ba8669 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java @@ -0,0 +1,44 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.integration; + +import com.yahoo.config.provision.DockerImage; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.container.ContainerName; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.HOST_HOSTNAME; +import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; + +/** + * Tests rebooting of Docker host + * + * @author musum + */ +public class RebootTest { + + private final String hostname = "host1.test.yahoo.com"; + private final DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image"); + + @Test + void test() { + try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { + tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build()); + + ContainerName host1 = new ContainerName("host1"); + tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any()); + + tester.setWantedState(NodeAdminStateUpdater.State.SUSPENDED); + + tester.inOrder(tester.orchestrator).suspend(eq(HOST_HOSTNAME.value()), eq(List.of(hostname, HOST_HOSTNAME.value()))); + tester.inOrder(tester.containerOperations).stopServices(containerMatcher(host1)); + assertTrue(tester.nodeAdmin.setFrozen(true)); + } + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java new file mode 100644 index 00000000000..1445546097a --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java @@ -0,0 +1,50 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.integration; + +import com.yahoo.config.provision.DockerImage; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.container.ContainerName; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; + +/** + * Tests that different wanted and current restart generation leads to execution of restart command + * + * @author musum + */ +public class RestartTest { + + @Test + void test() { + DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image:1.2.3"); + try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { + String hostname = "host1.test.yahoo.com"; + NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostname) + .wantedDockerImage(dockerImage) + .wantedVespaVersion(dockerImage.tagAsVersion()) + .build(); + tester.addChildNodeRepositoryNode(nodeSpec); + + ContainerName host1 = new ContainerName("host1"); + tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any()); + tester.inOrder(tester.nodeRepository).updateNodeAttributes( + eq(hostname), eq(new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()))); + + // Increment wantedRestartGeneration to 2 in node-repo + tester.addChildNodeRepositoryNode(new NodeSpec.Builder(tester.nodeRepository.getNode(hostname)) + .wantedRestartGeneration(2).build()); + + tester.inOrder(tester.orchestrator).suspend(eq(hostname)); + tester.inOrder(tester.containerOperations).restartVespa(containerMatcher(host1)); + tester.inOrder(tester.nodeRepository).updateNodeAttributes( + eq(hostname), eq(new NodeAttributes().withRestartGeneration(2))); + } + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java new file mode 100644 index 00000000000..51b3bb5e6c4 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -0,0 +1,178 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance; + +import com.yahoo.config.provision.NodeResources; +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; +import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup; +import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; +import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; +import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +/** + * @author dybis + */ +public class StorageMaintainerTest { + + private final TestTerminal terminal = new TestTerminal(); + private final CoredumpHandler coredumpHandler = mock(CoredumpHandler.class); + private final DiskCleanup diskCleanup = mock(DiskCleanup.class); + private final SyncClient syncClient = mock(SyncClient.class); + private final TestTimer timer = new TestTimer(Instant.ofEpochSecond(1234567890)); + private final FileSystem fileSystem = TestFileSystem.create(); + private final StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, coredumpHandler, diskCleanup, syncClient, timer, + fileSystem.getPath("/data/vespa/storage/container-archive")); + + @Test + void testDiskUsed() { + NodeAgentContext context = NodeAgentContextImpl.builder("host-1.domain.tld").fileSystem(fileSystem).build(); + + terminal.expectCommand("du -xsk /data/vespa/storage/host-1 2>&1", 0, "321\t/data/vespa/storage/host-1/"); + assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); + + // Value should still be cached, no new execution against the terminal + assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); + } + + @Test + void testNonExistingDiskUsed() { + DiskSize size = storageMaintainer.getDiskUsed(null, Path.of("/fake/path")); + assertEquals(DiskSize.ZERO, size); + } + + @Test + void archive_container_data_test() throws IOException { + // Create some files in containers + NodeAgentContext context1 = createNodeAgentContextAndContainerStorage(fileSystem, "container-1"); + createNodeAgentContextAndContainerStorage(fileSystem, "container-2"); + + Path pathToArchiveDir = fileSystem.getPath("/data/vespa/storage/container-archive"); + Files.createDirectories(pathToArchiveDir); + + Path containerStorageRoot = context1.paths().of("/").pathOnHost().getParent(); + Set<String> containerStorageRootContentsBeforeArchive = FileFinder.from(containerStorageRoot) + .maxDepth(1) + .stream() + .map(FileFinder.FileAttributes::filename) + .collect(Collectors.toSet()); + assertEquals(Set.of("container-archive", "container-1", "container-2"), containerStorageRootContentsBeforeArchive); + + + // Archive container-1 + storageMaintainer.archiveNodeStorage(context1); + + timer.advance(Duration.ofSeconds(3)); + storageMaintainer.archiveNodeStorage(context1); + + // container-1 should be gone from container-storage + Set<String> containerStorageRootContentsAfterArchive = FileFinder.from(containerStorageRoot) + .maxDepth(1) + .stream() + .map(FileFinder.FileAttributes::filename) + .collect(Collectors.toSet()); + assertEquals(Set.of("container-archive", "container-2"), containerStorageRootContentsAfterArchive); + + // container archive directory should contain exactly 1 directory - the one we just archived + List<FileFinder.FileAttributes> containerArchiveContentsAfterArchive = FileFinder.from(pathToArchiveDir).maxDepth(1).list(); + assertEquals(1, containerArchiveContentsAfterArchive.size()); + Path archivedContainerStoragePath = containerArchiveContentsAfterArchive.get(0).path(); + assertEquals("container-1_20090213233130", archivedContainerStoragePath.getFileName().toString()); + Set<String> archivedContainerStorageContents = FileFinder.files(archivedContainerStoragePath) + .stream() + .map(fileAttributes -> archivedContainerStoragePath.relativize(fileAttributes.path()).toString()) + .collect(Collectors.toSet()); + assertEquals(Set.of("opt/vespa/logs/vespa/vespa.log", "opt/vespa/logs/vespa/zookeeper.log"), archivedContainerStorageContents); + } + + private static NodeAgentContext createNodeAgentContextAndContainerStorage(FileSystem fileSystem, String containerName) throws IOException { + NodeAgentContext context = NodeAgentContextImpl.builder(containerName + ".domain.tld") + .fileSystem(fileSystem).build(); + + ContainerPath containerVespaHome = context.paths().underVespaHome(""); + Files.createDirectories(context.paths().of("/etc/something")); + Files.createFile(context.paths().of("/etc/something/conf")); + + Files.createDirectories(containerVespaHome.resolve("logs/vespa")); + Files.createFile(containerVespaHome.resolve("logs/vespa/vespa.log")); + Files.createFile(containerVespaHome.resolve("logs/vespa/zookeeper.log")); + + Files.createDirectories(containerVespaHome.resolve("var/db")); + Files.createFile(containerVespaHome.resolve("var/db/some-file")); + + Files.createDirectories(containerVespaHome.resolve("var/tmp")); + Files.createFile(containerVespaHome.resolve("var/tmp/some-file")); + + ContainerPath containerRoot = context.paths().of("/"); + Set<String> actualContents = FileFinder.files(containerRoot) + .stream() + .map(fileAttributes -> containerRoot.relativize(fileAttributes.path()).toString()) + .collect(Collectors.toSet()); + Set<String> expectedContents = Set.of( + "etc/something/conf", + "opt/vespa/logs/vespa/vespa.log", + "opt/vespa/logs/vespa/zookeeper.log", + "opt/vespa/var/tmp/some-file", + "opt/vespa/var/db/some-file"); + assertEquals(expectedContents, actualContents); + return context; + } + + @Test + void not_run_if_not_enough_used() { + NodeAgentContext context = NodeAgentContextImpl.builder( + NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build()) + .fileSystem(fileSystem).build(); + mockDiskUsage(500L); + + storageMaintainer.cleanDiskIfFull(context); + verifyNoInteractions(diskCleanup); + } + + @Test + void deletes_correct_amount() { + NodeAgentContext context = NodeAgentContextImpl.builder( + NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build()) + .fileSystem(fileSystem).build(); + + mockDiskUsage(950_000L); + + storageMaintainer.cleanDiskIfFull(context); + // Allocated size: 1 GB, usage: 950_000 kiB (972.8 MB). Wanted usage: 70% => 700 MB + verify(diskCleanup).cleanup(eq(context), any(), eq(272_800_000L)); + } + + @AfterEach + public void after() { + terminal.verifyAllCommandsExecuted(); + } + + private void mockDiskUsage(long kBytes) { + terminal.expectCommand("du -xsk /data/vespa/storage/h123a 2>&1", 0, kBytes + "\t/path"); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java new file mode 100644 index 00000000000..063e8cb3f77 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java @@ -0,0 +1,351 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.acl; + +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; +import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock; +import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; +import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; +import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.nio.file.FileSystem; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.endsWith; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class AclMaintainerTest { + + private static final String EMPTY_FILTER_TABLE = "-P INPUT ACCEPT\n-P FORWARD ACCEPT\n-P OUTPUT ACCEPT\n"; + private static final String EMPTY_NAT_TABLE = "-P PREROUTING ACCEPT\n-P INPUT ACCEPT\n-P OUTPUT ACCEPT\n-P POSTROUTING ACCEPT\n"; + + private final ContainerOperations containerOperations = mock(ContainerOperations.class); + private final IPAddressesMock ipAddresses = new IPAddressesMock(); + private final AclMaintainer aclMaintainer = new AclMaintainer(containerOperations, ipAddresses); + + private final FileSystem fileSystem = TestFileSystem.create(); + private final Function<Acl, NodeAgentContext> contextGenerator = + acl -> NodeAgentContextImpl.builder("container1.host.com").fileSystem(fileSystem).acl(acl).build(); + private final List<String> writtenFileContents = new ArrayList<>(); + + @Test + void configures_full_container_acl_from_empty() { + Acl acl = new Acl.Builder().withTrustedPorts(22, 4443) + .withTrustedNode("hostname1", "3001::abcd") + .withTrustedNode("hostname2", "3001::1234") + .withTrustedNode("hostname1", "192.168.0.5") + .withTrustedNode("hostname4", "172.16.5.234").build(); + NodeAgentContext context = contextGenerator.apply(acl); + + ipAddresses.addAddress(context.hostname().value(), "2001::1"); + ipAddresses.addAddress(context.hostname().value(), "10.0.0.1"); + + whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); + whenListRules(context, "filter", IPVersion.IPv6, EMPTY_FILTER_TABLE); + whenListRules(context, "nat", IPVersion.IPv4, EMPTY_NAT_TABLE); + whenListRules(context, "nat", IPVersion.IPv6, EMPTY_NAT_TABLE); + + aclMaintainer.converge(context); + + verify(containerOperations, times(4)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); + verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); + verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); + verifyNoMoreInteractions(containerOperations); + + List<String> expected = List.of( + // IPv4 filter table restore + "*filter\n" + + "-P INPUT ACCEPT\n" + + "-P FORWARD ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + + "-A INPUT -i lo -j ACCEPT\n" + + "-A INPUT -p icmp -j ACCEPT\n" + + "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + + "-A INPUT -s 172.16.5.234/32 -j ACCEPT\n" + + "-A INPUT -s 192.168.0.5/32 -j ACCEPT\n" + + "-A INPUT -j REJECT --reject-with icmp-port-unreachable\n" + + "COMMIT\n", + + // IPv6 filter table restore + "*filter\n" + + "-P INPUT ACCEPT\n" + + "-P FORWARD ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + + "-A INPUT -i lo -j ACCEPT\n" + + "-A INPUT -p ipv6-icmp -j ACCEPT\n" + + "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + + "-A INPUT -s 3001::1234/128 -j ACCEPT\n" + + "-A INPUT -s 3001::abcd/128 -j ACCEPT\n" + + "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n" + + "COMMIT\n", + + // IPv4 nat table restore + "*nat\n" + + "-P PREROUTING ACCEPT\n" + + "-P INPUT ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-P POSTROUTING ACCEPT\n" + + "-A OUTPUT -d 10.0.0.1/32 -j REDIRECT\n" + + "COMMIT\n", + + // IPv6 nat table restore + "*nat\n" + + "-P PREROUTING ACCEPT\n" + + "-P INPUT ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-P POSTROUTING ACCEPT\n" + + "-A OUTPUT -d 2001::1/128 -j REDIRECT\n" + + "COMMIT\n"); + assertEquals(expected, writtenFileContents); + } + + @Test + void configures_minimal_container_acl_from_empty() { + // The ACL spec is empty and our this node's addresses do not resolve + Acl acl = new Acl.Builder().withTrustedPorts().build(); + NodeAgentContext context = contextGenerator.apply(acl); + + whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); + whenListRules(context, "filter", IPVersion.IPv6, EMPTY_FILTER_TABLE); + whenListRules(context, "nat", IPVersion.IPv4, EMPTY_NAT_TABLE); + whenListRules(context, "nat", IPVersion.IPv6, EMPTY_NAT_TABLE); + + aclMaintainer.converge(context); + + verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); + verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); + verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); + verifyNoMoreInteractions(containerOperations); + + List<String> expected = List.of( + // IPv4 filter table restore + "*filter\n" + + "-P INPUT ACCEPT\n" + + "-P FORWARD ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + + "-A INPUT -i lo -j ACCEPT\n" + + "-A INPUT -p icmp -j ACCEPT\n" + + "-A INPUT -j REJECT --reject-with icmp-port-unreachable\n" + + "COMMIT\n", + + // IPv6 filter table restore + "*filter\n" + + "-P INPUT ACCEPT\n" + + "-P FORWARD ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + + "-A INPUT -i lo -j ACCEPT\n" + + "-A INPUT -p ipv6-icmp -j ACCEPT\n" + + "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n" + + "COMMIT\n"); + assertEquals(expected, writtenFileContents); + } + + @Test + void only_configure_iptables_for_ipversion_that_differs() { + Acl acl = new Acl.Builder().withTrustedPorts(22, 4443).withTrustedNode("hostname1", "3001::abcd").build(); + NodeAgentContext context = contextGenerator.apply(acl); + + ipAddresses.addAddress(context.hostname().value(), "2001::1"); + + whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); + whenListRules(context, "filter", IPVersion.IPv6, + "-P INPUT ACCEPT\n" + + "-P FORWARD ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + + "-A INPUT -i lo -j ACCEPT\n" + + "-A INPUT -p ipv6-icmp -j ACCEPT\n" + + "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + + "-A INPUT -s 3001::abcd/128 -j ACCEPT\n" + + "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n"); + whenListRules(context, "nat", IPVersion.IPv6, + "-P PREROUTING ACCEPT\n" + + "-P INPUT ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-P POSTROUTING ACCEPT\n" + + "-A OUTPUT -d 2001::1/128 -j REDIRECT\n"); + + aclMaintainer.converge(context); + + verify(containerOperations, times(3)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); + verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); + verify(containerOperations, never()).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); //we don't have a ip4 address for the container so no redirect + verifyNoMoreInteractions(containerOperations); + + List<String> expected = List.of( + "*filter\n" + + "-P INPUT ACCEPT\n" + + "-P FORWARD ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + + "-A INPUT -i lo -j ACCEPT\n" + + "-A INPUT -p icmp -j ACCEPT\n" + + "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + + "-A INPUT -j REJECT --reject-with icmp-port-unreachable\n" + + "COMMIT\n"); + assertEquals(expected, writtenFileContents); + } + + @Test + void rollback_is_attempted_when_applying_acl_fail() { + Acl acl = new Acl.Builder().withTrustedPorts(22, 4443).withTrustedNode("hostname1", "3001::abcd").build(); + NodeAgentContext context = contextGenerator.apply(acl); + + ipAddresses.addAddress(context.hostname().value(), "2001::1"); + + whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); + whenListRules(context, "filter", IPVersion.IPv6, + "-P INPUT ACCEPT\n" + + "-P FORWARD ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + + "-A INPUT -i lo -j ACCEPT\n" + + "-A INPUT -p ipv6-icmp -j ACCEPT\n" + + "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + + "-A INPUT -s 3001::abcd/128 -j ACCEPT\n" + + "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n"); + whenListRules(context, "nat", IPVersion.IPv6, + "-P PREROUTING ACCEPT\n" + + "-P INPUT ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-P POSTROUTING ACCEPT\n" + + "-A OUTPUT -d 2001::1/128 -j REDIRECT\n"); + + when(containerOperations.executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any())) + .thenThrow(new RuntimeException("iptables restore failed")); + + aclMaintainer.converge(context); + + verify(containerOperations, times(3)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); + verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); + verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables"), eq("-F"), eq("-t"), eq("filter")); + verifyNoMoreInteractions(containerOperations); + + aclMaintainer.converge(context); + } + + @Test + public void config_server_acl() { + Acl acl = new Acl.Builder().withTrustedPorts(22, 4443) + .withTrustedNode("cfg1", "2001:db8::1") + .withTrustedNode("cfg2", "2001:db8::2") + .withTrustedNode("cfg3", "2001:db8::3") + .withTrustedNode("cfg1", "172.17.0.41") + .withTrustedNode("cfg2", "172.17.0.42") + .withTrustedNode("cfg3", "172.17.0.43") + .build(); + NodeAgentContext context = NodeAgentContextImpl.builder("cfg3.example.com") + .fileSystem(fileSystem) + .acl(acl) + .nodeSpecBuilder(builder -> builder.type(NodeType.config)) + .build(); + + ipAddresses.addAddress(context.hostname().value(), "2001:db8::3"); + ipAddresses.addAddress(context.hostname().value(), "172.17.0.43"); + + whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); + whenListRules(context, "filter", IPVersion.IPv6, EMPTY_FILTER_TABLE); + whenListRules(context, "nat", IPVersion.IPv4, EMPTY_NAT_TABLE); + whenListRules(context, "nat", IPVersion.IPv6, EMPTY_NAT_TABLE); + + aclMaintainer.converge(context); + + verify(containerOperations, times(4)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); + verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); + verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); + verifyNoMoreInteractions(containerOperations); + + List<String> expected = List.of( + // IPv4 filter table restore + """ + *filter + -P INPUT ACCEPT + -P FORWARD ACCEPT + -P OUTPUT ACCEPT + -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + -A INPUT -i lo -j ACCEPT + -A INPUT -p icmp -j ACCEPT + -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT + -A INPUT -s 172.17.0.41/32 -j ACCEPT + -A INPUT -s 172.17.0.42/32 -j ACCEPT + -A INPUT -s 172.17.0.43/32 -j ACCEPT + -A INPUT -j REJECT --reject-with icmp-port-unreachable + COMMIT + """, + // IPv6 filter table restore + """ + *filter + -P INPUT ACCEPT + -P FORWARD ACCEPT + -P OUTPUT ACCEPT + -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + -A INPUT -i lo -j ACCEPT + -A INPUT -p ipv6-icmp -j ACCEPT + -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT + -A INPUT -s 2001:db8::1/128 -j ACCEPT + -A INPUT -s 2001:db8::2/128 -j ACCEPT + -A INPUT -s 2001:db8::3/128 -j ACCEPT + -A INPUT -j REJECT --reject-with icmp6-port-unreachable + COMMIT + """, + // IPv4 nat table restore + """ + *nat + -P PREROUTING ACCEPT + -P INPUT ACCEPT + -P OUTPUT ACCEPT + -P POSTROUTING ACCEPT + -A OUTPUT -d 172.17.0.43/32 -j REDIRECT + COMMIT + """, + // IPv6 nat table restore + """ + *nat + -P PREROUTING ACCEPT + -P INPUT ACCEPT + -P OUTPUT ACCEPT + -P POSTROUTING ACCEPT + -A OUTPUT -d 2001:db8::3/128 -j REDIRECT + COMMIT + """); + assertEquals(expected, writtenFileContents); + } + + @BeforeEach + public void setup() { + doAnswer(invoc -> { + String path = invoc.getArgument(2); + writtenFileContents.add(new UnixPath(path).readUtf8File()); + return new CommandResult(null, 0, ""); + }).when(containerOperations).executeCommandInNetworkNamespace(any(), endsWith("-restore"), any()); + } + + private void whenListRules(NodeAgentContext context, String table, IPVersion ipVersion, String output) { + when(containerOperations.executeCommandInNetworkNamespace( + eq(context), any(CommandLine.Options.class), eq(ipVersion.iptablesCmd()), eq("-S"), eq("-t"), eq(table))) + .thenReturn(new CommandResult(null, 0, output)); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java new file mode 100644 index 00000000000..52eac44fbc3 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java @@ -0,0 +1,88 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.acl; + +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; +import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; +import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +public class FilterTableLineEditorTest { + + @Test + void filter_set_wanted_rules() { + Acl acl = new Acl.Builder().withTrustedPorts(22).withTrustedNode("hostname", "3001::1").build(); + + assertFilterTableLineEditorResult( + acl, IPVersion.IPv6, + + "-P INPUT ACCEPT\n" + + "-P FORWARD ACCEPT\n" + + "-P OUTPUT ACCEPT\n", + + "-P INPUT ACCEPT\n" + + "-P FORWARD ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + + "-A INPUT -i lo -j ACCEPT\n" + + "-A INPUT -p ipv6-icmp -j ACCEPT\n" + + "-A INPUT -p tcp -m multiport --dports 22 -j ACCEPT\n" + + "-A INPUT -s 3001::1/128 -j ACCEPT\n" + + "-A INPUT -j REJECT --reject-with icmp6-port-unreachable"); + } + + @Test + void produces_minimal_diff_simple() { + assertFilterTableDiff(List.of(2, 5, 3, 6, 1, 4), List.of(2, 5, 6, 1, 4), + "Patching file table:\n" + + "--A INPUT -s 2001::3/128 -j ACCEPT\n"); + } + + @Test + void produces_minimal_diff_complex() { + assertFilterTableDiff(List.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), List.of(5, 11, 6, 3, 10, 4, 8, 12), + "Patching file table:\n" + + "--A INPUT -s 2001::1/128 -j ACCEPT\n" + + "--A INPUT -s 2001::2/128 -j ACCEPT\n" + + "+-A INPUT -s 2001::11/128 -j ACCEPT\n" + + "+-A INPUT -s 2001::12/128 -j ACCEPT\n" + + "--A INPUT -s 2001::7/128 -j ACCEPT\n" + + "--A INPUT -s 2001::9/128 -j ACCEPT\n"); + } + + private static void assertFilterTableLineEditorResult( + Acl acl, IPVersion ipVersion, String currentFilterTable, String expectedRestoreFileContent) { + FilterTableLineEditor filterLineEditor = FilterTableLineEditor.from(acl, ipVersion); + Editor editor = new Editor( + "nat-table", + () -> List.of(currentFilterTable.split("\n")), + result -> assertEquals(expectedRestoreFileContent, String.join("\n", result)), + filterLineEditor); + editor.edit(m -> {}); + } + + private static void assertFilterTableDiff(List<Integer> currentIpSuffix, List<Integer> wantedIpSuffix, String diff) { + Acl.Builder currentAcl = new Acl.Builder(); + NodeType nodeType = NodeType.tenant; + currentIpSuffix.forEach(i -> currentAcl.withTrustedNode("host" + i, "2001::" + i)); + List<String> currentTable = new ArrayList<>(); + + Acl.Builder wantedAcl = new Acl.Builder(); + wantedIpSuffix.forEach(i -> wantedAcl.withTrustedNode("host" + i, "2001::" + i)); + + new Editor("table", List::of, currentTable::addAll, FilterTableLineEditor.from(currentAcl.build(), IPVersion.IPv6)) + .edit(log -> {}); + + new Editor("table", () -> currentTable, result -> {}, FilterTableLineEditor.from(wantedAcl.build(), IPVersion.IPv6)) + .edit(log -> assertEquals(diff, log)); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java new file mode 100644 index 00000000000..d8d526050d7 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java @@ -0,0 +1,96 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.acl; + +import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +public class NatTableLineEditorTest { + + @Test + void nat_set_redirect_rule_without_touching_docker_rules() { + assertNatTableLineEditorResult( + "-A OUTPUT -d 3001::1/128 -j REDIRECT", + + "-P PREROUTING ACCEPT\n" + + "-P INPUT ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-P POSTROUTING ACCEPT\n" + + "-N DOCKER_OUTPUT\n" + + "-N DOCKER_POSTROUTING\n" + + "-A OUTPUT -d 127.0.0.11/32 -j DOCKER_OUTPUT\n" + + "-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING\n" + + "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p tcp -m tcp --dport 53 -j DNAT --to-destination 127.0.0.11:43500\n" + + "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp -m udp --dport 53 -j DNAT --to-destination 127.0.0.11:57392\n" + + "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p tcp -m tcp --sport 43500 -j SNAT --to-source :53\n" + + "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p udp -m udp --sport 57392 -j SNAT --to-source :53\n", + + "-P PREROUTING ACCEPT\n" + + "-P INPUT ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-P POSTROUTING ACCEPT\n" + + "-N DOCKER_OUTPUT\n" + + "-N DOCKER_POSTROUTING\n" + + "-A OUTPUT -d 127.0.0.11/32 -j DOCKER_OUTPUT\n" + + "-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING\n" + + "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p tcp -m tcp --dport 53 -j DNAT --to-destination 127.0.0.11:43500\n" + + "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp -m udp --dport 53 -j DNAT --to-destination 127.0.0.11:57392\n" + + "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p tcp -m tcp --sport 43500 -j SNAT --to-source :53\n" + + "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p udp -m udp --sport 57392 -j SNAT --to-source :53\n" + + "-A OUTPUT -d 3001::1/128 -j REDIRECT"); + } + + @Test + void nat_cleanup_wrong_redirect_rules() { + assertNatTableLineEditorResult( + "-A OUTPUT -d 3001::1/128 -j REDIRECT", + + "-P PREROUTING ACCEPT\n" + + "-P INPUT ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-P POSTROUTING ACCEPT\n" + + "-A OUTPUT -d 3001::2/128 -j REDIRECT\n", + + "-P PREROUTING ACCEPT\n" + + "-P INPUT ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-P POSTROUTING ACCEPT\n" + + "-A OUTPUT -d 3001::1/128 -j REDIRECT"); + } + + @Test + void nat_delete_duplicate_rules() { + assertNatTableLineEditorResult( + "-A OUTPUT -d 3001::1/128 -j REDIRECT", + + "-P PREROUTING ACCEPT\n" + + "-P INPUT ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-P POSTROUTING ACCEPT\n" + + "-A OUTPUT -d 3001::2/128 -j REDIRECT\n" + + "-A OUTPUT -d 3001::1/128 -j REDIRECT\n" + + "-A OUTPUT -d 3001::4/128 -j REDIRECT\n", + + "-P PREROUTING ACCEPT\n" + + "-P INPUT ACCEPT\n" + + "-P OUTPUT ACCEPT\n" + + "-P POSTROUTING ACCEPT\n" + + "-A OUTPUT -d 3001::1/128 -j REDIRECT"); + } + + private static void assertNatTableLineEditorResult(String redirectRule, String currentNatTable, String expectedNatTable) { + NatTableLineEditor natLineEditor = NatTableLineEditor.from(redirectRule); + Editor editor = new Editor( + "nat-table", + () -> List.of(currentNatTable.split("\n")), + result -> assertEquals(expectedNatTable, String.join("\n", result)), + natLineEditor); + editor.edit(m -> {}); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java new file mode 100644 index 00000000000..9487affd376 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -0,0 +1,234 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; + +import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; +import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; +import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.List; + +import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author freva + */ +public class CoreCollectorTest { + private static final Instant CORE_CREATED = Instant.ofEpochMilli(2233445566L); + + private final ContainerOperations docker = mock(ContainerOperations.class); + private final CoreCollector coreCollector = new CoreCollector(docker); + private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld") + .fileSystem(TestFileSystem.create()).build(); + + private final ContainerPath TEST_CORE_PATH = (ContainerPath) new UnixPath(context.paths().of("/tmp/core.1234")) + .createParents() + .createNewFile() + .setLastModifiedTime(CORE_CREATED) + .toPath(); + private final String TEST_BIN_PATH = "/usr/bin/program"; + private final List<String> GDB_BACKTRACE = List.of("[New Thread 2703]", + "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.", + "#0 0x00000000004004d8 in main (argv=...) at main.c:4", "4\t printf(argv[3]);", + "#0 0x00000000004004d8 in main (argv=...) at main.c:4"); + + @Test + void extractsBinaryPathTest() { + final String[] cmd = {"file", TEST_CORE_PATH.pathInContainer()}; + + mockExec(cmd, + "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + + "'/usr/bin/program'"); + assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH)); + + mockExec(cmd, + "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + + "'/usr/bin/program --foo --bar baz'"); + assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH)); + + mockExec(cmd, + "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, " + + "from 'program', real uid: 0, effective uid: 0, real gid: 0, effective gid: 0, " + + "execfn: '/usr/bin/program', platform: 'x86_64"); + assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH)); + + String fallbackResponse = "/response/from/fallback"; + mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-batch", "-core", "/tmp/core.1234"}, + """ + GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.2) 7.7.1 + Type “apropos word” to search for commands related to “word”… + Reading symbols from abc…(no debugging symbols found)…done. + [New LWP 23678] + Core was generated by `/response/from/fallback'. \s + Program terminated with signal SIGSEGV, Segmentation fault. \s + #0 0x0000000000400541 in main () + #0 0x0000000000400541 in main () + (gdb) bt + #0 0x0000000000400541 in main () + (gdb) + """); + mockExec(cmd, + "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style"); + assertEquals(fallbackResponse, coreCollector.readBinPath(context, TEST_CORE_PATH)); + + mockExec(cmd, "", "Error code 1234"); + assertEquals(fallbackResponse, coreCollector.readBinPath(context, TEST_CORE_PATH)); + } + + @Test + void extractsBinaryPathUsingGdbTest() { + String[] cmd = new String[]{GDB_PATH_RHEL8, "-n", "-batch", "-core", "/tmp/core.1234"}; + + mockExec(cmd, "Core was generated by `/usr/bin/program-from-gdb --identity foo/search/cluster.content_'."); + assertEquals("/usr/bin/program-from-gdb", coreCollector.readBinPathFallback(context, TEST_CORE_PATH)); + + mockExec(cmd, "", "Error 123"); + try { + coreCollector.readBinPathFallback(context, TEST_CORE_PATH); + fail("Expected not to be able to get bin path"); + } catch (RuntimeException e) { + assertEquals("Failed to extract binary path from GDB, result: exit status 1, output 'Error 123', command: " + + "[/opt/rh/gcc-toolset-12/root/bin/gdb, -n, -batch, -core, /tmp/core.1234]", e.getMessage()); + } + } + + @Test + void extractsBacktraceUsingGdb() { + mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", + "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, + String.join("\n", GDB_BACKTRACE)); + assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false)); + + mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", + "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, + "", "Failure"); + try { + coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false); + fail("Expected not to be able to read backtrace"); + } catch (RuntimeException e) { + assertEquals("Failed to read backtrace exit status 1, output 'Failure', Command: " + + "[" + GDB_PATH_RHEL8 + ", -n, -ex, set print frame-arguments none, -ex, bt, -batch, " + + "/usr/bin/program, /tmp/core.1234]", e.getMessage()); + } + } + + @Test + void extractsBacktraceFromAllThreadsUsingGdb() { + mockExec(new String[]{GDB_PATH_RHEL8, "-n", + "-ex", "set print frame-arguments none", + "-ex", "thread apply all bt", "-batch", + "/usr/bin/program", "/tmp/core.1234"}, + String.join("\n", GDB_BACKTRACE)); + assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, true)); + } + + @Test + void collectsDataTest() { + mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, + "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + + "'/usr/bin/program'"); + mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", + "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, + String.join("\n", GDB_BACKTRACE)); + mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", + "-ex", "thread apply all bt", "-batch", + "/usr/bin/program", "/tmp/core.1234"}, + String.join("\n", GDB_BACKTRACE)); + + var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH) + .setCreated(CORE_CREATED) + .setType(CoreDumpMetadata.Type.CORE_DUMP) + .setBacktrace(GDB_BACKTRACE) + .setBacktraceAllThreads(GDB_BACKTRACE); + assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); + } + + @Test + void collectsDataRelativePath() { + mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, + "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from 'sbin/distributord-bin'"); + String absolutePath = "/opt/vespa/sbin/distributord-bin"; + mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", + "-ex", "bt", "-batch", absolutePath, "/tmp/core.1234"}, + String.join("\n", GDB_BACKTRACE)); + mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", + "-ex", "thread apply all bt", "-batch", absolutePath, "/tmp/core.1234"}, + String.join("\n", GDB_BACKTRACE)); + + var expected = new CoreDumpMetadata() + .setBinPath(absolutePath) + .setCreated(CORE_CREATED) + .setType(CoreDumpMetadata.Type.CORE_DUMP) + .setBacktrace(GDB_BACKTRACE) + .setBacktraceAllThreads(GDB_BACKTRACE); + assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); + } + + @Test + void collectsPartialIfBacktraceFailsTest() { + mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, + "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + + "'/usr/bin/program'"); + mockExec(new String[]{GDB_PATH_RHEL8 + " -n -ex set print frame-arguments none -ex bt -batch /usr/bin/program /tmp/core.1234"}, + "", "Failure"); + + var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH).setCreated(CORE_CREATED).setType(CoreDumpMetadata.Type.CORE_DUMP); + assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); + } + + @Test + void reportsJstackInsteadOfGdbForJdkCores() { + mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, + "dump.core.5954: ELF 64-bit LSB core file x86-64, version 1 (SYSV), too many program header sections (33172)"); + + String jdkPath = "/path/to/jdk/java"; + mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-batch", "-core", "/tmp/core.1234"}, + "Core was generated by `" + jdkPath + " -Dconfig.id=default/container.11 -XX:+Pre'."); + + String jstack = "jstack11"; + mockExec(new String[]{"jhsdb", "jstack", "--exe", jdkPath, "--core", "/tmp/core.1234"}, + jstack); + + var expected = new CoreDumpMetadata().setBinPath(jdkPath) + .setCreated(CORE_CREATED) + .setType(CoreDumpMetadata.Type.CORE_DUMP) + .setBacktraceAllThreads(List.of(jstack)); + assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); + } + + @Test + void metadata_for_java_heap_dump() { + var expected = new CoreDumpMetadata().setBinPath("java") + .setType(CoreDumpMetadata.Type.JVM_HEAP) + .setCreated(CORE_CREATED) + .setBacktrace(List.of("Heap dump, no backtrace available")); + + assertEquals(expected, coreCollector.collect(context, (ContainerPath) new UnixPath(context.paths().of("/dump_java_pid123.hprof")) + .createNewFile() + .setLastModifiedTime(CORE_CREATED) + .toPath())); + } + + private void mockExec(String[] cmd, String output) { + mockExec(cmd, output, ""); + } + + private void mockExec(String[] cmd, String output, String error) { + mockExec(context, cmd, output, error); + } + + private void mockExec(NodeAgentContext context, String[] cmd, String output, String error) { + when(docker.executeCommandInContainer(context, context.users().root(), cmd)) + .thenReturn(new CommandResult(null, error.isEmpty() ? 0 : 1, error.isEmpty() ? output : error)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java new file mode 100644 index 00000000000..e65a226b789 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -0,0 +1,300 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; + +import com.yahoo.config.provision.DockerImage; +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.security.KeyId; +import com.yahoo.security.SealedSharedKey; +import com.yahoo.security.SecretSharedKey; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; +import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores; +import com.yahoo.vespa.hosted.node.admin.container.metrics.DimensionMetrics; +import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import javax.crypto.spec.SecretKeySpec; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.yolean.Exceptions.uncheck; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author freva + */ +public class CoredumpHandlerTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld") + .fileSystem(fileSystem).build(); + private final ContainerPath containerCrashPath = context.paths().of("/var/crash"); + private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps"); + + private final CoreCollector coreCollector = mock(CoreCollector.class); + private final Cores cores = mock(Cores.class); + private final Metrics metrics = new Metrics(); + private final TestTimer timer = new TestTimer(); + @SuppressWarnings("unchecked") + private final Supplier<String> coredumpIdSupplier = mock(Supplier.class); + private final SecretSharedKeySupplier secretSharedKeySupplier = mock(SecretSharedKeySupplier.class); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private final CoredumpHandler coredumpHandler = + new CoredumpHandler(coreCollector, cores, containerCrashPath.pathInContainer(), + doneCoredumpsPath, metrics, timer, coredumpIdSupplier, secretSharedKeySupplier, + flagSource); + + @Test + void coredump_enqueue_test() throws IOException { + ContainerPath crashPath = context.paths().of("/some/crash/path"); + ContainerPath processingDir = context.paths().of("/some/other/processing"); + + Files.createDirectories(crashPath); + createFileAged(crashPath.resolve("bash.core.431"), Duration.ZERO); + + assertFolderContents(crashPath, "bash.core.431"); + Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); + assertEquals(Optional.empty(), enqueuedPath); + + // bash.core.431 finished writing... and 2 more have since been written + timer.advance(Duration.ofMinutes(3)); + createFileAged(crashPath.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); + createFileAged(crashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); + + when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321"); + enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); + assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); + assertFolderContents(crashPath, "bash.core.431", "vespa-slobrok.core.673"); + assertFolderContents(processingDir, "id-123"); + assertFolderContents(processingDir.resolve("id-123"), "dump_vespa-proton.core.119"); + verify(coredumpIdSupplier, times(1)).get(); + + // Enqueue another + enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); + assertEquals(Optional.of(processingDir.resolve("id-321")), enqueuedPath); + assertFolderContents(crashPath, "bash.core.431"); + assertFolderContents(processingDir, "id-123", "id-321"); + assertFolderContents(processingDir.resolve("id-321"), "dump_vespa-slobrok.core.673"); + verify(coredumpIdSupplier, times(2)).get(); + } + + @Test + void enqueue_with_hs_err_files() throws IOException { + ContainerPath crashPath = context.paths().of("/some/crash/path"); + ContainerPath processingDir = context.paths().of("/some/other/processing"); + Files.createDirectories(crashPath); + + createFileAged(crashPath.resolve("java.core.69"), Duration.ofSeconds(515)); + createFileAged(crashPath.resolve("hs_err_pid69.log"), Duration.ofSeconds(520)); + + createFileAged(crashPath.resolve("java.core.2420"), Duration.ofSeconds(540)); + createFileAged(crashPath.resolve("hs_err_pid2420.log"), Duration.ofSeconds(549)); + createFileAged(crashPath.resolve("hs_err_pid2421.log"), Duration.ofSeconds(550)); + + when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321"); + Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); + assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); + assertFolderContents(crashPath, "hs_err_pid69.log", "java.core.69"); + assertFolderContents(processingDir, "id-123"); + assertFolderContents(processingDir.resolve("id-123"), "hs_err_pid2420.log", "hs_err_pid2421.log", "dump_java.core.2420"); + } + + @Test + void coredump_to_process_test() throws IOException { + ContainerPath processingDir = context.paths().of("/some/other/processing"); + + // Initially there are no core dumps + Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, containerCrashPath, processingDir); + assertEquals(Optional.empty(), enqueuedPath); + + // 3 core dumps occur + Files.createDirectories(containerCrashPath); + createFileAged(containerCrashPath.resolve("bash.core.431"), Duration.ZERO); + createFileAged(containerCrashPath.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); + createFileAged(containerCrashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); + + when(coredumpIdSupplier.get()).thenReturn("id-123"); + enqueuedPath = coredumpHandler.getCoredumpToProcess(context, containerCrashPath, processingDir); + assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); + + // Running this again wont enqueue new core dumps as we are still processing the one enqueued previously + enqueuedPath = coredumpHandler.getCoredumpToProcess(context, containerCrashPath, processingDir); + assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); + verify(coredumpIdSupplier, times(1)).get(); + } + + @Test + void gather_metadata_test() throws IOException { + var metadata = new CoreDumpMetadata().setKernelVersion("3.10.0-862.9.1.el7.x86_64") + .setBacktrace(List.of("call 1", "function 2", "something something")) + .setVespaVersion("6.48.4") + .setBinPath("/bin/bash") + .setCoreDumpPath(context.paths().of("/home/docker/dumps/container-123/id-123/dump_core.456")) + .setDockerImage(DockerImage.fromString("example.com/vespa/ci:6.48.4")); + + new UnixPath(fileSystem.getPath("/proc/cpuinfo")).createParents().writeUtf8File("microcode\t: 0xf0"); + + ContainerPath coredumpDirectory = context.paths().of("/var/crash/id-123"); + Files.createDirectories(coredumpDirectory.pathOnHost()); + Files.createFile(coredumpDirectory.resolve("dump_core.456")); + when(coreCollector.collect(eq(context), eq(coredumpDirectory.resolve("dump_core.456")))) + .thenReturn(metadata); + + assertEquals(metadata, coredumpHandler.gatherMetadata(context, coredumpDirectory)); + verify(coreCollector, times(1)).collect(any(), any()); + + // On second invocation the test already exist, so times(1) is not incremented + assertEquals(metadata, coredumpHandler.gatherMetadata(context, coredumpDirectory)); + doThrow(new IllegalStateException("Should not be invoked")) + .when(coreCollector).collect(any(), any()); + verify(coreCollector, times(1)).collect(any(), any()); + } + + @Test + void cant_get_metadata_if_no_core_file() { + assertThrows(IllegalStateException.class, () -> { + coredumpHandler.gatherMetadata(context, context.paths().of("/fake/path")); + }); + } + + @Test + void fails_to_get_core_file_if_only_compressed_or_encrypted() { + assertThrows(IllegalStateException.class, () -> { + ContainerPath coredumpDirectory = context.paths().of("/path/to/coredump/proccessing/id-123"); + Files.createDirectories(coredumpDirectory); + Files.createFile(coredumpDirectory.resolve("dump_bash.core.431.zst")); + Files.createFile(coredumpDirectory.resolve("dump_bash.core.543.zst.enc")); + coredumpHandler.findCoredumpFileInProcessingDirectory(coredumpDirectory); + }); + } + + void do_process_single_coredump_test(String expectedCoreFileName) throws IOException { + ContainerPath coredumpDirectory = context.paths().of("/path/to/coredump/proccessing/id-123"); + Files.createDirectories(coredumpDirectory); + Files.write(coredumpDirectory.resolve("metadata2.json"), "{\"test-metadata\":{}}".getBytes()); + Files.createFile(coredumpDirectory.resolve("dump_bash.core.431")); + assertFolderContents(coredumpDirectory, "metadata2.json", "dump_bash.core.431"); + CoreDumpMetadata expectedMetadata = new CoreDumpMetadata(); + expectedMetadata.setDecryptionToken("131Q0MMF3hBuMVnXg1WnSFexZGrcwa9ZhfHlegLNwPIN6hQJnBxq5srLf3aZbYdlRVE"); + + coredumpHandler.processAndReportSingleCoreDump(context, coredumpDirectory, Optional.empty()); + verify(coreCollector, never()).collect(any(), any()); + verify(cores, times(1)).report(eq(context.hostname()), eq("id-123"), eq(expectedMetadata)); + assertFalse(Files.exists(coredumpDirectory)); + assertFolderContents(doneCoredumpsPath.resolve("container-123"), "id-123"); + assertFolderContents(doneCoredumpsPath.resolve("container-123").resolve("id-123"), "metadata2.json", expectedCoreFileName); + } + + @Test + void processing_single_coredump_test_without_encryption_throws() throws IOException { + assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst")); + } + + @Test + void process_single_coredump_test_with_encryption() throws IOException { + flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), "bar-key"); + when(secretSharedKeySupplier.create(KeyId.ofString("bar-key"))).thenReturn(Optional.of(makeFixedSecretSharedKey())); + do_process_single_coredump_test("dump_bash.core.431.zst.enc"); + } + + @Test + void processing_throws_when_no_public_key_set_in_feature_flag() throws IOException { + flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), ""); // empty -> not set + verify(secretSharedKeySupplier, never()).create(any()); + assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst")); + } + + @Test + void processing_throws_when_no_key_returned_for_key_id_specified_by_feature_flag() throws IOException { + flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), "baz-key"); + when(secretSharedKeySupplier.create(KeyId.ofString("baz-key"))).thenReturn(Optional.empty()); + assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst")); + } + + @Test + void report_enqueued_and_processed_metrics() throws IOException { + Path processingPath = containerCrashPath.resolve("processing"); + Files.createFile(containerCrashPath.resolve("dump-1")); + Files.createFile(containerCrashPath.resolve("dump-2")); + Files.createFile(containerCrashPath.resolve("hs_err_pid2.log")); + Files.createDirectory(processingPath); + Files.createFile(processingPath.resolve("metadata2.json")); + Files.createFile(processingPath.resolve("dump-3")); + + new UnixPath(doneCoredumpsPath.resolve("container-123").resolve("dump-3-folder").resolve("dump-3")) + .createParents() + .createNewFile(); + + coredumpHandler.updateMetrics(context, containerCrashPath); + List<DimensionMetrics> updatedMetrics = metrics.getMetricsByType(Metrics.DimensionType.PRETAGGED); + assertEquals(1, updatedMetrics.size()); + Map<String, Number> values = updatedMetrics.get(0).getMetrics(); + assertEquals(3, values.get("coredumps.enqueued").intValue()); + assertEquals(1, values.get("coredumps.processed").intValue()); + } + + @BeforeEach + public void setup() throws IOException { + Files.createDirectories(containerCrashPath.pathOnHost()); + } + + private static void assertFolderContents(Path pathToFolder, String... filenames) { + Set<String> expectedContentsOfFolder = Set.of(filenames); + Set<String> actualContentsOfFolder; + try (Stream<UnixPath> paths = new UnixPath(pathToFolder).listContentsOfDirectory()) { + actualContentsOfFolder = paths.map(unixPath -> unixPath.toPath().getFileName().toString()) + .collect(Collectors.toSet()); + } + assertEquals(expectedContentsOfFolder, actualContentsOfFolder); + } + + private Path createFileAged(Path path, Duration age) { + return uncheck(() -> Files.setLastModifiedTime( + Files.createFile(path), + FileTime.from(timer.currentTime().minus(age)))); + } + + private static byte[] bytesOf(String str) { + return str.getBytes(StandardCharsets.UTF_8); + } + + private static SecretSharedKey makeFixedSecretSharedKey() { + byte[] keyBytes = bytesOf("very secret yes!"); // 128 bits + var secretKey = new SecretKeySpec(keyBytes, "AES"); + var keyId = KeyId.ofString("the shiniest key"); + // We don't parse any of these fields in the test, so just use dummy contents. + byte[] enc = bytesOf("hello world"); + byte[] ciphertext = bytesOf("imaginary ciphertext"); + return new SecretSharedKey(secretKey, new SealedSharedKey(SealedSharedKey.CURRENT_TOKEN_VERSION, keyId, enc, ciphertext)); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java new file mode 100644 index 00000000000..0e20d3965a0 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java @@ -0,0 +1,103 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +public class CoredumpCleanupRuleTest { + + private final FileSystem fileSystem = TestFileSystem.create(); + + @Test + void for_container_test() throws IOException { + Path path = fileSystem.getPath("/test/path"); + DiskCleanupRule rule = CoredumpCleanupRule.forContainer(path); + + assertPriorities(rule, Map.of()); + + createFile(path.resolve("core1"), Instant.ofEpochSecond(232)); + assertPriorities(rule, Map.of("/test/path/core1", Priority.MEDIUM)); + + createFile(path.resolve("core2"), Instant.ofEpochSecond(123)); + assertPriorities(rule, Map.of( + "/test/path/core2", Priority.MEDIUM, + "/test/path/core1", Priority.HIGHEST)); + + createFile(path.resolve("vespa-proton-bin.core.325"), Instant.ofEpochSecond(456)); + createFile(path.resolve("vespa-distributor.core.764"), Instant.ofEpochSecond(256)); + var expected = Map.of( + "/test/path/core2", Priority.HIGHEST, + "/test/path/core1", Priority.HIGHEST, + "/test/path/vespa-proton-bin.core.325", Priority.HIGHEST, + "/test/path/vespa-distributor.core.764", Priority.MEDIUM); + assertPriorities(rule, expected); + + // processing core has no effect on this + Files.createDirectories(path.resolve("processing/abcd-1234")); + createFile(path.resolve("processing/abcd-1234/core5"), Instant.ofEpochSecond(67)); + assertPriorities(rule, expected); + } + + @Test + void for_host_test() throws IOException { + Path path = fileSystem.getPath("/test/path"); + DiskCleanupRule rule = CoredumpCleanupRule.forHost(path); + + assertPriorities(rule, Map.of()); + + createFile(path.resolve("h123a/abcd-1234/dump_core1"), Instant.parse("2020-04-21T19:21:00Z")); + createFile(path.resolve("h123a/abcd-1234/metadata.json"), Instant.parse("2020-04-21T19:26:00Z")); + assertPriorities(rule, Map.of("/test/path/h123a/abcd-1234/dump_core1", Priority.MEDIUM)); + + createFile(path.resolve("h123a/abcd-efgh/dump_core1"), Instant.parse("2020-04-21T07:13:00Z")); + createFile(path.resolve("h123a/56ad-af42/dump_vespa-distributor.321"), Instant.parse("2020-04-21T23:37:00Z")); + createFile(path.resolve("h123a/4324-a23d/dump_core2"), Instant.parse("2020-04-22T04:56:00Z")); + createFile(path.resolve("h123a/8534-7da3/dump_vespa-proton-bin.123"), Instant.parse("2020-04-19T15:35:00Z")); + + // Also create a core for a second container: h123b + createFile(path.resolve("h123b/db1a-ab34/dump_core1"), Instant.parse("2020-04-21T07:01:00Z")); + createFile(path.resolve("h123b/7392-59ad/dump_vespa-proton-bin.342"), Instant.parse("2020-04-22T12:05:00Z")); + + assertPriorities(rule, Map.of( + "/test/path/h123a/abcd-1234/dump_core1", Priority.HIGH, + "/test/path/h123a/abcd-efgh/dump_core1", Priority.HIGH, + + // Although it is the oldest core of the day for h123a, it is the first one that starts with vespa- + "/test/path/h123a/56ad-af42/dump_vespa-distributor.321", Priority.MEDIUM, + "/test/path/h123a/4324-a23d/dump_core2", Priority.MEDIUM, + "/test/path/h123a/8534-7da3/dump_vespa-proton-bin.123", Priority.MEDIUM, + "/test/path/h123b/db1a-ab34/dump_core1", Priority.MEDIUM, + "/test/path/h123b/7392-59ad/dump_vespa-proton-bin.342", Priority.MEDIUM + )); + } + + private static void createFile(Path path, Instant instant) throws IOException { + Files.createDirectories(path.getParent()); + Files.createFile(path); + Files.setLastModifiedTime(path, FileTime.from(instant)); + } + + private static void assertPriorities(DiskCleanupRule rule, Map<String, Priority> expected) { + Map<String, Priority> actual = rule.prioritize().stream() + .collect(Collectors.toMap(pfa -> pfa.fileAttributes().path().toString(), PrioritizedFileAttributes::priority)); + + assertEquals(new TreeMap<>(expected), new TreeMap<>(actual)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java new file mode 100644 index 00000000000..390501a4530 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java @@ -0,0 +1,129 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFileAttributes; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author freva + */ +public class DiskCleanupTest { + + private final TestTaskContext context = new TestTaskContext(); + private final DiskCleanupTester tester = new DiskCleanupTester(); + private final DiskCleanup diskCleanup = new DiskCleanup(); + + @Test + void nothing_deleted() throws IOException { + assertFalse(diskCleanup.cleanup(context, List.of(), 0)); + assertFalse(diskCleanup.cleanup(context, List.of(), 10)); + + DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock(); + DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock(); + assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 0)); + assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + + tester.createFile("/path/that-should-not-be-deleted", 5); + assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + tester.assertAllFilesExistExcept(); + + // Create a file and let rule return it, but before cleanup is run, the file is deleted + rule1.addFile(tester.createFile("/path/file-does-not-exist", 1), Priority.HIGHEST); + Files.delete(tester.path("/path/file-does-not-exist")); + assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + } + + @Test + void delete_test() throws IOException { + tester.createFile("/opt/vespa/var/db/do-not-delete-1.db", 1); + tester.createFile("/opt/vespa/var/db/do-not-delete-2.db", 1); + tester.createFile("/opt/vespa/var/zookeeper/do-not-delete-3", 1); + tester.createFile("/opt/vespa/var/index/something-important", 1); + + DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock() + .addFile(tester.createFile("/opt/vespa/logs/vespa-1.log", 10), Priority.MEDIUM) + .addFile(tester.createFile("/opt/vespa/logs/vespa-2.log", 8), Priority.HIGH) + .addFile(tester.createFile("/opt/vespa/logs/vespa-3.log", 13), Priority.HIGHEST) + .addFile(tester.createFile("/opt/vespa/logs/vespa-4.log", 10), Priority.HIGHEST); + DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock() + .addFile(tester.createFile("/opt/vespa/var/crash/core1", 105), Priority.LOW) + .addFile(tester.createFile("/opt/vespa/var/crash/vespa-proton-bin.core-232", 190), Priority.HIGH) + .addFile(tester.createFile("/opt/vespa/var/crash/core3", 54), Priority.MEDIUM) + .addFile(tester.createFile("/opt/vespa/var/crash/core4", 300), Priority.HIGH); + + // 2 files with HIGHEST priority, tie broken by the largest size which is won by "vespa-3.log", since + // it is >= 10 bytes, no more files are deleted + assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log"); + + // Called with the same arguments, but vespa-3.log is still missing... + assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log"); + + assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 500)); + tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log", // from before + // 300 + 190 + 8 + 54 + "/opt/vespa/var/crash/core4", "/opt/vespa/var/crash/vespa-proton-bin.core-232", "/opt/vespa/logs/vespa-2.log", "/opt/vespa/var/crash/core3"); + } + + private static class DiskCleanupRuleMock implements DiskCleanupRule { + private final ArrayList<PrioritizedFileAttributes> pfa = new ArrayList<>(); + + private DiskCleanupRuleMock addFile(Path path, Priority priority) throws IOException { + PosixFileAttributes attributes = Files.getFileAttributeView(path, PosixFileAttributeView.class).readAttributes(); + pfa.add(new PrioritizedFileAttributes(new FileAttributes(path, attributes), priority)); + return this; + } + + @Override + public Collection<PrioritizedFileAttributes> prioritize() { + return Collections.unmodifiableList(pfa); + } + } + + private static class DiskCleanupTester { + private final FileSystem fileSystem = TestFileSystem.create(); + private final Set<String> files = new HashSet<>(); + + private Path path(String path) { + return fileSystem.getPath(path); + } + + private Path createFile(String pathStr, int size) throws IOException { + Path path = path(pathStr); + Files.createDirectories(path.getParent()); + Files.write(path, new byte[size]); + files.add(path.toString()); + return path; + } + + private void assertAllFilesExistExcept(String... deletedPaths) { + Set<String> actual = FileFinder.files(path("/")).stream().map(fa -> fa.path().toString()).collect(Collectors.toSet()); + Set<String> expected = new HashSet<>(files); + expected.removeAll(Set.of(deletedPaths)); + assertEquals(expected, actual); + } + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java new file mode 100644 index 00000000000..c85ddf41906 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java @@ -0,0 +1,58 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; +import static org.mockito.Mockito.mock; + +/** + * @author freva + */ +public class LinearCleanupRuleTest { + + @Test + void basic() { + assertRule(Map.of(), Priority.LOWEST, Priority.HIGHEST); + + assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.LOW, 1.0, Priority.LOW), Priority.LOW, Priority.LOW); + assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.MEDIUM, 1.0, Priority.MEDIUM), Priority.LOW, Priority.MEDIUM); + + assertRule(Map.of( + -5.0, Priority.LOW, + 0.0, Priority.LOW, + 0.2, Priority.LOW, + 0.35, Priority.MEDIUM, + 0.65, Priority.MEDIUM, + 0.8, Priority.HIGH, + 1.0, Priority.HIGH, + 5.0, Priority.HIGH), + Priority.LOW, Priority.HIGH); + } + + @Test + void fail_if_high_priority_lower_than_low() { + assertThrows(IllegalArgumentException.class, () -> { + assertRule(Map.of(), Priority.HIGHEST, Priority.LOWEST); + }); + } + + private static void assertRule(Map<Double, Priority> expectedPriorities, Priority low, Priority high) { + Map<FileAttributes, Double> fileAttributesByScore = expectedPriorities.keySet().stream() + .collect(Collectors.toMap(score -> mock(FileAttributes.class), score -> score)); + LinearCleanupRule rule = new LinearCleanupRule( + () -> List.copyOf(fileAttributesByScore.keySet()), fileAttributesByScore::get, low, high); + + Map<Double, Priority> actualPriorities = rule.prioritize().stream() + .collect(Collectors.toMap(pfa -> fileAttributesByScore.get(pfa.fileAttributes()), PrioritizedFileAttributes::priority)); + assertEquals(expectedPriorities, actualPriorities); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java new file mode 100644 index 00000000000..607efa9771a --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java @@ -0,0 +1,31 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; + +import com.yahoo.yolean.concurrent.Sleeper; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author bjorncs + */ +class ArtifactProducersTest { + + @Test + void generates_exception_on_unknown_artifact() { + ArtifactProducers instance = ArtifactProducers.createDefault(Sleeper.NOOP); + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, () -> instance.resolve(List.of("unknown-artifact"))); + String expectedMsg = + "Invalid artifact type 'unknown-artifact'. Valid types are " + + "['config-dump', 'jvm-heap-dump', 'jvm-jfr', 'jvm-jmap', 'jvm-jstack', 'jvm-jstat', 'perf-report', 'pmap', " + + "'vespa-log', 'zookeeper-snapshot'] " + + "and valid aliases are " + + "['jvm-dump': ['jvm-heap-dump', 'jvm-jmap', 'jvm-jstack', 'jvm-jstat', 'vespa-log']]"; + assertEquals(expectedMsg, exception.getMessage()); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java new file mode 100644 index 00000000000..db19d6b0074 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java @@ -0,0 +1,319 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; +import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; +import com.yahoo.vespa.hosted.node.admin.integration.NodeRepoMock; +import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient; +import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; +import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; +import com.yahoo.vespa.test.file.TestFileSystem; +import com.yahoo.yolean.concurrent.Sleeper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; + +import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.ServiceDumpReport.DumpOptions; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author bjorncs + */ +class VespaServiceDumperImplTest { + + private static final String HOSTNAME = "host-1.domain.tld"; + + private final FileSystem fileSystem = TestFileSystem.create(); + private final Path tmpDirectory = fileSystem.getPath("/data/vespa/storage/host-1/opt/vespa/var/tmp"); + + @BeforeEach + void create_tmp_directory() throws IOException { + // Create temporary directory in container + Files.createDirectories(tmpDirectory); + } + + @Test + void creates_valid_dump_id_from_dump_request() { + long nowMillis = Instant.now().toEpochMilli(); + ServiceDumpReport request = new ServiceDumpReport( + nowMillis, null, null, null, null, "default/container.3", null, null, List.of("perf-report"), null); + String dumpId = VespaServiceDumperImpl.createDumpId(request); + assertEquals("default-container-3-" + nowMillis, dumpId); + } + + @Test + void invokes_perf_commands_when_generating_perf_report() { + // Setup mocks + ContainerOperations operations = mock(ContainerOperations.class); + when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) + .thenReturn(new CommandResult(null, 0, "12345")) + .thenReturn(new CommandResult(null, 0, "")) + .thenReturn(new CommandResult(null, 0, "")); + SyncClient syncClient = createSyncClientMock(); + NodeRepoMock nodeRepository = new NodeRepoMock(); + TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); + NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("perf-report"), new ServiceDumpReport.DumpOptions(true, 45.0, null)); + + VespaServiceDumper reporter = new VespaServiceDumperImpl( + ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); + NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) + .fileSystem(fileSystem) + .build(); + reporter.processServiceDumpRequest(context); + + verify(operations).executeCommandInContainer( + context, context.users().vespa(), "/opt/vespa/libexec/vespa/find-pid", "default/container.1"); + verify(operations).executeCommandInContainer( + context, context.users().vespa(), "perf", "record", "-g", "--output=/opt/vespa/var/tmp/vespa-service-dump-1600000000000/perf-record.bin", + "--pid=12345", "sleep", "45"); + verify(operations).executeCommandInContainer( + context, context.users().vespa(), "bash", "-c", "perf report --input=/opt/vespa/var/tmp/vespa-service-dump-1600000000000/perf-record.bin" + + " > /opt/vespa/var/tmp/vespa-service-dump-1600000000000/perf-report.txt"); + + String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," + + "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + + "\"configId\":\"default/container.1\",\"artifacts\":[\"perf-report\"]," + + "\"dumpOptions\":{\"callGraphRecording\":true,\"duration\":45.0}}"; + assertReportEquals(nodeRepository, expectedJson); + + List<URI> expectedUris = List.of( + URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-record.bin.zst"), + URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-report.txt")); + assertSyncedFiles(context, syncClient, expectedUris); + } + + @Test + void invokes_jcmd_commands_when_creating_jfr_recording() { + // Setup mocks + ContainerOperations operations = mock(ContainerOperations.class); + when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) + .thenReturn(new CommandResult(null, 0, "12345")) + .thenReturn(new CommandResult(null, 0, "ok")) + .thenReturn(new CommandResult(null, 0, "name=host-admin success")); + SyncClient syncClient = createSyncClientMock(); + NodeRepoMock nodeRepository = new NodeRepoMock(); + TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); + NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("jvm-jfr")); + + VespaServiceDumper reporter = new VespaServiceDumperImpl( + ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); + NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) + .fileSystem(fileSystem) + .build(); + reporter.processServiceDumpRequest(context); + + verify(operations).executeCommandInContainer( + context, context.users().vespa(), "/opt/vespa/libexec/vespa/find-pid", "default/container.1"); + verify(operations).executeCommandInContainer( + context, context.users().vespa(), "jcmd", "12345", "JFR.start", "name=host-admin", "path-to-gc-roots=true", "settings=profile", + "filename=/opt/vespa/var/tmp/vespa-service-dump-1600000000000/recording.jfr", "duration=30s"); + verify(operations).executeCommandInContainer(context, context.users().vespa(), "jcmd", "12345", "JFR.check", "name=host-admin"); + + String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000," + + "\"completedAt\":1600001000000," + + "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + + "\"configId\":\"default/container.1\",\"artifacts\":[\"jvm-jfr\"],\"dumpOptions\":{}}"; + assertReportEquals(nodeRepository, expectedJson); + + List<URI> expectedUris = List.of( + URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/recording.jfr.zst")); + assertSyncedFiles(context, syncClient, expectedUris); + } + + @Test + void invokes_zookeeper_backup_command_when_generating_snapshot() { + // Setup mocks + ContainerOperations operations = mock(ContainerOperations.class); + when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) + .thenReturn(new CommandResult(null, 0, "12345")); + SyncClient syncClient = createSyncClientMock(); + NodeRepoMock nodeRepository = new NodeRepoMock(); + TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); + NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("zookeeper-snapshot")); + + VespaServiceDumper reporter = new VespaServiceDumperImpl( + ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); + NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) + .fileSystem(fileSystem) + .build(); + reporter.processServiceDumpRequest(context); + + verify(operations).executeCommandInContainer( + context, + context.users().vespa(), + "bash", + "-c", + "/opt/vespa/bin/vespa-backup-zk-data.sh -o /opt/vespa/var/tmp/vespa-service-dump-1600000000000/zookeeper-snapshot.tgz -k -f"); + + String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," + + "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + + "\"configId\":\"default/container.1\",\"artifacts\":[\"zookeeper-snapshot\"],\"dumpOptions\":{}}"; + assertReportEquals(nodeRepository, expectedJson); + + List<URI> expectedUris = List.of( + URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/zookeeper-snapshot.tgz")); + assertSyncedFiles(context, syncClient, expectedUris); + } + + @Test + void invokes_config_proxy_command_whn_invoking_config_dump() { + // Setup mocks + ContainerOperations operations = mock(ContainerOperations.class); + when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) + .thenReturn(new CommandResult(null, 0, "12345")); + SyncClient syncClient = createSyncClientMock(); + NodeRepoMock nodeRepository = new NodeRepoMock(); + TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); + NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("config-dump")); + + VespaServiceDumper reporter = new VespaServiceDumperImpl( + ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); + NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) + .fileSystem(fileSystem) + .build(); + reporter.processServiceDumpRequest(context); + + verify(operations).executeCommandInContainer( + context, + context.users().vespa(), + "bash", + "-c", + "mkdir -p /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config;" + + " /opt/vespa/bin/vespa-configproxy-cmd -m dumpcache /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config;" + + " tar cvf /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config.tar /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config;" + + " zstd /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config.tar -o /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config-dump.tar.zst"); + + String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," + + "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + + "\"configId\":\"default/container.1\",\"artifacts\":[\"config-dump\"],\"dumpOptions\":{}}"; + assertReportEquals(nodeRepository, expectedJson); + + List<URI> expectedUris = List.of( + URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/config-dump.tar.zst")); + assertSyncedFiles(context, syncClient, expectedUris); + } + + @Test + void handles_multiple_artifact_types() { + // Setup mocks + ContainerOperations operations = mock(ContainerOperations.class); + when(operations.executeCommandInContainer( + any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) + // For perf report: + .thenReturn(new CommandResult(null, 0, "12345")) + .thenReturn(new CommandResult(null, 0, "")) + .thenReturn(new CommandResult(null, 0, "")) + // For jfr recording: + .thenReturn(new CommandResult(null, 0, "12345")) + .thenReturn(new CommandResult(null, 0, "ok")) + .thenReturn(new CommandResult(null, 0, "name=host-admin success")); + SyncClient syncClient = createSyncClientMock(); + NodeRepoMock nodeRepository = new NodeRepoMock(); + TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); + NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("perf-report", "jvm-jfr"), + new ServiceDumpReport.DumpOptions(true, 20.0, null)); + VespaServiceDumper reporter = new VespaServiceDumperImpl( + ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); + NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) + .fileSystem(fileSystem) + .build(); + reporter.processServiceDumpRequest(context); + + List<URI> expectedUris = List.of( + URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-record.bin.zst"), + URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-report.txt"), + URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/recording.jfr.zst")); + assertSyncedFiles(context, syncClient, expectedUris); + } + + @Test + void fails_gracefully_on_invalid_request_json() { + // Setup mocks + ContainerOperations operations = mock(ContainerOperations.class); + SyncClient syncClient = createSyncClientMock(); + NodeRepoMock nodeRepository = new NodeRepoMock(); + TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); + JsonNodeFactory fac = new ObjectMapper().getNodeFactory(); + ObjectNode invalidRequest = new ObjectNode(fac) + .set("dumpOptions", new ObjectNode(fac).put("duration", "invalidDurationDataType")); + NodeSpec spec = NodeSpec.Builder + .testSpec(HOSTNAME, NodeState.active) + .report(ServiceDumpReport.REPORT_ID, invalidRequest) + .build(); + nodeRepository.updateNodeSpec(spec); + VespaServiceDumper reporter = new VespaServiceDumperImpl( + ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); + NodeAgentContextImpl context = NodeAgentContextImpl.builder(spec) + .fileSystem(fileSystem) + .build(); + reporter.processServiceDumpRequest(context); + String expectedJson = "{\"createdMillis\":1600001000000,\"startedAt\":1600001000000,\"failedAt\":1600001000000," + + "\"configId\":\"unknown\",\"error\":\"Invalid JSON in service dump request\",\"artifacts\":[]}"; + assertReportEquals(nodeRepository, expectedJson); + } + + private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, List<String> artifacts) { + return createNodeSpecWithDumpRequest(repository, artifacts, new DumpOptions(null, null, null)); + } + + private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, List<String> artifacts, DumpOptions options) { + ServiceDumpReport request = ServiceDumpReport.createRequestReport( + Instant.ofEpochMilli(1600000000000L), null, "default/container.1", artifacts, options); + NodeSpec spec = NodeSpec.Builder + .testSpec(HOSTNAME, NodeState.active) + .report(ServiceDumpReport.REPORT_ID, request.toJsonNode()) + .archiveUri(URI.create("s3://uri-1/tenant1/")) + .build(); + repository.updateNodeSpec(spec); + return spec; + } + + private static void assertReportEquals(NodeRepoMock nodeRepository, String expectedJson) { + ServiceDumpReport report = nodeRepository.getNode(HOSTNAME).reports() + .getReport(ServiceDumpReport.REPORT_ID, ServiceDumpReport.class).get(); + String actualJson = report.toJson(); + assertEquals(expectedJson, actualJson); + } + + @SuppressWarnings("unchecked") + private static void assertSyncedFiles(NodeAgentContextImpl context, SyncClient client, List<URI> expectedDestinations) { + ArgumentCaptor<List<SyncFileInfo>> filesCaptor = ArgumentCaptor.forClass(List.class); + verify(client).sync(eq(context), filesCaptor.capture(), eq(Integer.MAX_VALUE)); + List<SyncFileInfo> actualFiles = filesCaptor.getValue(); + List<URI> actualFilenames = actualFiles.stream() + .map(SyncFileInfo::destination) + .sorted() + .toList(); + assertEquals(expectedDestinations, actualFilenames); + } + + private SyncClient createSyncClientMock() { + SyncClient client = mock(SyncClient.class); + when(client.sync(any(TaskContext.class), anyList(), anyInt())) + .thenReturn(true); + return client; + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java new file mode 100644 index 00000000000..8e56741274e --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -0,0 +1,134 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.NONE; +import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.ZSTD; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +public class SyncFileInfoTest { + + private static final FileSystem fileSystem = TestFileSystem.create(); + + private static final URI nodeArchiveUri = URI.create("s3://vespa-data-bucket/vespa/music/main/h432a/"); + private static final Path accessLogPath1 = fileSystem.getPath("/opt/vespa/logs/access/access.log.20210211"); + private static final Path accessLogPath2 = fileSystem.getPath("/opt/vespa/logs/access/access.log.20210212.zst"); + private static final Path accessLogPath3 = fileSystem.getPath("/opt/vespa/logs/access/access-json.log.20210213.zst"); + private static final Path accessLogPath4 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.20210214.zst"); + private static final Path accessLogPath5 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.container.20210214.zst"); + private static final Path accessLogPath6 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.metrics-proxy.20210214.zst"); + private static final Path connectionLogPath1 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.20210210"); + private static final Path connectionLogPath2 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.20210212.zst"); + private static final Path connectionLogPath3 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.metrics-proxy.20210210"); + private static final Path vespaLogPath1 = fileSystem.getPath("/opt/vespa/logs/vespa.log"); + private static final Path vespaLogPath2 = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12"); + private static final Path zkLogPath0 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.0.log"); + private static final Path zkLogPath1 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.1.log"); + private static final Path startServicesPath1 = fileSystem.getPath("/opt/vespa/logs/start-services.out"); + private static final Path startServicesPath2 = fileSystem.getPath("/opt/vespa/logs/start-services.out-20230808100143"); + private static final Path rotatedNginxErrorLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-error.log.20231019-1234555"); + private static final Path currentNginxErrorLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-error.log"); + private static final Path nginxAccessLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-access.log.20231019-1234"); + + @Test + void access_logs() { + assertForLogFile(accessLogPath1, null, null, true); + assertForLogFile(accessLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210211.zst", ZSTD, false); + + assertForLogFile(accessLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210212.zst", NONE, true); + assertForLogFile(accessLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210212.zst", NONE, false); + + assertForLogFile(accessLogPath3, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access-json.log.20210213.zst", NONE, true); + assertForLogFile(accessLogPath3, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access-json.log.20210213.zst", NONE, false); + + assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.20210214.zst", NONE, true); + assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.20210214.zst", NONE, false); + + assertForLogFile(accessLogPath5, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.container.20210214.zst", NONE, true); + assertForLogFile(accessLogPath5, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.container.20210214.zst", NONE, false); + + assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, accessLogPath6, true, ApplicationId.defaultId())); + assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, accessLogPath6, false, ApplicationId.defaultId())); + } + + @Test + void connection_logs() { + assertForLogFile(connectionLogPath1, null, null, true); + assertForLogFile(connectionLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210210.zst", ZSTD, false); + + assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210212.zst", NONE, true); + assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210212.zst", NONE, false); + + assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, connectionLogPath3, true, ApplicationId.defaultId())); + assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, connectionLogPath3, false, ApplicationId.defaultId())); + } + + @Test + void vespa_logs() { + new UnixPath(vespaLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ofHours(1), true); + assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2022-05-09.14-22-11.zst", ZSTD, Duration.ZERO, false); + + assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, true); + assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, false); + } + + @Test + void zookeeper_logs() { + new UnixPath(zkLogPath0).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-13T13:13:45Z")); + assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ofHours(1), true); + assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-13.13-13-45.zst", ZSTD, Duration.ZERO, false); + + new UnixPath(zkLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, true); + assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, false); + } + + @Test + void nginx_error_logs() { + new UnixPath(currentNginxErrorLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + assertForLogFile(currentNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.zst", ZSTD, Duration.ofHours(1),true); + assertForLogFile(currentNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.zst", ZSTD, Duration.ZERO,false); + + new UnixPath(rotatedNginxErrorLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + assertForLogFile(rotatedNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.20231019-1234555.zst", ZSTD, true); + assertForLogFile(rotatedNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.20231019-1234555.zst", ZSTD, false); + + // Does not sync access logs + new UnixPath(nginxAccessLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + Optional<SyncFileInfo> sfi = SyncFileInfo.forLogFile(nodeArchiveUri, nginxAccessLog, false, ApplicationId.defaultId()); + assertEquals(Optional.empty(), sfi); + } + + @Test + void start_services() { + assertForLogFile(startServicesPath1, null, null, true); + assertForLogFile(startServicesPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/start-services/start-services.out-20230808100143.zst", ZSTD, true); + } + + private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, boolean rotatedOnly) { + assertForLogFile(srcPath, destination, compression, null, rotatedOnly); + } + + private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, Duration minDurationBetweenSync, boolean rotatedOnly) { + Optional<SyncFileInfo> sfi = SyncFileInfo.forLogFile(nodeArchiveUri, srcPath, rotatedOnly, ApplicationId.defaultId()); + assertEquals(destination, sfi.map(SyncFileInfo::destination).map(URI::toString).orElse(null)); + assertEquals(compression, sfi.map(SyncFileInfo::uploadCompression).orElse(null)); + assertEquals(minDurationBetweenSync, sfi.flatMap(SyncFileInfo::minDurationBetweenSync).orElse(null)); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java new file mode 100644 index 00000000000..616100363e9 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java @@ -0,0 +1,58 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import com.yahoo.compress.ZstdCompressor; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +public class ZstdCompressingInputStreamTest { + + @Test + void compression_test() { + Random rnd = new Random(); + byte[] data = new byte[(int) (100_000 * (10 + rnd.nextDouble()))]; + rnd.nextBytes(data); + assertCompression(data, 1 << 14); + } + + @Test + void compress_empty_file_test() { + byte[] compressedData = compress(new byte[0], 1 << 10); + assertEquals(13, compressedData.length, "zstd compressing an empty file results in a 13 bytes file"); + } + + private static void assertCompression(byte[] data, int bufferSize) { + byte[] compressedData = compress(data, bufferSize); + byte[] decompressedData = new byte[data.length]; + var compressor = new ZstdCompressor(); + compressor.decompress(compressedData, 0, compressedData.length, decompressedData, 0, decompressedData.length); + + assertArrayEquals(data, decompressedData); + } + + private static byte[] compress(byte[] data, int bufferSize) { + ByteArrayInputStream bais = new ByteArrayInputStream(data); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(bais, bufferSize)) { + byte[] buffer = new byte[bufferSize]; + for (int nRead; (nRead = zcis.read(buffer, 0, buffer.length)) != -1; ) + baos.write(buffer, 0, nRead); + baos.flush(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + return baos.toByteArray(); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java new file mode 100644 index 00000000000..355c997a3e0 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -0,0 +1,166 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeadmin; + +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithScheduler; +import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithSchedulerFactory; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +/** + * @author bakksjo + */ +public class NodeAdminImplTest { + + private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory = mock(NodeAgentWithSchedulerFactory.class); + private final TestTimer timer = new TestTimer(); + private final ProcMeminfoReader procMeminfoReader = mock(ProcMeminfoReader.class); + private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentWithSchedulerFactory, + new Metrics(), timer, Duration.ZERO, Duration.ZERO, procMeminfoReader); + + @Test + void nodeAgentsAreProperlyLifeCycleManaged() { + final NodeAgentContext context1 = createNodeAgentContext("host1.test.yahoo.com"); + final NodeAgentContext context2 = createNodeAgentContext("host2.test.yahoo.com"); + final NodeAgentWithScheduler nodeAgent1 = mockNodeAgentWithSchedulerFactory(context1); + final NodeAgentWithScheduler nodeAgent2 = mockNodeAgentWithSchedulerFactory(context2); + + final InOrder inOrder = inOrder(nodeAgentWithSchedulerFactory, nodeAgent1, nodeAgent2); + nodeAdmin.refreshContainersToRun(Set.of()); + verifyNoMoreInteractions(nodeAgentWithSchedulerFactory); + + nodeAdmin.refreshContainersToRun(Set.of(context1)); + inOrder.verify(nodeAgent1).start(); + inOrder.verify(nodeAgent2, never()).start(); + inOrder.verify(nodeAgent1, never()).stopForRemoval(); + + nodeAdmin.refreshContainersToRun(Set.of(context1)); + inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any()); + inOrder.verify(nodeAgent1, never()).start(); + inOrder.verify(nodeAgent1, never()).stopForRemoval(); + + nodeAdmin.refreshContainersToRun(Set.of()); + inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any()); + verify(nodeAgent1).stopForRemoval(); + + nodeAdmin.refreshContainersToRun(Set.of(context2)); + inOrder.verify(nodeAgent2).start(); + inOrder.verify(nodeAgent2, never()).stopForRemoval(); + inOrder.verify(nodeAgent1, never()).stopForRemoval(); + + nodeAdmin.refreshContainersToRun(Set.of()); + inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any()); + inOrder.verify(nodeAgent2, never()).start(); + inOrder.verify(nodeAgent2).stopForRemoval(); + inOrder.verify(nodeAgent1, never()).start(); + inOrder.verify(nodeAgent1, never()).stopForRemoval(); + } + + @Test + void testSetFrozen() { + Set<NodeAgentContext> contexts = new HashSet<>(); + List<NodeAgentWithScheduler> nodeAgents = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + NodeAgentContext context = createNodeAgentContext("host" + i + ".test.yahoo.com"); + NodeAgentWithScheduler nodeAgent = mockNodeAgentWithSchedulerFactory(context); + + contexts.add(context); + nodeAgents.add(nodeAgent); + } + + nodeAdmin.refreshContainersToRun(contexts); + + assertTrue(nodeAdmin.isFrozen()); // Initially everything is frozen to force convergence + mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); + assertTrue(nodeAdmin.setFrozen(false)); // Unfreeze everything + + + mockNodeAgentSetFrozenResponse(nodeAgents, false, false, false); + assertFalse(nodeAdmin.setFrozen(true)); // NodeAdmin freezes only when all the NodeAgents are frozen + + mockNodeAgentSetFrozenResponse(nodeAgents, false, true, true); + assertFalse(nodeAdmin.setFrozen(true)); + assertFalse(nodeAdmin.isFrozen()); + + mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); + assertTrue(nodeAdmin.setFrozen(true)); + assertTrue(nodeAdmin.isFrozen()); + + mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); + assertTrue(nodeAdmin.setFrozen(true)); + assertTrue(nodeAdmin.isFrozen()); + + mockNodeAgentSetFrozenResponse(nodeAgents, false, false, false); + assertFalse(nodeAdmin.setFrozen(false)); + assertFalse(nodeAdmin.isFrozen()); // NodeAdmin unfreezes instantly + + mockNodeAgentSetFrozenResponse(nodeAgents, false, false, true); + assertFalse(nodeAdmin.setFrozen(false)); + assertFalse(nodeAdmin.isFrozen()); + + mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); + assertTrue(nodeAdmin.setFrozen(false)); + assertFalse(nodeAdmin.isFrozen()); + } + + @Test + void testSubsystemFreezeDuration() { + // Initially everything is frozen to force convergence + assertTrue(nodeAdmin.isFrozen()); + assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); + timer.advance(Duration.ofSeconds(1)); + assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration()); + + // Unfreezing floors freeze duration + assertTrue(nodeAdmin.setFrozen(false)); // Unfreeze everything + assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); + timer.advance(Duration.ofSeconds(1)); + assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); + + // Advancing time now will make freeze duration proceed according to clock + assertTrue(nodeAdmin.setFrozen(true)); + assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); + timer.advance(Duration.ofSeconds(1)); + assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration()); + } + + private void mockNodeAgentSetFrozenResponse(List<NodeAgentWithScheduler> nodeAgents, boolean... responses) { + for (int i = 0; i < nodeAgents.size(); i++) { + NodeAgentWithScheduler nodeAgent = nodeAgents.get(i); + when(nodeAgent.setFrozen(anyBoolean(), any())).thenReturn(responses[i]); + } + } + + private NodeAgentContext createNodeAgentContext(String hostname) { + return NodeAgentContextImpl.builder(hostname).fileSystem(TestFileSystem.create()).build(); + } + + private NodeAgentWithScheduler mockNodeAgentWithSchedulerFactory(NodeAgentContext context) { + NodeAgentWithScheduler nodeAgentWithScheduler = mock(NodeAgentWithScheduler.class); + when(nodeAgentWithSchedulerFactory.create(eq(context))).thenReturn(nodeAgentWithScheduler); + return nodeAgentWithScheduler; + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java new file mode 100644 index 00000000000..420146b52f0 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java @@ -0,0 +1,277 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeadmin; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.OrchestratorStatus; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.integration.NodeRepoMock; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.RESUMED; +import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED; +import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Basic test of NodeAdminStateUpdater + * + * @author freva + */ +public class NodeAdminStateUpdaterTest { + private final NodeAgentContextFactory nodeAgentContextFactory = mock(NodeAgentContextFactory.class); + private final NodeRepoMock nodeRepository = spy(new NodeRepoMock()); + private final Orchestrator orchestrator = mock(Orchestrator.class); + private final NodeAdmin nodeAdmin = mock(NodeAdmin.class); + private final HostName hostHostname = HostName.of("basehost1.test.yahoo.com"); + + private final NodeAdminStateUpdater updater = spy(new NodeAdminStateUpdater( + nodeAgentContextFactory, nodeRepository, orchestrator, nodeAdmin, hostHostname)); + + + @Test + void state_convergence() { + mockNodeRepo(NodeState.active, 4); + List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream() + .map(NodeSpec::hostname) + .toList(); + List<String> suspendHostnames = new ArrayList<>(activeHostnames); + suspendHostnames.add(hostHostname.value()); + when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); + + { + // Initially everything is frozen to force convergence + assertConvergeError(RESUMED, "NodeAdmin is not yet unfrozen"); + when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); + updater.converge(RESUMED); + verify(orchestrator, times(1)).resume(hostHostname.value()); + + // We are already resumed, so this should return without resuming again + updater.converge(RESUMED); + verify(orchestrator, times(1)).resume(hostHostname.value()); + verify(nodeAdmin, times(2)).setFrozen(eq(false)); + + // Host is externally suspended in orchestrator, should be resumed by node-admin + setHostOrchestratorStatus(hostHostname, OrchestratorStatus.ALLOWED_TO_BE_DOWN); + updater.converge(RESUMED); + verify(orchestrator, times(2)).resume(hostHostname.value()); + verify(nodeAdmin, times(3)).setFrozen(eq(false)); + setHostOrchestratorStatus(hostHostname, OrchestratorStatus.NO_REMARKS); + + // Lets try to suspend node admin only + when(nodeAdmin.setFrozen(eq(true))).thenReturn(false); + assertConvergeError(SUSPENDED_NODE_ADMIN, "NodeAdmin is not yet frozen"); + verify(nodeAdmin, times(3)).setFrozen(eq(false)); + } + + { + // First orchestration failure happens within the freeze convergence timeout, + // and so should not call setFrozen(false) + final String exceptionMessage = "Cannot allow to suspend because some reason"; + when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); + doThrow(new RuntimeException(exceptionMessage)).doNothing() + .when(orchestrator).suspend(eq(hostHostname.value())); + assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMessage); + verify(nodeAdmin, times(3)).setFrozen(eq(false)); + + updater.converge(SUSPENDED_NODE_ADMIN); + verify(nodeAdmin, times(3)).setFrozen(eq(false)); + verify(orchestrator, times(2)).suspend(hostHostname.value()); + setHostOrchestratorStatus(hostHostname, OrchestratorStatus.ALLOWED_TO_BE_DOWN); + + // Already suspended, no changes + updater.converge(SUSPENDED_NODE_ADMIN); + verify(nodeAdmin, times(3)).setFrozen(eq(false)); + verify(orchestrator, times(2)).suspend(hostHostname.value()); + + // Host is externally resumed + setHostOrchestratorStatus(hostHostname, OrchestratorStatus.NO_REMARKS); + updater.converge(SUSPENDED_NODE_ADMIN); + verify(nodeAdmin, times(3)).setFrozen(eq(false)); + verify(orchestrator, times(3)).suspend(hostHostname.value()); + setHostOrchestratorStatus(hostHostname, OrchestratorStatus.ALLOWED_TO_BE_DOWN); + } + + { + // At this point orchestrator will say its OK to suspend, but something goes wrong when we try to stop services + final String exceptionMessage = "Failed to stop services"; + verify(orchestrator, times(0)).suspend(eq(hostHostname.value()), eq(suspendHostnames)); + doThrow(new RuntimeException(exceptionMessage)).doNothing().when(nodeAdmin).stopNodeAgentServices(); + assertConvergeError(SUSPENDED, exceptionMessage); + verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(suspendHostnames)); + // Make sure we dont roll back if we fail to stop services - we will try to stop again next tick + verify(nodeAdmin, times(3)).setFrozen(eq(false)); + + // Finally we are successful in transitioning to frozen + updater.converge(SUSPENDED); + } + } + + @Test + void half_transition_revert() { + final String exceptionMsg = "Cannot allow to suspend because some reason"; + mockNodeRepo(NodeState.active, 3); + + // Initially everything is frozen to force convergence + when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); + updater.converge(RESUMED); + verify(nodeAdmin, times(1)).setFrozen(eq(false)); + verify(nodeAdmin, times(1)).refreshContainersToRun(any()); + + // Let's start suspending, we are able to freeze the nodes, but orchestrator denies suspension + when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); + when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); + doThrow(new RuntimeException(exceptionMsg)).when(orchestrator).suspend(eq(hostHostname.value())); + + assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg); + verify(nodeAdmin, times(1)).setFrozen(eq(true)); + verify(orchestrator, times(1)).suspend(eq(hostHostname.value())); + assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg); + verify(nodeAdmin, times(2)).setFrozen(eq(true)); + verify(orchestrator, times(2)).suspend(eq(hostHostname.value())); + assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg); + verify(nodeAdmin, times(3)).setFrozen(eq(true)); + verify(orchestrator, times(3)).suspend(eq(hostHostname.value())); + + // No new unfreezes nor refresh while trying to freeze + verify(nodeAdmin, times(1)).setFrozen(eq(false)); + verify(nodeAdmin, times(1)).refreshContainersToRun(any()); + + // Only resume and fetch containers when subsystem freeze duration expires + when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofHours(1)); + assertConvergeError(SUSPENDED_NODE_ADMIN, "Timed out trying to freeze all nodes: will force an unfrozen tick"); + verify(nodeAdmin, times(2)).setFrozen(eq(false)); + verify(orchestrator, times(3)).suspend(eq(hostHostname.value())); // no new suspend calls + verify(nodeAdmin, times(2)).refreshContainersToRun(any()); + + // We change our mind, want to remain resumed + updater.converge(RESUMED); + verify(nodeAdmin, times(3)).setFrozen(eq(false)); // Make sure that we unfreeze! + } + + @Test + void do_not_orchestrate_host_when_not_active() { + when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofHours(1)); + when(nodeAdmin.setFrozen(anyBoolean())).thenReturn(true); + mockNodeRepo(NodeState.ready, 3); + + // Resume and suspend only require that node-agents are frozen and permission from + // orchestrator to resume/suspend host. Therefore, if host is not active, we only need to freeze. + updater.converge(RESUMED); + verify(orchestrator, never()).resume(eq(hostHostname.value())); + + updater.converge(SUSPENDED_NODE_ADMIN); + verify(orchestrator, never()).suspend(eq(hostHostname.value())); + + // When doing batch suspend, only suspend the containers if the host is not active + List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream() + .map(NodeSpec::hostname) + .toList(); + updater.converge(SUSPENDED); + verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(activeHostnames)); + } + + @Test + void node_spec_and_acl_aligned() { + Acl acl = new Acl.Builder().withTrustedPorts(22).build(); + mockNodeRepo(NodeState.active, 3); + mockAcl(acl, 1, 2, 3); + + updater.adjustNodeAgentsToRunFromNodeRepository(); + updater.adjustNodeAgentsToRunFromNodeRepository(); + updater.adjustNodeAgentsToRunFromNodeRepository(); + + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl)); + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl)); + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host3.yahoo.com")), eq(acl)); + verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value())); + verify(nodeRepository, times(3)).getAcls(eq(hostHostname.value())); + } + + @Test + void node_spec_and_acl_mismatch_missing_one_acl() { + Acl acl = new Acl.Builder().withTrustedPorts(22).build(); + mockNodeRepo(NodeState.active, 3); + mockAcl(acl, 1, 2); // Acl for 3 is missing + + updater.adjustNodeAgentsToRunFromNodeRepository(); + mockNodeRepo(NodeState.active, 2); // Next tick, the spec for 3 is no longer returned + updater.adjustNodeAgentsToRunFromNodeRepository(); + updater.adjustNodeAgentsToRunFromNodeRepository(); + + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl)); + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl)); + verify(nodeAgentContextFactory, times(1)).create(argThat(spec -> spec.hostname().equals("host3.yahoo.com")), eq(Acl.EMPTY)); + verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value())); + verify(nodeRepository, times(3)).getAcls(eq(hostHostname.value())); + } + + @Test + void node_spec_and_acl_mismatch_additional_acl() { + Acl acl = new Acl.Builder().withTrustedPorts(22).build(); + mockNodeRepo(NodeState.active, 2); + mockAcl(acl, 1, 2, 3); // Acl for 3 is extra + + updater.adjustNodeAgentsToRunFromNodeRepository(); + updater.adjustNodeAgentsToRunFromNodeRepository(); + updater.adjustNodeAgentsToRunFromNodeRepository(); + + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl)); + verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl)); + verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value())); + verify(nodeRepository, times(3)).getAcls(eq(hostHostname.value())); + } + + private void assertConvergeError(NodeAdminStateUpdater.State targetState, String reason) { + try { + updater.converge(targetState); + fail("Expected converging to " + targetState + " to fail with \"" + reason + "\", but it succeeded without error"); + } catch (RuntimeException e) { + assertEquals(reason, e.getMessage()); + } + } + + private void mockNodeRepo(NodeState hostState, int numberOfNodes) { + nodeRepository.resetNodeSpecs(); + + IntStream.rangeClosed(1, numberOfNodes) + .mapToObj(i -> NodeSpec.Builder.testSpec("host" + i + ".yahoo.com").parentHostname(hostHostname.value()).build()) + .forEach(nodeRepository::updateNodeSpec); + + nodeRepository.updateNodeSpec(NodeSpec.Builder.testSpec(hostHostname.value(), hostState).type(NodeType.host).build()); + } + + private void mockAcl(Acl acl, int... nodeIds) { + nodeRepository.setAcl(Arrays.stream(nodeIds) + .mapToObj(i -> "host" + i + ".yahoo.com") + .collect(Collectors.toMap(Function.identity(), h -> acl))); + } + + private void setHostOrchestratorStatus(HostName hostname, OrchestratorStatus orchestratorStatus) { + nodeRepository.updateNodeSpec(hostname.value(), node -> node.orchestratorStatus(orchestratorStatus)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java new file mode 100644 index 00000000000..589eceebb74 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java @@ -0,0 +1,103 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeagent; + +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author freva + */ +public class NodeAgentContextImplTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final NodeAgentContext context = NodeAgentContextImpl.builder("container-1.domain.tld") + .fileSystem(fileSystem).build(); + + @Test + void path_on_host_from_path_in_node_test() { + assertEquals( + "/data/vespa/storage/container-1", + context.paths().of("/").pathOnHost().toString()); + + assertEquals( + "/data/vespa/storage/container-1/dev/null", + context.paths().of("/dev/null").pathOnHost().toString()); + } + + @Test + void path_in_container_must_be_absolute() { + assertThrows(IllegalArgumentException.class, () -> { + context.paths().of("some/relative/path"); + }); + } + + @Test + void path_in_node_from_path_on_host_test() { + assertEquals( + "/dev/null", + context.paths().fromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-1/dev/null")).pathInContainer()); + } + + @Test + void path_on_host_must_be_absolute() { + assertThrows(IllegalArgumentException.class, () -> { + context.paths().fromPathOnHost(Path.of("some/relative/path")); + }); + } + + @Test + void path_on_host_must_be_inside_container_storage_of_context() { + assertThrows(IllegalArgumentException.class, () -> { + context.paths().fromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-2/dev/null")); + }); + } + + @Test + void path_on_host_must_be_inside_container_storage() { + assertThrows(IllegalArgumentException.class, () -> { + context.paths().fromPathOnHost(fileSystem.getPath("/home")); + }); + } + + @Test + void path_under_vespa_host_in_container_test() { + assertEquals( + "/opt/vespa", + context.paths().underVespaHome("").pathInContainer()); + + assertEquals( + "/opt/vespa/logs/vespa/vespa.log", + context.paths().underVespaHome("logs/vespa/vespa.log").pathInContainer()); + } + + @Test + void path_under_vespa_home_must_be_relative() { + assertThrows(IllegalArgumentException.class, () -> { + context.paths().underVespaHome("/home"); + }); + } + + @Test + void disabledTasksTest() { + NodeAgentContext context1 = createContextWithDisabledTasks(); + assertFalse(context1.isDisabled(NodeAgentTask.DiskCleanup)); + assertFalse(context1.isDisabled(NodeAgentTask.CoreDumps)); + + NodeAgentContext context2 = createContextWithDisabledTasks("root>UpgradeTask", "DiskCleanup", "node>CoreDumps"); + assertFalse(context2.isDisabled(NodeAgentTask.DiskCleanup)); + assertTrue(context2.isDisabled(NodeAgentTask.CoreDumps)); + } + + private NodeAgentContext createContextWithDisabledTasks(String... tasks) { + InMemoryFlagSource flagSource = new InMemoryFlagSource(); + flagSource.withListFlag(PermanentFlags.DISABLED_HOST_ADMIN_TASKS.id(), List.of(tasks), String.class); + return NodeAgentContextImpl.builder("node123").fileSystem(fileSystem).flagSource(flagSource).build(); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java new file mode 100644 index 00000000000..5e09c45d217 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java @@ -0,0 +1,152 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeagent; + +import com.yahoo.jdisc.core.SystemTimer; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; + +import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextSupplier.ContextSupplierInterruptedException; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author freva + */ +public class NodeAgentContextManagerTest { + + private static final int TIMEOUT = 10_000; + + private final SystemTimer timer = new SystemTimer(); + private final NodeAgentContext initialContext = generateContext(); + private final NodeAgentContextManager manager = new NodeAgentContextManager(timer, initialContext); + + @Test + @Timeout(TIMEOUT) + void context_is_ignored_unless_scheduled_while_waiting() { + NodeAgentContext context1 = generateContext(); + manager.scheduleTickWith(context1, timer.currentTime()); + assertSame(initialContext, manager.currentContext()); + + AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); + manager.waitUntilWaitingForNextContext(); + assertFalse(async.isCompleted()); + + NodeAgentContext context2 = generateContext(); + manager.scheduleTickWith(context2, timer.currentTime()); + + assertSame(context2, async.awaitResult().response.get()); + assertSame(context2, manager.currentContext()); + } + + @Test + @Timeout(TIMEOUT) + void returns_no_earlier_than_at_given_time() { + AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); + manager.waitUntilWaitingForNextContext(); + + NodeAgentContext context1 = generateContext(); + Instant returnAt = timer.currentTime().plusMillis(500); + manager.scheduleTickWith(context1, returnAt); + + assertSame(context1, async.awaitResult().response.get()); + assertSame(context1, manager.currentContext()); + // Is accurate to a millisecond + assertFalse(timer.currentTime().plusMillis(1).isBefore(returnAt)); + } + + @Test + @Timeout(TIMEOUT) + void blocks_in_nextContext_until_one_is_scheduled() { + AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); + manager.waitUntilWaitingForNextContext(); + assertFalse(async.isCompleted()); + + NodeAgentContext context1 = generateContext(); + manager.scheduleTickWith(context1, timer.currentTime()); + + async.awaitResult(); + assertEquals(Optional.of(context1), async.response); + assertFalse(async.exception.isPresent()); + } + + @Test + @Timeout(TIMEOUT) + void blocks_in_nextContext_until_interrupt() { + AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); + manager.waitUntilWaitingForNextContext(); + assertFalse(async.isCompleted()); + + manager.interrupt(); + + async.awaitResult(); + assertEquals(Optional.of(ContextSupplierInterruptedException.class), async.exception.map(Exception::getClass)); + assertFalse(async.response.isPresent()); + } + + @Test + @Timeout(TIMEOUT) + void setFrozen_does_not_block_with_no_timeout() { + assertFalse(manager.setFrozen(false, Duration.ZERO)); + + // Generate new context and get it from the supplier, this completes the unfreeze + NodeAgentContext context1 = generateContext(); + AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); + manager.waitUntilWaitingForNextContext(); + manager.scheduleTickWith(context1, timer.currentTime()); + assertSame(context1, async.awaitResult().response.get()); + + assertTrue(manager.setFrozen(false, Duration.ZERO)); + } + + @Test + @Timeout(TIMEOUT) + void setFrozen_blocks_at_least_for_duration_of_timeout() { + long wantedDurationMillis = 100; + long start = timer.currentTimeMillis(); + assertFalse(manager.setFrozen(false, Duration.ofMillis(wantedDurationMillis))); + long actualDurationMillis = timer.currentTimeMillis() - start; + + assertTrue(actualDurationMillis >= wantedDurationMillis); + } + + private static NodeAgentContext generateContext() { + return NodeAgentContextImpl.builder("container-123.domain.tld").fileSystem(TestFileSystem.create()).build(); + } + + private static class AsyncExecutor<T> { + private final CountDownLatch latch = new CountDownLatch(1); + private volatile Optional<T> response = Optional.empty(); + private volatile Optional<Exception> exception = Optional.empty(); + + private AsyncExecutor(Callable<T> supplier) { + new Thread(() -> { + try { + response = Optional.of(supplier.call()); + } catch (Exception e) { + exception = Optional.of(e); + } + latch.countDown(); + }).start(); + } + + private AsyncExecutor<T> awaitResult() { + try { + latch.await(); + } catch (InterruptedException ignored) { } + return this; + } + + private boolean isCompleted() { + return latch.getCount() == 0; + } + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java new file mode 100644 index 00000000000..709326cc3b8 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -0,0 +1,889 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeagent; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.OrchestratorStatus; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; +import com.yahoo.vespa.hosted.node.admin.container.Container; +import com.yahoo.vespa.hosted.node.admin.container.ContainerId; +import com.yahoo.vespa.hosted.node.admin.container.ContainerName; +import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; +import com.yahoo.vespa.hosted.node.admin.container.ContainerResources; +import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; + +import java.nio.file.FileSystem; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.BiFunction; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author Øyvind Bakksjø + */ +public class NodeAgentImplTest { + private static final NodeResources resources = new NodeResources(2, 16, 250, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local); + private static final Version vespaVersion = Version.fromString("1.2.3"); + private static final ContainerId containerId = new ContainerId("af23"); + private static final String hostName = "host1.test.yahoo.com"; + + private final NodeAgentContextSupplier contextSupplier = mock(NodeAgentContextSupplier.class); + private final DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image"); + private final ContainerOperations containerOperations = mock(ContainerOperations.class); + private final NodeRepository nodeRepository = mock(NodeRepository.class); + private final Orchestrator orchestrator = mock(Orchestrator.class); + private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); + private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); + private final HealthChecker healthChecker = mock(HealthChecker.class); + private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private final TestTimer timer = new TestTimer(Instant.now()); + private final FileSystem fileSystem = TestFileSystem.create(); + + @BeforeEach + public void setUp() { + when(containerOperations.suspendNode(any())).thenReturn(""); + when(containerOperations.resumeNode(any())).thenReturn(""); + when(containerOperations.restartVespa(any())).thenReturn(""); + when(containerOperations.startServices(any())).thenReturn(""); + when(containerOperations.stopServices(any())).thenReturn(""); + } + + @Test + void upToDateContainerIsUntouched() { + final NodeSpec node = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .orchestratorStatus(OrchestratorStatus.NO_REMARKS) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + + nodeAgent.doConverge(context); + + verify(containerOperations, never()).removeContainer(eq(context), any()); + verify(orchestrator, never()).suspend(any(String.class)); + verify(containerOperations, never()).pullImageAsyncIfNeeded(any(), any(), any()); + + final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository); + // TODO: Verify this isn't run unless 1st time + inOrder.verify(containerOperations, never()).startServices(eq(context)); + inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); + inOrder.verify(orchestrator, never()).resume(hostName); + } + + @Test + void verifyRemoveOldFilesIfDiskFull() { + final NodeSpec node = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + + nodeAgent.doConverge(context); + + verify(storageMaintainer, times(1)).cleanDiskIfFull(eq(context)); + } + + @Test + void startsAfterStoppingServices() { + final InOrder inOrder = inOrder(containerOperations); + final NodeSpec node = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + + nodeAgent.doConverge(context); + inOrder.verify(containerOperations, never()).startServices(eq(context)); + inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); + + nodeAgent.stopForHostSuspension(context); + nodeAgent.doConverge(context); + inOrder.verify(containerOperations, never()).startServices(eq(context)); + inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); // Expect a resume, but no start services + + // No new suspends/stops, so no need to resume/start + nodeAgent.doConverge(context); + inOrder.verify(containerOperations, never()).startServices(eq(context)); + inOrder.verify(containerOperations, never()).resumeNode(eq(context)); + + nodeAgent.stopForHostSuspension(context); + nodeAgent.doConverge(context); + inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any()); + inOrder.verify(containerOperations, times(1)).startContainer(eq(context)); + inOrder.verify(containerOperations, times(0)).startServices(eq(context)); // done as part of startContainer + inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); + } + + @Test + void absentContainerCausesStart() { + final NodeSpec node = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(null, false); + + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false); + + nodeAgent.doConverge(context); + + verify(containerOperations, never()).removeContainer(eq(context), any()); + verify(containerOperations, never()).startServices(any()); + verify(orchestrator, never()).suspend(any(String.class)); + + final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository, aclMaintainer, healthChecker); + inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(dockerImage), any()); + inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any()); + inOrder.verify(containerOperations, times(1)).startContainer(eq(context)); + inOrder.verify(aclMaintainer, times(1)).converge(eq(context)); + inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); + inOrder.verify(healthChecker, times(1)).verifyHealth(eq(context)); + inOrder.verify(nodeRepository).updateNodeAttributes( + hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(vespaVersion).withRebootGeneration(0)); + inOrder.verify(orchestrator, never()).resume(hostName); + } + + @Test + void containerIsNotStoppedIfNewImageMustBePulled() { + final DockerImage newDockerImage = DockerImage.fromString("registry.example.com/repo/new-image"); + final NodeSpec node = nodeBuilder(NodeState.active) + .wantedDockerImage(newDockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + when(containerOperations.pullImageAsyncIfNeeded(any(), any(), any())).thenReturn(true); + + nodeAgent.doConverge(context); + + verify(orchestrator, never()).suspend(any(String.class)); + verify(orchestrator, never()).resume(any(String.class)); + verify(containerOperations, never()).removeContainer(eq(context), any()); + + final InOrder inOrder = inOrder(containerOperations); + inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(newDockerImage), any()); + } + + @Test + void containerIsUpdatedIfCpuChanged() { + NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .orchestratorStatus(OrchestratorStatus.NO_REMARKS); + + NodeAgentContext firstContext = createContext(specBuilder.build()); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + + when(containerOperations.pullImageAsyncIfNeeded(any(), any(), any())).thenReturn(true); + + InOrder inOrder = inOrder(orchestrator, containerOperations); + + nodeAgent.doConverge(firstContext); + inOrder.verify(orchestrator, never()).resume(any(String.class)); + + NodeAgentContext secondContext = createContext(specBuilder.diskGb(200).build()); + nodeAgent.doConverge(secondContext); + inOrder.verify(orchestrator, never()).resume(any(String.class)); + + NodeAgentContext thirdContext = NodeAgentContextImpl.builder(specBuilder.vcpu(5).build()).fileSystem(fileSystem).cpuSpeedUp(1.25).build(); + nodeAgent.doConverge(thirdContext); + ContainerResources resourcesAfterThird = ContainerResources.from(0, 4, 16); + mockGetContainer(dockerImage, resourcesAfterThird, true); + + inOrder.verify(orchestrator, never()).suspend(any()); + inOrder.verify(containerOperations).updateContainer(eq(thirdContext), eq(containerId), eq(resourcesAfterThird)); + inOrder.verify(containerOperations, never()).removeContainer(any(), any()); + inOrder.verify(containerOperations, never()).startContainer(any()); + inOrder.verify(orchestrator, never()).resume(any()); + + // No changes + nodeAgent.converge(thirdContext); + inOrder.verify(orchestrator, never()).suspend(any()); + inOrder.verify(containerOperations, never()).updateContainer(eq(thirdContext), eq(containerId), any()); + inOrder.verify(containerOperations, never()).removeContainer(any(), any()); + inOrder.verify(orchestrator, never()).resume(any()); + + // Set the feature flag + flagSource.withDoubleFlag(PermanentFlags.CONTAINER_CPU_CAP.id(), 2.3); + + nodeAgent.doConverge(thirdContext); + inOrder.verify(containerOperations).updateContainer(eq(thirdContext), eq(containerId), eq(ContainerResources.from(9.2, 4, 16))); + inOrder.verify(orchestrator, never()).resume(any()); + } + + @Test + void containerIsRecreatedIfMemoryChanged() { + NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .wantedRestartGeneration(2).currentRestartGeneration(1); + + NodeAgentContext firstContext = createContext(specBuilder.build()); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + + when(containerOperations.pullImageAsyncIfNeeded(any(), any(), any())).thenReturn(true); + + nodeAgent.doConverge(firstContext); + NodeAgentContext secondContext = createContext(specBuilder.memoryGb(20).build()); + nodeAgent.doConverge(secondContext); + ContainerResources resourcesAfterThird = ContainerResources.from(0, 2, 20); + mockGetContainer(dockerImage, resourcesAfterThird, true); + + InOrder inOrder = inOrder(orchestrator, containerOperations, nodeRepository); + inOrder.verify(orchestrator).resume(any(String.class)); + inOrder.verify(containerOperations).removeContainer(eq(secondContext), any()); + inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); + inOrder.verify(containerOperations, never()).restartVespa(any()); + inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withRestartGeneration(2).withRebootGeneration(0))); + + nodeAgent.doConverge(secondContext); + inOrder.verify(orchestrator).resume(any(String.class)); + inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); + inOrder.verify(containerOperations, never()).removeContainer(any(), any()); + } + + @Test + void noRestartIfOrchestratorSuspendFails() { + final NodeSpec node = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .wantedRestartGeneration(2).currentRestartGeneration(1) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + + doThrow(new OrchestratorException("Denied")).when(orchestrator).suspend(eq(hostName)); + try { + nodeAgent.doConverge(context); + fail("Expected to throw an exception"); + } catch (OrchestratorException ignored) { + } + + verify(containerOperations, never()).createContainer(eq(context), any()); + verify(containerOperations, never()).startContainer(eq(context)); + verify(orchestrator, never()).resume(any(String.class)); + verify(nodeRepository, never()).updateNodeAttributes(any(String.class), any(NodeAttributes.class)); + + // Verify aclMaintainer is called even if suspension fails + verify(aclMaintainer, times(1)).converge(eq(context)); + } + + @Test + void recreatesContainerIfRebootWanted() { + final long wantedRebootGeneration = 2; + final NodeSpec node = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .wantedRebootGeneration(wantedRebootGeneration).currentRebootGeneration(1) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false); + doThrow(ConvergenceException.ofTransient("Connection refused")).doNothing() + .when(healthChecker).verifyHealth(eq(context)); + + try { + nodeAgent.doConverge(context); + } catch (ConvergenceException ignored) { + } + + // First time we fail to resume because health verification fails + verify(orchestrator, times(1)).suspend(eq(hostName)); + verify(containerOperations, times(1)).removeContainer(eq(context), any()); + verify(containerOperations, times(1)).createContainer(eq(context), any()); + verify(containerOperations, times(1)).startContainer(eq(context)); + verify(orchestrator, never()).resume(eq(hostName)); + verify(nodeRepository, never()).updateNodeAttributes(any(), any()); + + nodeAgent.doConverge(context); + + // Do not reboot the container again + verify(containerOperations, times(1)).removeContainer(eq(context), any()); + verify(containerOperations, times(1)).createContainer(eq(context), any()); + verify(orchestrator, times(1)).resume(eq(hostName)); + verify(nodeRepository, times(1)).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() + .withRebootGeneration(wantedRebootGeneration))); + } + + @Test + void failedNodeRunningContainerShouldStillBeRunning() { + final NodeSpec node = nodeBuilder(NodeState.failed) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + + nodeAgent.doConverge(context); + + verify(containerOperations, never()).removeContainer(eq(context), any()); + verify(orchestrator, never()).resume(any(String.class)); + verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); + } + + @Test + void readyNodeLeadsToNoAction() { + final NodeSpec node = nodeBuilder(NodeState.ready).build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(null, false); + + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + + nodeAgent.doConverge(context); + nodeAgent.doConverge(context); + nodeAgent.doConverge(context); + + // Should only be called once, when we initialize + verify(containerOperations, times(1)).getContainer(eq(context)); + verify(containerOperations, never()).removeContainer(eq(context), any()); + verify(containerOperations, never()).createContainer(eq(context), any()); + verify(containerOperations, never()).startContainer(eq(context)); + verify(orchestrator, never()).resume(any(String.class)); + verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); + } + + @Test + void inactiveNodeRunningContainerShouldStillBeRunning() { + final NodeSpec node = nodeBuilder(NodeState.inactive) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + + nodeAgent.doConverge(context); + + final InOrder inOrder = inOrder(storageMaintainer, containerOperations); + inOrder.verify(containerOperations, never()).removeContainer(eq(context), any()); + + verify(orchestrator, never()).resume(any(String.class)); + verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); + } + + @Test + void reservedNodeDoesNotUpdateNodeRepoWithVersion() { + final NodeSpec node = nodeBuilder(NodeState.reserved) + .wantedDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(null, false); + + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + + nodeAgent.doConverge(context); + + verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); + } + + private void nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState nodeState, Optional<Long> wantedRestartGeneration) { + NodeSpec.Builder builder = nodeBuilder(nodeState) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage); + wantedRestartGeneration.ifPresent(restartGeneration -> builder + .wantedRestartGeneration(restartGeneration).currentRestartGeneration(restartGeneration)); + + NodeSpec node = builder.build(); + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + + nodeAgent.doConverge(context); + + final InOrder inOrder = inOrder(storageMaintainer, containerOperations, nodeRepository); + inOrder.verify(containerOperations, times(1)).stopServices(eq(context)); + inOrder.verify(storageMaintainer, times(1)).handleCoreDumpsForContainer(eq(context), any(), eq(true)); + inOrder.verify(containerOperations, times(1)).removeContainer(eq(context), any()); + inOrder.verify(storageMaintainer, times(1)).archiveNodeStorage(eq(context)); + inOrder.verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(NodeState.ready)); + + verify(containerOperations, never()).createContainer(eq(context), any()); + verify(containerOperations, never()).startContainer(eq(context)); + verify(containerOperations, never()).suspendNode(eq(context)); + verify(containerOperations, times(1)).stopServices(eq(context)); + verify(orchestrator, never()).resume(any(String.class)); + verify(orchestrator, never()).suspend(any(String.class)); + // current Docker image and vespa version should be cleared + verify(nodeRepository, times(1)).updateNodeAttributes( + eq(hostName), eq(new NodeAttributes().withDockerImage(DockerImage.EMPTY).withVespaVersion(Version.emptyVersion))); + } + + @Test + void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycled() { + nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState.dirty, Optional.of(1L)); + } + + @Test + void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycledNoRestartGeneration() { + nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState.dirty, Optional.empty()); + } + + @Test + void testRestartDeadContainerAfterNodeAdminRestart() { + final NodeSpec node = nodeBuilder(NodeState.active) + .currentDockerImage(dockerImage).wantedDockerImage(dockerImage) + .currentVespaVersion(vespaVersion) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, false); + + when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); + + nodeAgent.doConverge(context); + + verify(containerOperations, times(1)).removeContainer(eq(context), any()); + verify(containerOperations, times(1)).createContainer(eq(context), any()); + verify(containerOperations, times(1)).startContainer(eq(context)); + } + + @Test + void resumeProgramRunsUntilSuccess() { + final NodeSpec node = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .currentVespaVersion(vespaVersion) + .wantedRestartGeneration(1).currentRestartGeneration(1) + .orchestratorStatus(OrchestratorStatus.ALLOWED_TO_BE_DOWN) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + + when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); + + final InOrder inOrder = inOrder(orchestrator, containerOperations, nodeRepository); + doThrow(new RuntimeException("Failed 1st time")) + .doReturn("") + .when(containerOperations).resumeNode(eq(context)); + + // 1st try + try { + nodeAgent.doConverge(context); + fail("Expected to throw an exception"); + } catch (RuntimeException ignored) { + } + + inOrder.verify(containerOperations, times(1)).resumeNode(any()); + inOrder.verifyNoMoreInteractions(); + + // 2nd try + nodeAgent.doConverge(context); + + inOrder.verify(containerOperations).resumeNode(any()); + inOrder.verify(orchestrator).resume(hostName); + inOrder.verifyNoMoreInteractions(); + } + + @Test + void start_container_subtask_failure_leads_to_container_restart() { + final NodeSpec node = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion) + .wantedRestartGeneration(1).currentRestartGeneration(1) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = spy(makeNodeAgent(null, false)); + + when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false); + doThrow(new RuntimeException("Failed to set up network")).doNothing().when(containerOperations).startContainer(eq(context)); + + try { + nodeAgent.doConverge(context); + fail("Expected to get RuntimeException"); + } catch (RuntimeException ignored) { + } + + verify(containerOperations, never()).removeContainer(eq(context), any()); + verify(containerOperations, times(1)).createContainer(eq(context), any()); + verify(containerOperations, times(1)).startContainer(eq(context)); + verify(nodeAgent, never()).resumeNodeIfNeeded(any()); + + // The docker container was actually started and is running, but subsequent exec calls to set up + // networking failed + mockGetContainer(dockerImage, true); + nodeAgent.doConverge(context); + + verify(containerOperations, times(1)).removeContainer(eq(context), any()); + verify(containerOperations, times(2)).createContainer(eq(context), any()); + verify(containerOperations, times(2)).startContainer(eq(context)); + verify(nodeAgent, times(1)).resumeNodeIfNeeded(any()); + } + + @Test + void testRunningConfigServer() { + final NodeSpec node = nodeBuilder(NodeState.active) + .type(NodeType.config) + .wantedDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion) + .orchestratorStatus(OrchestratorStatus.ALLOWED_TO_BE_DOWN) + .build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(null, false); + + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false); + + nodeAgent.doConverge(context); + + verify(containerOperations, never()).removeContainer(eq(context), any()); + verify(orchestrator, never()).suspend(any(String.class)); + + final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository, aclMaintainer); + inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(dockerImage), any()); + inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any()); + inOrder.verify(containerOperations, times(1)).startContainer(eq(context)); + inOrder.verify(aclMaintainer, times(1)).converge(eq(context)); + inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); + inOrder.verify(nodeRepository).updateNodeAttributes( + hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(vespaVersion).withRebootGeneration(0)); + inOrder.verify(orchestrator).resume(hostName); + } + + + // Tests that only containers without owners are stopped + @Test + void testThatStopContainerDependsOnOwnerPresent() { + verifyThatContainerIsStopped(NodeState.parked, Optional.empty()); + verifyThatContainerIsStopped(NodeState.parked, Optional.of(ApplicationId.defaultId())); + verifyThatContainerIsStopped(NodeState.failed, Optional.empty()); + verifyThatContainerIsStopped(NodeState.failed, Optional.of(ApplicationId.defaultId())); + verifyThatContainerIsStopped(NodeState.inactive, Optional.of(ApplicationId.defaultId())); + } + + @Test + void initial_cpu_cap_test() { + NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion); + + NodeAgentContext context = createContext(specBuilder.build()); + NodeAgentImpl nodeAgent = makeNodeAgent(null, false, Duration.ofSeconds(30)); + + InOrder inOrder = inOrder(orchestrator, containerOperations); + + ConvergenceException healthCheckException = ConvergenceException.ofTransient("Not yet up"); + doThrow(healthCheckException).when(healthChecker).verifyHealth(any()); + for (int i = 0; i < 3; i++) { + try { + nodeAgent.doConverge(context); + fail("Expected to fail with health check exception"); + } catch (ConvergenceException e) { + assertEquals(healthCheckException, e); + } + timer.advance(Duration.ofSeconds(30)); + } + + doNothing().when(healthChecker).verifyHealth(any()); + try { + nodeAgent.doConverge(context); + fail("Expected to fail due to warm up period not yet done"); + } catch (ConvergenceException e) { + assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage()); + } + inOrder.verify(orchestrator, never()).resume(any()); + inOrder.verify(orchestrator, never()).suspend(any()); + inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); + + + timer.advance(Duration.ofSeconds(31)); + nodeAgent.doConverge(context); + + inOrder.verify(orchestrator, never()).suspend(any()); + inOrder.verify(containerOperations).updateContainer(eq(context), eq(containerId), eq(ContainerResources.from(0, 2, 16))); + inOrder.verify(containerOperations, never()).removeContainer(any(), any()); + inOrder.verify(containerOperations, never()).startContainer(any()); + inOrder.verify(orchestrator, never()).resume(any()); + + // No changes + nodeAgent.converge(context); + inOrder.verify(orchestrator, never()).suspend(any()); + inOrder.verify(containerOperations, never()).updateContainer(eq(context), eq(containerId), any()); + inOrder.verify(containerOperations, never()).removeContainer(any(), any()); + inOrder.verify(orchestrator, never()).resume(any()); + } + + @Test + void resumes_normally_if_container_is_already_capped_on_start() { + NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .wantedRestartGeneration(1).currentRestartGeneration(1); + + NodeAgentContext context = createContext(specBuilder.build()); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); + mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); + + InOrder inOrder = inOrder(orchestrator, containerOperations); + + nodeAgent.doConverge(context); + + nodeAgent.converge(context); + inOrder.verify(orchestrator, never()).suspend(any(String.class)); + inOrder.verify(containerOperations, never()).updateContainer(eq(context), eq(containerId), any()); + inOrder.verify(containerOperations, never()).removeContainer(any(), any()); + inOrder.verify(orchestrator, never()).resume(any(String.class)); + } + + @Test + void uncaps_and_caps_cpu_for_services_restart() { + NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .wantedRestartGeneration(2).currentRestartGeneration(1); + + NodeAgentContext context = createContext(specBuilder.build()); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); + mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true); + + InOrder inOrder = inOrder(orchestrator, containerOperations); + + nodeAgent.converge(context); + inOrder.verify(orchestrator, times(1)).suspend(eq(hostName)); + inOrder.verify(containerOperations, times(1)).updateContainer(eq(context), eq(containerId), eq(ContainerResources.from(0, 0, 16))); + inOrder.verify(containerOperations, times(1)).restartVespa(eq(context)); + + mockGetContainer(dockerImage, ContainerResources.from(0, 0, 16), true); + doNothing().when(healthChecker).verifyHealth(any()); + try { + nodeAgent.doConverge(context); + fail("Expected to fail due to warm up period not yet done"); + } catch (ConvergenceException e) { + assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage()); + } + inOrder.verify(orchestrator, never()).resume(any()); + inOrder.verify(orchestrator, never()).suspend(any()); + inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); + + + timer.advance(Duration.ofSeconds(31)); + nodeAgent.doConverge(context); + inOrder.verify(orchestrator, times(1)).resume(eq(hostName)); + } + + @Test + void resume_during_first_warmup() { + InOrder inOrder = inOrder(orchestrator, nodeRepository); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); + mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true); + + // Warmup period prevents resume when node has a current docker image, i.e., already existed. + nodeAgent.converge(createContext(nodeBuilder(NodeState.active).wantedDockerImage(dockerImage).currentDockerImage(dockerImage).build())); + inOrder.verifyNoMoreInteractions(); + + nodeAgent.converge(createContext(nodeBuilder(NodeState.active).wantedDockerImage(dockerImage).build())); + inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withDockerImage(dockerImage) + .withRebootGeneration(0) + .withVespaVersion(Version.fromString("7.1.1")))); + inOrder.verifyNoMoreInteractions(); + } + + + @Test + void drop_all_documents() { + InOrder inOrder = inOrder(orchestrator, nodeRepository); + BiFunction<NodeState, DropDocumentsReport, NodeSpec> specBuilder = (state, report) -> (report == null ? + nodeBuilder(state) : nodeBuilder(state).report(DropDocumentsReport.reportId(), report.toJsonNode())) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .build(); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); + + NodeAgentContext context = createContext(specBuilder.apply(NodeState.active, null)); + UnixPath indexPath = new UnixPath(context.paths().underVespaHome("var/db/vespa/search/cluster.foo/0/doc")).createParents().createNewFile(); + mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true); + assertTrue(indexPath.exists()); + + // Initially no changes, index is not dropped + nodeAgent.converge(context); + assertTrue(indexPath.exists()); + inOrder.verifyNoMoreInteractions(); + + context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, null, null, null))); + nodeAgent.converge(context); + verify(containerOperations).removeContainer(eq(context), any()); + assertFalse(indexPath.exists()); + inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, timer.currentTimeMillis(), null, null).toJsonNode()))); + inOrder.verifyNoMoreInteractions(); + + // After droppedAt and before readiedAt are set, we cannot proceed + mockGetContainer(null, false); + context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, null, null))); + nodeAgent.converge(context); + verify(containerOperations, never()).removeContainer(eq(context), any()); + verify(containerOperations, never()).startContainer(eq(context)); + inOrder.verifyNoMoreInteractions(); + + context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, 3L, null))); + nodeAgent.converge(context); + verify(containerOperations).startContainer(eq(context)); + inOrder.verifyNoMoreInteractions(); + + mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); + timer.advance(Duration.ofSeconds(31)); + nodeAgent.converge(context); + verify(containerOperations, times(1)).startContainer(eq(context)); + verify(containerOperations, never()).removeContainer(eq(context), any()); + inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() + .withRebootGeneration(0) + .withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, 2L, 3L, timer.currentTimeMillis()).toJsonNode()))); + inOrder.verifyNoMoreInteractions(); + } + + private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) { + NodeSpec.Builder nodeBuilder = nodeBuilder(nodeState) + .type(NodeType.tenant) + .flavor("docker") + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage); + + owner.ifPresent(nodeBuilder::owner); + NodeSpec node = nodeBuilder.build(); + + NodeAgentContext context = createContext(node); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + + when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); + + nodeAgent.doConverge(context); + + verify(containerOperations, never()).removeContainer(eq(context), any()); + if (owner.isPresent()) { + verify(containerOperations, never()).stopServices(eq(context)); + } else { + verify(containerOperations, times(1)).stopServices(eq(context)); + nodeAgent.doConverge(context); + // Should not be called more than once, have already been stopped + verify(containerOperations, times(1)).stopServices(eq(context)); + } + } + + private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) { + return makeNodeAgent(dockerImage, isRunning, Duration.ofSeconds(-1)); + } + + private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning, Duration warmUpDuration) { + mockGetContainer(dockerImage, isRunning); + doAnswer(invoc -> { + NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class); + ContainerResources resources = invoc.getArgument(1, ContainerResources.class); + mockGetContainer(context.node().wantedDockerImage().get(), resources, true); + return null; + }).when(containerOperations).createContainer(any(), any()); + + doAnswer(invoc -> { + NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class); + ContainerResources resources = invoc.getArgument(2, ContainerResources.class); + mockGetContainer(context.node().wantedDockerImage().get(), resources, true); + return null; + }).when(containerOperations).updateContainer(any(), any(), any()); + + return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, containerOperations, + () -> RegistryCredentials.none, storageMaintainer, flagSource, + List.of(credentialsMaintainer), Optional.of(aclMaintainer), Optional.of(healthChecker), + timer, warmUpDuration, VespaServiceDumper.DUMMY_INSTANCE, List.of()); + } + + private void mockGetContainer(DockerImage dockerImage, boolean isRunning) { + mockGetContainer(dockerImage, ContainerResources.from(0, resources.vcpu(), resources.memoryGb()), isRunning); + } + + private void mockGetContainer(DockerImage dockerImage, ContainerResources containerResources, boolean isRunning) { + doAnswer(invoc -> { + NodeAgentContext context = invoc.getArgument(0); + if (!hostName.equals(context.hostname().value())) + throw new IllegalArgumentException(); + return dockerImage != null ? + Optional.of(new Container( + containerId, + ContainerName.fromHostname(hostName), + timer.currentTime(), + isRunning ? Container.State.running : Container.State.exited, + "image-id-1", + dockerImage, + Map.of(), + 42, + 43, + hostName, + containerResources, + List.of(), + true)) : + Optional.empty(); + }).when(containerOperations).getContainer(any()); + } + + private NodeAgentContext createContext(NodeSpec nodeSpec) { + return NodeAgentContextImpl.builder(nodeSpec).fileSystem(fileSystem).build(); + } + + private NodeSpec.Builder nodeBuilder(NodeState state) { + return NodeSpec.Builder.testSpec(hostName, state).realResources(resources); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java new file mode 100644 index 00000000000..c45d9ab4b61 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java @@ -0,0 +1,29 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeagent; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author freva + */ +class UserNamespaceTest { + + private final UserNamespace userNamespace = new UserNamespace(1000, 2000, 10000); + + @Test + public void translates_between_ids() { + assertEquals(1001, userNamespace.userIdOnHost(1)); + assertEquals(2001, userNamespace.groupIdOnHost(1)); + assertEquals(1, userNamespace.userIdInContainer(1001)); + assertEquals(1, userNamespace.groupIdInContainer(2001)); + + assertEquals(userNamespace.overflowId(), userNamespace.userIdInContainer(1)); + assertEquals(userNamespace.overflowId(), userNamespace.userIdInContainer(999999)); + + assertThrows(IllegalArgumentException.class, () -> userNamespace.userIdOnHost(-1)); + assertThrows(IllegalArgumentException.class, () -> userNamespace.userIdOnHost(70_000)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java new file mode 100644 index 00000000000..eddc7edd597 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java @@ -0,0 +1,28 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.provider; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class DebugHandlerHelperTest { + @Test + void trivial() { + DebugHandlerHelper helper = new DebugHandlerHelper(); + helper.addConstant("constant-key", "constant-value"); + + NodeAdminDebugHandler handler = () -> Map.of("handler-value-key", "handler-value-value"); + helper.addHandler("handler-key", handler); + + helper.addThreadSafeSupplier("supplier-key", () -> "supplier-value"); + + assertEquals("{" + + "supplier-key=supplier-value, " + + "handler-key={handler-value-key=handler-value-value}, " + + "constant-key=constant-value" + + "}", + helper.getDebugPage().toString()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java new file mode 100644 index 00000000000..115969c5ded --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java @@ -0,0 +1,68 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.logging.Logger; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * @author bjorncs + */ +public class DefaultEnvWriterTest { + + @TempDir + public File temporaryFolder; + + private static final Path EXAMPLE_FILE = Path.of("src/test/resources/default-env-example.txt"); + private static final Path EXPECTED_RESULT_FILE = Path.of("src/test/resources/default-env-rewritten.txt"); + + private final TaskContext context = mock(TaskContext.class); + + @Test + void default_env_is_correctly_rewritten() throws IOException { + Path tempFile = File.createTempFile("junit", null, temporaryFolder).toPath(); + Files.copy(EXAMPLE_FILE, tempFile, REPLACE_EXISTING); + + DefaultEnvWriter writer = new DefaultEnvWriter(); + writer.addOverride("VESPA_HOSTNAME", "my-new-hostname"); + writer.addFallback("VESPA_CONFIGSERVER", "new-fallback-configserver"); + writer.addOverride("VESPA_TLS_CONFIG_FILE", "/override/path/to/config.file"); + + boolean modified = writer.updateFile(context, tempFile); + + assertTrue(modified); + assertEquals(Files.readString(EXPECTED_RESULT_FILE), Files.readString(tempFile)); + verify(context, times(1)).log(any(Logger.class), any(String.class)); + + modified = writer.updateFile(context, tempFile); + assertFalse(modified); + assertEquals(Files.readString(EXPECTED_RESULT_FILE), Files.readString(tempFile)); + verify(context, times(1)).log(any(Logger.class), any(String.class)); + } + + @Test + void generates_default_env_content() throws IOException { + DefaultEnvWriter writer = new DefaultEnvWriter(); + writer.addOverride("VESPA_HOSTNAME", "my-new-hostname"); + writer.addFallback("VESPA_CONFIGSERVER", "new-fallback-configserver"); + writer.addOverride("VESPA_TLS_CONFIG_FILE", "/override/path/to/config.file"); + writer.addUnset("VESPA_LEGACY_OPTION"); + String generatedContent = writer.generateContent(); + assertEquals(Files.readString(EXPECTED_RESULT_FILE), generatedContent); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java new file mode 100644 index 00000000000..76676739613 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java @@ -0,0 +1,148 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.editor; + +import org.junit.jupiter.api.Test; + +import java.util.Optional; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.*; + +public class StringEditorTest { + private final StringEditor editor = new StringEditor(); + private final Cursor cursor = editor.cursor(); + + @Test + void testBasics() { + assertCursor(0, 0, ""); + + cursor.write("hello"); + assertCursor(0, 5, "hello"); + + cursor.write("one\ntwo"); + assertCursor(1, 3, "helloone\ntwo"); + + cursor.deleteAll(); + assertCursor(0, 0, ""); + + cursor.moveForward(); + assertCursor(0, 0, ""); + + cursor.writeLine("foo"); + assertCursor(1, 0, "foo\n"); + + cursor.writeLines("one", "two"); + assertCursor(3, 0, "foo\none\ntwo\n"); + + cursor.deleteBackward(); + assertCursor(2, 3, "foo\none\ntwo"); + + cursor.deleteBackward(2); + assertCursor(2, 1, "foo\none\nt"); + + Mark mark = cursor.createMark(); + + cursor.moveToStartOfPreviousLine().moveBackward(2); + assertCursor(0, 2, "foo\none\nt"); + + assertEquals("o\none\nt", cursor.getTextTo(mark)); + + cursor.deleteTo(mark); + assertCursor(0, 2, "fo"); + + cursor.deleteBackward(2); + assertCursor(0, 0, ""); + + cursor.writeLines("one", "two", "three").moveToStartOfBuffer(); + assertCursor(0, 0, "one\ntwo\nthree\n"); + + Pattern pattern = Pattern.compile("t(.)"); + Optional<Match> match = cursor.moveForwardToEndOfMatch(pattern); + assertCursor(1, 2, "one\ntwo\nthree\n"); + assertTrue(match.isPresent()); + assertEquals("tw", match.get().match()); + assertEquals("", match.get().prefix()); + assertEquals("o", match.get().suffix()); + assertEquals(new Position(1, 0), match.get().startOfMatch()); + assertEquals(new Position(1, 2), match.get().endOfMatch()); + assertEquals(1, match.get().groupCount()); + assertEquals("w", match.get().group(1)); + + match = cursor.moveForwardToEndOfMatch(pattern); + assertCursor(2, 2, "one\ntwo\nthree\n"); + assertTrue(match.isPresent()); + assertEquals("th", match.get().match()); + assertEquals(1, match.get().groupCount()); + assertEquals("h", match.get().group(1)); + + match = cursor.moveForwardToEndOfMatch(pattern); + assertCursor(2, 2, "one\ntwo\nthree\n"); + assertFalse(match.isPresent()); + + assertTrue(cursor.skipBackward("h")); + assertCursor(2, 1, "one\ntwo\nthree\n"); + assertFalse(cursor.skipBackward("x")); + + assertTrue(cursor.skipForward("hre")); + assertCursor(2, 4, "one\ntwo\nthree\n"); + assertFalse(cursor.skipForward("x")); + + try { + cursor.moveTo(mark); + fail(); + } catch (IllegalArgumentException e) { + // expected + } + + mark = cursor.createMark(); + cursor.moveToStartOfBuffer(); + assertEquals(new Position(0, 0), cursor.getPosition()); + cursor.moveTo(mark); + assertEquals(new Position(2, 4), cursor.getPosition()); + + cursor.moveTo(1, 2); + assertCursor(1, 2, "one\ntwo\nthree\n"); + + cursor.deleteSuffix(); + assertCursor(1, 2, "one\ntw\nthree\n"); + + cursor.deletePrefix(); + assertCursor(1, 0, "one\n\nthree\n"); + + cursor.deleteLine(); + assertCursor(1, 0, "one\nthree\n"); + + cursor.deleteLine(); + assertCursor(1, 0, "one\n"); + + cursor.deleteLine(); + assertCursor(1, 0, "one\n"); + + cursor.moveToStartOfBuffer().moveForward().writeNewlineAfter(); + assertCursor(0, 1, "o\nne\n"); + + cursor.deleteAll().writeLines("one", "two", "three", "four"); + cursor.moveToStartOfBuffer().moveToStartOfNextLine(); + assertCursor(1, 0, "one\ntwo\nthree\nfour\n"); + Pattern pattern2 = Pattern.compile("(o)(.)?"); + int count = cursor.replaceMatches(pattern2, m -> { + String prefix = m.group(2) == null ? "" : m.group(2); + return prefix + m.match() + m.group(1); + }); + assertCursor(3, 5, "one\ntwoo\nthree\nfuouor\n"); + assertEquals(2, count); + + cursor.moveToStartOfBuffer().moveToEndOfLine(); + Pattern pattern3 = Pattern.compile("o"); + count = cursor.replaceMatches(pattern3, m -> "a"); + assertEquals(4, count); + assertCursor(3, 5, "one\ntwaa\nthree\nfuauar\n"); + } + + private void assertCursor(int lineIndex, int columnIndex, String text) { + assertEquals(text, cursor.getBufferText()); + assertEquals(new Position(lineIndex, columnIndex), cursor.getPosition()); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java new file mode 100644 index 00000000000..15fb36dc3d5 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java @@ -0,0 +1,59 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.editor; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TextBufferImplTest { + private final TextBufferImpl textBuffer = new TextBufferImpl(); + + @Test + void testWrite() { + assertEquals("", textBuffer.getString()); + assertWrite(2, 0, "foo\nbar\n", + 0, 0, "foo\nbar\n"); + + assertWrite(1, 6, "fofirst\nsecondo\nbar\n", + 0, 2, "first\nsecond"); + + assertWrite(3, 1, "fofirst\nsecondo\nbar\na", + 3, 0, "a"); + assertWrite(4, 0, "fofirst\nsecondo\nbar\na\n", + 3, 1, "\n"); + } + + @Test + void testDelete() { + write(0, 0, "foo\nbar\nzoo\n"); + delete(0, 2, 2, 1); + assertEquals("fooo\n", textBuffer.getString()); + + delete(0, 4, 1, 0); + assertEquals("fooo", textBuffer.getString()); + + delete(0, 0, 0, 4); + assertEquals("", textBuffer.getString()); + + delete(0, 0, 0, 0); + assertEquals("", textBuffer.getString()); + } + + private void assertWrite(int expectedLineIndex, int expectedColumnIndex, String expectedString, + int lineIndex, int columnIndex, String text) { + Position position = write(lineIndex, columnIndex, text); + assertEquals(new Position(expectedLineIndex, expectedColumnIndex), position); + assertEquals(expectedString, textBuffer.getString()); + } + + private Position write(int lineIndex, int columnIndex, String text) { + return textBuffer.write(new Position(lineIndex, columnIndex), text); + } + + private void delete(int startLineIndex, int startColumnIndex, + int endLineIndex, int endColumnIndex) { + textBuffer.delete(new Position(startLineIndex, startColumnIndex), + new Position(endLineIndex, endColumnIndex)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java new file mode 100644 index 00000000000..507bf706484 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java @@ -0,0 +1,26 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +public class DiskSizeTest { + + @Test + void bytes_to_display_count_test() { + assertEquals("-1 bytes", DiskSize.of(-1).asString()); + assertEquals("123 bytes", DiskSize.of(123).asString()); + assertEquals("1 kB", DiskSize.of(1_000).asString()); + assertEquals("15 MB", DiskSize.of(15_000_000).asString()); + assertEquals("123 GB", DiskSize.of(123_456_789_012L).asString()); + assertEquals("988 TB", DiskSize.of(987_654_321_098_765L).asString()); + assertEquals("987.7 TB", DiskSize.of(987_654_321_098_765L).asString(1)); + assertEquals("987.65 TB", DiskSize.of(987_654_321_098_765L).asString(2)); + assertEquals("2 PB", DiskSize.of(2_000_000_000_000_000L).asString()); + assertEquals("9 EB", DiskSize.of(Long.MAX_VALUE).asString()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java new file mode 100644 index 00000000000..9a651494854 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java @@ -0,0 +1,122 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.nio.file.FileSystem; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class EditorTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final UnixPath path = new UnixPath(fileSystem.getPath("/file")); + + @Test + void testEdit() { + path.writeUtf8File(joinLines("first", "second", "third")); + + LineEditor lineEditor = mock(LineEditor.class); + when(lineEditor.edit(any())).thenReturn( + LineEdit.none(), // don't edit the first line + LineEdit.remove(), // remove the second + LineEdit.replaceWith("replacement")); // replace the third + + Editor editor = new Editor(path.toPath(), lineEditor); + TaskContext context = mock(TaskContext.class); + + assertTrue(editor.converge(context)); + + verify(lineEditor, times(3)).edit(any()); + + // Verify the system modification message + ArgumentCaptor<String> modificationMessage = ArgumentCaptor.forClass(String.class); + verify(context).recordSystemModification(any(), modificationMessage.capture()); + assertEquals( + "Patching file /file:\n-second\n-third\n+replacement\n", + modificationMessage.getValue()); + + // Verify the new contents of the file: + assertEquals(joinLines("first", "replacement"), path.readUtf8File()); + } + + @Test + void testInsert() { + path.writeUtf8File(joinLines("second", "eight", "fifth", "seventh")); + + LineEditor lineEditor = mock(LineEditor.class); + when(lineEditor.edit(any())).thenReturn( + LineEdit.insertBefore("first"), // insert first, and keep the second line + LineEdit.replaceWith("third", "fourth"), // remove eight, and replace with third and fourth instead + LineEdit.none(), // Keep fifth + LineEdit.insert(List.of("sixth"), // insert sixth before seventh + List.of("eight"))); // add eight after seventh + + Editor editor = new Editor(path.toPath(), lineEditor); + TaskContext context = mock(TaskContext.class); + + assertTrue(editor.converge(context)); + + // Verify the system modification message + ArgumentCaptor<String> modificationMessage = ArgumentCaptor.forClass(String.class); + verify(context).recordSystemModification(any(), modificationMessage.capture()); + assertEquals( + "Patching file /file:\n" + + "+first\n" + + "-eight\n" + + "+third\n" + + "+fourth\n" + + "+sixth\n" + + "+eight\n", + modificationMessage.getValue()); + + // Verify the new contents of the file: + assertEquals(joinLines("first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eight"), + path.readUtf8File()); + } + + @Test + void noop() { + path.writeUtf8File("line\n"); + + LineEditor lineEditor = mock(LineEditor.class); + when(lineEditor.edit(any())).thenReturn(LineEdit.none()); + + Editor editor = new Editor(path.toPath(), lineEditor); + TaskContext context = mock(TaskContext.class); + + assertFalse(editor.converge(context)); + + verify(lineEditor, times(1)).edit(any()); + + // Verify the system modification message + verify(context, times(0)).recordSystemModification(any(), any()); + + // Verify same contents + assertEquals("line\n", path.readUtf8File()); + } + + @Test + void testMissingFile() { + LineEditor lineEditor = mock(LineEditor.class); + when(lineEditor.onComplete()).thenReturn(List.of("line")); + + TaskContext context = mock(TaskContext.class); + var editor = new Editor(path.toPath(), lineEditor); + editor.converge(context); + + assertEquals("line\n", path.readUtf8File()); + } + + private static String joinLines(String... lines) { + return String.join("\n", lines) + "\n"; + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java new file mode 100644 index 00000000000..8559e36fe8b --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java @@ -0,0 +1,39 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class FileAttributesCacheTest { + @Test + void exists() { + UnixPath unixPath = mock(UnixPath.class); + FileAttributesCache cache = new FileAttributesCache(unixPath); + + when(unixPath.getAttributesIfExists()).thenReturn(Optional.empty()); + assertFalse(cache.get().isPresent()); + verify(unixPath, times(1)).getAttributesIfExists(); + verifyNoMoreInteractions(unixPath); + + FileAttributes attributes = new FileAttributes(Instant.EPOCH, 0, 0, "", false, false, 0, 0, 0); + when(unixPath.getAttributesIfExists()).thenReturn(Optional.of(attributes)); + when(unixPath.getAttributesIfExists()).thenReturn(Optional.of(attributes)); + assertTrue(cache.get().isPresent()); + verify(unixPath, times(1 + 1)).getAttributesIfExists(); + verifyNoMoreInteractions(unixPath); + + assertEquals(attributes, cache.getOrThrow()); + verifyNoMoreInteractions(unixPath); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java new file mode 100644 index 00000000000..ed183738ef0 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java @@ -0,0 +1,20 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.jupiter.api.Test; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileAttributes.deviceMajor; +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileAttributes.deviceMinor; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +class FileAttributesTest { + + @Test + void parse_dev_t() { + assertEquals(0x12345BCD, deviceMajor(0x1234567890ABCDEFL)); + assertEquals(0x67890AEF, deviceMinor(0x1234567890ABCDEFL)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java new file mode 100644 index 00000000000..e1cea37ccbc --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java @@ -0,0 +1,62 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class FileContentCacheTest { + private final UnixPath unixPath = mock(UnixPath.class); + private final FileContentCache cache = new FileContentCache(unixPath); + + private final byte[] content = "content".getBytes(StandardCharsets.UTF_8); + private final byte[] newContent = "new-content".getBytes(StandardCharsets.UTF_8); + + @Test + void get() { + when(unixPath.readBytes()).thenReturn(content); + assertArrayEquals(content, cache.get(Instant.ofEpochMilli(0))); + verify(unixPath, times(1)).readBytes(); + verifyNoMoreInteractions(unixPath); + + // cache hit + assertArrayEquals(content, cache.get(Instant.ofEpochMilli(0))); + verify(unixPath, times(1)).readBytes(); + verifyNoMoreInteractions(unixPath); + + // cache miss + when(unixPath.readBytes()).thenReturn(newContent); + assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(1))); + verify(unixPath, times(1 + 1)).readBytes(); + verifyNoMoreInteractions(unixPath); + + // cache hit both at times 0 and 1 + assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(0))); + verify(unixPath, times(1 + 1)).readBytes(); + verifyNoMoreInteractions(unixPath); + assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(1))); + verify(unixPath, times(1 + 1)).readBytes(); + verifyNoMoreInteractions(unixPath); + } + + @Test + void updateWith() { + cache.updateWith(content, Instant.ofEpochMilli(2)); + assertArrayEquals(content, cache.get(Instant.ofEpochMilli(2))); + verifyNoMoreInteractions(unixPath); + + cache.updateWith(newContent, Instant.ofEpochMilli(4)); + assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(4))); + verifyNoMoreInteractions(unixPath); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java new file mode 100644 index 00000000000..f7fb66fca94 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java @@ -0,0 +1,27 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.nio.file.FileSystem; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +public class FileDeleterTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final UnixPath path = new UnixPath(fileSystem.getPath("/tmp/foo")); + private final FileDeleter deleter = new FileDeleter(path.toPath()); + private final TaskContext context = mock(TaskContext.class); + + @Test + void deleteExisting() { + assertFalse(deleter.converge(context)); + path.createParents().writeUtf8File("bar"); + assertTrue(deleter.converge(context)); + assertFalse(deleter.converge(context)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java new file mode 100644 index 00000000000..76941d3333b --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java @@ -0,0 +1,238 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.time.Duration; +import java.time.Instant; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.Set.of; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + + +/** + * @author freva + */ + +public class FileFinderTest { + + @Nested + public class GeneralLogicTests { + + private final FileSystem fileSystem = TestFileSystem.create(); + + @Test + void all_files_non_recursive() { + assertFileHelper(FileFinder.files(testRoot()) + .maxDepth(1), + + of("file-1.json", "test.json", "test.txt"), + of("test", "test/file.txt", "test/data.json", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); + } + + @Test + void all_files_recursive() { + assertFileHelper(FileFinder.files(testRoot()), + + of("file-1.json", "test.json", "test.txt", "test/file.txt", "test/data.json", "test/subdir-1/test"), + of("test", "test/subdir-1", "test/subdir-2")); + } + + @Test + void all_files_recursive_with_prune_relative() { + assertFileHelper(FileFinder.files(testRoot()).prune(fileSystem.getPath("test")), + + of("file-1.json", "test.json", "test.txt"), + of("test", "test/file.txt", "test/data.json", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); + } + + @Test + void all_files_recursive_with_prune_absolute() { + assertFileHelper(FileFinder.files(testRoot()).prune(testRoot().resolve("test/subdir-1")), + + of("file-1.json", "test.json", "test.txt", "test/file.txt", "test/data.json"), + of("test", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); + } + + @Test + void throws_if_prune_path_not_under_base_path() { + assertThrows(IllegalArgumentException.class, () -> { + FileFinder.files(Path.of("/some/path")).prune(Path.of("/other/path")); + }); + } + + @Test + void with_file_filter_recursive() { + assertFileHelper(FileFinder.files(testRoot()) + .match(FileFinder.nameEndsWith(".json")), + + of("file-1.json", "test.json", "test/data.json"), + of("test.txt", "test", "test/file.txt", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); + } + + @Test + void all_files_limited_depth() { + assertFileHelper(FileFinder.files(testRoot()) + .maxDepth(2), + + of("test.txt", "file-1.json", "test.json", "test/file.txt", "test/data.json"), + of("test", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); + } + + @Test + void directory_with_filter() { + assertFileHelper(FileFinder.directories(testRoot()) + .match(FileFinder.nameStartsWith("subdir")) + .maxDepth(2), + + of("test/subdir-1", "test/subdir-2"), + of("file-1.json", "test.json", "test.txt", "test", "test/file.txt", "test/data.json")); + } + + @Test + void match_file_and_directory_with_same_name() { + assertFileHelper(FileFinder.from(testRoot()) + .match(FileFinder.nameEndsWith("test")), + + of("test", "test/subdir-1/test"), + of("file-1.json", "test.json", "test.txt")); + } + + @Test + void all_contents() { + assertFileHelper(FileFinder.from(testRoot()) + .maxDepth(1), + + of("file-1.json", "test.json", "test.txt", "test"), + of()); + + assertTrue(Files.exists(testRoot())); + } + + @BeforeEach + public void setup() throws IOException { + Path root = testRoot(); + Files.createDirectories(root); + + Files.createFile(root.resolve("file-1.json")); + Files.createFile(root.resolve("test.json")); + Files.createFile(root.resolve("test.txt")); + + Files.createDirectories(root.resolve("test")); + Files.createFile(root.resolve("test/file.txt")); + Files.createFile(root.resolve("test/data.json")); + + Files.createDirectories(root.resolve("test/subdir-1")); + Files.createFile(root.resolve("test/subdir-1/test")); + + Files.createDirectories(root.resolve("test/subdir-2")); + } + + private Path testRoot() { + return fileSystem.getPath("/file-finder"); + } + + private void assertFileHelper(FileFinder fileFinder, Set<String> expectedList, Set<String> expectedContentsAfterDelete) { + Set<String> actualList = fileFinder.stream() + .map(FileFinder.FileAttributes::path) + .map(testRoot()::relativize) + .map(Path::toString) + .collect(Collectors.toSet()); + assertEquals(expectedList, actualList); + + fileFinder.deleteRecursively(mock(TaskContext.class)); + Set<String> actualContentsAfterDelete = recursivelyListContents(testRoot()).stream() + .map(testRoot()::relativize) + .map(Path::toString) + .collect(Collectors.toSet()); + assertEquals(expectedContentsAfterDelete, actualContentsAfterDelete); + } + + private List<Path> recursivelyListContents(Path basePath) { + try (Stream<Path> pathStream = Files.list(basePath)) { + List<Path> paths = new LinkedList<>(); + pathStream.forEach(path -> { + paths.add(path); + if (Files.isDirectory(path)) + paths.addAll(recursivelyListContents(path)); + }); + return paths; + } catch (NoSuchFileException e) { + return List.of(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + @Nested + public class FilterUnitTests { + + private final BasicFileAttributes attributes = mock(BasicFileAttributes.class); + + @Test + void age_filter_test() { + Path path = Path.of("/my/fake/path"); + when(attributes.lastModifiedTime()).thenReturn(FileTime.from(Instant.now().minus(Duration.ofHours(1)))); + FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes); + + assertFalse(FileFinder.olderThan(Duration.ofMinutes(61)).test(fileAttributes)); + assertTrue(FileFinder.olderThan(Duration.ofMinutes(59)).test(fileAttributes)); + + assertTrue(FileFinder.youngerThan(Duration.ofMinutes(61)).test(fileAttributes)); + assertFalse(FileFinder.youngerThan(Duration.ofMinutes(59)).test(fileAttributes)); + } + + @Test + void size_filters() { + Path path = Path.of("/my/fake/path"); + when(attributes.size()).thenReturn(100L); + FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes); + + assertFalse(FileFinder.largerThan(101).test(fileAttributes)); + assertTrue(FileFinder.largerThan(99).test(fileAttributes)); + + assertTrue(FileFinder.smallerThan(101).test(fileAttributes)); + assertFalse(FileFinder.smallerThan(99).test(fileAttributes)); + } + + @Test + void filename_filters() { + Path path = Path.of("/my/fake/path/some-12352-file.json"); + FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes); + + assertTrue(FileFinder.nameStartsWith("some-").test(fileAttributes)); + assertFalse(FileFinder.nameStartsWith("som-").test(fileAttributes)); + + assertTrue(FileFinder.nameEndsWith(".json").test(fileAttributes)); + assertFalse(FileFinder.nameEndsWith("file").test(fileAttributes)); + + assertTrue(FileFinder.nameMatches(Pattern.compile("some-[0-9]+-file.json")).test(fileAttributes)); + assertTrue(FileFinder.nameMatches(Pattern.compile("^some-[0-9]+-file.json$")).test(fileAttributes)); + assertFalse(FileFinder.nameMatches(Pattern.compile("some-[0-9]-file.json")).test(fileAttributes)); + } + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java new file mode 100644 index 00000000000..e418833ab50 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java @@ -0,0 +1,73 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; +import java.nio.file.NoSuchFileException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.mock; + +/** + * @author hakonhall + */ +class FileMoverTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final TaskContext context = mock(TaskContext.class); + private final UnixPath source = new UnixPath(fileSystem.getPath("/from/source")); + private final UnixPath destination = new UnixPath(fileSystem.getPath("/to/destination")); + private final FileMover mover = new FileMover(source.toPath(), destination.toPath()); + + @Test + void movingRegularFile() { + assertConvergeThrows(() -> mover.converge(context), NoSuchFileException.class, "/from/source"); + + source.createParents().writeUtf8File("content"); + assertConvergeThrows(() -> mover.converge(context), NoSuchFileException.class, "/to/destination"); + + destination.createParents(); + assertTrue(mover.converge(context)); + assertFalse(source.exists()); + assertTrue(destination.exists()); + assertEquals("content", destination.readUtf8File()); + + assertFalse(mover.converge(context)); + + source.writeUtf8File("content 2"); + assertConvergeThrows(() -> mover.converge(context), FileAlreadyExistsException.class, "/to/destination"); + + mover.replaceExisting(); + assertTrue(mover.converge(context)); + + source.writeUtf8File("content 3"); + destination.deleteIfExists(); + destination.createDirectory(); + assertTrue(mover.converge(context)); + } + + private void assertConvergeThrows(Runnable runnable, Class<?> expectedRootExceptionClass, String expectedMessage) { + try { + runnable.run(); + fail(); + } catch (Throwable t) { + Throwable rootCause = t; + do { + Throwable cause = rootCause.getCause(); + if (cause == null) break; + rootCause = cause; + } while (true); + + assertTrue(expectedRootExceptionClass.isInstance(rootCause), "Unexpected root cause: " + rootCause); + assertEquals(expectedMessage, rootCause.getMessage()); + } + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java new file mode 100644 index 00000000000..b0992e9826a --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java @@ -0,0 +1,60 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.nio.file.FileSystem; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author hakonhall + */ +public class FileSnapshotTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final UnixPath path = new UnixPath(fileSystem.getPath("/var/lib/file.txt")); + + private FileSnapshot fileSnapshot = FileSnapshot.forPath(path.toPath()); + + @Test + void fileDoesNotExist() { + assertFalse(fileSnapshot.exists()); + assertFalse(fileSnapshot.attributes().isPresent()); + assertFalse(fileSnapshot.content().isPresent()); + assertEquals(path.toPath(), fileSnapshot.path()); + } + + @Test + void directory() { + path.createParents().createDirectory(); + fileSnapshot = fileSnapshot.snapshot(); + assertTrue(fileSnapshot.exists()); + assertTrue(fileSnapshot.attributes().isPresent()); + assertTrue(fileSnapshot.attributes().get().isDirectory()); + } + + @Test + void regularFile() { + path.createParents().writeUtf8File("file content"); + fileSnapshot = fileSnapshot.snapshot(); + assertTrue(fileSnapshot.exists()); + assertTrue(fileSnapshot.attributes().isPresent()); + assertTrue(fileSnapshot.attributes().get().isRegularFile()); + assertTrue(fileSnapshot.utf8Content().isPresent()); + assertEquals("file content", fileSnapshot.utf8Content().get()); + + FileSnapshot newFileSnapshot = fileSnapshot.snapshot(); + assertSame(fileSnapshot, newFileSnapshot); + } + + @Test + void fileRemoval() { + path.createParents().writeUtf8File("file content"); + fileSnapshot = fileSnapshot.snapshot(); + assertTrue(fileSnapshot.exists()); + path.deleteIfExists(); + fileSnapshot = fileSnapshot.snapshot(); + assertFalse(fileSnapshot.exists()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java new file mode 100644 index 00000000000..c60de78bf8c --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java @@ -0,0 +1,79 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class FileSyncTest { + private final TestTaskContext taskContext = new TestTaskContext(); + private final FileSystem fileSystem = TestFileSystem.create(); + + private final Path path = fileSystem.getPath("/dir/file.txt"); + private final UnixPath unixPath = new UnixPath(path); + private final FileSync fileSync = new FileSync(path); + + private String content = "content"; + private int ownerId = 123; // default is 1 + private int groupId = 456; // default is 2 + private String permissions = "rw-r-xr--"; + + @Test + void trivial() { + assertConvergence("Creating file /dir/file.txt with permissions rw-r-xr--", + "Changing user ID of /dir/file.txt from 1 to 123", + "Changing group ID of /dir/file.txt from 2 to 456"); + + content = "new-content"; + assertConvergence("Patching file /dir/file.txt"); + + ownerId = 124; + assertConvergence("Changing user ID of /dir/file.txt from 123 to 124"); + + groupId = 457; + assertConvergence("Changing group ID of /dir/file.txt from 456 to 457"); + + permissions = "rwxr--rwx"; + assertConvergence("Changing permissions of /dir/file.txt from rw-r-xr-- to " + + permissions); + } + + private void assertConvergence(String... systemModificationMessages) { + PartialFileData fileData = PartialFileData.builder() + .withContent(content) + .withOwnerId(ownerId) + .withGroupId(groupId) + .withPermissions(permissions) + .create(); + taskContext.clearSystemModificationLog(); + assertTrue(fileSync.convergeTo(taskContext, fileData)); + + assertTrue(Files.isRegularFile(path)); + fileData.getContent().ifPresent(content -> assertArrayEquals(content, unixPath.readBytes())); + fileData.getOwnerId().ifPresent(owner -> assertEquals((int) owner, unixPath.getOwnerId())); + fileData.getGroupId().ifPresent(group -> assertEquals((int) group, unixPath.getGroupId())); + fileData.getPermissions().ifPresent(permissions -> assertEquals(permissions, unixPath.getPermissions())); + + List<String> actualMods = taskContext.getSystemModificationLog(); + List<String> expectedMods = List.of(systemModificationMessages); + assertEquals(expectedMods, actualMods); + + UnixPath unixPath = new UnixPath(path); + Instant lastModifiedTime = unixPath.getLastModifiedTime(); + taskContext.clearSystemModificationLog(); + assertFalse(fileSync.convergeTo(taskContext, fileData)); + assertEquals(lastModifiedTime, unixPath.getLastModifiedTime()); + + actualMods = taskContext.getSystemModificationLog(); + assertEquals(new ArrayList<>(), actualMods); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java new file mode 100644 index 00000000000..1264206bef3 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java @@ -0,0 +1,62 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.test.file.TestFileSystem; +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import org.junit.jupiter.api.Test; + +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.time.Instant; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class FileWriterTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final TaskContext context = mock(TaskContext.class); + + @Test + void testWrite() { + final String content = "content"; + final String permissions = "rwxr-xr-x"; + final int owner = 123; + final int group = 456; + + Path path = fileSystem.getPath("/opt/vespa/tmp/file.txt"); + FileWriter writer = new FileWriter(path, () -> content) + .withPermissions(permissions) + .withOwnerId(owner) + .withGroupId(group) + .onlyIfFileDoesNotAlreadyExist(); + assertTrue(writer.converge(context)); + verify(context, times(1)).recordSystemModification(any(), eq("Creating file " + path + " with permissions rwxr-xr-x")); + + UnixPath unixPath = new UnixPath(path); + assertEquals(content, unixPath.readUtf8File()); + assertEquals(permissions, unixPath.getPermissions()); + assertEquals(owner, unixPath.getOwnerId()); + assertEquals(group, unixPath.getGroupId()); + Instant fileTime = unixPath.getLastModifiedTime(); + + // Second time is a no-op. + assertFalse(writer.converge(context)); + assertEquals(fileTime, unixPath.getLastModifiedTime()); + } + + @Test + void testAtomicWrite() { + FileWriter writer = new FileWriter(fileSystem.getPath("/foo/bar")) + .atomicWrite(true); + + assertTrue(writer.converge(context, "content")); + + verify(context).recordSystemModification(any(), eq("Creating file /foo/bar")); + assertEquals("content", new UnixPath(writer.path()).readUtf8File()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java new file mode 100644 index 00000000000..11675bbe46f --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java @@ -0,0 +1,87 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.io.UncheckedIOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author hakonhall + */ +public class MakeDirectoryTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final TestTaskContext context = new TestTaskContext(); + + private final String path = "/parent/dir"; + private String permissions = "rwxr----x"; + private int ownerId = 123; + private int groupId = 456; + + @Test + void newDirectory() { + verifySystemModifications( + "Creating directory " + path, + "Changing user ID of /parent/dir from 1 to 123", + "Changing group ID of /parent/dir from 2 to 456"); + + ownerId = 124; + verifySystemModifications("Changing user ID of /parent/dir from 123 to 124"); + + groupId = 457; + verifySystemModifications("Changing group ID of /parent/dir from 456 to 457"); + + permissions = "--x---r--"; + verifySystemModifications("Changing permissions of /parent/dir from rwxr----x to --x---r--"); + } + + private void verifySystemModifications(String... modifications) { + context.clearSystemModificationLog(); + MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path)) + .createParents() + .withPermissions(permissions) + .withOwnerId(ownerId) + .withGroupId(groupId); + assertTrue(makeDirectory.converge(context)); + + assertEquals(List.of(modifications), context.getSystemModificationLog()); + + context.clearSystemModificationLog(); + assertFalse(makeDirectory.converge(context)); + assertEquals(List.of(), context.getSystemModificationLog()); + } + + @Test + void exceptionIfMissingParent() { + String path = "/parent/dir"; + MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path)); + + try { + makeDirectory.converge(context); + } catch (UncheckedIOException e) { + if (e.getCause() instanceof NoSuchFileException) { + return; + } + throw e; + } + fail(); + } + + @Test + void okIfParentExists() { + String path = "/dir"; + MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path)); + assertTrue(makeDirectory.converge(context)); + assertTrue(Files.isDirectory(fileSystem.getPath(path))); + + MakeDirectory makeDirectory2 = new MakeDirectory(fileSystem.getPath(path)); + assertFalse(makeDirectory2.converge(context)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java new file mode 100644 index 00000000000..79fa1cf6ea2 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java @@ -0,0 +1,52 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * @author hakonhall + */ +public class StoredBooleanTest { + private final TaskContext context = mock(TaskContext.class); + private final FileSystem fileSystem = TestFileSystem.create(); + private final Path path = fileSystem.getPath("/foo"); + private final StoredBoolean storedBoolean = new StoredBoolean(path); + + @Test + void storedBoolean() { + assertFalse(storedBoolean.value()); + storedBoolean.set(context); + assertTrue(storedBoolean.value()); + storedBoolean.clear(context); + assertFalse(storedBoolean.value()); + } + + @Test + void testCompatibility() throws IOException { + StoredInteger storedInteger = new StoredInteger(path); + assertFalse(storedBoolean.value()); + + storedInteger.write(context, 1); + assertTrue(storedBoolean.value()); + + storedInteger.write(context, 2); + assertTrue(storedBoolean.value()); + + storedInteger.write(context, 0); + assertFalse(storedBoolean.value()); + + Files.delete(path); + assertFalse(storedBoolean.value()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java new file mode 100644 index 00000000000..d9dfcefc7e3 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java @@ -0,0 +1,39 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.nio.file.FileSystem; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +public class TemplateTest { + + @Test + void basic() { + FileSystem fileSystem = TestFileSystem.create(); + Path templatePath = fileSystem.getPath("/example.vm"); + String templateContent = "a $x, $y b"; + new UnixPath(templatePath).writeUtf8File(templateContent); + + Path toPath = fileSystem.getPath("/example"); + TaskContext taskContext = mock(TaskContext.class); + boolean converged = Template.at(templatePath) + .set("x", "foo") + .set("y", "bar") + .getFileWriterTo(toPath) + .converge(taskContext); + + assertTrue(converged); + + String actualContent = new UnixPath(toPath).readUtf8File(); + assertEquals("a foo, bar b", actualContent); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java new file mode 100644 index 00000000000..5892a9b9f53 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java @@ -0,0 +1,199 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * @author hakonhall + */ +public class UnixPathTest { + + private final FileSystem fs = TestFileSystem.create(); + + @Test + void createParents() { + Path parentDirectory = fs.getPath("/a/b/c"); + Path filePath = parentDirectory.resolve("bar"); + UnixPath path = new UnixPath(filePath); + + assertFalse(Files.exists(fs.getPath("/a"))); + path.createParents(); + assertTrue(Files.exists(parentDirectory)); + } + + @Test + void utf8File() { + String original = "foo\nbar\n"; + UnixPath path = new UnixPath(fs.getPath("example.txt")); + path.writeUtf8File(original); + String fromFile = path.readUtf8File(); + assertEquals(original, fromFile); + assertEquals(List.of("foo", "bar"), path.readLines()); + } + + @Test + void touch() { + UnixPath path = new UnixPath(fs.getPath("example.txt")); + assertTrue(path.create()); + assertEquals("", path.readUtf8File()); + assertFalse(path.create()); + } + + @Test + void permissions() { + String expectedPermissions = "rwxr-x---"; + UnixPath path = new UnixPath(fs.getPath("file.txt")); + path.writeUtf8File("foo"); + path.setPermissions(expectedPermissions); + assertEquals(expectedPermissions, path.getPermissions()); + } + + @Test + void badPermissionsString() { + assertThrows(IllegalArgumentException.class, () -> { + new UnixPath(fs.getPath("file.txt")).setPermissions("abcdefghi"); + }); + } + + @Test + void owner() { + Path path = fs.getPath("file.txt"); + UnixPath unixPath = new UnixPath(path); + unixPath.writeUtf8File("foo"); + + unixPath.setOwnerId(123); + assertEquals(123, unixPath.getOwnerId()); + + unixPath.setGroupId(456); + assertEquals(456, unixPath.getGroupId()); + } + + @Test + void createDirectoryWithPermissions() { + Path path = fs.getPath("dir"); + UnixPath unixPath = new UnixPath(path); + String permissions = "rwxr-xr--"; + assertTrue(unixPath.createDirectory(permissions)); + assertTrue(unixPath.isDirectory()); + assertEquals(permissions, unixPath.getPermissions()); + assertFalse(unixPath.createDirectory(permissions)); + } + + @Test + void createSymbolicLink() { + String original = "foo\nbar\n"; + UnixPath path = new UnixPath(fs.getPath("example.txt")); + path.writeUtf8File(original); + String fromFile = path.readUtf8File(); + assertEquals(original, fromFile); + + UnixPath link = path.createSymbolicLink(fs.getPath("link-to-example.txt")); + assertEquals(original, link.readUtf8File()); + } + + @Test + void readBytesIfExists() { + UnixPath path = new UnixPath(fs.getPath("example.txt")); + assertFalse(path.readBytesIfExists().isPresent()); + path.writeBytes(new byte[]{42}); + assertArrayEquals(new byte[]{42}, path.readBytesIfExists().get()); + } + + @Test + void deleteRecursively() throws Exception { + // Create the following file tree: + // + // /dir1 + // |--- dir2 + // |--- file1 + // /link1 -> /dir1/dir2 + // + var dir1 = fs.getPath("/dir1"); + var dir2 = dir1.resolve("dir2"); + var file1 = dir2.resolve("file1"); + Files.createDirectories(dir2); + Files.writeString(file1, "file1"); + var link1 = Files.createSymbolicLink(fs.getPath("/link1"), dir2); + + new UnixPath(link1).deleteRecursively(); + assertTrue(Files.exists(dir2), "Deleting " + link1 + " recursively does not remove " + dir2); + assertTrue(Files.exists(file1), "Deleting " + link1 + " recursively does not remove " + file1); + + new UnixPath(dir1).deleteRecursively(); + assertFalse(Files.exists(file1), dir1 + " deleted recursively"); + assertFalse(Files.exists(dir2), dir1 + " deleted recursively"); + assertFalse(Files.exists(dir1), dir1 + " deleted recursively"); + } + + @Test + void isEmptyDirectory() { + var path = new UnixPath((fs.getPath("/foo"))); + assertFalse(path.isEmptyDirectory()); + + path.writeUtf8File(""); + assertFalse(path.isEmptyDirectory()); + + path.deleteIfExists(); + path.createDirectory(); + assertTrue(path.isEmptyDirectory()); + + path.resolve("bar").writeUtf8File(""); + assertFalse(path.isEmptyDirectory()); + } + + @Test + void atomicWrite() { + var path = new UnixPath(fs.getPath("/dir/foo")); + path.createParents(); + path.writeUtf8File("bar"); + path.atomicWriteBytes("bar v2".getBytes(StandardCharsets.UTF_8)); + assertEquals("bar v2", path.readUtf8File()); + } + + @Test + void testParentAndFilename() { + var absolutePath = new UnixPath("/foo/bar"); + assertEquals("/foo", absolutePath.getParent().toString()); + assertEquals("bar", absolutePath.getFilename()); + + var pathWithoutSlash = new UnixPath("foo"); + assertRuntimeException(IllegalStateException.class, "Path has no parent directory: 'foo'", pathWithoutSlash::getParent); + assertEquals("foo", pathWithoutSlash.getFilename()); + + var pathWithSlash = new UnixPath("/foo"); + assertEquals("/", pathWithSlash.getParent().toString()); + assertEquals("foo", pathWithSlash.getFilename()); + + assertRuntimeException(IllegalStateException.class, "Path has no parent directory: '/'", () -> new UnixPath("/").getParent()); + assertRuntimeException(IllegalStateException.class, "Path has no filename: '/'", () -> new UnixPath("/").getFilename()); + } + + private <T extends RuntimeException> void assertRuntimeException(Class<T> baseClass, String message, Runnable runnable) { + try { + runnable.run(); + fail("No exception was thrown"); + } catch (RuntimeException e) { + if (!baseClass.isInstance(e)) { + fail("Exception class mismatch " + baseClass.getName() + " != " + e.getClass().getName()); + } + + assertEquals(message, e.getMessage()); + } + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java new file mode 100644 index 00000000000..37fe90209ea --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java @@ -0,0 +1,211 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.fs; + +import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace; +import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author freva + */ +class ContainerFileSystemTest { + + private final FileSystem fileSystem = TestFileSystem.create(); + private final UnixPath containerRootOnHost = new UnixPath(fileSystem.getPath("/data/storage/ctr1")); + private final UserScope userScope = UserScope.create(new UserNamespace(10_000, 11_000, 10000)); + private final ContainerFileSystem containerFs = ContainerFileSystem.create(containerRootOnHost.createDirectories().toPath(), userScope); + + @Test + public void creates_files_and_directories_with_container_root_as_owner() throws IOException { + ContainerPath containerPath = ContainerPath.fromPathInContainer(containerFs, Path.of("/opt/vespa/logs/file"), userScope.root()); + UnixPath unixPath = new UnixPath(containerPath).createParents().writeUtf8File("hello world"); + + for (ContainerPath p = containerPath; p.getParent() != null; p = p.getParent()) + assertOwnership(p, 0, 0, 10000, 11000); + + unixPath.setOwnerId(500).setGroupId(1000); + assertOwnership(containerPath, 500, 1000, 10500, 12000); + + UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); + ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/copy1"), userScope.root()); + Files.copy(hostFile.toPath(), destination); + assertOwnership(destination, 0, 0, 10000, 11000); + } + + @Test + public void file_write_and_read() throws IOException { + ContainerPath containerPath = ContainerPath.fromPathInContainer(containerFs, Path.of("/file"), userScope.root()); + UnixPath unixPath = new UnixPath(containerPath); + unixPath.writeUtf8File("hello"); + assertOwnership(containerPath, 0, 0, 10000, 11000); + + unixPath.setOwnerId(500).setGroupId(200); + assertOwnership(containerPath, 500, 200, 10500, 11200); + Files.writeString(containerPath, " world", StandardOpenOption.APPEND); + assertOwnership(containerPath, 500, 200, 10500, 11200); // Owner should not have been updated as the file already existed + + assertEquals("hello world", unixPath.readUtf8File()); + + unixPath.deleteIfExists(); + new UnixPath(containerPath.withUser(userScope.vespa())).writeUtf8File("test123"); + assertOwnership(containerPath, 1000, 1000, 11000, 12000); + } + + @Test + public void copy() throws IOException { + UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); + ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest"), userScope.root()); + + // If file is copied to JimFS path, the UID/GIDs are not fixed + Files.copy(hostFile.toPath(), destination.pathOnHost()); + assertEquals(String.valueOf(userScope.namespace().overflowId()), Files.getOwner(destination).getName()); + Files.delete(destination); + + Files.copy(hostFile.toPath(), destination); + assertOwnership(destination, 0, 0, 10000, 11000); + + // Set owner + group on both source host file and destination container file + hostFile.setOwnerId(5).setGroupId(10); + new UnixPath(destination).setOwnerId(500).setGroupId(200); + assertOwnership(destination, 500, 200, 10500, 11200); + // Copy the host file to destination again with COPY_ATTRIBUTES and REPLACE_EXISTING + Files.copy(hostFile.toPath(), destination, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); + // The destination is recreated, so the owner should be root + assertOwnership(destination, 0, 0, 10000, 11000); + + // Set owner + group and copy within ContainerFS + new UnixPath(destination).setOwnerId(500).setGroupId(200); + ContainerPath destination2 = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest2"), userScope.root()); + Files.copy(destination, destination2, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); + assertOwnership(destination2, 500, 200, 10500, 11200); + } + + @Test + public void move() throws IOException { + UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); + ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest"), userScope.root()); + + // If file is moved to JimFS path, the UID/GIDs are not fixed + Files.move(hostFile.toPath(), destination.pathOnHost()); + assertEquals(String.valueOf(userScope.namespace().overflowId()), Files.getOwner(destination).getName()); + Files.delete(destination); + + hostFile.createNewFile(); + Files.move(hostFile.toPath(), destination); + assertOwnership(destination, 0, 0, 10000, 11000); + + // Set owner + group on both source host file and destination container file + hostFile.createNewFile(); + hostFile.setOwnerId(5).setGroupId(10); + new UnixPath(destination).setOwnerId(500).setGroupId(200); + assertOwnership(destination, 500, 200, 10500, 11200); + // Move the host file to destination again with COPY_ATTRIBUTES and REPLACE_EXISTING + Files.move(hostFile.toPath(), destination, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); + // The destination is recreated, so the owner should be root + assertOwnership(destination, 0, 0, 10000, 11000); + + // Set owner + group and move within ContainerFS + new UnixPath(destination).setOwnerId(500).setGroupId(200); + ContainerPath destination2 = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest2"), userScope.root()); + Files.move(destination, destination2, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); + assertOwnership(destination2, 500, 200, 10500, 11200); + } + + @Test + public void symlink() throws IOException { + ContainerPath source = ContainerPath.fromPathInContainer(containerFs, Path.of("/src"), userScope.root()); + // Symlink from ContainerPath to some relative path (different FS provider) + Files.createSymbolicLink(source, fileSystem.getPath("../relative/target")); + assertEquals(fileSystem.getPath("../relative/target"), Files.readSymbolicLink(source)); + Files.delete(source); + + // Symlinks from ContainerPath to a ContainerPath: Target is resolved within container with base FS provider + Files.createSymbolicLink(source, ContainerPath.fromPathInContainer(containerFs, Path.of("/path/in/container"), userScope.root())); + assertEquals(fileSystem.getPath("/path/in/container"), Files.readSymbolicLink(source)); + assertOwnership(source, 0, 0, 10000, 11000); + } + + @Test + public void disallow_operations_on_symlink() throws IOException { + Path destination = fileSystem.getPath("/dir/file"); + Files.createDirectories(destination.getParent()); + + ContainerPath link = containerFs.getPath("/link"); + Files.createSymbolicLink(link, destination); + + // Cannot write file via symlink + assertThrows(IOException.class, () -> Files.writeString(link, "hello")); + + assertOwnership(link, 0, 0, 10_000, 11_000); + Files.setAttribute(link, "unix:uid", 10); // This succeeds because attribute is set on the link (destination does not exist) + assertFalse(Files.exists(destination)); + assertOwnership(link, 10, 0, 10_010, 11_000); + } + + @Test + public void disallow_operations_on_parent_symlink() throws IOException { + Path destination = fileSystem.getPath("/dir/sub/folder"); + Files.createDirectories(destination.getParent()); + + // Create symlink /some/dir/link -> /dir/sub + ContainerPath link = containerFs.getPath("/some/dir/link"); + Files.createDirectories(link.getParent()); + Files.createSymbolicLink(link, destination.getParent()); + + ContainerPath file = link.resolve("file"); + assertThrows(IOException.class, () -> Files.writeString(file, "hello")); + Files.writeString(file.pathOnHost(), "hello"); // Writing through host FS works + } + + @Test + public void permissions() throws IOException { + assertPermissions(Files.createDirectory(containerFs.getPath("/dir1")), "rwxr-x---"); + assertPermissions(Files.createDirectory(containerFs.getPath("/dir2"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-"); + + assertPermissions(Files.createDirectories(containerFs.getPath("/sub/dir/leaf"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-"); + assertPermissions(containerFs.getPath("/sub/dir"), "r-x-w-rw-"); // Non-leafs get the same permission as the leaf + + // TODO: Uncomment when JimFS forwards attributes for SecureDirectoryStream::newByteChannel +// assertPermissions(Files.createFile(containerFs.getPath("/file1")), "rw-r-----"); +// assertPermissions(Files.createFile(containerFs.getPath("/file2"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-"); + } + + private static void assertOwnership(ContainerPath path, int contUid, int contGid, int hostUid, int hostGid) throws IOException { + assertOwnership(path, contUid, contGid); + assertOwnership(path.pathOnHost(), hostUid, hostGid); + } + + private static void assertOwnership(Path path, int uid, int gid) throws IOException { + Map<String, Object> attrs = Files.readAttributes(path, "unix:*", LinkOption.NOFOLLOW_LINKS); + assertEquals(uid, attrs.get("uid")); + assertEquals(gid, attrs.get("gid")); + } + + private static void assertPermissions(Path path, String expected) throws IOException { + String actual = PosixFilePermissions.toString(Files.getPosixFilePermissions(path)); + assertEquals(expected, actual); + } + + private static FileAttribute<?> permissionsFromString(String permissions) { + return PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString(permissions)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java new file mode 100644 index 00000000000..eb7a8e13925 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java @@ -0,0 +1,120 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.fs; + +import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.Executable; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * @author freva + */ +class ContainerPathTest { + + private final FileSystem baseFs = TestFileSystem.create(); + private final ContainerFileSystem containerFs = ContainerFileSystem.create(baseFs.getPath("/data/storage/ctr1"), mock(UserScope.class)); + + @Test + public void create_new_container_path() { + ContainerPath path = fromPathInContainer(Path.of("/opt/vespa//logs/./file")); + assertPaths(path, "/data/storage/ctr1/opt/vespa/logs/file", "/opt/vespa/logs/file"); + + path = fromPathOnHost(baseFs.getPath("/data/storage/ctr1/opt/vespa/logs/file")); + assertPaths(path, "/data/storage/ctr1/opt/vespa/logs/file", "/opt/vespa/logs/file"); + + path = fromPathOnHost(baseFs.getPath("/data/storage/ctr2/..////./ctr1/./opt")); + assertPaths(path, "/data/storage/ctr1/opt", "/opt"); + + assertThrows(() -> fromPathInContainer(Path.of("relative/path")), "Path in container must be absolute: relative/path"); + assertThrows(() -> fromPathOnHost(baseFs.getPath("relative/path")), "Paths have different roots: /data/storage/ctr1, relative/path"); + assertThrows(() -> fromPathOnHost(baseFs.getPath("/data/storage/ctr2")), "Path /data/storage/ctr2 is not under container root /data/storage/ctr1"); + assertThrows(() -> fromPathOnHost(baseFs.getPath("/data/storage/ctr1/../ctr2")), "Path /data/storage/ctr2 is not under container root /data/storage/ctr1"); + } + + @Test + public void container_path_operations() { + ContainerPath path = fromPathInContainer(Path.of("/opt/vespa/logs/file")); + ContainerPath parent = path.getParent(); + assertPaths(path.getRoot(), "/data/storage/ctr1", "/"); + assertPaths(parent, "/data/storage/ctr1/opt/vespa/logs", "/opt/vespa/logs"); + assertNull(path.getRoot().getParent()); + + assertEquals(Path.of("file"), path.getFileName()); + assertEquals(Path.of("logs"), path.getName(2)); + assertEquals(4, path.getNameCount()); + assertEquals(Path.of("vespa/logs"), path.subpath(1, 3)); + + assertTrue(path.startsWith(path)); + assertTrue(path.startsWith(parent)); + assertFalse(parent.startsWith(path)); + assertFalse(path.startsWith(Path.of(path.toString()))); + + assertTrue(path.endsWith(Path.of(path.pathInContainer()))); + assertTrue(path.endsWith(Path.of("logs/file"))); + assertFalse(path.endsWith(Path.of("/logs/file"))); + } + + @Test + public void resolution() { + ContainerPath path = fromPathInContainer(Path.of("/opt/vespa/logs")); + assertPaths(path.resolve(Path.of("/root")), "/data/storage/ctr1/root", "/root"); + assertPaths(path.resolve(Path.of("relative")), "/data/storage/ctr1/opt/vespa/logs/relative", "/opt/vespa/logs/relative"); + assertPaths(path.resolve(Path.of("/../../../dir2/../../../dir2")), "/data/storage/ctr1/dir2", "/dir2"); + assertPaths(path.resolve(Path.of("/some/././///path")), "/data/storage/ctr1/some/path", "/some/path"); + + assertPaths(path.resolve(Path.of("../dir")), "/data/storage/ctr1/opt/vespa/dir", "/opt/vespa/dir"); + assertEquals(path.resolve(Path.of("../dir")), path.resolveSibling("dir")); + } + + @Test + public void resolves_real_paths() throws IOException { + ContainerPath path = fromPathInContainer(Path.of("/opt/vespa/logs")); + Files.createDirectories(path.pathOnHost().getParent()); + + Files.createFile(baseFs.getPath("/data/storage/ctr1/opt/vespa/target1")); + Files.createSymbolicLink(path.pathOnHost(), path.pathOnHost().resolveSibling("target1")); + assertPaths(path.toRealPath(LinkOption.NOFOLLOW_LINKS), "/data/storage/ctr1/opt/vespa/logs", "/opt/vespa/logs"); + assertPaths(path.toRealPath(), "/data/storage/ctr1/opt/vespa/target1", "/opt/vespa/target1"); + + Files.delete(path.pathOnHost()); + Files.createFile(baseFs.getPath("/data/storage/ctr1/opt/target2")); + Files.createSymbolicLink(path.pathOnHost(), baseFs.getPath("../target2")); + assertPaths(path.toRealPath(), "/data/storage/ctr1/opt/target2", "/opt/target2"); + + Files.delete(path.pathOnHost()); + Files.createFile(baseFs.getPath("/data/storage/ctr2")); + Files.createSymbolicLink(path.pathOnHost(), path.getRoot().pathOnHost().resolveSibling("ctr2")); + assertThrows(path::toRealPath, "Path /data/storage/ctr2 is not under container root /data/storage/ctr1"); + } + + private ContainerPath fromPathInContainer(Path pathInContainer) { + return ContainerPath.fromPathInContainer(containerFs, pathInContainer, UnixUser.ROOT); + } + private ContainerPath fromPathOnHost(Path pathOnHost) { + return ContainerPath.fromPathOnHost(containerFs, pathOnHost, UnixUser.ROOT); + } + + private static void assertPaths(ContainerPath actual, String expectedPathOnHost, String expectedPathInContainer) { + assertEquals(expectedPathOnHost, actual.pathOnHost().toString()); + assertEquals(expectedPathInContainer, actual.pathInContainer()); + } + + private static void assertThrows(Executable executable, String expectedMsg) { + String actualMsg = Assertions.assertThrows(IllegalArgumentException.class, executable).getMessage(); + assertEquals(expectedMsg, actualMsg); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java new file mode 100644 index 00000000000..525c6d9162c --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java @@ -0,0 +1,41 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.fs; + +import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace; +import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.attribute.UserPrincipalNotFoundException; + +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal; +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author freva + */ +class ContainerUserPrincipalLookupServiceTest { + + private final UserScope userScope = UserScope.create(new UserNamespace(10_000, 11_000, 10000)); + private final ContainerUserPrincipalLookupService userPrincipalLookupService = + new ContainerUserPrincipalLookupService(TestFileSystem.create().getUserPrincipalLookupService(), userScope); + + @Test + public void correctly_resolves_ids() throws IOException { + ContainerUserPrincipal user = userPrincipalLookupService.lookupPrincipalByName("1000"); + assertEquals("vespa", user.getName()); + assertEquals("11000", user.baseFsPrincipal().getName()); + assertEquals(user, userPrincipalLookupService.lookupPrincipalByName("vespa")); + + ContainerGroupPrincipal group = userPrincipalLookupService.lookupPrincipalByGroupName("1000"); + assertEquals("vespa", group.getName()); + assertEquals("12000", group.baseFsPrincipal().getName()); + assertEquals(group, userPrincipalLookupService.lookupPrincipalByGroupName("vespa")); + + assertThrows(UserPrincipalNotFoundException.class, () -> userPrincipalLookupService.lookupPrincipalByName("test")); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java new file mode 100644 index 00000000000..299d3e4b441 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java @@ -0,0 +1,32 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.network; + +import com.google.common.net.InetAddresses; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author smorgrav + */ +public class IPAddressesMock implements IPAddresses { + + private final Map<String, List<InetAddress>> otherAddresses = new HashMap<>(); + + public IPAddressesMock addAddress(String hostname, String ip) { + List<InetAddress> addresses = otherAddresses.getOrDefault(hostname, new ArrayList<>()); + addresses.add(InetAddresses.forString(ip)); + otherAddresses.put(hostname, addresses); + return this; + } + + @Override + public InetAddress[] getAddresses(String hostname) { + List<InetAddress> addresses = otherAddresses.get(hostname); + if (addresses == null) return new InetAddress[0]; + return addresses.toArray(new InetAddress[addresses.size()]); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java new file mode 100644 index 00000000000..59ddc1f6c8d --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java @@ -0,0 +1,78 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.network; + +import com.google.common.net.InetAddresses; +import org.junit.jupiter.api.Test; + +import java.net.Inet6Address; +import java.net.InetAddress; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author smorgrav + */ +public class IPAddressesTest { + + private final IPAddressesMock mock = new IPAddressesMock(); + + @Test + void choose_sitelocal_ipv4_over_public() { + mock.addAddress("localhost", "38.3.4.2") + .addAddress("localhost", "10.0.2.2") + .addAddress("localhost", "fe80::1") + .addAddress("localhost", "2001::1"); + + assertEquals(InetAddresses.forString("10.0.2.2"), mock.getIPv4Address("localhost").get()); + } + + @Test + void choose_ipv6_public_over_local() { + mock.addAddress("localhost", "38.3.4.2") + .addAddress("localhost", "10.0.2.2") + .addAddress("localhost", "fe80::1") + .addAddress("localhost", "2001::1"); + + assertEquals(InetAddresses.forString("2001::1"), mock.getIPv6Address("localhost").get()); + } + + @Test + void throws_when_multiple_ipv6_addresses() { + assertThrows(RuntimeException.class, () -> { + mock.addAddress("localhost", "2001::1") + .addAddress("localhost", "2001::2"); + mock.getIPv6Address("localhost"); + }); + } + + @Test + void throws_when_multiple_private_ipv4_addresses() { + assertThrows(RuntimeException.class, () -> { + mock.addAddress("localhost", "38.3.4.2") + .addAddress("localhost", "10.0.2.2") + .addAddress("localhost", "10.0.2.3"); + mock.getIPv4Address("localhost"); + }); + } + + @Test + void translator_with_valid_parameters() { + + // Test simplest possible address + Inet6Address original = (Inet6Address) InetAddresses.forString("2001:db8::1"); + Inet6Address prefix = (Inet6Address) InetAddresses.forString("fd00::"); + InetAddress translated = IPAddresses.prefixTranslate(original, prefix, 8); + assertEquals("fd00:0:0:0:0:0:0:1", translated.getHostAddress()); + + + // Test an actual aws address we use + original = (Inet6Address) InetAddresses.forString("2600:1f16:f34:5300:ccc6:1703:b7c2:369d"); + translated = IPAddresses.prefixTranslate(original, prefix, 8); + assertEquals("fd00:0:0:0:ccc6:1703:b7c2:369d", translated.getHostAddress()); + + // Test different subnet size + translated = IPAddresses.prefixTranslate(original, prefix, 6); + assertEquals("fd00:0:0:5300:ccc6:1703:b7c2:369d", translated.getHostAddress()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java new file mode 100644 index 00000000000..69d5c6f2c31 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java @@ -0,0 +1,69 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.network; + +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +/** + * @author gjoranv + */ +public class VersionedIpAddressTest { + + @Test + void ip4_address_can_be_generated_from_string() { + var ip4 = VersionedIpAddress.from("10.0.0.1"); + assertEquals(IPVersion.IPv4, ip4.version()); + assertEquals("10.0.0.1", ip4.asString()); + } + + @Test + void ip6_address_can_be_generated_from_string() { + var ip6 = VersionedIpAddress.from("::1"); + assertEquals(IPVersion.IPv6, ip6.version()); + assertEquals("::1", ip6.asString()); + } + + @Test + void they_are_sorted_by_version_then_by_address() { + var ip4 = VersionedIpAddress.from("10.0.0.1"); + var ip4_2 = VersionedIpAddress.from("127.0.0.1"); + var ip6 = VersionedIpAddress.from("::1"); + var ip6_2 = VersionedIpAddress.from("::2"); + + var sorted = Stream.of(ip4_2, ip6, ip4, ip6_2) + .sorted() + .toList(); + assertEquals(List.of(ip6, ip6_2, ip4, ip4_2), sorted); + } + + @Test + void endpoint_with_port_is_generated_correctly_for_both_versions() { + var ip4 = VersionedIpAddress.from("10.0.0.1"); + var ip6 = VersionedIpAddress.from("::1"); + + assertEquals("10.0.0.1:8080", ip4.asEndpoint(8080)); + assertEquals("[::1]:8080", ip6.asEndpoint(8080)); + } + + @Test + void equals_and_hashCode_are_implemented() { + var one = VersionedIpAddress.from("::1"); + var two = VersionedIpAddress.from("::2"); + var local = VersionedIpAddress.from("127.0.0.1"); + var ten = VersionedIpAddress.from("10.0.0.1"); + assertEquals(one, VersionedIpAddress.from("::1")); + assertNotEquals(one, two); + assertNotEquals(one, local); + assertNotEquals(one, ten); + + assertEquals(local, VersionedIpAddress.from("127.0.0.1")); + assertNotEquals(local, two); + assertNotEquals(local, 10); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java new file mode 100644 index 00000000000..19bc2d59bb2 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java @@ -0,0 +1,147 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.jdisc.Timer; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author hakonhall + */ +public class ChildProcess2ImplTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final Timer timer = mock(Timer.class); + private final CommandLine commandLine = mock(CommandLine.class); + private final ProcessApi2 processApi = mock(ProcessApi2.class); + private Path temporaryFile; + + @BeforeEach + public void setUp() throws IOException { + temporaryFile = Files.createTempFile(fileSystem.getPath("/"), "", ""); + } + + @Test + void testSuccess() throws Exception { + when(commandLine.getTimeout()).thenReturn(Duration.ofHours(1)); + when(commandLine.getMaxOutputBytes()).thenReturn(10L); + when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); + when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); + when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); + when(commandLine.toString()).thenReturn("program arg"); + + when(timer.currentTime()).thenReturn( + Instant.ofEpochMilli(1), + Instant.ofEpochMilli(2)); + + when(processApi.waitFor(anyLong(), any())).thenReturn(true); + + try (ChildProcess2Impl child = + new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { + child.waitForTermination(); + } + } + + @Test + void testTimeout() throws Exception { + when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1)); + when(commandLine.getMaxOutputBytes()).thenReturn(10L); + when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); + when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); + when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); + when(commandLine.toString()).thenReturn("program arg"); + + when(timer.currentTime()).thenReturn( + Instant.ofEpochSecond(0), + Instant.ofEpochSecond(2)); + + when(processApi.waitFor(anyLong(), any())).thenReturn(true); + + try (ChildProcess2Impl child = + new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { + try { + child.waitForTermination(); + fail(); + } catch (TimeoutChildProcessException e) { + assertEquals( + "Command 'program arg' timed out after PT1S: stdout/stderr: ''", + e.getMessage()); + } + } + } + + @Test + void testMaxOutputBytes() throws Exception { + when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1)); + when(commandLine.getMaxOutputBytes()).thenReturn(10L); + when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); + when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); + when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); + when(commandLine.toString()).thenReturn("program arg"); + + when(timer.currentTime()).thenReturn( + Instant.ofEpochMilli(0), + Instant.ofEpochMilli(1)); + + when(processApi.waitFor(anyLong(), any())).thenReturn(true); + + Files.writeString(temporaryFile, "1234567890123"); + + try (ChildProcess2Impl child = + new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { + try { + child.waitForTermination(); + fail(); + } catch (LargeOutputChildProcessException e) { + assertEquals( + "Command 'program arg' output more than 13 bytes: stdout/stderr: '1234567890123'", + e.getMessage()); + } + } + } + + @Test + void testUnkillable() throws Exception { + when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1)); + when(commandLine.getMaxOutputBytes()).thenReturn(10L); + when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); + when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); + when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); + when(commandLine.toString()).thenReturn("program arg"); + + when(timer.currentTime()).thenReturn( + Instant.ofEpochMilli(0), + Instant.ofEpochMilli(1)); + + when(processApi.waitFor(anyLong(), any())).thenReturn(false); + + Files.writeString(temporaryFile, "1234567890123"); + + try (ChildProcess2Impl child = + new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { + try { + child.waitForTermination(); + fail(); + } catch (UnkillableChildProcessException e) { + assertEquals( + "Command 'program arg' did not terminate even after SIGTERM, +PT2M, SIGKILL, and +PT3M: stdout/stderr: '1234567890123'", + e.getMessage()); + } + } + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java new file mode 100644 index 00000000000..fead96404a5 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java @@ -0,0 +1,190 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; + +import static org.junit.jupiter.api.Assertions.*; + +public class CommandLineTest { + private final TestTerminal terminal = new TestTerminal(); + private final TestTaskContext context = new TestTaskContext(); + private final CommandLine commandLine = terminal.newCommandLine(context); + + @AfterEach + public void tearDown() { + terminal.verifyAllCommandsExecuted(); + } + + @Test + void testStrings() { + terminal.expectCommand( + "/bin/bash \"with space\" \"speci&l\" \"\" \"double\\\"quote\" 2>&1", + 0, + ""); + commandLine.add("/bin/bash", "with space", "speci&l", "", "double\"quote").execute(); + assertEquals("bash", commandLine.programName()); + } + + @Test + void testBasicExecute() { + terminal.expectCommand("foo bar 2>&1", 0, "line1\nline2\n\n"); + CommandResult result = commandLine.add("foo", "bar").execute(); + assertEquals(0, result.getExitCode()); + assertEquals("line1\nline2", result.getOutput()); + assertEquals("line1\nline2\n\n", result.getUntrimmedOutput()); + assertEquals(List.of("line1", "line2"), result.getOutputLines()); + assertEquals(1, context.getSystemModificationLog().size()); + assertEquals("Executing command: foo bar 2>&1", context.getSystemModificationLog().get(0)); + + List<CommandLine> commandLines = terminal.getTestProcessFactory().getMutableCommandLines(); + assertEquals(1, commandLines.size()); + assertEquals(commandLine, commandLines.get(0)); + + int lines = result.map(r -> r.getOutputLines().size()); + assertEquals(2, lines); + } + + @Test + void verifyDefaults() { + assertEquals(CommandLine.DEFAULT_TIMEOUT, commandLine.getTimeout()); + assertEquals(CommandLine.DEFAULT_MAX_OUTPUT_BYTES, commandLine.getMaxOutputBytes()); + assertEquals(CommandLine.DEFAULT_SIGTERM_GRACE_PERIOD, commandLine.getSigTermGracePeriod()); + assertEquals(CommandLine.DEFAULT_SIGKILL_GRACE_PERIOD, commandLine.getSigKillGracePeriod()); + assertEquals(0, commandLine.getArguments().size()); + assertEquals(Optional.empty(), commandLine.getOutputFile()); + assertEquals(StandardCharsets.UTF_8, commandLine.getOutputEncoding()); + assertTrue(commandLine.getRedirectStderrToStdoutInsteadOfDiscard()); + Predicate<Integer> defaultExitCodePredicate = commandLine.getSuccessfulExitCodePredicate(); + assertTrue(defaultExitCodePredicate.test(0)); + assertFalse(defaultExitCodePredicate.test(1)); + } + + @Test + void executeSilently() { + terminal.ignoreCommand(""); + commandLine.add("foo", "bar").executeSilently(); + assertEquals(0, context.getSystemModificationLog().size()); + commandLine.recordSilentExecutionAsSystemModification(); + assertEquals(1, context.getSystemModificationLog().size()); + assertEquals("Executed command: foo bar 2>&1", context.getSystemModificationLog().get(0)); + } + + @Test + void processFactorySpawnFails() { + assertThrows(NegativeArraySizeException.class, () -> { + terminal.interceptCommand( + commandLine.toString(), + command -> { + throw new NegativeArraySizeException(); + }); + commandLine.add("foo").execute(); + }); + } + + @Test + void waitingForTerminationExceptionStillClosesChild() { + TestChildProcess2 child = new TestChildProcess2(0, ""); + child.throwInWaitForTermination(new NegativeArraySizeException()); + terminal.interceptCommand(commandLine.toString(), command -> child); + assertFalse(child.closeCalled()); + try { + commandLine.add("foo").execute(); + fail(); + } catch (NegativeArraySizeException e) { + // OK + } + + assertTrue(child.closeCalled()); + } + + @Test + void programFails() { + terminal.expectCommand("foo 2>&1", 1, ""); + try { + commandLine.add("foo").execute(); + fail(); + } catch (ChildProcessFailureException e) { + assertEquals( + "Command 'foo 2>&1' terminated with exit code 1: stdout/stderr: ''", + e.getMessage()); + } + } + + @Test + void mapException() { + terminal.ignoreCommand("output"); + CommandResult result = terminal.newCommandLine(context).add("program").execute(); + IllegalArgumentException exception = new IllegalArgumentException("foo"); + try { + result.mapOutput(output -> { + throw exception; + }); + fail(); + } catch (UnexpectedOutputException e) { + assertEquals("Command 'program 2>&1' output was not of the expected format: " + + "Failed to map output: stdout/stderr: 'output'", e.getMessage()); + assertEquals(e.getCause(), exception); + } + } + + @Test + void testMapEachLine() { + assertEquals( + 1 + 2 + 3, + terminal.ignoreCommand("1\n2\n3\n") + .newCommandLine(context) + .add("foo") + .execute() + .mapEachLine(Integer::valueOf) + .stream() + .mapToInt(i -> i) + .sum()); + } + + @Test + void addTokensWithMultipleWhiteSpaces() { + terminal.expectCommand("iptables -L 2>&1"); + commandLine.addTokens("iptables -L").execute(); + + terminal.verifyAllCommandsExecuted(); + } + + @Test + void addTokensWithSpecialCharacters() { + terminal.expectCommand("find . ! -name hei 2>&1"); + commandLine.addTokens("find . ! -name hei").execute(); + + terminal.verifyAllCommandsExecuted(); + } + + @Test + void testEnvironment() { + terminal.expectCommand("env k1=v1 -u k2 \"key 3=value 3\" programname 2>&1"); + commandLine.add("programname") + .setEnvironmentVariable("key 3", "value 3") + .removeEnvironmentVariable("k2") + .setEnvironmentVariable("k1", "v1") + .execute(); + terminal.verifyAllCommandsExecuted(); + } + + @Test + public void testToString() { + commandLine.add("bash", "-c", "echo", "$MY_SECRET"); + assertEquals("bash -c echo \"$MY_SECRET\" 2>&1", commandLine.toString()); + commandLine.censorArgument(); + assertEquals("bash -c echo <censored> 2>&1", commandLine.toString()); + + terminal.expectCommand("bash -c echo \"$MY_SECRET\" 2>&1"); + commandLine.execute(); + terminal.verifyAllCommandsExecuted(); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java new file mode 100644 index 00000000000..58429f9f084 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java @@ -0,0 +1,88 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.jdisc.test.TestTimer; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static com.yahoo.yolean.Exceptions.uncheck; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ProcessFactoryImplTest { + private final ProcessStarter starter = mock(ProcessStarter.class); + private final TestTimer timer = new TestTimer(); + private final ProcessFactoryImpl processFactory = new ProcessFactoryImpl(starter, timer); + + @Test + void testSpawn() { + CommandLine commandLine = mock(CommandLine.class); + when(commandLine.getArguments()).thenReturn(List.of("program")); + when(commandLine.getRedirectStderrToStdoutInsteadOfDiscard()).thenReturn(true); + when(commandLine.programName()).thenReturn("program"); + Path outputPath; + try (ChildProcess2Impl child = processFactory.spawn(commandLine)) { + outputPath = child.getOutputPath(); + assertTrue(Files.exists(outputPath)); + assertEquals("rw-------", new UnixPath(outputPath).getPermissions()); + ArgumentCaptor<ProcessBuilder> processBuilderCaptor = + ArgumentCaptor.forClass(ProcessBuilder.class); + verify(starter).start(processBuilderCaptor.capture()); + ProcessBuilder processBuilder = processBuilderCaptor.getValue(); + assertTrue(processBuilder.redirectErrorStream()); + ProcessBuilder.Redirect redirect = processBuilder.redirectOutput(); + assertEquals(ProcessBuilder.Redirect.Type.WRITE, redirect.type()); + assertEquals(outputPath.toFile(), redirect.file()); + } + + assertFalse(Files.exists(outputPath)); + } + + @Test + void testSpawnWithPersistentOutputFile() { + + class TemporaryFile implements AutoCloseable { + private final Path path; + + private TemporaryFile() { + String outputFileName = ProcessFactoryImplTest.class.getSimpleName() + "-temporary-test-file.out"; + FileAttribute<Set<PosixFilePermission>> fileAttribute = PosixFilePermissions.asFileAttribute( + PosixFilePermissions.fromString("rw-------")); + path = uncheck(() -> Files.createTempFile(outputFileName, ".out", fileAttribute)); + } + + @Override + public void close() { + uncheck(() -> Files.deleteIfExists(path)); + } + } + + try (TemporaryFile outputPath = new TemporaryFile()) { + CommandLine commandLine = mock(CommandLine.class); + when(commandLine.getArguments()).thenReturn(List.of("program")); + when(commandLine.programName()).thenReturn("program"); + when(commandLine.getOutputFile()).thenReturn(Optional.of(outputPath.path)); + try (ChildProcess2Impl child = processFactory.spawn(commandLine)) { + assertEquals(outputPath.path, child.getOutputPath()); + assertTrue(Files.exists(outputPath.path)); + assertEquals("rw-------", new UnixPath(outputPath.path).getPermissions()); + } + + assertTrue(Files.exists(outputPath.path)); + } + + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java new file mode 100644 index 00000000000..f6a695ea003 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java @@ -0,0 +1,149 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.systemd; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcessFailureException; +import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; + +/** + * @author hakonhall + */ +public class SystemCtlTest { + + private final TaskContext taskContext = mock(TaskContext.class); + private final TestTerminal terminal = new TestTerminal(); + + @Test + void enable() { + terminal.expectCommand("systemctl --quiet is-enabled docker 2>&1", 1, "") + .expectCommand("systemctl enable docker 2>&1") + .expectCommand("systemctl --quiet is-enabled docker 2>&1"); + + SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(terminal).enable("docker"); + assertTrue(enableDockerService.converge(taskContext)); + assertFalse(enableDockerService.converge(taskContext), "Already converged"); + } + + @Test + void enableCommandFailure() { + terminal.expectCommand("systemctl --quiet is-enabled docker 2>&1", 1, "") + .expectCommand("systemctl enable docker 2>&1", 1, "error enabling service"); + SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(terminal).enable("docker"); + try { + enableDockerService.converge(taskContext); + fail(); + } catch (ChildProcessFailureException e) { + // success + } + } + + + @Test + void start() { + terminal.expectCommand( + "systemctl show docker 2>&1", + 0, + "a=b\n" + + "ActiveState=failed\n" + + "bar=zoo\n") + .expectCommand("systemctl start docker 2>&1", 0, ""); + + SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker"); + assertTrue(startDockerService.converge(taskContext)); + } + + @Test + void startIsNoop() { + terminal.expectCommand( + "systemctl show docker 2>&1", + 0, + "a=b\n" + + "ActiveState=active\n" + + "bar=zoo\n") + .expectCommand("systemctl start docker 2>&1", 0, ""); + + SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker"); + assertFalse(startDockerService.converge(taskContext)); + } + + + @Test + void startCommandFailre() { + terminal.expectCommand("systemctl show docker 2>&1", 1, "error"); + SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker"); + try { + startDockerService.converge(taskContext); + fail(); + } catch (ChildProcessFailureException e) { + // success + } + } + + + @Test + void disable() { + terminal.expectCommand("systemctl --quiet is-enabled docker 2>&1") + .expectCommand("systemctl disable docker 2>&1") + .expectCommand("systemctl --quiet is-enabled docker 2>&1", 1, ""); + + assertTrue(new SystemCtl(terminal).disable("docker").converge(taskContext)); + assertFalse(new SystemCtl(terminal).disable("docker").converge(taskContext), "Already converged"); + } + + @Test + void stop() { + terminal.expectCommand( + "systemctl show docker 2>&1", + 0, + "a=b\n" + + "ActiveState=active\n" + + "bar=zoo\n") + .expectCommand("systemctl stop docker 2>&1", 0, ""); + + assertTrue(new SystemCtl(terminal).stop("docker").converge(taskContext)); + } + + @Test + void restart() { + terminal.expectCommand("systemctl restart docker 2>&1", 0, ""); + assertTrue(new SystemCtl(terminal).restart("docker").converge(taskContext)); + } + + @Test + void testUnitExists() { + SystemCtl systemCtl = new SystemCtl(terminal); + + terminal.expectCommand("systemctl list-unit-files foo.service 2>&1", 0, + "UNIT FILE STATE\n" + + "\n" + + "0 unit files listed.\n"); + assertFalse(systemCtl.serviceExists(taskContext, "foo")); + + terminal.expectCommand("systemctl list-unit-files foo.service 2>&1", 0, + "UNIT FILE STATE \n" + + "foo.service enabled\n" + + "\n" + + "1 unit files listed.\n"); + assertTrue(systemCtl.serviceExists(taskContext, "foo")); + + terminal.expectCommand("systemctl list-unit-files foo.service 2>&1", 0, "garbage"); + try { + systemCtl.serviceExists(taskContext, "foo"); + fail(); + } catch (Exception e) { + assertTrue(e.getMessage().contains("garbage")); + } + } + + @Test + void withSudo() { + SystemCtl systemCtl = new SystemCtl(terminal).withSudo(); + terminal.expectCommand("sudo systemctl restart docker 2>&1", 0, ""); + assertTrue(systemCtl.restart("docker").converge(taskContext)); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java new file mode 100644 index 00000000000..3fc10a38a99 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java @@ -0,0 +1,52 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.systemd; + +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +public class SystemCtlTesterTest { + + private static final String unit = "my-unit"; + private final TestTerminal terminal = new TestTerminal(); + private final SystemCtlTester systemCtl = new SystemCtlTester(terminal); + private final TestTaskContext context = new TestTaskContext(); + + @Test + void return_expectations() { + assertSystemCtlMethod(sct -> sct.expectEnable(unit), sc -> sc.enable(unit).converge(context)); + assertSystemCtlMethod(sct -> sct.expectDisable(unit), sc -> sc.disable(unit).converge(context)); + assertSystemCtlMethod(sct -> sct.expectStart(unit), sc -> sc.start(unit).converge(context)); + assertSystemCtlMethod(sct -> sct.expectStop(unit), sc -> sc.stop(unit).converge(context)); + assertSystemCtlMethod(sct -> sct.expectServiceExists(unit), sc -> sc.serviceExists(context, unit)); + assertSystemCtlMethod(sct -> sct.expectIsActive(unit), sc -> sc.isActive(context, unit)); + } + + @Test + void void_tests() { + systemCtl.expectRestart(unit); + systemCtl.restart(unit).converge(context); + terminal.verifyAllCommandsExecuted(); + + systemCtl.expectDaemonReload(); + systemCtl.daemonReload(context); + terminal.verifyAllCommandsExecuted(); + } + + private void assertSystemCtlMethod(Function<SystemCtlTester, SystemCtlTester.Expectation> systemCtlTesterExpectationFunction, + Function<SystemCtl, Boolean> systemCtlFunction) { + List.of(true, false).forEach(wantedReturnValue -> { + systemCtlTesterExpectationFunction.apply(systemCtl).andReturn(wantedReturnValue); + assertEquals(wantedReturnValue, systemCtlFunction.apply(systemCtl)); + terminal.verifyAllCommandsExecuted(); + }); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java new file mode 100644 index 00000000000..1e2f69d7bc8 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java @@ -0,0 +1,218 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author hakonhall + */ +public class TemplateTest { + @Test + void verifyNewlineRemoval() { + Template template = Template.from("a%{list a}\n" + + "b%{end}\n" + + "c%{list c-}\n" + + "d%{end-}\n" + + "e\n", + new TemplateDescriptor().setRemoveNewline(false)); + template.add("a"); + template.add("c"); + + assertEquals("a\n" + + "b\n" + + "cde\n", + template.render()); + } + + @Test + void verifyIfSection() { + Template template = Template.from("Hello%{if cond} world%{end}!"); + assertEquals("Hello world!", template.snapshot().set("cond", true).render()); + assertEquals("Hello!", template.snapshot().set("cond", false).render()); + } + + @Test + void verifyComplexIfSection() { + Template template = Template.from("%{if cond}\n" + + "var: %{=varname}\n" + + "if: %{if !inner}inner is false%{end-}\n" + + "list: %{list formname}element%{end-}\n" + + "%{end}\n"); + + assertEquals("", template.snapshot().set("cond", false).render()); + + assertEquals("var: varvalue\n" + + "if: \n" + + "list: \n", + template.snapshot() + .set("cond", true) + .set("varname", "varvalue") + .set("inner", true) + .render()); + + Template template2 = template.snapshot() + .set("cond", true) + .set("varname", "varvalue") + .set("inner", false); + template2.add("formname"); + + assertEquals("var: varvalue\n" + + "if: inner is false\n" + + "list: element\n", template2.render()); + } + + @Test + void verifyElse() { + var template = Template.from("%{if cond}\n" + + "if body\n" + + "%{else}\n" + + "else body\n" + + "%{end}\n"); + assertEquals("if body\n", template.snapshot().set("cond", true).render()); + assertEquals("else body\n", template.snapshot().set("cond", false).render()); + } + + @Test + void verifySnapshotPreservesList() { + var template = Template.from("%{list foo}hello %{=area}%{end}"); + template.add("foo") + .set("area", "world"); + + assertEquals("hello world", template.render()); + assertEquals("hello world", template.snapshot().render()); + + Template snapshot = template.snapshot(); + snapshot.add("foo") + .set("area", "Norway"); + assertEquals("hello worldhello Norway", snapshot.render()); + } + + @Test + void verifyVariableSection() { + Template template = getTemplate("template1.tmp"); + template.set("varname", "varvalue"); + assertEquals("variable section 'varvalue'\n" + + "end of text\n", template.render()); + } + + @Test + void verifySimpleListSection() { + Template template = getTemplate("template1.tmp"); + template.set("varname", "varvalue") + .add("listname") + .set("varname", "different varvalue") + .set("varname2", "varvalue2"); + assertEquals("variable section 'varvalue'\n" + + "same variable section 'different varvalue'\n" + + "different variable section 'varvalue2'\n" + + "between ends\n" + + "end of text\n", template.render()); + } + + @Test + void verifyNestedListSection() { + Template template = getTemplate("template2.tmp"); + ListElement A0 = template.add("listA"); + ListElement A0B0 = A0.add("listB"); + ListElement A0B1 = A0.add("listB"); + + ListElement A1 = template.add("listA"); + ListElement A1B0 = A1.add("listB"); + assertEquals("body A\n" + + "body B\n" + + "body B\n" + + "body A\n" + + "body B\n", + template.render()); + } + + @Test + void verifyVariableReferences() { + Template template = getTemplate("template3.tmp"); + template.set("varname", "varvalue") + .set("innerVarSetAtTop", "val2"); + template.add("l"); + template.add("l") + .set("varname", "varvalue2"); + assertEquals("varvalue\n" + + "varvalue\n" + + "inner varvalue\n" + + "val2\n" + + "inner varvalue2\n" + + "val2\n", + template.render()); + } + + @Test + void badTemplates() { + assertException(BadTemplateException.class, "Unknown section 'zoo' at line 2 and column 6", + () -> Template.from("foo\nbar%{zoo}")); + + assertException(BadTemplateException.class, "Expected identifier at line 1 and column 4", + () -> Template.from("%{=")); + + assertException(BadTemplateException.class, "Expected identifier at line 1 and column 4", + () -> Template.from("%{=¬atoken}")); + + assertException(BadTemplateException.class, "Expected identifier at line 1 and column 8", + () -> Template.from("%{list ¬atoken}")); + + assertException(BadTemplateException.class, "Missing end directive for section started at line 1 and column 12", + () -> Template.from("%{list foo}missing end")); + + assertException(BadTemplateException.class, "Stray 'end' at line 1 and column 3", + () -> Template.from("%{end}stray end")); + + assertException(TemplateNameNotSetException.class, "Variable at line 1 and column 4 has not been set: notset", + () -> Template.from("%{=notset}").render()); + + assertException(TemplateNameNotSetException.class, "Variable at line 1 and column 6 has not been set: cond", + () -> Template.from("%{if cond}%{end}").render()); + + assertException(NotBooleanValueTemplateException.class, "cond was set to a non-boolean value: must be true or false", + () -> Template.from("%{if cond}%{end}").set("cond", 1).render()); + + assertException(NoSuchNameTemplateException.class, "No such element 'listname' in the template section starting at " + + "line 1 and column 1, and ending at line 1 and column 4", + () -> Template.from("foo").add("listname")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the list section at line 1 and column 16 is in conflict with the identically " + + "named list section at line 1 and column 1", + () -> Template.from("%{list a}%{end}%{list a}%{end}")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the list section at line 1 and column 6 is in conflict with the identically " + + "named variable section at line 1 and column 1", + () -> Template.from("%{=a}%{list a}%{end}")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the variable section at line 1 and column 16 is in conflict with the identically " + + "named list section at line 1 and column 1", + () -> Template.from("%{list a}%{end}%{=a}")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the list section at line 1 and column 14 is in conflict with the identically " + + "named if section at line 1 and column 1", + () -> Template.from("%{if a}%{end}%{list a}%{end}")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the if section at line 1 and column 16 is in conflict with the identically " + + "named list section at line 1 and column 1", + () -> Template.from("%{list a}%{end}%{if a}%{end}")); + } + + private <T extends Throwable> void assertException(Class<T> class_, String message, Runnable runnable) { + T exception = assertThrows(class_, runnable::run); + assertEquals(message, exception.getMessage()); + } + + private Template getTemplate(String filename) { + return Template.at(Path.of("src/test/resources/" + filename)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java new file mode 100644 index 00000000000..505cf807116 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java @@ -0,0 +1,194 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.yum; + +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author hakonhall + */ +public class YumPackageNameTest { + + @Test + void testBuilder() { + YumPackageName yumPackage = new YumPackageName.Builder("docker") + .setEpoch("2") + .setVersion("1.12.6") + .setRelease("71.git3e8e77d.el7.centos.1") + .setArchitecture("x86_64") + .build(); + assertEquals("docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", yumPackage.toName()); + } + + @Test + void testAllValidFormats() { + // name + verifyPackageName( + "docker-engine-selinux", + null, + "docker-engine-selinux", + null, + null, + null, + "docker-engine-selinux", + null); + + // name with parenthesis + verifyPackageName( + "dnf-command(versionlock)", + null, + "dnf-command(versionlock)", + null, + null, + null, + "dnf-command(versionlock)", + null); + + // name.arch + verifyPackageName( + "docker-engine-selinux.x86_64", + null, + "docker-engine-selinux", + null, + null, + "x86_64", + "docker-engine-selinux.x86_64", + null); + + // name-ver + verifyPackageName("docker-engine-selinux-1.12.6", + null, + "docker-engine-selinux", + "1.12.6", + null, + null, + "docker-engine-selinux-0:1.12.6", + null); + + // name-ver-rel + verifyPackageName("docker-engine-selinux-1.12.6-1.el7", + null, + "docker-engine-selinux", + "1.12.6", + "1.el7", + null, + "docker-engine-selinux-0:1.12.6-1.el7", + "docker-engine-selinux-0:1.12.6-1.el7.*"); + + // name-ver-rel.arch + verifyPackageName("docker-engine-selinux-1.12.6-1.el7.x86_64", + null, + "docker-engine-selinux", + "1.12.6", + "1.el7", + "x86_64", + "docker-engine-selinux-0:1.12.6-1.el7.x86_64", + "docker-engine-selinux-0:1.12.6-1.el7.*"); + + // name-epoch:ver-rel.arch + verifyPackageName( + "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", + "2", + "docker", + "1.12.6", + "71.git3e8e77d.el7.centos.1", + "x86_64", + "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", + "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.*"); + + // epoch:name-ver-rel.arch + verifyPackageName( + "2:docker-1.12.6-71.git3e8e77d.el7.centos.1.x86_64", + "2", + "docker", + "1.12.6", + "71.git3e8e77d.el7.centos.1", + "x86_64", + "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", + "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.*"); + } + + private void verifyPackageName(String input, + String epoch, + String name, + String version, + String release, + String architecture, + String toName, + String toVersionName) { + YumPackageName yumPackageName = YumPackageName.fromString(input); + assertPackageField("epoch", epoch, yumPackageName.getEpoch()); + assertPackageField("name", name, Optional.of(yumPackageName.getName())); + assertPackageField("version", version, yumPackageName.getVersion()); + assertPackageField("release", release, yumPackageName.getRelease()); + assertPackageField("architecture", architecture, yumPackageName.getArchitecture()); + assertPackageField("toName()", toName, Optional.of(yumPackageName.toName())); + + if (toVersionName == null) { + try { + yumPackageName.toVersionLockName(); + fail(); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("Version is missing ") || + e.getMessage().contains("Release is missing "), + "Exception message contains expected substring: " + e.getMessage()); + } + } else { + assertEquals(toVersionName, yumPackageName.toVersionLockName()); + } + } + + private void assertPackageField(String field, String expected, Optional<String> actual) { + if (expected == null) { + assertFalse(actual.isPresent(), field + " is not present"); + } else { + assertEquals(expected, actual.get(), field + " has expected value"); + } + } + + @Test + void testArchitectures() { + assertEquals("x86_64", YumPackageName.fromString("docker.x86_64").getArchitecture().get()); + assertEquals("i686", YumPackageName.fromString("docker.i686").getArchitecture().get()); + assertEquals("noarch", YumPackageName.fromString("docker.noarch").getArchitecture().get()); + } + + @Test + void unrecognizedArchitectureGetsGobbledUp() { + YumPackageName packageName = YumPackageName.fromString("docker-engine-selinux-1.12.6-1.el7.i486"); + // This is not a great feature - please use YumPackageName.Builder instead. + assertEquals("1.el7.i486", packageName.getRelease().get()); + } + + @Test + void failParsingOfPackageNameWithEpochAndArchitecture() { + try { + YumPackageName.fromString("epoch:docker-engine-selinux-1.12.6-1.el7.x86_64"); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().toLowerCase().contains("epoch")); + } + } + + @Test + void testSubset() { + YumPackageName yumPackage = new YumPackageName.Builder("docker") + .setVersion("1.12.6") + .build(); + + assertTrue(yumPackage.isSubsetOf(yumPackage)); + assertTrue(yumPackage.isSubsetOf(new YumPackageName.Builder("docker") + .setVersion("1.12.6") + .setEpoch("2") + .setRelease("71.git3e8e77d.el7.centos.1") + .setArchitecture("x86_64") + .build())); + assertFalse(yumPackage.isSubsetOf(new YumPackageName.Builder("docker") + .setVersion("1.13.1") + .build())); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java new file mode 100644 index 00000000000..27b23d26b24 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java @@ -0,0 +1,335 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.yum; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcessFailureException; +import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; + +/** + * @author hakonhall + */ +public class YumTest { + + private final TaskContext taskContext = mock(TaskContext.class); + private final TestTerminal terminal = new TestTerminal(); + private final Yum yum = new Yum(terminal); + + @AfterEach + public void after() { + terminal.verifyAllCommandsExecuted(); + } + + @Test + void testQueryInstalled() { + terminal.expectCommand( + "rpm -q docker --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", + 0, + "docker\n2\n1.13.1\n74.git6e3bb8e.el7.centos\nx86_64"); + + Optional<YumPackageName> installed = yum.queryInstalled(taskContext, "docker"); + + assertTrue(installed.isPresent()); + assertEquals("docker", installed.get().getName()); + assertEquals("2", installed.get().getEpoch().get()); + assertEquals("1.13.1", installed.get().getVersion().get()); + assertEquals("74.git6e3bb8e.el7.centos", installed.get().getRelease().get()); + assertEquals("x86_64", installed.get().getArchitecture().get()); + } + + @Test + void testQueryInstalledPartial() { + terminal.expectCommand( + "rpm -q vespa-node-admin --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", + 0, + "vespa-node-admin\n(none)\n6.283.62\n1.el7\nnoarch"); + + Optional<YumPackageName> installed = yum.queryInstalled(taskContext, "vespa-node-admin"); + + assertTrue(installed.isPresent()); + assertEquals("vespa-node-admin", installed.get().getName()); + assertEquals("0", installed.get().getEpoch().get()); + assertEquals("6.283.62", installed.get().getVersion().get()); + assertEquals("1.el7", installed.get().getRelease().get()); + assertEquals("noarch", installed.get().getArchitecture().get()); + } + + @Test + void testQueryNotInstalled() { + terminal.expectCommand( + "rpm -q fake-package --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", + 1, + "package fake-package is not installed"); + + Optional<YumPackageName> installed = yum.queryInstalled(taskContext, "fake-package"); + + assertFalse(installed.isPresent()); + } + + @Test + void testQueryInstalledMultiplePackages() { + terminal.expectCommand( + "rpm -q kernel-devel --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", + 0, + "kernel-devel\n" + + "(none)\n" + + "4.18.0\n" + + "305.7.1.el8_4\n" + + "x86_64\n" + + "kernel-devel\n" + + "(none)\n" + + "4.18.0\n" + + "240.15.1.el8_3\n" + + "x86_64\n"); + try { + yum.queryInstalled(taskContext, "kernel-devel"); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("Found multiple installed packages for 'kernel-devel'. Version is required to match package exactly", e.getMessage()); + } + } + + @Test + void testAlreadyInstalled() { + mockRpmQuery("package-1", null); + terminal.expectCommand( + "yum install --assumeyes --enablerepo=repo1 --enablerepo=repo2 --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", + 0, + "foobar\nNothing to do.\n"); // Note trailing dot + assertFalse(yum.install("package-1", "package-2") + .enableRepo("repo1", "repo2") + .converge(taskContext)); + } + + @Test + void testAlreadyUpgraded() { + terminal.expectCommand( + "yum upgrade --assumeyes --setopt skip_missing_names_on_update=False package-1 package-2 2>&1", + 0, + "foobar\nNothing to do.\n"); // Same message as yum install no-op + + assertFalse(yum.upgrade("package-1", "package-2") + .converge(taskContext)); + } + + @Test + void testAlreadyRemoved() { + mockRpmQuery("package-1", YumPackageName.fromString("package-1-1.2.3-1")); + terminal.expectCommand( + "yum remove --assumeyes package-1 package-2 2>&1", + 0, + "foobar\nNo packages marked for removal.\n"); // Different output + + assertFalse(yum.remove("package-1", "package-2") + .converge(taskContext)); + } + + @Test + void skipsYumRemoveNotInRpm() { + mockRpmQuery("package-1", null); + mockRpmQuery("package-2", null); + assertFalse(yum.remove("package-1", "package-2").converge(taskContext)); + } + + @Test + void testInstall() { + mockRpmQuery("package-1", null); + terminal.expectCommand( + "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", + 0, + "installing, installing"); + + assertTrue(yum + .install("package-1", "package-2") + .converge(taskContext)); + } + + @Test + void skipsYumInstallIfInRpm() { + mockRpmQuery("package-1-0:1.2.3-1", YumPackageName.fromString("package-1-1.2.3-1")); + mockRpmQuery("package-2", YumPackageName.fromString("1:package-2-1.2.3-1.el7.x86_64")); + assertFalse(yum.install("package-1-1.2.3-1", "package-2").converge(taskContext)); + } + + @Test + void testInstallWithEnablerepo() { + mockRpmQuery("package-1", null); + terminal.expectCommand( + "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", + 0, + "installing, installing"); + + assertTrue(yum + .install("package-1", "package-2") + .enableRepo("repo-name") + .converge(taskContext)); + } + + @Test + void testInstallWithEnablerepoDisablerepo() { + mockRpmQuery("package-1", null); + terminal.expectCommand( + "yum install --assumeyes \"--disablerepo=*\" --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", + 0, + "installing, installing"); + + assertTrue(yum + .install("package-1", "package-2") + .enableRepo("repo-name") + .disableRepo("*") + .converge(taskContext)); + } + + @Test + void testWithVersionLock() { + terminal.expectCommand("yum versionlock list 2>&1", + 0, + "Last metadata expiration check: 0:51:26 ago on Thu 14 Jan 2021 09:39:24 AM UTC.\n"); + terminal.expectCommand("yum versionlock add --assumeyes \"openssh-0:8.0p1-4.el8_1.*\" 2>&1"); + terminal.expectCommand( + "yum install --assumeyes openssh-0:8.0p1-4.el8_1.x86_64 2>&1", + 0, + "installing"); + + YumPackageName pkg = new YumPackageName + .Builder("openssh") + .setVersion("8.0p1") + .setRelease("4.el8_1") + .setArchitecture("x86_64") + .build(); + assertTrue(yum.installFixedVersion(pkg).converge(taskContext)); + } + + @Test + void testWithDifferentVersionLock() { + terminal.expectCommand("yum versionlock list 2>&1", + 0, + "Repository chef_rpms-release is listed more than once in the configuration\n" + + "chef-0:12.21.1-1.el7.*\n" + + "package-0:0.1-8.el7.*\n"); + + terminal.expectCommand("yum versionlock delete \"package-0:0.1-8.el7.*\" 2>&1"); + + terminal.expectCommand("yum versionlock add --assumeyes --enablerepo=somerepo \"package-0:0.10-654.el7.*\" 2>&1"); + + terminal.expectCommand( + "yum install --assumeyes --enablerepo=somerepo package-0:0.10-654.el7 2>&1", + 0, + "Nothing to do\n"); + + + assertTrue(yum + .installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")) + .enableRepo("somerepo") + .converge(taskContext)); + } + + @Test + void testWithExistingVersionLock() { + terminal.expectCommand("yum versionlock list 2>&1", + 0, + "Repository chef_rpms-release is listed more than once in the configuration\n" + + "chef-0:12.21.1-1.el7.*\n" + + "package-0:0.10-654.el7.*\n"); + terminal.expectCommand( + "yum install --assumeyes package-0:0.10-654.el7 2>&1", + 0, + "Nothing to do\n"); + + assertFalse(yum.installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")).converge(taskContext)); + } + + @Test + void testWithDowngrade() { + terminal.expectCommand("yum versionlock list 2>&1", + 0, + "Repository chef_rpms-release is listed more than once in the configuration\n" + + "chef-0:12.21.1-1.el7.*\n" + + "package-0:0.10-654.el7.*\n"); + + terminal.expectCommand( + "yum install --assumeyes package-0:0.10-654.el7 2>&1", + 0, + "Package matching package-=.0.10-654.el7 already installed. Checking for update.\n" + + "Nothing to do\n"); + + terminal.expectCommand("yum downgrade --assumeyes package-0:0.10-654.el7 2>&1"); + + assertTrue(yum.installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")).converge(taskContext)); + } + + @Test + void testFailedInstall() { + assertThrows(ChildProcessFailureException.class, () -> { + mockRpmQuery("package-1", null); + terminal.expectCommand( + "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", + 1, + "error"); + + yum + .install("package-1", "package-2") + .enableRepo("repo-name") + .converge(taskContext); + fail(); + }); + } + + @Test + void testUnknownPackages() { + mockRpmQuery("package-1", null); + terminal.expectCommand( + "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 package-3 2>&1", + 0, + "Loaded plugins: fastestmirror, langpacks\n" + + "Loading mirror speeds from cached hostfile\n" + + "No package package-1 available.\n" + + "No package package-2 available.\n" + + "Nothing to do\n"); + + var command = yum.install("package-1", "package-2", "package-3"); + try { + command.converge(taskContext); + fail(); + } catch (Exception e) { + assertNotNull(e.getCause()); + assertEquals("Unknown package: package-1", e.getCause().getMessage()); + } + } + + @Test + void throwIfNoPackagesSpecified() { + assertThrows(IllegalArgumentException.class, () -> { + yum.install(); + }); + } + + @Test + void allowToCallUpgradeWithNoPackages() { + terminal.expectCommand("yum upgrade --assumeyes 2>&1", 0, "OK"); + yum.upgrade().converge(taskContext); + } + + @Test + void testDeleteVersionLock() { + terminal.expectCommand("yum versionlock delete openssh-0:8.0p1-4.el8_1.x86_64 2>&1"); + + YumPackageName pkg = new YumPackageName + .Builder("openssh") + .setVersion("8.0p1") + .setRelease("4.el8_1") + .setArchitecture("x86_64") + .build(); + assertTrue(yum.deleteVersionLock(pkg).converge(taskContext)); + } + + private void mockRpmQuery(String packageName, YumPackageName installedOrNull) { + new YumTester(terminal).expectQueryInstalled(packageName).andReturn(installedOrNull); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java new file mode 100644 index 00000000000..aafa0fcfd72 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java @@ -0,0 +1,80 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.yum; + +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +public class YumTesterTest { + + private static final String[] packages = {"pkg1", "pkg2"}; + private static final String[] repos = {"repo1", "repo2"}; + private static final String[] disablerepos = {"disablerepo1", "disablerepo2"}; + private static final YumPackageName minimalPackage = YumPackageName.fromString("pkg-1.13.1-0.el7"); + private static final YumPackageName fullPackage = YumPackageName.fromString("2:pkg-1.13.1-0.el7.x86_64"); + + private final TestTerminal terminal = new TestTerminal(); + private final YumTester yum = new YumTester(terminal); + private final TestTaskContext context = new TestTaskContext(); + + @Test + void generic_yum_methods() { + assertYumMethod(yum -> yum.expectInstall(packages).withDisableRepo(disablerepos).withEnableRepo(repos), + yum -> yum.install(List.of(packages)).disableRepo(disablerepos).enableRepo(repos).converge(context)); + + assertYumMethod(yum -> yum.expectUpdate(packages).withDisableRepo(disablerepos).withEnableRepo(repos), + yum -> yum.upgrade(List.of(packages)).disableRepo(disablerepos).enableRepo(repos).converge(context)); + + assertYumMethod(yum -> yum.expectRemove(packages).withDisableRepo(disablerepos).withEnableRepo(repos), + yum -> yum.remove(List.of(packages)).disableRepo(disablerepos).enableRepo(repos).converge(context)); + + assertYumMethod(yum -> yum.expectInstallFixedVersion(minimalPackage.toName()).withDisableRepo(disablerepos).withEnableRepo(repos), + yum -> yum.installFixedVersion(minimalPackage).disableRepo(disablerepos).enableRepo(repos).converge(context)); + + // versionlock always returns success + assertYumMethodAlwaysSuccess(yum -> yum.expectDeleteVersionLock(minimalPackage.toName()), + yum -> yum.deleteVersionLock(minimalPackage).converge(context)); + + } + + @Test + void disable_other_repos() { + assertYumMethod(yum -> yum.expectInstall(packages).withDisableRepo("*").withEnableRepo(repos), + yum -> yum.install(List.of(packages)).disableRepo("*").enableRepo(repos).converge(context)); + } + + @Test + void expect_query_installed() { + yum.expectQueryInstalled(packages[0]).andReturn(fullPackage); + assertEquals(Optional.of(fullPackage), yum.queryInstalled(context, packages[0])); + terminal.verifyAllCommandsExecuted(); + } + + private void assertYumMethod(Function<YumTester, YumTester.GenericYumCommandExpectation> yumTesterExpectationFunction, + Function<Yum, Boolean> yumFunction) { + List.of(true, false).forEach(wantedReturnValue -> { + yumTesterExpectationFunction.apply(yum).andReturn(wantedReturnValue); + assertEquals(wantedReturnValue, yumFunction.apply(yum)); + terminal.verifyAllCommandsExecuted(); + }); + } + + private void assertYumMethodAlwaysSuccess(Function<YumTester, YumTester.GenericYumCommandExpectation> yumTesterExpectationFunction, + Function<Yum, Boolean> yumFunction) { + List.of(true, false).forEach(wantedReturnValue -> { + yumTesterExpectationFunction.apply(yum).andReturn(wantedReturnValue); + assertEquals(true, yumFunction.apply(yum)); + terminal.verifyAllCommandsExecuted(); + }); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java new file mode 100644 index 00000000000..7ac47aad1fa --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java @@ -0,0 +1,39 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.wireguard; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; +import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author gjoranv + */ +public class WireguardPeerTest { + + @Test + void peers_are_sorted_by_hostname_ascending() { + List<WireguardPeer> peers = Stream.of( + peer("b"), + peer("a"), + peer("c") + ).sorted().toList(); + + assertEquals("a", peers.get(0).hostname().value()); + assertEquals("b", peers.get(1).hostname().value()); + assertEquals("c", peers.get(2).hostname().value()); + } + + private static WireguardPeer peer(String hostname) { + return new WireguardPeer(HostName.of(hostname), List.of(VersionedIpAddress.from("::1:1")), + new WireguardKeyWithTimestamp(WireguardKey.generateRandomForTesting(), Instant.EPOCH)); + } + +} |