diff options
Diffstat (limited to 'node-admin/src/test/java')
12 files changed, 115 insertions, 203 deletions
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java index a2dbfe0db4b..8683c2cb417 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java @@ -6,7 +6,6 @@ import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; import java.util.List; @@ -34,7 +33,7 @@ public class ContainerFailTest { .build(); tester.addChildNodeRepositoryNode(nodeSpec); - NodeAgentContext context = NodeAgentContextImpl.builder(nodeSpec).fileSystem(TestFileSystem.create()).build(); + NodeAgentContext context = NodeAgentContextImpl.builder(nodeSpec).build(); tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any(), any()); tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index cd020d81450..b7b247bbf87 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.After; @@ -51,8 +50,9 @@ public class StorageMaintainerTest { fileSystem.getPath("/data/vespa/storage/container-archive")); @Test - public void testDiskUsed() { + public void testDiskUsed() throws IOException { NodeAgentContext context = NodeAgentContextImpl.builder("host-1.domain.tld").fileSystem(fileSystem).build(); + Files.createDirectories(context.pathOnHostFromPathInNode("/")); terminal.expectCommand("du -xsk /data/vespa/storage/host-1 2>&1", 0, "321\t/data/vespa/storage/host-1/"); assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); @@ -76,7 +76,7 @@ public class StorageMaintainerTest { Path pathToArchiveDir = fileSystem.getPath("/data/vespa/storage/container-archive"); Files.createDirectories(pathToArchiveDir); - Path containerStorageRoot = context1.containerPath("/").pathOnHost().getParent(); + Path containerStorageRoot = context1.pathOnHostFromPathInNode("/").getParent(); Set<String> containerStorageRootContentsBeforeArchive = FileFinder.from(containerStorageRoot) .maxDepth(1) .stream() @@ -115,21 +115,21 @@ public class StorageMaintainerTest { NodeAgentContext context = NodeAgentContextImpl.builder(containerName + ".domain.tld") .fileSystem(fileSystem).build(); - ContainerPath containerVespaHome = context.containerPathUnderVespaHome(""); - Files.createDirectories(context.containerPath("/etc/something")); - Files.createFile(context.containerPath("/etc/something/conf")); + Path containerVespaHomeOnHost = context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("")); + Files.createDirectories(context.pathOnHostFromPathInNode("/etc/something")); + Files.createFile(context.pathOnHostFromPathInNode("/etc/something/conf")); - Files.createDirectories(containerVespaHome.resolve("logs/vespa")); - Files.createFile(containerVespaHome.resolve("logs/vespa/vespa.log")); - Files.createFile(containerVespaHome.resolve("logs/vespa/zookeeper.log")); + Files.createDirectories(containerVespaHomeOnHost.resolve("logs/vespa")); + Files.createFile(containerVespaHomeOnHost.resolve("logs/vespa/vespa.log")); + Files.createFile(containerVespaHomeOnHost.resolve("logs/vespa/zookeeper.log")); - Files.createDirectories(containerVespaHome.resolve("var/db")); - Files.createFile(containerVespaHome.resolve("var/db/some-file")); + Files.createDirectories(containerVespaHomeOnHost.resolve("var/db")); + Files.createFile(containerVespaHomeOnHost.resolve("var/db/some-file")); - ContainerPath containerRoot = context.containerPath("/"); - Set<String> actualContents = FileFinder.files(containerRoot) + Path containerRootOnHost = context.pathOnHostFromPathInNode("/"); + Set<String> actualContents = FileFinder.files(containerRootOnHost) .stream() - .map(fileAttributes -> containerRoot.relativize(fileAttributes.path()).toString()) + .map(fileAttributes -> containerRootOnHost.relativize(fileAttributes.path()).toString()) .collect(Collectors.toSet()); Set<String> expectedContents = Set.of( "etc/something/conf", @@ -145,6 +145,7 @@ public class StorageMaintainerTest { NodeAgentContext context = NodeAgentContextImpl.builder( NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build()) .fileSystem(fileSystem).build(); + Files.createDirectories(context.pathOnHostFromPathInNode("/")); mockDiskUsage(500L); storageMaintainer.cleanDiskIfFull(context); @@ -157,6 +158,7 @@ public class StorageMaintainerTest { NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build()) .fileSystem(fileSystem).build(); + Files.createDirectories(context.pathOnHostFromPathInNode("/")); mockDiskUsage(950_000L); storageMaintainer.cleanDiskIfFull(context); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java index ded99cf3778..eacaa038194 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -4,11 +4,10 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; -import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; +import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -28,10 +27,9 @@ public class CoreCollectorTest { private final String JDK_PATH = "/path/to/jdk/java"; private final ContainerOperations docker = mock(ContainerOperations.class); private final CoreCollector coreCollector = new CoreCollector(docker); - private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld") - .fileSystem(TestFileSystem.create()).build(); + private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld").build(); - private final ContainerPath TEST_CORE_PATH = context.containerPath("/tmp/core.1234"); + private final Path TEST_CORE_PATH = Path.of("/tmp/core.1234"); private final String TEST_BIN_PATH = "/usr/bin/program"; private final List<String> GDB_BACKTRACE = List.of("[New Thread 2703]", "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.", @@ -40,7 +38,7 @@ public class CoreCollectorTest { @Test public void extractsBinaryPathTest() { - final String[] cmd = {"file", TEST_CORE_PATH.pathInContainer()}; + final String[] cmd = {"file", TEST_CORE_PATH.toString()}; mockExec(cmd, "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + @@ -128,7 +126,7 @@ public class CoreCollectorTest { @Test public void collectsDataTest() { - mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, + mockExec(new String[]{"file", TEST_CORE_PATH.toString()}, "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + "'/usr/bin/program'"); mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT9}, "", "stat: No such file or directory"); @@ -148,7 +146,7 @@ public class CoreCollectorTest { @Test public void collectsPartialIfBacktraceFailsTest() { - mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, + mockExec(new String[]{"file", TEST_CORE_PATH.toString()}, "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + "'/usr/bin/program'"); mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT9}, "The stat output"); @@ -161,7 +159,7 @@ public class CoreCollectorTest { @Test public void reportsJstackInsteadOfGdbForJdkCores() { - mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, + mockExec(new String[]{"file", TEST_CORE_PATH.toString()}, "dump.core.5954: ELF 64-bit LSB core file x86-64, version 1 (SYSV), too many program header sections (33172)"); mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT9}, "", "stat: No such file or directory"); @@ -182,7 +180,7 @@ public class CoreCollectorTest { @Test public void metadata_for_java_heap_dump() { - assertEquals(JAVA_HEAP_DUMP_METADATA, coreCollector.collect(context, context.containerPath("/dump_java_pid123.hprof"))); + assertEquals(JAVA_HEAP_DUMP_METADATA, coreCollector.collect(context, Path.of("dump_java_pid123.hprof"))); } private void mockExec(String[] cmd, String output) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index 9f507f451b9..31883311e33 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import com.yahoo.vespa.test.file.TestFileSystem; @@ -48,7 +47,7 @@ public class CoredumpHandlerTest { private final FileSystem fileSystem = TestFileSystem.create(); private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld") .fileSystem(fileSystem).build(); - private final ContainerPath containerCrashPath = context.containerPath("/var/crash"); + private final Path crashPathInContainer = fileSystem.getPath("/var/crash"); private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps"); private final TestTerminal terminal = new TestTerminal(); @@ -59,38 +58,38 @@ public class CoredumpHandlerTest { @SuppressWarnings("unchecked") private final Supplier<String> coredumpIdSupplier = mock(Supplier.class); private final CoredumpHandler coredumpHandler = new CoredumpHandler(terminal, coreCollector, coredumpReporter, - containerCrashPath.pathInContainer(), doneCoredumpsPath, 100, metrics, clock, coredumpIdSupplier); + crashPathInContainer.toString(), doneCoredumpsPath, 100, metrics, clock, coredumpIdSupplier); @Test public void coredump_enqueue_test() throws IOException { - ContainerPath crashPath = context.containerPath("/some/crash/path"); - ContainerPath processingDir = context.containerPath("/some/other/processing"); + final Path crashPathOnHost = fileSystem.getPath("/home/docker/container-1/some/crash/path"); + final Path processingDir = fileSystem.getPath("/home/docker/container-1/some/other/processing"); - Files.createDirectories(crashPath); - createFileAged(crashPath.resolve("bash.core.431"), Duration.ZERO); + Files.createDirectories(crashPathOnHost); + createFileAged(crashPathOnHost.resolve("bash.core.431"), Duration.ZERO); - assertFolderContents(crashPath, "bash.core.431"); - Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(crashPath, processingDir); + assertFolderContents(crashPathOnHost, "bash.core.431"); + Optional<Path> enqueuedPath = coredumpHandler.enqueueCoredump(crashPathOnHost, processingDir); assertEquals(Optional.empty(), enqueuedPath); // bash.core.431 finished writing... and 2 more have since been written clock.advance(Duration.ofMinutes(3)); - createFileAged(crashPath.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); - createFileAged(crashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); + createFileAged(crashPathOnHost.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); + createFileAged(crashPathOnHost.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321"); - enqueuedPath = coredumpHandler.enqueueCoredump(crashPath, processingDir); + enqueuedPath = coredumpHandler.enqueueCoredump(crashPathOnHost, processingDir); assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); - assertFolderContents(crashPath, "bash.core.431", "vespa-slobrok.core.673"); + assertFolderContents(crashPathOnHost, "bash.core.431", "vespa-slobrok.core.673"); assertFolderContents(processingDir, "id-123"); assertFolderContents(processingDir.resolve("id-123"), "dump_vespa-proton.core.119"); verify(coredumpIdSupplier, times(1)).get(); // Enqueue another - enqueuedPath = coredumpHandler.enqueueCoredump(crashPath, processingDir); + enqueuedPath = coredumpHandler.enqueueCoredump(crashPathOnHost, processingDir); assertEquals(Optional.of(processingDir.resolve("id-321")), enqueuedPath); - assertFolderContents(crashPath, "bash.core.431"); + assertFolderContents(crashPathOnHost, "bash.core.431"); assertFolderContents(processingDir, "id-123", "id-321"); assertFolderContents(processingDir.resolve("id-321"), "dump_vespa-slobrok.core.673"); verify(coredumpIdSupplier, times(2)).get(); @@ -98,45 +97,46 @@ public class CoredumpHandlerTest { @Test public void enqueue_with_hs_err_files() throws IOException { - ContainerPath crashPath = context.containerPath("/some/crash/path"); - ContainerPath processingDir = context.containerPath("/some/other/processing"); - Files.createDirectories(crashPath); + final Path crashPathOnHost = fileSystem.getPath("/home/docker/container-1/some/crash/path"); + final Path processingDir = fileSystem.getPath("/home/docker/container-1/some/other/processing"); + Files.createDirectories(crashPathOnHost); - createFileAged(crashPath.resolve("java.core.69"), Duration.ofSeconds(515)); - createFileAged(crashPath.resolve("hs_err_pid69.log"), Duration.ofSeconds(520)); + createFileAged(crashPathOnHost.resolve("java.core.69"), Duration.ofSeconds(515)); + createFileAged(crashPathOnHost.resolve("hs_err_pid69.log"), Duration.ofSeconds(520)); - createFileAged(crashPath.resolve("java.core.2420"), Duration.ofSeconds(540)); - createFileAged(crashPath.resolve("hs_err_pid2420.log"), Duration.ofSeconds(549)); - createFileAged(crashPath.resolve("hs_err_pid2421.log"), Duration.ofSeconds(550)); + createFileAged(crashPathOnHost.resolve("java.core.2420"), Duration.ofSeconds(540)); + createFileAged(crashPathOnHost.resolve("hs_err_pid2420.log"), Duration.ofSeconds(549)); + createFileAged(crashPathOnHost.resolve("hs_err_pid2421.log"), Duration.ofSeconds(550)); when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321"); - Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(crashPath, processingDir); + Optional<Path> enqueuedPath = coredumpHandler.enqueueCoredump(crashPathOnHost, processingDir); assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); - assertFolderContents(crashPath, "hs_err_pid69.log", "java.core.69"); + assertFolderContents(crashPathOnHost, "hs_err_pid69.log", "java.core.69"); assertFolderContents(processingDir, "id-123"); assertFolderContents(processingDir.resolve("id-123"), "hs_err_pid2420.log", "hs_err_pid2421.log", "dump_java.core.2420"); } @Test public void coredump_to_process_test() throws IOException { - ContainerPath processingDir = context.containerPath("/some/other/processing"); + final Path crashPathOnHost = fileSystem.getPath("/home/docker/container-1/some/crash/path"); + final Path processingDir = fileSystem.getPath("/home/docker/container-1/some/other/processing"); // Initially there are no core dumps - Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(containerCrashPath, processingDir); + Optional<Path> enqueuedPath = coredumpHandler.enqueueCoredump(crashPathOnHost, processingDir); assertEquals(Optional.empty(), enqueuedPath); // 3 core dumps occur - Files.createDirectories(containerCrashPath); - createFileAged(containerCrashPath.resolve("bash.core.431"), Duration.ZERO); - createFileAged(containerCrashPath.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); - createFileAged(containerCrashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); + Files.createDirectories(crashPathOnHost); + createFileAged(crashPathOnHost.resolve("bash.core.431"), Duration.ZERO); + createFileAged(crashPathOnHost.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); + createFileAged(crashPathOnHost.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); when(coredumpIdSupplier.get()).thenReturn("id-123"); - enqueuedPath = coredumpHandler.getCoredumpToProcess(containerCrashPath, processingDir); + enqueuedPath = coredumpHandler.getCoredumpToProcess(crashPathOnHost, processingDir); assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); // Running this again wont enqueue new core dumps as we are still processing the one enqueued previously - enqueuedPath = coredumpHandler.getCoredumpToProcess(containerCrashPath, processingDir); + enqueuedPath = coredumpHandler.getCoredumpToProcess(crashPathOnHost, processingDir); assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); verify(coredumpIdSupplier, times(1)).get(); } @@ -164,10 +164,11 @@ public class CoredumpHandlerTest { "}}"; - ContainerPath coredumpDirectory = context.containerPath("/var/crash/id-123"); - Files.createDirectories(coredumpDirectory.pathOnHost()); + Path coredumpDirectoryInContainer = fileSystem.getPath("/var/crash/id-123"); + Path coredumpDirectory = context.pathOnHostFromPathInNode(coredumpDirectoryInContainer); + Files.createDirectories(coredumpDirectory); Files.createFile(coredumpDirectory.resolve("dump_core.456")); - when(coreCollector.collect(eq(context), eq(coredumpDirectory.resolve("dump_core.456")))) + when(coreCollector.collect(eq(context), eq(coredumpDirectoryInContainer.resolve("dump_core.456")))) .thenReturn(metadata); assertEquals(expectedMetadataStr, coredumpHandler.getMetadata(context, coredumpDirectory, () -> attributes)); @@ -180,12 +181,12 @@ public class CoredumpHandlerTest { @Test(expected = IllegalStateException.class) public void cant_get_metadata_if_no_core_file() throws IOException { - coredumpHandler.getMetadata(context, context.containerPath("/fake/path"), Map::of); + coredumpHandler.getMetadata(context, fileSystem.getPath("/fake/path"), Map::of); } @Test(expected = IllegalStateException.class) public void fails_to_get_core_file_if_only_compressed() throws IOException { - ContainerPath coredumpDirectory = context.containerPath("/path/to/coredump/proccessing/id-123"); + Path coredumpDirectory = fileSystem.getPath("/path/to/coredump/proccessing/id-123"); Files.createDirectories(coredumpDirectory); Files.createFile(coredumpDirectory.resolve("dump_bash.core.431.lz4")); coredumpHandler.findCoredumpFileInProcessingDirectory(coredumpDirectory); @@ -193,14 +194,14 @@ public class CoredumpHandlerTest { @Test public void process_single_coredump_test() throws IOException { - ContainerPath coredumpDirectory = context.containerPath("/path/to/coredump/proccessing/id-123"); + Path coredumpDirectory = fileSystem.getPath("/path/to/coredump/proccessing/id-123"); Files.createDirectories(coredumpDirectory); Files.write(coredumpDirectory.resolve("metadata.json"), "metadata".getBytes()); Files.createFile(coredumpDirectory.resolve("dump_bash.core.431")); assertFolderContents(coredumpDirectory, "metadata.json", "dump_bash.core.431"); - terminal.interceptCommand("/usr/bin/lz4 -f /data/vespa/storage/container-123/path/to/coredump/proccessing/id-123/dump_bash.core.431 " + - "/data/vespa/storage/container-123/path/to/coredump/proccessing/id-123/dump_bash.core.431.lz4 2>&1", + terminal.interceptCommand("/usr/bin/lz4 -f /path/to/coredump/proccessing/id-123/dump_bash.core.431 " + + "/path/to/coredump/proccessing/id-123/dump_bash.core.431.lz4 2>&1", commandLine -> { uncheck(() -> Files.createFile(fileSystem.getPath(commandLine.getArguments().get(3)))); return new TestChildProcess2(0, ""); @@ -215,10 +216,10 @@ public class CoredumpHandlerTest { @Test public void report_enqueued_and_processed_metrics() throws IOException { - Path processingPath = containerCrashPath.resolve("processing"); - Files.createFile(containerCrashPath.resolve("dump-1")); - Files.createFile(containerCrashPath.resolve("dump-2")); - Files.createFile(containerCrashPath.resolve("hs_err_pid2.log")); + Path processingPath = crashPathInContainer.resolve("processing"); + Files.createFile(crashPathInContainer.resolve("dump-1")); + Files.createFile(crashPathInContainer.resolve("dump-2")); + Files.createFile(crashPathInContainer.resolve("hs_err_pid2.log")); Files.createDirectory(processingPath); Files.createFile(processingPath.resolve("metadata.json")); Files.createFile(processingPath.resolve("dump-3")); @@ -227,7 +228,7 @@ public class CoredumpHandlerTest { .createParents() .createNewFile(); - coredumpHandler.updateMetrics(context, containerCrashPath); + coredumpHandler.updateMetrics(context, crashPathInContainer); List<DimensionMetrics> updatedMetrics = metrics.getMetricsByType(Metrics.DimensionType.PRETAGGED); assertEquals(1, updatedMetrics.size()); Map<String, Number> values = updatedMetrics.get(0).getMetrics(); @@ -237,7 +238,7 @@ public class CoredumpHandlerTest { @Before public void setup() throws IOException { - Files.createDirectories(containerCrashPath.pathOnHost()); + Files.createDirectories(crashPathInContainer); } @After diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index f8dc05b5eda..232b95f5ede 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -5,7 +5,6 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; import org.mockito.InOrder; @@ -155,7 +154,7 @@ public class NodeAdminImplTest { } private NodeAgentContext createNodeAgentContext(String hostname) { - return NodeAgentContextImpl.builder(hostname).fileSystem(TestFileSystem.create()).build(); + return NodeAgentContextImpl.builder(hostname).build(); } private NodeAgentWithScheduler mockNodeAgentWithSchedulerFactory(NodeAgentContext context) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java index 1c439234a34..b91afd0bfef 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; import java.nio.file.FileSystem; -import java.nio.file.Path; import java.util.List; import static org.junit.Assert.assertEquals; @@ -26,54 +25,54 @@ public class NodeAgentContextImplTest { public void path_on_host_from_path_in_node_test() { assertEquals( "/data/vespa/storage/container-1", - context.containerPath("/").pathOnHost().toString()); + context.pathOnHostFromPathInNode("/").toString()); assertEquals( "/data/vespa/storage/container-1/dev/null", - context.containerPath("/dev/null").pathOnHost().toString()); + context.pathOnHostFromPathInNode("/dev/null").toString()); } @Test(expected=IllegalArgumentException.class) public void path_in_container_must_be_absolute() { - context.containerPath("some/relative/path"); + context.pathOnHostFromPathInNode("some/relative/path"); } @Test public void path_in_node_from_path_on_host_test() { assertEquals( "/dev/null", - context.containerPathFromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-1/dev/null")).pathInContainer()); + context.pathInNodeFromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-1/dev/null")).toString()); } @Test(expected=IllegalArgumentException.class) public void path_on_host_must_be_absolute() { - context.containerPathFromPathOnHost(Path.of("some/relative/path")); + context.pathInNodeFromPathOnHost("some/relative/path"); } @Test(expected=IllegalArgumentException.class) public void path_on_host_must_be_inside_container_storage_of_context() { - context.containerPathFromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-2/dev/null")); + context.pathInNodeFromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-2/dev/null")); } @Test(expected=IllegalArgumentException.class) public void path_on_host_must_be_inside_container_storage() { - context.containerPathFromPathOnHost(fileSystem.getPath("/home")); + context.pathInNodeFromPathOnHost(fileSystem.getPath("/home")); } @Test public void path_under_vespa_host_in_container_test() { assertEquals( "/opt/vespa", - context.containerPathUnderVespaHome("").pathInContainer()); + context.pathInNodeUnderVespaHome("").toString()); assertEquals( "/opt/vespa/logs/vespa/vespa.log", - context.containerPathUnderVespaHome("logs/vespa/vespa.log").pathInContainer()); + context.pathInNodeUnderVespaHome("logs/vespa/vespa.log").toString()); } @Test(expected=IllegalArgumentException.class) public void path_under_vespa_home_must_be_relative() { - context.containerPathUnderVespaHome("/home"); + context.pathInNodeUnderVespaHome("/home"); } @Test @@ -87,9 +86,9 @@ public class NodeAgentContextImplTest { assertTrue(context2.isDisabled(NodeAgentTask.CoreDumps)); } - private NodeAgentContext createContextWithDisabledTasks(String... tasks) { + private static NodeAgentContext createContextWithDisabledTasks(String... tasks) { InMemoryFlagSource flagSource = new InMemoryFlagSource(); flagSource.withListFlag(PermanentFlags.DISABLED_HOST_ADMIN_TASKS.id(), List.of(tasks), String.class); - return NodeAgentContextImpl.builder("node123").fileSystem(fileSystem).flagSource(flagSource).build(); + return NodeAgentContextImpl.builder("node123").flagSource(flagSource).build(); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java index 197b1c81b57..51fedd54381 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; -import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; import java.time.Clock; @@ -143,7 +142,7 @@ public class NodeAgentContextManagerTest { } private static NodeAgentContext generateContext() { - return NodeAgentContextImpl.builder("container-123.domain.tld").fileSystem(TestFileSystem.create()).build(); + return NodeAgentContextImpl.builder("container-123.domain.tld").build(); } private static class AsyncExecutor<T> { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 1c6831e6379..8764db502bb 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -27,12 +27,10 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Before; import org.junit.Test; import org.mockito.InOrder; -import java.nio.file.FileSystem; import java.time.Duration; import java.time.Instant; import java.util.List; @@ -74,7 +72,6 @@ public class NodeAgentImplTest { private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final ManualClock clock = new ManualClock(Instant.now()); - private final FileSystem fileSystem = TestFileSystem.create(); @Before public void setUp() { @@ -236,7 +233,7 @@ public class NodeAgentImplTest { nodeAgent.doConverge(secondContext); inOrder.verify(orchestrator, never()).resume(any(String.class)); - NodeAgentContext thirdContext = NodeAgentContextImpl.builder(specBuilder.vcpu(5).build()).fileSystem(fileSystem).cpuSpeedUp(1.25).build(); + NodeAgentContext thirdContext = NodeAgentContextImpl.builder(specBuilder.vcpu(5).build()).cpuSpeedUp(1.25).build(); nodeAgent.doConverge(thirdContext); ContainerResources resourcesAfterThird = ContainerResources.from(0, 4, 16); mockGetContainer(dockerImage, resourcesAfterThird, true); @@ -780,7 +777,7 @@ public class NodeAgentImplTest { } private NodeAgentContext createContext(NodeSpec nodeSpec) { - return NodeAgentContextImpl.builder(nodeSpec).fileSystem(fileSystem).build(); + return NodeAgentContextImpl.builder(nodeSpec).build(); } private NodeSpec.Builder nodeBuilder(NodeState state) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java deleted file mode 100644 index 73b59a17c37..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author valerijf - */ -class UserNamespaceTest { - - private final UserNamespace userNamespace = new UserNamespace(1000, 2000, "vespa", "users", 1000, 100); - - @Test - public void translates_between_ids() { - assertEquals(1001, userNamespace.userIdOnHost(1)); - assertEquals(2001, userNamespace.groupIdOnHost(1)); - assertEquals(1, userNamespace.userIdInContainer(1001)); - assertEquals(1, userNamespace.groupIdInContainer(2001)); - - assertEquals(userNamespace.overflowId(), userNamespace.userIdInContainer(1)); - assertEquals(userNamespace.overflowId(), userNamespace.userIdInContainer(999999)); - - assertThrows(IllegalArgumentException.class, () -> userNamespace.userIdOnHost(-1)); - assertThrows(IllegalArgumentException.class, () -> userNamespace.userIdOnHost(70_000)); - } -}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java index a5fc6a1373f..eb15450b756 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java @@ -1,20 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.fs; -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; import java.util.Map; +import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.OVERFLOW_ID; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -24,8 +21,7 @@ class ContainerFileSystemTest { private final FileSystem fileSystem = TestFileSystem.create(); private final UnixPath containerRootOnHost = new UnixPath(fileSystem.getPath("/data/storage/ctr1")); - private final UserNamespace userNamespace = new UserNamespace(10_000, 11_000, "vespa", "users", 1000, 100); - private final ContainerFileSystem containerFs = ContainerFileSystem.create(containerRootOnHost.createDirectories().toPath(), userNamespace); + private final ContainerFileSystem containerFs = ContainerFileSystem.create(containerRootOnHost.createDirectories().toPath(), 10_000, 11_000); @Test public void creates_files_and_directories_with_container_root_as_owner() throws IOException { @@ -45,47 +41,17 @@ class ContainerFileSystemTest { } @Test - public void file_write_and_read() throws IOException { - ContainerPath containerPath = ContainerPath.fromPathInContainer(containerFs, Path.of("/file")); - UnixPath unixPath = new UnixPath(containerPath); - unixPath.writeUtf8File("hello"); - assertOwnership(containerPath, 0, 0, 10000, 11000); - - unixPath.setOwnerId(500).setGroupId(200); - assertOwnership(containerPath, 500, 200, 10500, 11200); - Files.write(containerPath, " world".getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); - assertOwnership(containerPath, 500, 200, 10500, 11200); // Owner should not have been updated as the file already existed - - assertEquals("hello world", unixPath.readUtf8File()); - } - - @Test public void copy() throws IOException { UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest")); // If file is copied to JimFS path, the UID/GIDs are not fixed Files.copy(hostFile.toPath(), destination.pathOnHost()); - assertEquals(String.valueOf(userNamespace.overflowId()), Files.getOwner(destination).getName()); + assertEquals(String.valueOf(OVERFLOW_ID), Files.getOwner(destination).getName()); Files.delete(destination); Files.copy(hostFile.toPath(), destination); assertOwnership(destination, 0, 0, 10000, 11000); - - // Set owner + group on both source host file and destination container file - hostFile.setOwnerId(5).setGroupId(10); - new UnixPath(destination).setOwnerId(500).setGroupId(200); - assertOwnership(destination, 500, 200, 10500, 11200); - // Copy the host file to destination again with COPY_ATTRIBUTES and REPLACE_EXISTING - Files.copy(hostFile.toPath(), destination, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - // The destination is recreated, so the owner should be root - assertOwnership(destination, 0, 0, 10000, 11000); - - // Set owner + group and copy within ContainerFS - new UnixPath(destination).setOwnerId(500).setGroupId(200); - ContainerPath destination2 = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest2")); - Files.copy(destination, destination2, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - assertOwnership(destination2, 0, 0, 10000, 11000); } @Test @@ -95,41 +61,12 @@ class ContainerFileSystemTest { // If file is moved to JimFS path, the UID/GIDs are not fixed Files.move(hostFile.toPath(), destination.pathOnHost()); - assertEquals(String.valueOf(userNamespace.overflowId()), Files.getOwner(destination).getName()); + assertEquals(String.valueOf(OVERFLOW_ID), Files.getOwner(destination).getName()); Files.delete(destination); hostFile.createNewFile(); Files.move(hostFile.toPath(), destination); assertOwnership(destination, 0, 0, 10000, 11000); - - // Set owner + group on both source host file and destination container file - hostFile.createNewFile(); - hostFile.setOwnerId(5).setGroupId(10); - new UnixPath(destination).setOwnerId(500).setGroupId(200); - assertOwnership(destination, 500, 200, 10500, 11200); - // Move the host file to destination again with COPY_ATTRIBUTES and REPLACE_EXISTING - Files.move(hostFile.toPath(), destination, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - // The destination is recreated, so the owner should be root - assertOwnership(destination, 0, 0, 10000, 11000); - - // Set owner + group and move within ContainerFS - new UnixPath(destination).setOwnerId(500).setGroupId(200); - ContainerPath destination2 = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest2")); - Files.move(destination, destination2, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - assertOwnership(destination2, 0, 0, 10000, 11000); - } - - @Test - public void symlink() throws IOException { - ContainerPath source = ContainerPath.fromPathInContainer(containerFs, Path.of("/src")); - // Symlink from ContainerPath to some relative path (different FS provider) - Files.createSymbolicLink(source, fileSystem.getPath("../relative/target")); - assertEquals(fileSystem.getPath("../relative/target"), Files.readSymbolicLink(source)); - Files.delete(source); - - // Symlinks from ContainerPath to a ContainerPath: Target is resolved within container with base FS provider - Files.createSymbolicLink(source, ContainerPath.fromPathInContainer(containerFs, Path.of("/path/in/container"))); - assertEquals(fileSystem.getPath("/path/in/container"), Files.readSymbolicLink(source)); } private static void assertOwnership(ContainerPath path, int contUid, int contGid, int hostUid, int hostGid) throws IOException { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java index 6bca8c2f0b1..ebbbaf3b525 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.fs; -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -13,7 +12,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; import java.io.IOException; import java.nio.file.FileSystem; @@ -27,7 +25,7 @@ import java.nio.file.Path; class ContainerPathTest { private final FileSystem baseFs = TestFileSystem.create(); - private final ContainerFileSystem containerFs = ContainerFileSystem.create(baseFs.getPath("/data/storage/ctr1"), mock(UserNamespace.class)); + private final ContainerFileSystem containerFs = ContainerFileSystem.create(baseFs.getPath("/data/storage/ctr1"), 0, 0); @Test public void create_new_container_path() { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java index bc26cfa73f3..a459c24049e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.fs; -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Test; @@ -18,22 +17,35 @@ import static org.junit.jupiter.api.Assertions.assertThrows; */ class ContainerUserPrincipalLookupServiceTest { - private final UserNamespace userNamespace = new UserNamespace(10_000, 11_000, "vespa", "users", 1000, 100); private final ContainerUserPrincipalLookupService userPrincipalLookupService = - new ContainerUserPrincipalLookupService(TestFileSystem.create().getUserPrincipalLookupService(), userNamespace); + new ContainerUserPrincipalLookupService(TestFileSystem.create().getUserPrincipalLookupService(), 1000, 2000); @Test public void correctly_resolves_ids() throws IOException { ContainerUserPrincipal user = userPrincipalLookupService.lookupPrincipalByName("1000"); assertEquals("vespa", user.getName()); - assertEquals("11000", user.baseFsPrincipal().getName()); + assertEquals("2000", user.baseFsPrincipal().getName()); assertEquals(user, userPrincipalLookupService.lookupPrincipalByName("vespa")); - ContainerGroupPrincipal group = userPrincipalLookupService.lookupPrincipalByGroupName("100"); - assertEquals("users", group.getName()); - assertEquals("11100", group.baseFsPrincipal().getName()); - assertEquals(group, userPrincipalLookupService.lookupPrincipalByGroupName("users")); + ContainerGroupPrincipal group = userPrincipalLookupService.lookupPrincipalByGroupName("1000"); + assertEquals("vespa", group.getName()); + assertEquals("3000", group.baseFsPrincipal().getName()); + assertEquals(group, userPrincipalLookupService.lookupPrincipalByGroupName("vespa")); assertThrows(UserPrincipalNotFoundException.class, () -> userPrincipalLookupService.lookupPrincipalByName("test")); } -} + + @Test + public void translates_between_ids() { + assertEquals(1001, userPrincipalLookupService.containerUidToHostUid(1)); + assertEquals(2001, userPrincipalLookupService.containerGidToHostGid(1)); + assertEquals(1, userPrincipalLookupService.hostUidToContainerUid(1001)); + assertEquals(1, userPrincipalLookupService.hostGidToContainerGid(2001)); + + assertEquals(65_534, userPrincipalLookupService.hostUidToContainerUid(1)); + assertEquals(65_534, userPrincipalLookupService.hostUidToContainerUid(999999)); + + assertThrows(IllegalArgumentException.class, () -> userPrincipalLookupService.containerUidToHostUid(-1)); + assertThrows(IllegalArgumentException.class, () -> userPrincipalLookupService.containerUidToHostUid(70_000)); + } +}
\ No newline at end of file |