diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2021-02-17 15:13:31 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-02-17 15:13:31 +0100 |
commit | 49adf2dd1f96f8e5636f5b3ee80fdf56c96aff86 (patch) | |
tree | d304949e26ffedd95f797985079d37c20eb9f294 /node-admin/src/test | |
parent | 444da656dd308fe6f44cc73aa0638eac47532968 (diff) | |
parent | dc02c144a1c8218e499b0b074beb30f799c726cd (diff) |
Merge pull request #16540 from vespa-engine/freva/sync-logs-s3
Sync tenant node logs to S3
Diffstat (limited to 'node-admin/src/test')
2 files changed, 152 insertions, 177 deletions
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index cb615a94615..a17ffadcb45 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -3,8 +3,11 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; import com.yahoo.config.provision.NodeResources; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup; +import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; @@ -13,8 +16,6 @@ import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.After; import org.junit.Test; -import org.junit.experimental.runners.Enclosed; -import org.junit.runner.RunWith; import java.io.IOException; import java.nio.file.FileSystem; @@ -38,150 +39,141 @@ import static org.mockito.Mockito.verifyNoInteractions; /** * @author dybis */ -@RunWith(Enclosed.class) public class StorageMaintainerTest { - public static class DiskUsageTests { - - private final TestTerminal terminal = new TestTerminal(); - - @Test - public void testDiskUsed() throws IOException { - StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, null, null); - FileSystem fileSystem = TestFileSystem.create(); - NodeAgentContext context = new NodeAgentContextImpl.Builder("host-1.domain.tld").fileSystem(fileSystem).build(); - Files.createDirectories(context.pathOnHostFromPathInNode("/")); + private final TestTerminal terminal = new TestTerminal(); + private final CoredumpHandler coredumpHandler = mock(CoredumpHandler.class); + private final DiskCleanup diskCleanup = mock(DiskCleanup.class); + private final SyncClient syncClient = mock(SyncClient.class); + private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(1234567890)); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private final FileSystem fileSystem = TestFileSystem.create(); + private final StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, coredumpHandler, diskCleanup, syncClient, clock, + fileSystem.getPath("/home/docker/container-storage/container-archive"), flagSource); + + @Test + public void testDiskUsed() throws IOException { + NodeAgentContext context = new NodeAgentContextImpl.Builder("host-1.domain.tld").fileSystem(fileSystem).build(); + Files.createDirectories(context.pathOnHostFromPathInNode("/")); + + terminal.expectCommand("du -xsk /home/docker/container-storage/host-1 2>&1", 0, "321\t/home/docker/container-storage/host-1/"); + assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); + + // Value should still be cached, no new execution against the terminal + assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); + } - terminal.expectCommand("du -xsk /home/docker/container-storage/host-1 2>&1", 0, "321\t/home/docker/container-storage/host-1/"); - assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); + @Test + public void testNonExistingDiskUsed() { + DiskSize size = storageMaintainer.getDiskUsed(null, Paths.get("/fake/path")); + assertEquals(DiskSize.ZERO, size); + } - // Value should still be cached, no new execution against the terminal - assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); - } + @Test + public void archive_container_data_test() throws IOException { + // Create some files in containers + NodeAgentContext context1 = createNodeAgentContextAndContainerStorage(fileSystem, "container-1"); + createNodeAgentContextAndContainerStorage(fileSystem, "container-2"); + + Path pathToArchiveDir = fileSystem.getPath("/home/docker/container-storage/container-archive"); + Files.createDirectories(pathToArchiveDir); + + Path containerStorageRoot = context1.pathOnHostFromPathInNode("/").getParent(); + Set<String> containerStorageRootContentsBeforeArchive = FileFinder.from(containerStorageRoot) + .maxDepth(1) + .stream() + .map(FileFinder.FileAttributes::filename) + .collect(Collectors.toSet()); + assertEquals(Set.of("container-archive", "container-1", "container-2"), containerStorageRootContentsBeforeArchive); + + + // Archive container-1 + storageMaintainer.archiveNodeStorage(context1); + + clock.advance(Duration.ofSeconds(3)); + storageMaintainer.archiveNodeStorage(context1); + + // container-1 should be gone from container-storage + Set<String> containerStorageRootContentsAfterArchive = FileFinder.from(containerStorageRoot) + .maxDepth(1) + .stream() + .map(FileFinder.FileAttributes::filename) + .collect(Collectors.toSet()); + assertEquals(Set.of("container-archive", "container-2"), containerStorageRootContentsAfterArchive); + + // container archive directory should contain exactly 1 directory - the one we just archived + List<FileFinder.FileAttributes> containerArchiveContentsAfterArchive = FileFinder.from(pathToArchiveDir).maxDepth(1).list(); + assertEquals(1, containerArchiveContentsAfterArchive.size()); + Path archivedContainerStoragePath = containerArchiveContentsAfterArchive.get(0).path(); + assertEquals("container-1_20090213233130", archivedContainerStoragePath.getFileName().toString()); + Set<String> archivedContainerStorageContents = FileFinder.files(archivedContainerStoragePath) + .stream() + .map(fileAttributes -> archivedContainerStoragePath.relativize(fileAttributes.path()).toString()) + .collect(Collectors.toSet()); + assertEquals(Set.of("opt/vespa/logs/vespa/vespa.log", "opt/vespa/logs/vespa/zookeeper.log"), archivedContainerStorageContents); + } - @Test - public void testNonExistingDiskUsed() { - StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, null, null); - DiskSize size = storageMaintainer.getDiskUsed(null, Paths.get("/fake/path")); - assertEquals(DiskSize.ZERO, size); - } + private static NodeAgentContext createNodeAgentContextAndContainerStorage(FileSystem fileSystem, String containerName) throws IOException { + NodeAgentContext context = new NodeAgentContextImpl.Builder(containerName + ".domain.tld") + .fileSystem(fileSystem).build(); - @After - public void after() { - terminal.verifyAllCommandsExecuted(); - } + Path containerVespaHomeOnHost = context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("")); + Files.createDirectories(context.pathOnHostFromPathInNode("/etc/something")); + Files.createFile(context.pathOnHostFromPathInNode("/etc/something/conf")); + + Files.createDirectories(containerVespaHomeOnHost.resolve("logs/vespa")); + Files.createFile(containerVespaHomeOnHost.resolve("logs/vespa/vespa.log")); + Files.createFile(containerVespaHomeOnHost.resolve("logs/vespa/zookeeper.log")); + + Files.createDirectories(containerVespaHomeOnHost.resolve("var/db")); + Files.createFile(containerVespaHomeOnHost.resolve("var/db/some-file")); + + Path containerRootOnHost = context.pathOnHostFromPathInNode("/"); + Set<String> actualContents = FileFinder.files(containerRootOnHost) + .stream() + .map(fileAttributes -> containerRootOnHost.relativize(fileAttributes.path()).toString()) + .collect(Collectors.toSet()); + Set<String> expectedContents = Set.of( + "etc/something/conf", + "opt/vespa/logs/vespa/vespa.log", + "opt/vespa/logs/vespa/zookeeper.log", + "opt/vespa/var/db/some-file"); + assertEquals(expectedContents, actualContents); + return context; } - public static class ArchiveContainerDataTests { - @Test - public void archive_container_data_test() throws IOException { - // Create some files in containers - FileSystem fileSystem = TestFileSystem.create(); - NodeAgentContext context1 = createNodeAgentContextAndContainerStorage(fileSystem, "container-1"); - createNodeAgentContextAndContainerStorage(fileSystem, "container-2"); - - Path pathToArchiveDir = fileSystem.getPath("/home/docker/container-storage/container-archive"); - Files.createDirectories(pathToArchiveDir); - - Path containerStorageRoot = context1.pathOnHostFromPathInNode("/").getParent(); - Set<String> containerStorageRootContentsBeforeArchive = FileFinder.from(containerStorageRoot) - .maxDepth(1) - .stream() - .map(FileFinder.FileAttributes::filename) - .collect(Collectors.toSet()); - assertEquals(Set.of("container-archive", "container-1", "container-2"), containerStorageRootContentsBeforeArchive); - - - // Archive container-1 - ManualClock clock = new ManualClock(Instant.ofEpochSecond(1234567890)); - StorageMaintainer storageMaintainer = new StorageMaintainer(null, null, pathToArchiveDir, null, clock); - storageMaintainer.archiveNodeStorage(context1); - - clock.advance(Duration.ofSeconds(3)); - storageMaintainer.archiveNodeStorage(context1); - - // container-1 should be gone from container-storage - Set<String> containerStorageRootContentsAfterArchive = FileFinder.from(containerStorageRoot) - .maxDepth(1) - .stream() - .map(FileFinder.FileAttributes::filename) - .collect(Collectors.toSet()); - assertEquals(Set.of("container-archive", "container-2"), containerStorageRootContentsAfterArchive); - - // container archive directory should contain exactly 1 directory - the one we just archived - List<FileFinder.FileAttributes> containerArchiveContentsAfterArchive = FileFinder.from(pathToArchiveDir).maxDepth(1).list(); - assertEquals(1, containerArchiveContentsAfterArchive.size()); - Path archivedContainerStoragePath = containerArchiveContentsAfterArchive.get(0).path(); - assertEquals("container-1_20090213233130", archivedContainerStoragePath.getFileName().toString()); - Set<String> archivedContainerStorageContents = FileFinder.files(archivedContainerStoragePath) - .stream() - .map(fileAttributes -> archivedContainerStoragePath.relativize(fileAttributes.path()).toString()) - .collect(Collectors.toSet()); - assertEquals(Set.of("opt/vespa/logs/vespa/vespa.log", "opt/vespa/logs/vespa/zookeeper.log"), archivedContainerStorageContents); - } - - private NodeAgentContext createNodeAgentContextAndContainerStorage(FileSystem fileSystem, String containerName) throws IOException { - NodeAgentContext context = new NodeAgentContextImpl.Builder(containerName + ".domain.tld") - .fileSystem(fileSystem).build(); - - Path containerVespaHomeOnHost = context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("")); - Files.createDirectories(context.pathOnHostFromPathInNode("/etc/something")); - Files.createFile(context.pathOnHostFromPathInNode("/etc/something/conf")); - - Files.createDirectories(containerVespaHomeOnHost.resolve("logs/vespa")); - Files.createFile(containerVespaHomeOnHost.resolve("logs/vespa/vespa.log")); - Files.createFile(containerVespaHomeOnHost.resolve("logs/vespa/zookeeper.log")); - - Files.createDirectories(containerVespaHomeOnHost.resolve("var/db")); - Files.createFile(containerVespaHomeOnHost.resolve("var/db/some-file")); - - Path containerRootOnHost = context.pathOnHostFromPathInNode("/"); - Set<String> actualContents = FileFinder.files(containerRootOnHost) - .stream() - .map(fileAttributes -> containerRootOnHost.relativize(fileAttributes.path()).toString()) - .collect(Collectors.toSet()); - Set<String> expectedContents = Set.of( - "etc/something/conf", - "opt/vespa/logs/vespa/vespa.log", - "opt/vespa/logs/vespa/zookeeper.log", - "opt/vespa/var/db/some-file"); - assertEquals(expectedContents, actualContents); - return context; - } - } + @Test + public void not_run_if_not_enough_used() throws IOException { + NodeAgentContext context = new NodeAgentContextImpl.Builder( + NodeSpec.Builder.testSpec("h123a.domain.tld").resources(new NodeResources(1, 1, 1, 1)).build()) + .fileSystem(fileSystem).build(); + Files.createDirectories(context.pathOnHostFromPathInNode("/")); + mockDiskUsage(500L); - public static class CleanupTests { + storageMaintainer.cleanDiskIfFull(context); + verifyNoInteractions(diskCleanup); + } - private final TestTerminal terminal = new TestTerminal(); - private final DiskCleanup diskCleanup = mock(DiskCleanup.class); - private final ManualClock clock = new ManualClock(); - private final StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, null, null, diskCleanup, clock); - private final FileSystem fileSystem = TestFileSystem.create(); - private final NodeAgentContext context = new NodeAgentContextImpl - .Builder(NodeSpec.Builder.testSpec("h123a.domain.tld").resources(new NodeResources(1, 1, 1, 1)).build()) + @Test + public void deletes_correct_amount() throws IOException { + NodeAgentContext context = new NodeAgentContextImpl.Builder( + NodeSpec.Builder.testSpec("h123a.domain.tld").resources(new NodeResources(1, 1, 1, 1)).build()) .fileSystem(fileSystem).build(); - @Test - public void not_run_if_not_enough_used() throws IOException { - Files.createDirectories(context.pathOnHostFromPathInNode("/")); - mockDiskUsage(500L); + Files.createDirectories(context.pathOnHostFromPathInNode("/")); + mockDiskUsage(950_000L); - storageMaintainer.cleanDiskIfFull(context); - verifyNoInteractions(diskCleanup); - } - - @Test - public void deletes_correct_amount() throws IOException { - Files.createDirectories(context.pathOnHostFromPathInNode("/")); - mockDiskUsage(950_000L); + storageMaintainer.cleanDiskIfFull(context); + // Allocated size: 1 GB, usage: 950_000 kiB (972.8 MB). Wanted usage: 70% => 700 MB + verify(diskCleanup).cleanup(eq(context), any(), eq(272_800_000L)); + } - storageMaintainer.cleanDiskIfFull(context); - // Allocated size: 1 GB, usage: 950_000 kiB (972.8 MB). Wanted usage: 70% => 700 MB - verify(diskCleanup).cleanup(eq(context), any(), eq(272_800_000L)); - } + @After + public void after() { + terminal.verifyAllCommandsExecuted(); + } - private void mockDiskUsage(long kBytes) { - terminal.expectCommand("du -xsk /home/docker/container-storage/h123a 2>&1", 0, kBytes + "\t/path"); - } + private void mockDiskUsage(long kBytes) { + terminal.expectCommand("du -xsk /home/docker/container-storage/h123a 2>&1", 0, kBytes + "\t/path"); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java index 0d596a46d77..9bca907983e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -4,19 +4,15 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.sync; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.BeforeClass; import org.junit.Test; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; import java.nio.file.FileSystem; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; +import java.util.Optional; +import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.NONE; +import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.ZSTD; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; /** * @author freva @@ -28,52 +24,39 @@ public class SyncFileInfoTest { private static final String bucket = "logs-region-acdf21"; private static final ApplicationId application = ApplicationId.from("tenant", "application", "instance"); private static final HostName hostname = HostName.from("h12352a.env.region-1.vespa.domain.example"); - private static final Path accessLogPath = fileSystem.getPath("/opt/vespa/logs/qrs/access.json-20210212.zst"); - private static final Path vespaLogPath = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12"); + private static final Path accessLogPath1 = fileSystem.getPath("/opt/vespa/logs/qrs/access.log.20210211"); + private static final Path accessLogPath2 = fileSystem.getPath("/opt/vespa/logs/qrs/access.log.20210212.zst"); + private static final Path accessLogPath3 = fileSystem.getPath("/opt/vespa/logs/qrs/access-json.log.20210213.zst"); + private static final Path accessLogPath4 = fileSystem.getPath("/opt/vespa/logs/qrs/JsonAccessLog.default.20210214.zst"); + private static final Path connectionLogPath1 = fileSystem.getPath("/opt/vespa/logs/qrs/ConnectionLog.default.20210210"); + private static final Path connectionLogPath2 = fileSystem.getPath("/opt/vespa/logs/qrs/ConnectionLog.default.20210212.zst"); + private static final Path vespaLogPath1 = fileSystem.getPath("/opt/vespa/logs/vespa.log"); + private static final Path vespaLogPath2 = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12"); @Test - public void tenant_access_log() { - SyncFileInfo sfi = SyncFileInfo.tenantAccessLog(bucket, application, hostname, accessLogPath); - assertEquals(Paths.get("/tenant.application.instance/h12352a/logs/access/access.json-20210212.zst"), sfi.destPath()); - assertEquals(bucket, sfi.bucketName()); - assertNotEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi)); - } + public void tenant_log() { + assertTenantSyncFileInfo(accessLogPath1, null, null); + assertTenantSyncFileInfo(accessLogPath2, "tenant.application.instance/h12352a/logs/access/access.log.20210212.zst", NONE); + assertTenantSyncFileInfo(accessLogPath3, "tenant.application.instance/h12352a/logs/access/access-json.log.20210213.zst", NONE); + assertTenantSyncFileInfo(accessLogPath4, "tenant.application.instance/h12352a/logs/access/JsonAccessLog.default.20210214.zst", NONE); - @Test - public void tenant_vespa_log() { - SyncFileInfo sfi = SyncFileInfo.tenantVespaLog(bucket, application, hostname, vespaLogPath); - assertEquals(Paths.get("/tenant.application.instance/h12352a/logs/vespa/vespa.log-2021-02-12.zst"), sfi.destPath()); - assertEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi)); - } + assertTenantSyncFileInfo(connectionLogPath1, null, null); + assertTenantSyncFileInfo(connectionLogPath2, "tenant.application.instance/h12352a/logs/connection/ConnectionLog.default.20210212.zst", NONE); - @Test - public void infra_access_log() { - SyncFileInfo sfi = SyncFileInfo.infrastructureAccessLog(bucket, hostname, accessLogPath); - assertEquals(Paths.get("/infrastructure/h12352a/logs/access/access.json-20210212.zst"), sfi.destPath()); - assertNotEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi)); + assertTenantSyncFileInfo(vespaLogPath1, null, null); + assertTenantSyncFileInfo(vespaLogPath2, "tenant.application.instance/h12352a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD); } @Test public void infra_vespa_log() { - SyncFileInfo sfi = SyncFileInfo.infrastructureVespaLog(bucket, hostname, vespaLogPath); - assertEquals(Paths.get("/infrastructure/h12352a/logs/vespa/vespa.log-2021-02-12.zst"), sfi.destPath()); - assertEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi)); + SyncFileInfo sfi = SyncFileInfo.infrastructureVespaLog(bucket, hostname, vespaLogPath2); + assertEquals("infrastructure/h12352a/logs/vespa/vespa.log-2021-02-12.zst", sfi.destPath().toString()); + assertEquals(ZSTD, sfi.uploadCompression()); } - @BeforeClass - public static void setup() throws IOException { - Files.createDirectories(vespaLogPath.getParent()); - Files.createFile(vespaLogPath); - Files.createDirectories(accessLogPath.getParent()); - Files.createFile(accessLogPath); + private static void assertTenantSyncFileInfo(Path srcPath, String destPath, SyncFileInfo.Compression compression) { + Optional<SyncFileInfo> sfi = SyncFileInfo.tenantLog(bucket, application, hostname, srcPath); + assertEquals(destPath, sfi.map(SyncFileInfo::destPath).map(Path::toString).orElse(null)); + assertEquals(compression, sfi.map(SyncFileInfo::uploadCompression).orElse(null)); } - - private static Class<? extends InputStream> getInputStreamType(SyncFileInfo syncFileInfo) { - try (InputStream inputStream = syncFileInfo.inputStream()) { - return inputStream.getClass(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - -}
\ No newline at end of file +} |