diff options
author | Bjørn Christian Seime <bjorncs@verizonmedia.com> | 2021-09-13 16:06:23 +0200 |
---|---|---|
committer | Bjørn Christian Seime <bjorncs@verizonmedia.com> | 2021-09-13 16:42:30 +0200 |
commit | d706f11a6565042f647c9810cda9dfa8516bd798 (patch) | |
tree | 1a2436980abb1bb7c6b19205bf2f4d3c40883bdb /node-admin | |
parent | b111dff0f9b4b1bd699c5f5670b5c37cada0dcf0 (diff) |
Add artifact type and producer for perf record/report
Diffstat (limited to 'node-admin')
3 files changed, 111 insertions, 29 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PerfReportProducer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PerfReportProducer.java new file mode 100644 index 00000000000..f2725f4ba45 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PerfReportProducer.java @@ -0,0 +1,45 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; + +import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** + * @author bjorncs + */ +class PerfReportProducer extends AbstractProducer { + + public static String NAME = "perf-report"; + + PerfReportProducer(ContainerOperations container) { super(container); } + + @Override public String name() { return NAME; } + + @Override + public void produceArtifact(NodeAgentContext context, String configId, ServiceDumpReport.DumpOptions options, + UnixPath resultDirectoryInNode) throws IOException { + Path findPidBinary = context.pathInNodeUnderVespaHome("libexec/vespa/find-pid"); + CommandResult findPidResult = executeCommand(context, List.of(findPidBinary.toString(), configId), true); + String pid = findPidResult.getOutput(); + int duration = options != null && options.duration() != null && options.duration() > 0 + ? options.duration().intValue() : 30; + List<String> perfRecordCommand = new ArrayList<>(List.of("perf", "record")); + if (options != null && Boolean.TRUE.equals(options.callGraphRecording())) { + perfRecordCommand.add("-g"); + } + String recordFile = resultDirectoryInNode.resolve("perf-record.bin").toString(); + perfRecordCommand.addAll( + List.of("--output=" + recordFile, + "--pid=" + pid, "sleep", Integer.toString(duration))); + executeCommand(context, perfRecordCommand, true); + String perfReportFile = resultDirectoryInNode.resolve("perf-report.txt").toString(); + executeCommand(context, List.of("bash", "-c", "perf report --input=" + recordFile + " > " + perfReportFile), true); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java index 82616e9fc86..a5b45a46b65 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java @@ -52,8 +52,10 @@ public class VespaServiceDumperImpl implements VespaServiceDumper { this.syncClient = syncClient; this.nodeRepository = nodeRepository; this.clock = clock; - this.artifactProducers = List.of(new JvmDumpProducer(container)) - .stream() + List<AbstractProducer> producers = List.of( + new JvmDumpProducer(container), + new PerfReportProducer(container)); + this.artifactProducers = producers.stream() .collect(Collectors.toMap(ArtifactProducer::name, Function.identity())); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java index 29ded05c64a..3489c08e346 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -37,6 +38,17 @@ class VespaServiceDumperImplTest { private static final String HOSTNAME = "host-1.domain.tld"; + private Path tmpDirectory; + private FileSystem fileSystem; + + @BeforeEach + void create_test_file_system() throws IOException { + // Create temporary directory in container + fileSystem = TestFileSystem.create(); + tmpDirectory = fileSystem.getPath("/home/docker/container-storage/host-1/opt/vespa/tmp"); + Files.createDirectories(tmpDirectory); + } + @Test void creates_valid_dump_id_from_dump_request() { long nowMillis = Instant.now().toEpochMilli(); @@ -47,25 +59,20 @@ class VespaServiceDumperImplTest { } @Test - public void generates_dump_from_request() throws IOException { - // Create temporary directory in container - FileSystem fileSystem = TestFileSystem.create(); - Path tmpDirectory = fileSystem.getPath("/home/docker/container-storage/host-1/opt/vespa/tmp"); - Files.createDirectories(tmpDirectory); - + void generates_jvm_dump_from_request() throws IOException { // Setup mocks - ContainerOperations operations = createContainerMock(tmpDirectory); + ContainerOperations operations = mock(ContainerOperations.class); + when(operations.executeCommandInContainerAsRoot(any(), any())) + .thenAnswer(invocation -> { + // Create dummy files to simulate vespa-jvm-dumper + Files.createFile(tmpDirectory.resolve("vespa-service-dump/" + JvmDumpProducer.NAME + "/heap.bin")); + Files.createFile(tmpDirectory.resolve("vespa-service-dump/" + JvmDumpProducer.NAME + "/jstack")); + return new CommandResult(null, 0, "result"); + }); SyncClient syncClient = createSyncClientMock(); NodeRepoMock nodeRepository = new NodeRepoMock(); ManualClock clock = new ManualClock(Instant.ofEpochMilli(1600001000000L)); - ServiceDumpReport request = new ServiceDumpReport( - 1600000000000L, null, null, null, null, "default/container.1", null, null, List.of(JvmDumpProducer.NAME), null); - NodeSpec initialSpec = NodeSpec.Builder - .testSpec(HOSTNAME, NodeState.active) - .report(ServiceDumpReport.REPORT_ID, request.toJsonNode()) - .archiveUri(URI.create("s3://uri-1/tenant1/")) - .build(); - nodeRepository.updateNodeSpec(initialSpec); + NodeSpec initialSpec = createNodeSpecWithDumpRequest(nodeRepository, JvmDumpProducer.NAME, null); // Create dumper and invoke tested method VespaServiceDumper reporter = new VespaServiceDumperImpl(operations, syncClient, nodeRepository, clock); @@ -86,7 +93,47 @@ class VespaServiceDumperImplTest { URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/jvm-dump/heap.bin.zst"), URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/jvm-dump/jstack")); assertSyncedFiles(context, syncClient, expectedUris); + } + @Test + void invokes_perf_commands_when_generating_perf_report() { + // Setup mocks + ContainerOperations operations = mock(ContainerOperations.class); + when(operations.executeCommandInContainerAsRoot(any(), any())) + .thenReturn(new CommandResult(null, 0, "12345")) + .thenReturn(new CommandResult(null, 0, "")) + .thenReturn(new CommandResult(null, 0, "")); + SyncClient syncClient = createSyncClientMock(); + NodeRepoMock nodeRepository = new NodeRepoMock(); + ManualClock clock = new ManualClock(Instant.ofEpochMilli(1600001000000L)); + NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, PerfReportProducer.NAME, new ServiceDumpReport.DumpOptions(true, 45.0)); + + VespaServiceDumper reporter = new VespaServiceDumperImpl(operations, syncClient, nodeRepository, clock); + NodeAgentContextImpl context = new NodeAgentContextImpl.Builder(nodeSpec) + .fileSystem(fileSystem) + .build(); + reporter.processServiceDumpRequest(context); + + verify(operations).executeCommandInContainerAsRoot( + context, "/opt/vespa/libexec/vespa/find-pid", "default/container.1"); + verify(operations).executeCommandInContainerAsRoot( + context, "perf", "record", "-g", "--output=/opt/vespa/tmp/vespa-service-dump/perf-report/perf-record.bin", + "--pid=12345", "sleep", "45"); + verify(operations).executeCommandInContainerAsRoot( + context, "bash", "-c", "perf report --input=/opt/vespa/tmp/vespa-service-dump/perf-report/perf-record.bin" + + " > /opt/vespa/tmp/vespa-service-dump/perf-report/perf-report.txt"); + } + + private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, String artifactName, ServiceDumpReport.DumpOptions options) { + ServiceDumpReport request = new ServiceDumpReport( + 1600000000000L, null, null, null, null, "default/container.1", null, null, List.of(artifactName), options); + NodeSpec spec = NodeSpec.Builder + .testSpec(HOSTNAME, NodeState.active) + .report(ServiceDumpReport.REPORT_ID, request.toJsonNode()) + .archiveUri(URI.create("s3://uri-1/tenant1/")) + .build(); + repository.updateNodeSpec(spec); + return spec; } private static void assertReportEquals(NodeRepoMock nodeRepository, String expectedJson) { @@ -108,18 +155,6 @@ class VespaServiceDumperImplTest { assertEquals(expectedDestinations, actualFilenames); } - private static ContainerOperations createContainerMock(Path tmpDirectory) { - ContainerOperations operations = mock(ContainerOperations.class); - when(operations.executeCommandInContainerAsRoot(any(), any())) - .thenAnswer(invocation -> { - // Create dummy files to simulate vespa-jvm-dumper - Files.createFile(tmpDirectory.resolve("vespa-service-dump/" + JvmDumpProducer.NAME + "/heap.bin")); - Files.createFile(tmpDirectory.resolve("vespa-service-dump/" + JvmDumpProducer.NAME + "/jstack")); - return new CommandResult(null, 0, "result"); - }); - return operations; - } - private SyncClient createSyncClientMock() { SyncClient client = mock(SyncClient.class); when(client.sync(any(), any(), anyInt())) |