diff options
Diffstat (limited to 'node-admin')
8 files changed, 265 insertions, 33 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java index f43afc0ae4c..7367a254b4a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.DockerImage; import java.nio.file.Path; import java.util.List; +import java.util.Objects; import java.util.Optional; /** @@ -15,6 +16,7 @@ public class CoreDumpMetadata { private List<String> backtrace; private List<String> backtraceAllThreads; private Path coreDumpPath; + private String decryptionToken; private String kernelVersion; private String cpuMicrocodeVersion; private DockerImage dockerImage; @@ -26,6 +28,7 @@ public class CoreDumpMetadata { public Optional<List<String>> backtrace() { return Optional.ofNullable(backtrace); }; public Optional<List<String>> backtraceAllThreads() { return Optional.ofNullable(backtraceAllThreads); }; public Optional<Path> coredumpPath() { return Optional.ofNullable(coreDumpPath); }; + public Optional<String> decryptionToken() { return Optional.ofNullable(decryptionToken); } public Optional<String> kernelVersion() { return Optional.ofNullable(kernelVersion); }; public Optional<String> cpuMicrocodeVersion() { return Optional.ofNullable(cpuMicrocodeVersion); }; public Optional<DockerImage> dockerImage() { return Optional.ofNullable(dockerImage); }; @@ -35,6 +38,7 @@ public class CoreDumpMetadata { public CoreDumpMetadata setBacktrace(List<String> backtrace) { this.backtrace = backtrace; return this; }; public CoreDumpMetadata setBacktraceAllThreads(List<String> backtraceAllThreads) { this.backtraceAllThreads = backtraceAllThreads; return this; }; public CoreDumpMetadata setCoreDumpPath(Path coreDumpPath) { this.coreDumpPath = coreDumpPath; return this; }; + public CoreDumpMetadata setDecryptionToken(String decryptionToken) { this.decryptionToken = decryptionToken; return this; } public CoreDumpMetadata setKernelVersion(String kernelVersion) { this.kernelVersion = kernelVersion; return this; }; public CoreDumpMetadata setCpuMicrocodeVersion(String cpuMicrocodeVersion) { this.cpuMicrocodeVersion = cpuMicrocodeVersion; return this; }; public CoreDumpMetadata setDockerImage(DockerImage dockerImage) { this.dockerImage = dockerImage; return this; }; @@ -47,6 +51,7 @@ public class CoreDumpMetadata { ", backtrace=" + backtrace + ", backtraceAllThreads=" + backtraceAllThreads + ", coreDumpPath=" + coreDumpPath + + ", decryptionToken=" + decryptionToken + ", kernelVersion='" + kernelVersion + '\'' + ", cpuMicrocodeVersion='" + cpuMicrocodeVersion + '\'' + ", dockerImage=" + dockerImage + @@ -54,6 +59,25 @@ public class CoreDumpMetadata { '}'; } - @Override public boolean equals(Object o) { throw new UnsupportedOperationException(); } - @Override public int hashCode() { throw new UnsupportedOperationException(); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CoreDumpMetadata metadata = (CoreDumpMetadata) o; + return Objects.equals(binPath, metadata.binPath) && + Objects.equals(backtrace, metadata.backtrace) && + Objects.equals(backtraceAllThreads, metadata.backtraceAllThreads) && + Objects.equals(coreDumpPath, metadata.coreDumpPath) && + Objects.equals(decryptionToken, metadata.decryptionToken) && + Objects.equals(kernelVersion, metadata.kernelVersion) && + Objects.equals(cpuMicrocodeVersion, metadata.cpuMicrocodeVersion) && + Objects.equals(dockerImage, metadata.dockerImage) && + Objects.equals(vespaVersion, metadata.vespaVersion); + } + + @Override + public int hashCode() { + return Objects.hash(binPath, backtrace, backtraceAllThreads, coreDumpPath, decryptionToken, kernelVersion, + cpuMicrocodeVersion, dockerImage, vespaVersion); + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java index f6393d3726c..1b68e2f9467 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java @@ -6,8 +6,6 @@ import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.StandardConfigServerResponse; import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest; -import java.util.List; - /** * @author hakonhall */ @@ -20,16 +18,7 @@ public class CoresImpl implements Cores { @Override public void report(HostName hostname, String id, CoreDumpMetadata metadata) { - var request = new ReportCoreDumpRequest(); - metadata.binPath().ifPresent(binPath -> request.bin_path = binPath); - metadata.backtrace().ifPresent(backtrace -> request.backtrace = List.copyOf(backtrace)); - metadata.backtraceAllThreads().ifPresent(backtraceAllThreads -> request.backtrace_all_threads = List.copyOf(backtraceAllThreads)); - metadata.coredumpPath().ifPresent(coredumpPath -> request.coredump_path = coredumpPath.toString()); - metadata.kernelVersion().ifPresent(kernelVersion -> request.kernel_version = kernelVersion); - metadata.cpuMicrocodeVersion().ifPresent(cpuMicrocodeVersion -> request.cpu_microcode_version = cpuMicrocodeVersion); - metadata.dockerImage().ifPresent(dockerImage -> request.docker_image = dockerImage.asString()); - metadata.vespaVersion().ifPresent(vespaVersion -> request.vespa_version = vespaVersion); - + var request = new ReportCoreDumpRequest().fillFrom(metadata); String uriPath = "/cores/v1/report/" + hostname.value() + "/" + id; configServerApi.post(uriPath, request, StandardConfigServerResponse.class) .throwOnError("Failed to report core dump at " + metadata.coredumpPath()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java index f31c1e71bae..27cf28b8e1e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java @@ -1,10 +1,23 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.util.List; +import java.util.Optional; + +import static com.yahoo.yolean.Exceptions.uncheck; /** * Jackson class of JSON request, with names of fields verified in unit test. @@ -14,14 +27,64 @@ import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) public class ReportCoreDumpRequest { + private static final ObjectMapper objectMapper = new ObjectMapper(); + public List<String> backtrace; public List<String> backtrace_all_threads; public String bin_path; public String coredump_path; public String cpu_microcode_version; + public String decryption_token; public String docker_image; public String kernel_version; public String vespa_version; public ReportCoreDumpRequest() {} + + /** Fill this from metadata and return this. */ + @JsonIgnore + public ReportCoreDumpRequest fillFrom(CoreDumpMetadata metadata) { + metadata.binPath().ifPresent(binPath -> this.bin_path = binPath); + metadata.backtrace().ifPresent(backtrace -> this.backtrace = List.copyOf(backtrace)); + metadata.backtraceAllThreads().ifPresent(backtraceAllThreads -> this.backtrace_all_threads = List.copyOf(backtraceAllThreads)); + metadata.coredumpPath().ifPresent(coredumpPath -> this.coredump_path = coredumpPath.toString()); + metadata.decryptionToken().ifPresent(decryptionToken -> this.decryption_token = decryptionToken); + metadata.kernelVersion().ifPresent(kernelVersion -> this.kernel_version = kernelVersion); + metadata.cpuMicrocodeVersion().ifPresent(cpuMicrocodeVersion -> this.cpu_microcode_version = cpuMicrocodeVersion); + metadata.dockerImage().ifPresent(dockerImage -> this.docker_image = dockerImage.asString()); + metadata.vespaVersion().ifPresent(vespaVersion -> this.vespa_version = vespaVersion); + return this; + } + + @JsonIgnore + public void populateMetadata(CoreDumpMetadata metadata, FileSystem fileSystem) { + if (bin_path != null) metadata.setBinPath(bin_path); + if (backtrace != null) metadata.setBacktrace(backtrace); + if (backtrace_all_threads != null) metadata.setBacktraceAllThreads(backtrace_all_threads); + if (coredump_path != null) metadata.setCoreDumpPath(fileSystem.getPath(coredump_path)); + if (decryption_token != null) metadata.setDecryptionToken(decryption_token); + if (kernel_version != null) metadata.setKernelVersion(kernel_version); + if (cpu_microcode_version != null) metadata.setCpuMicrocodeVersion(cpu_microcode_version); + if (docker_image != null) metadata.setDockerImage(DockerImage.fromString(docker_image)); + if (vespa_version != null) metadata.setVespaVersion(vespa_version); + } + + @JsonIgnore + public void save(Path path) { + String serialized = uncheck(() -> objectMapper.writeValueAsString(this)); + uncheck(() -> Files.writeString(path, serialized)); + } + + @JsonIgnore + public static Optional<ReportCoreDumpRequest> load(Path path) { + final String serialized; + try { + serialized = Files.readString(path); + } catch (NoSuchFileException e) { + return Optional.empty(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return Optional.of(uncheck(() -> objectMapper.readValue(serialized, ReportCoreDumpRequest.class))); + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 49d00c03d23..5ae0620fa7d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -169,7 +169,8 @@ public class StorageMaintainer { /** Checks if container has any new coredumps, reports and archives them if so */ public void handleCoreDumpsForContainer(NodeAgentContext context, Optional<Container> container, boolean throwIfCoreBeingWritten) { if (context.isDisabled(NodeAgentTask.CoreDumps)) return; - coredumpHandler.converge(context, () -> getCoredumpNodeAttributes(context, container), throwIfCoreBeingWritten); + coredumpHandler.converge(context, () -> getCoredumpNodeAttributes(context, container), + container.map(Container::image), throwIfCoreBeingWritten); } private Map<String, Object> getCoredumpNodeAttributes(NodeAgentContext context, Optional<Container> container) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java index b5a47e0b3be..d99b45dbbbc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; +import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; @@ -128,6 +129,33 @@ public class CoreCollector { return data; } + CoreDumpMetadata collect2(NodeAgentContext context, ContainerPath coredumpPath) { + var metadata = new CoreDumpMetadata(); + + if (JAVA_HEAP_DUMP_PATTERN.matcher(coredumpPath.getFileName().toString()).find()) { + metadata.setBinPath("java") + .setBacktrace(List.of("Heap dump, no backtrace available")); + return metadata; + } + + try { + String binPath = readBinPath(context, coredumpPath); + metadata.setBinPath(binPath); + + if (Path.of(binPath).getFileName().toString().equals("java")) { + metadata.setBacktraceAllThreads(readJstack(context, coredumpPath, binPath)); + } else { + metadata.setBacktrace(readBacktrace(context, coredumpPath, binPath, false)); + metadata.setBacktraceAllThreads(readBacktrace(context, coredumpPath, binPath, true)); + } + } catch (ConvergenceException e) { + context.log(logger, Level.WARNING, "Failed to extract backtrace: " + e.getMessage()); + } catch (RuntimeException e) { + context.log(logger, Level.WARNING, "Failed to extract backtrace", e); + } + return metadata; + } + private String asString(CommandResult result) { return "exit status " + result.getExitCode() + ", output '" + result.getOutput() + "'"; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 5113b8d9e98..40606aade3a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -4,8 +4,16 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.config.provision.DockerImage; import com.yahoo.security.SecretSharedKey; import com.yahoo.security.SharedKeyGenerator; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; +import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores; +import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.container.metrics.Dimensions; import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; @@ -15,14 +23,15 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; import javax.crypto.CipherOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.UncheckedIOException; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Clock; import java.util.Comparator; import java.util.HashMap; @@ -50,6 +59,7 @@ public class CoredumpHandler { private static final Pattern HS_ERR_PATTERN = Pattern.compile("hs_err_pid[0-9]+\\.log"); private static final String PROCESSING_DIRECTORY_NAME = "processing"; private static final String METADATA_FILE_NAME = "metadata.json"; + private static final String METADATA2_FILE_NAME = "metadata2.json"; private static final String COMPRESSED_EXTENSION = ".zst"; private static final String ENCRYPTED_EXTENSION = ".enc"; public static final String COREDUMP_FILENAME_PREFIX = "dump_"; @@ -58,6 +68,7 @@ public class CoredumpHandler { private final ObjectMapper objectMapper = new ObjectMapper(); private final CoreCollector coreCollector; + private final Cores cores; private final CoredumpReporter coredumpReporter; private final String crashPatchInContainer; private final Path doneCoredumpsPath; @@ -65,22 +76,26 @@ public class CoredumpHandler { private final Clock clock; private final Supplier<String> coredumpIdSupplier; private final Supplier<SecretSharedKey> secretSharedKeySupplier; + private final BooleanFlag reportCoresViaCfgFlag; /** * @param crashPathInContainer path inside the container where core dump are dumped - * @param doneCoredumpsPath path on host where processed core dumps are stored + * @param doneCoredumpsPath path on host where processed core dumps are stored */ - public CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter, - String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics) { - this(coreCollector, coredumpReporter, crashPathInContainer, doneCoredumpsPath, - metrics, Clock.systemUTC(), () -> UUID.randomUUID().toString(), () -> null /*TODO*/); + public CoredumpHandler(CoreCollector coreCollector, Cores cores, CoredumpReporter coredumpReporter, + String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics, + FlagSource flagSource) { + this(coreCollector, cores, coredumpReporter, crashPathInContainer, doneCoredumpsPath, + metrics, Clock.systemUTC(), () -> UUID.randomUUID().toString(), () -> null /*TODO*/, + flagSource); } - CoredumpHandler(CoreCollector coreCollector, CoredumpReporter coredumpReporter, + CoredumpHandler(CoreCollector coreCollector, Cores cores, CoredumpReporter coredumpReporter, String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics, Clock clock, Supplier<String> coredumpIdSupplier, - Supplier<SecretSharedKey> secretSharedKeySupplier) { + Supplier<SecretSharedKey> secretSharedKeySupplier, FlagSource flagSource) { this.coreCollector = coreCollector; + this.cores = cores; this.coredumpReporter = coredumpReporter; this.crashPatchInContainer = crashPathInContainer; this.doneCoredumpsPath = doneCoredumpsPath; @@ -88,10 +103,12 @@ public class CoredumpHandler { this.clock = clock; this.coredumpIdSupplier = coredumpIdSupplier; this.secretSharedKeySupplier = secretSharedKeySupplier; + this.reportCoresViaCfgFlag = Flags.REPORT_CORES_VIA_CFG.bindTo(flagSource); } - public void converge(NodeAgentContext context, Supplier<Map<String, Object>> nodeAttributesSupplier, boolean throwIfCoreBeingWritten) { + public void converge(NodeAgentContext context, Supplier<Map<String, Object>> nodeAttributesSupplier, + Optional<DockerImage> dockerImage, boolean throwIfCoreBeingWritten) { ContainerPath containerCrashPath = context.paths().of(crashPatchInContainer, context.users().vespa()); ContainerPath containerProcessingPath = containerCrashPath.resolve(PROCESSING_DIRECTORY_NAME); @@ -110,7 +127,13 @@ public class CoredumpHandler { // Check if we have already started to process a core dump or we can enqueue a new core one getCoredumpToProcess(containerCrashPath, containerProcessingPath) - .ifPresent(path -> processAndReportSingleCoredump(context, path, nodeAttributesSupplier)); + .ifPresent(path -> { + if (reportCoresViaCfgFlag.with(FetchVector.Dimension.NODE_TYPE, context.nodeType().name()).value()) { + processAndReportSingleCoreDump2(context, path, dockerImage); + } else { + processAndReportSingleCoredump(context, path, nodeAttributesSupplier); + } + }); } /** @return path to directory inside processing directory that contains a core dump file to process */ @@ -224,7 +247,7 @@ public class CoredumpHandler { * Compresses and, if a key is provided, encrypts core file (and deletes the uncompressed core), then moves * the entire core dump processing directory to {@link #doneCoredumpsPath} for archive */ - private void finishProcessing(NodeAgentContext context, ContainerPath coredumpDirectory, Optional<SecretSharedKey> sharedCoreKey) throws IOException { + private void finishProcessing(NodeAgentContext context, ContainerPath coredumpDirectory, Optional<SecretSharedKey> sharedCoreKey) { ContainerPath coreFile = findCoredumpFileInProcessingDirectory(coredumpDirectory); String extension = COMPRESSED_EXTENSION + (sharedCoreKey.isPresent() ? ENCRYPTED_EXTENSION : ""); ContainerPath compressedCoreFile = coreFile.resolveSibling(coreFile.getFileName() + extension); @@ -235,12 +258,12 @@ public class CoredumpHandler { } catch (IOException e) { throw new UncheckedIOException(e); } - Files.delete(coreFile); + uncheck(() -> Files.delete(coreFile)); Path newCoredumpDirectory = doneCoredumpsPath.resolve(context.containerName().asString()); uncheck(() -> Files.createDirectories(newCoredumpDirectory)); // Files.move() does not support moving non-empty directories across providers, move using host paths - Files.move(coredumpDirectory.pathOnHost(), newCoredumpDirectory.resolve(coredumpDirectory.getFileName().toString())); + uncheck(() -> Files.move(coredumpDirectory.pathOnHost(), newCoredumpDirectory.resolve(coredumpDirectory.getFileName().toString()))); } ContainerPath findCoredumpFileInProcessingDirectory(ContainerPath coredumpProccessingDirectory) { @@ -314,4 +337,60 @@ public class CoredumpHandler { return clock.instant().minusSeconds(60).isAfter(fileAttributes.lastModifiedTime()); } + void processAndReportSingleCoreDump2(NodeAgentContext context, ContainerPath coreDumpDirectory, + Optional<DockerImage> dockerImage) { + CoreDumpMetadata metadata = gatherMetadata(context, coreDumpDirectory); + dockerImage.ifPresent(metadata::setDockerImage); + dockerImage.flatMap(DockerImage::tag).ifPresent(metadata::setVespaVersion); + dockerImage.ifPresent(metadata::setDockerImage); + Optional<SecretSharedKey> sharedCoreKey = Optional.ofNullable(secretSharedKeySupplier.get()); + sharedCoreKey.map(key -> key.sealedSharedKey().toTokenString()).ifPresent(metadata::setDecryptionToken); + + String coreDumpId = coreDumpDirectory.getFileName().toString(); + cores.report(context.hostname(), coreDumpId, metadata); + finishProcessing(context, coreDumpDirectory, sharedCoreKey); + context.log(logger, "Successfully reported core dump " + coreDumpId); + } + + private CoreDumpMetadata gatherMetadata(NodeAgentContext context, ContainerPath coreDumpDirectory) { + ContainerPath metadataPath = coreDumpDirectory.resolve(METADATA2_FILE_NAME); + Optional<ReportCoreDumpRequest> request = ReportCoreDumpRequest.load(metadataPath); + if (request.isPresent()) { + return request.map(requestInstance -> { + var metadata = new CoreDumpMetadata(); + requestInstance.populateMetadata(metadata, FileSystems.getDefault()); + return metadata; + }) + .get(); + } + + ContainerPath coreDumpFile = findCoredumpFileInProcessingDirectory(coreDumpDirectory); + CoreDumpMetadata metadata = coreCollector.collect2(context, coreDumpFile); + metadata.setCpuMicrocodeVersion(getMicrocodeVersion()) + .setKernelVersion(System.getProperty("os.version")) + .setCoreDumpPath(doneCoredumpsPath.resolve(context.containerName().asString()) + .resolve(coreDumpDirectory.getFileName().toString()) + .resolve(coreDumpFile.getFileName().toString())); + + ReportCoreDumpRequest requestInstance = new ReportCoreDumpRequest(); + requestInstance.fillFrom(metadata); + requestInstance.save(metadataPath); + context.log(logger, "Wrote " + metadataPath); + return metadata; + } + + private String getMicrocodeVersion() { + String output = uncheck(() -> Files.readAllLines(Paths.get("/proc/cpuinfo")).stream() + .filter(line -> line.startsWith("microcode")) + .findFirst() + .orElse("microcode : UNKNOWN")); + + String[] results = output.split(":"); + if (results.length != 2) { + throw ConvergenceException.ofError("Result from detect microcode command not as expected: " + output); + } + + return results[1].trim(); + } + } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java index 66b6e9cc67d..f49dd2e705b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java @@ -8,15 +8,21 @@ import com.yahoo.test.json.JsonTestHelper; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerException; import com.yahoo.vespa.hosted.node.admin.configserver.StandardConfigServerResponse; +import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; +import java.nio.file.FileSystem; import java.nio.file.Path; import java.util.List; +import java.util.Optional; import static com.yahoo.yolean.Exceptions.uncheck; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -27,6 +33,7 @@ import static org.mockito.Mockito.when; * @author hakonhall */ class CoresTest { + private final FileSystem fileSystem = TestFileSystem.create(); private final ObjectMapper mapper = new ObjectMapper(); private final ConfigServerApi configServerApi = mock(ConfigServerApi.class); private final Cores cores = new CoresImpl(configServerApi); @@ -35,7 +42,8 @@ class CoresTest { private final CoreDumpMetadata metadata = new CoreDumpMetadata() .setKernelVersion("4.18.0-372.26.1.el8_6.x86_64") .setCpuMicrocodeVersion("0x1000065") - .setCoreDumpPath(Path.of("/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813")) + .setCoreDumpPath(fileSystem.getPath("/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813")) + .setDecryptionToken("987def") .setDockerImage(DockerImage.fromString("us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8")) .setBinPath("/usr/bin/java") .setVespaVersion("8.68.8") @@ -75,6 +83,7 @@ class CoresTest { "bin_path": "/usr/bin/java", "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813", "cpu_microcode_version": "0x1000065", + "decryption_token": "987def", "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8", "kernel_version": "4.18.0-372.26.1.el8_6.x86_64", "vespa_version": "8.68.8" @@ -97,4 +106,39 @@ class CoresTest { var bodyJsonPojoCaptor = ArgumentCaptor.forClass(Object.class); verify(configServerApi).post(pathCaptor.capture(), bodyJsonPojoCaptor.capture(), any()); } + + @Test + void serialization() { + Path path = fileSystem.getPath("/foo.json"); + ReportCoreDumpRequest request = new ReportCoreDumpRequest().fillFrom(metadata); + request.save(path); + assertEquals(""" + { + "backtrace": [ + "Example", + "of", + "backtrace" + ], + "backtrace_all_threads": [ + "Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...", + "Debugger attached successfully.", + " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)", + " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)" + ], + "bin_path": "/usr/bin/java", + "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813", + "cpu_microcode_version": "0x1000065", + "decryption_token": "987def", + "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8", + "kernel_version": "4.18.0-372.26.1.el8_6.x86_64", + "vespa_version": "8.68.8" + }""", + JsonTestHelper.normalize(new UnixPath(path).readUtf8File())); + + Optional<ReportCoreDumpRequest> loaded = ReportCoreDumpRequest.load(path); + assertTrue(loaded.isPresent()); + var meta = new CoreDumpMetadata(); + loaded.get().populateMetadata(meta, fileSystem); + assertEquals(metadata, meta); + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index be4ee657292..1d53f0974ab 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; import com.yahoo.security.SealedSharedKey; import com.yahoo.security.SecretSharedKey; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores; import com.yahoo.vespa.hosted.node.admin.container.metrics.DimensionMetrics; import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; @@ -52,6 +54,7 @@ public class CoredumpHandlerTest { private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps"); private final CoreCollector coreCollector = mock(CoreCollector.class); + private final Cores cores = mock(Cores.class); private final CoredumpReporter coredumpReporter = mock(CoredumpReporter.class); private final Metrics metrics = new Metrics(); private final ManualClock clock = new ManualClock(); @@ -59,10 +62,11 @@ public class CoredumpHandlerTest { private final Supplier<String> coredumpIdSupplier = mock(Supplier.class); @SuppressWarnings("unchecked") private final Supplier<SecretSharedKey> secretSharedKeySupplier = mock(Supplier.class); - private final CoredumpHandler coredumpHandler = new CoredumpHandler(coreCollector, coredumpReporter, - containerCrashPath.pathInContainer(), doneCoredumpsPath, metrics, clock, coredumpIdSupplier, - secretSharedKeySupplier); - + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private final CoredumpHandler coredumpHandler = + new CoredumpHandler(coreCollector, cores, coredumpReporter, containerCrashPath.pathInContainer(), + doneCoredumpsPath, metrics, clock, coredumpIdSupplier, secretSharedKeySupplier, + flagSource); @Test void coredump_enqueue_test() throws IOException { |