Guard core dump upload via cfg

8 files changed, 265 insertions, 33 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java
index f43afc0ae4c..7367a254b4a 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java
@@ -5,6 +5,7 @@ import com.yahoo.config.provision.DockerImage;
 
 import java.nio.file.Path;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 
 /**
@@ -15,6 +16,7 @@ public class CoreDumpMetadata {
     private List<String> backtrace;
     private List<String> backtraceAllThreads;
     private Path coreDumpPath;
+    private String decryptionToken;
     private String kernelVersion;
     private String cpuMicrocodeVersion;
     private DockerImage dockerImage;
@@ -26,6 +28,7 @@ public class CoreDumpMetadata {
     public Optional<List<String>> backtrace() { return Optional.ofNullable(backtrace); };
     public Optional<List<String>> backtraceAllThreads() { return Optional.ofNullable(backtraceAllThreads); };
     public Optional<Path> coredumpPath() { return Optional.ofNullable(coreDumpPath); };
+    public Optional<String> decryptionToken() { return Optional.ofNullable(decryptionToken); }
     public Optional<String> kernelVersion() { return Optional.ofNullable(kernelVersion); };
     public Optional<String> cpuMicrocodeVersion() { return Optional.ofNullable(cpuMicrocodeVersion); };
     public Optional<DockerImage> dockerImage() { return Optional.ofNullable(dockerImage); };
@@ -35,6 +38,7 @@ public class CoreDumpMetadata {
     public CoreDumpMetadata setBacktrace(List<String> backtrace) { this.backtrace = backtrace; return this; };
     public CoreDumpMetadata setBacktraceAllThreads(List<String> backtraceAllThreads) { this.backtraceAllThreads = backtraceAllThreads; return this; };
     public CoreDumpMetadata setCoreDumpPath(Path coreDumpPath) { this.coreDumpPath = coreDumpPath; return this; };
+    public CoreDumpMetadata setDecryptionToken(String decryptionToken) { this.decryptionToken = decryptionToken; return this; }
     public CoreDumpMetadata setKernelVersion(String kernelVersion) { this.kernelVersion = kernelVersion; return this; };
     public CoreDumpMetadata setCpuMicrocodeVersion(String cpuMicrocodeVersion) { this.cpuMicrocodeVersion = cpuMicrocodeVersion; return this; };
     public CoreDumpMetadata setDockerImage(DockerImage dockerImage) { this.dockerImage = dockerImage; return this; };
@@ -47,6 +51,7 @@ public class CoreDumpMetadata {
                ", backtrace=" + backtrace +
                ", backtraceAllThreads=" + backtraceAllThreads +
                ", coreDumpPath=" + coreDumpPath +
+               ", decryptionToken=" + decryptionToken +
                ", kernelVersion='" + kernelVersion + '\'' +
                ", cpuMicrocodeVersion='" + cpuMicrocodeVersion + '\'' +
                ", dockerImage=" + dockerImage +
@@ -54,6 +59,25 @@ public class CoreDumpMetadata {
                '}';
     }
 
-    @Override public boolean equals(Object o) { throw new UnsupportedOperationException(); }
-    @Override public int hashCode() { throw new UnsupportedOperationException(); }
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        CoreDumpMetadata metadata = (CoreDumpMetadata) o;
+        return Objects.equals(binPath, metadata.binPath) &&
+               Objects.equals(backtrace, metadata.backtrace) &&
+               Objects.equals(backtraceAllThreads, metadata.backtraceAllThreads) &&
+               Objects.equals(coreDumpPath, metadata.coreDumpPath) &&
+               Objects.equals(decryptionToken, metadata.decryptionToken) &&
+               Objects.equals(kernelVersion, metadata.kernelVersion) &&
+               Objects.equals(cpuMicrocodeVersion, metadata.cpuMicrocodeVersion) &&
+               Objects.equals(dockerImage, metadata.dockerImage) &&
+               Objects.equals(vespaVersion, metadata.vespaVersion);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(binPath, backtrace, backtraceAllThreads, coreDumpPath, decryptionToken, kernelVersion,
+                            cpuMicrocodeVersion, dockerImage, vespaVersion);
+    }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java
index f6393d3726c..1b68e2f9467 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java
@@ -6,8 +6,6 @@ import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
 import com.yahoo.vespa.hosted.node.admin.configserver.StandardConfigServerResponse;
 import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest;
 
-import java.util.List;
-
 /**
  * @author hakonhall
  */
@@ -20,16 +18,7 @@ public class CoresImpl implements Cores {
 
     @Override
     public void report(HostName hostname, String id, CoreDumpMetadata metadata) {
-        var request = new ReportCoreDumpRequest();
-        metadata.binPath().ifPresent(binPath -> request.bin_path = binPath);
-        metadata.backtrace().ifPresent(backtrace -> request.backtrace = List.copyOf(backtrace));
-        metadata.backtraceAllThreads().ifPresent(backtraceAllThreads -> request.backtrace_all_threads = List.copyOf(backtraceAllThreads));
-        metadata.coredumpPath().ifPresent(coredumpPath -> request.coredump_path = coredumpPath.toString());
-        metadata.kernelVersion().ifPresent(kernelVersion -> request.kernel_version = kernelVersion);
-        metadata.cpuMicrocodeVersion().ifPresent(cpuMicrocodeVersion -> request.cpu_microcode_version = cpuMicrocodeVersion);
-        metadata.dockerImage().ifPresent(dockerImage -> request.docker_image = dockerImage.asString());
-        metadata.vespaVersion().ifPresent(vespaVersion -> request.vespa_version = vespaVersion);
-
+        var request = new ReportCoreDumpRequest().fillFrom(metadata);
         String uriPath = "/cores/v1/report/" + hostname.value() + "/" + id;
         configServerApi.post(uriPath, request, StandardConfigServerResponse.class)
                        .throwOnError("Failed to report core dump at " + metadata.coredumpPath());
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java
index f31c1e71bae..27cf28b8e1e 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java
@@ -1,10 +1,23 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
 import java.util.List;
+import java.util.Optional;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
 
 /**
  * Jackson class of JSON request, with names of fields verified in unit test.
@@ -14,14 +27,64 @@ import java.util.List;
 @JsonIgnoreProperties(ignoreUnknown = true)
 @JsonInclude(JsonInclude.Include.NON_NULL)
 public class ReportCoreDumpRequest {
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+
     public List<String> backtrace;
     public List<String> backtrace_all_threads;
     public String bin_path;
     public String coredump_path;
     public String cpu_microcode_version;
+    public String decryption_token;
     public String docker_image;
     public String kernel_version;
     public String vespa_version;
 
     public ReportCoreDumpRequest() {}
+
+    /** Fill this from metadata and return this. */
+    @JsonIgnore
+    public ReportCoreDumpRequest fillFrom(CoreDumpMetadata metadata) {
+        metadata.binPath().ifPresent(binPath -> this.bin_path = binPath);
+        metadata.backtrace().ifPresent(backtrace -> this.backtrace = List.copyOf(backtrace));
+        metadata.backtraceAllThreads().ifPresent(backtraceAllThreads -> this.backtrace_all_threads = List.copyOf(backtraceAllThreads));
+        metadata.coredumpPath().ifPresent(coredumpPath -> this.coredump_path = coredumpPath.toString());
+        metadata.decryptionToken().ifPresent(decryptionToken -> this.decryption_token = decryptionToken);
+        metadata.kernelVersion().ifPresent(kernelVersion -> this.kernel_version = kernelVersion);
+        metadata.cpuMicrocodeVersion().ifPresent(cpuMicrocodeVersion -> this.cpu_microcode_version = cpuMicrocodeVersion);
+        metadata.dockerImage().ifPresent(dockerImage -> this.docker_image = dockerImage.asString());
+        metadata.vespaVersion().ifPresent(vespaVersion -> this.vespa_version = vespaVersion);
+        return this;
+    }
+
+    @JsonIgnore
+    public void populateMetadata(CoreDumpMetadata metadata, FileSystem fileSystem) {
+        if (bin_path != null) metadata.setBinPath(bin_path);
+        if (backtrace != null) metadata.setBacktrace(backtrace);
+        if (backtrace_all_threads != null) metadata.setBacktraceAllThreads(backtrace_all_threads);
+        if (coredump_path != null) metadata.setCoreDumpPath(fileSystem.getPath(coredump_path));
+        if (decryption_token != null) metadata.setDecryptionToken(decryption_token);
+        if (kernel_version != null) metadata.setKernelVersion(kernel_version);
+        if (cpu_microcode_version != null) metadata.setCpuMicrocodeVersion(cpu_microcode_version);
+        if (docker_image != null) metadata.setDockerImage(DockerImage.fromString(docker_image));
+        if (vespa_version != null) metadata.setVespaVersion(vespa_version);
+    }
+
+    @JsonIgnore
+    public void save(Path path) {
+        String serialized = uncheck(() -> objectMapper.writeValueAsString(this));
+        uncheck(() -> Files.writeString(path, serialized));
+    }
+
+    @JsonIgnore
+    public static Optional<ReportCoreDumpRequest> load(Path path) {
+        final String serialized;
+        try {
+            serialized = Files.readString(path);
+        } catch (NoSuchFileException e) {
+            return Optional.empty();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+        return Optional.of(uncheck(() -> objectMapper.readValue(serialized, ReportCoreDumpRequest.class)));
+    }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index 49d00c03d23..5ae0620fa7d 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -169,7 +169,8 @@ public class StorageMaintainer {
     /** Checks if container has any new coredumps, reports and archives them if so */
     public void handleCoreDumpsForContainer(NodeAgentContext context, Optional<Container> container, boolean throwIfCoreBeingWritten) {
         if (context.isDisabled(NodeAgentTask.CoreDumps)) return;
-        coredumpHandler.converge(context, () -> getCoredumpNodeAttributes(context, container), throwIfCoreBeingWritten);
+        coredumpHandler.converge(context, () -> getCoredumpNodeAttributes(context, container),
+                                 container.map(Container::image), throwIfCoreBeingWritten);
     }
 
     private Map<String, Object> getCoredumpNodeAttributes(NodeAgentContext context, Optional<Container> container) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
index b5a47e0b3be..d99b45dbbbc 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
 
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata;
 import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
@@ -128,6 +129,33 @@ public class CoreCollector {
         return data;
     }
 
+    CoreDumpMetadata collect2(NodeAgentContext context, ContainerPath coredumpPath) {
+        var metadata = new CoreDumpMetadata();
+
+        if (JAVA_HEAP_DUMP_PATTERN.matcher(coredumpPath.getFileName().toString()).find()) {
+            metadata.setBinPath("java")
+                    .setBacktrace(List.of("Heap dump, no backtrace available"));
+            return metadata;
+        }
+
+        try {
+            String binPath = readBinPath(context, coredumpPath);
+            metadata.setBinPath(binPath);
+
+            if (Path.of(binPath).getFileName().toString().equals("java")) {
+                metadata.setBacktraceAllThreads(readJstack(context, coredumpPath, binPath));
+            } else {
+                metadata.setBacktrace(readBacktrace(context, coredumpPath, binPath, false));
+                metadata.setBacktraceAllThreads(readBacktrace(context, coredumpPath, binPath, true));
+            }
+        } catch (ConvergenceException e) {
+            context.log(logger, Level.WARNING, "Failed to extract backtrace: " + e.getMessage());
+        } catch (RuntimeException e) {
+            context.log(logger, Level.WARNING, "Failed to extract backtrace", e);
+        }
+        return metadata;
+    }
+
     private String asString(CommandResult result) {
         return "exit status " + result.getExitCode() + ", output '" + result.getOutput() + "'";
     }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
index 5113b8d9e98..40606aade3a 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
@@ -4,8 +4,16 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.yahoo.config.provision.DockerImage;
 import com.yahoo.security.SecretSharedKey;
 import com.yahoo.security.SharedKeyGenerator;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.container.metrics.Dimensions;
 import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics;
@@ -15,14 +23,15 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
 import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
-import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
 
 import javax.crypto.CipherOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.UncheckedIOException;
+import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.time.Clock;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -50,6 +59,7 @@ public class CoredumpHandler {
     private static final Pattern HS_ERR_PATTERN = Pattern.compile("hs_err_pid[0-9]+\\.log");
     private static final String PROCESSING_DIRECTORY_NAME = "processing";
     private static final String METADATA_FILE_NAME = "metadata.json";
+    private static final String METADATA2_FILE_NAME = "metadata2.json";
     private static final String COMPRESSED_EXTENSION = ".zst";
     private static final String ENCRYPTED_EXTENSION = ".enc";
     public static final String COREDUMP_FILENAME_PREFIX = "dump_";
@@ -58,6 +68,7 @@ public class CoredumpHandler {
     private final ObjectMapper objectMapper = new ObjectMapper();
 
     private final CoreCollector coreCollector;
+    private final Cores cores;
     private final CoredumpReporter coredumpReporter;
     private final String crashPatchInContainer;
     private final Path doneCoredumpsPath;
@@ -65,22 +76,26 @@ public class CoredumpHandler {
     private final Clock clock;
     private final Supplier<String> coredumpIdSupplier;
     private final Supplier<SecretSharedKey> secretSharedKeySupplier;
+    private final BooleanFlag reportCoresViaCfgFlag;
 
     /**
      * @param crashPathInContainer path inside the container where core dump are dumped
-     * @param doneCoredumpsPath path on host where processed core dumps are stored
+     * @param doneCoredumpsPath    path on host where processed core dumps are stored
      */
-    public CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter,
-                           String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics) {
-        this(coreCollector, coredumpReporter, crashPathInContainer, doneCoredumpsPath,
-                metrics, Clock.systemUTC(), () -> UUID.randomUUID().toString(), () -> null /*TODO*/);
+    public CoredumpHandler(CoreCollector coreCollector, Cores cores, CoredumpReporter coredumpReporter,
+                           String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics,
+                           FlagSource flagSource) {
+        this(coreCollector, cores, coredumpReporter, crashPathInContainer, doneCoredumpsPath,
+             metrics, Clock.systemUTC(), () -> UUID.randomUUID().toString(), () -> null /*TODO*/,
+             flagSource);
     }
 
-    CoredumpHandler(CoreCollector coreCollector, CoredumpReporter coredumpReporter,
+    CoredumpHandler(CoreCollector coreCollector, Cores cores, CoredumpReporter coredumpReporter,
                     String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics,
                     Clock clock, Supplier<String> coredumpIdSupplier,
-                    Supplier<SecretSharedKey> secretSharedKeySupplier) {
+                    Supplier<SecretSharedKey> secretSharedKeySupplier, FlagSource flagSource) {
         this.coreCollector = coreCollector;
+        this.cores = cores;
         this.coredumpReporter = coredumpReporter;
         this.crashPatchInContainer = crashPathInContainer;
         this.doneCoredumpsPath = doneCoredumpsPath;
@@ -88,10 +103,12 @@ public class CoredumpHandler {
         this.clock = clock;
         this.coredumpIdSupplier = coredumpIdSupplier;
         this.secretSharedKeySupplier = secretSharedKeySupplier;
+        this.reportCoresViaCfgFlag = Flags.REPORT_CORES_VIA_CFG.bindTo(flagSource);
     }
 
 
-    public void converge(NodeAgentContext context, Supplier<Map<String, Object>> nodeAttributesSupplier, boolean throwIfCoreBeingWritten) {
+    public void converge(NodeAgentContext context, Supplier<Map<String, Object>> nodeAttributesSupplier,
+                         Optional<DockerImage> dockerImage, boolean throwIfCoreBeingWritten) {
         ContainerPath containerCrashPath = context.paths().of(crashPatchInContainer, context.users().vespa());
         ContainerPath containerProcessingPath = containerCrashPath.resolve(PROCESSING_DIRECTORY_NAME);
 
@@ -110,7 +127,13 @@ public class CoredumpHandler {
 
         // Check if we have already started to process a core dump or we can enqueue a new core one
         getCoredumpToProcess(containerCrashPath, containerProcessingPath)
-                .ifPresent(path -> processAndReportSingleCoredump(context, path, nodeAttributesSupplier));
+                .ifPresent(path -> {
+                    if (reportCoresViaCfgFlag.with(FetchVector.Dimension.NODE_TYPE, context.nodeType().name()).value()) {
+                        processAndReportSingleCoreDump2(context, path, dockerImage);
+                    } else {
+                        processAndReportSingleCoredump(context, path, nodeAttributesSupplier);
+                    }
+                });
     }
 
     /** @return path to directory inside processing directory that contains a core dump file to process */
@@ -224,7 +247,7 @@ public class CoredumpHandler {
      * Compresses and, if a key is provided, encrypts core file (and deletes the uncompressed core), then moves
      * the entire core dump processing directory to {@link #doneCoredumpsPath} for archive
      */
-    private void finishProcessing(NodeAgentContext context, ContainerPath coredumpDirectory, Optional<SecretSharedKey> sharedCoreKey) throws IOException {
+    private void finishProcessing(NodeAgentContext context, ContainerPath coredumpDirectory, Optional<SecretSharedKey> sharedCoreKey) {
         ContainerPath coreFile = findCoredumpFileInProcessingDirectory(coredumpDirectory);
         String extension = COMPRESSED_EXTENSION + (sharedCoreKey.isPresent() ? ENCRYPTED_EXTENSION : "");
         ContainerPath compressedCoreFile = coreFile.resolveSibling(coreFile.getFileName() + extension);
@@ -235,12 +258,12 @@ public class CoredumpHandler {
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
-        Files.delete(coreFile);
+        uncheck(() -> Files.delete(coreFile));
 
         Path newCoredumpDirectory = doneCoredumpsPath.resolve(context.containerName().asString());
         uncheck(() -> Files.createDirectories(newCoredumpDirectory));
         // Files.move() does not support moving non-empty directories across providers, move using host paths
-        Files.move(coredumpDirectory.pathOnHost(), newCoredumpDirectory.resolve(coredumpDirectory.getFileName().toString()));
+        uncheck(() -> Files.move(coredumpDirectory.pathOnHost(), newCoredumpDirectory.resolve(coredumpDirectory.getFileName().toString())));
     }
 
     ContainerPath findCoredumpFileInProcessingDirectory(ContainerPath coredumpProccessingDirectory) {
@@ -314,4 +337,60 @@ public class CoredumpHandler {
         return clock.instant().minusSeconds(60).isAfter(fileAttributes.lastModifiedTime());
     }
 
+    void processAndReportSingleCoreDump2(NodeAgentContext context, ContainerPath coreDumpDirectory,
+                                         Optional<DockerImage> dockerImage) {
+        CoreDumpMetadata metadata = gatherMetadata(context, coreDumpDirectory);
+        dockerImage.ifPresent(metadata::setDockerImage);
+        dockerImage.flatMap(DockerImage::tag).ifPresent(metadata::setVespaVersion);
+        dockerImage.ifPresent(metadata::setDockerImage);
+        Optional<SecretSharedKey> sharedCoreKey = Optional.ofNullable(secretSharedKeySupplier.get());
+        sharedCoreKey.map(key -> key.sealedSharedKey().toTokenString()).ifPresent(metadata::setDecryptionToken);
+
+        String coreDumpId = coreDumpDirectory.getFileName().toString();
+        cores.report(context.hostname(), coreDumpId, metadata);
+        finishProcessing(context, coreDumpDirectory, sharedCoreKey);
+        context.log(logger, "Successfully reported core dump " + coreDumpId);
+    }
+
+    private CoreDumpMetadata gatherMetadata(NodeAgentContext context, ContainerPath coreDumpDirectory) {
+        ContainerPath metadataPath = coreDumpDirectory.resolve(METADATA2_FILE_NAME);
+        Optional<ReportCoreDumpRequest> request = ReportCoreDumpRequest.load(metadataPath);
+        if (request.isPresent()) {
+            return request.map(requestInstance -> {
+                              var metadata = new CoreDumpMetadata();
+                              requestInstance.populateMetadata(metadata, FileSystems.getDefault());
+                              return metadata;
+                          })
+                          .get();
+        }
+
+        ContainerPath coreDumpFile = findCoredumpFileInProcessingDirectory(coreDumpDirectory);
+        CoreDumpMetadata metadata = coreCollector.collect2(context, coreDumpFile);
+        metadata.setCpuMicrocodeVersion(getMicrocodeVersion())
+                .setKernelVersion(System.getProperty("os.version"))
+                .setCoreDumpPath(doneCoredumpsPath.resolve(context.containerName().asString())
+                                                  .resolve(coreDumpDirectory.getFileName().toString())
+                                                  .resolve(coreDumpFile.getFileName().toString()));
+
+        ReportCoreDumpRequest requestInstance = new ReportCoreDumpRequest();
+        requestInstance.fillFrom(metadata);
+        requestInstance.save(metadataPath);
+        context.log(logger, "Wrote " + metadataPath);
+        return metadata;
+    }
+
+    private String getMicrocodeVersion() {
+        String output = uncheck(() -> Files.readAllLines(Paths.get("/proc/cpuinfo")).stream()
+                                           .filter(line -> line.startsWith("microcode"))
+                                           .findFirst()
+                                           .orElse("microcode : UNKNOWN"));
+
+        String[] results = output.split(":");
+        if (results.length != 2) {
+            throw ConvergenceException.ofError("Result from detect microcode command not as expected: " + output);
+        }
+
+        return results[1].trim();
+    }
+
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java
index 66b6e9cc67d..f49dd2e705b 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java
@@ -8,15 +8,21 @@ import com.yahoo.test.json.JsonTestHelper;
 import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
 import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.node.admin.configserver.StandardConfigServerResponse;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
+import com.yahoo.vespa.test.file.TestFileSystem;
 import org.junit.jupiter.api.Test;
 import org.mockito.ArgumentCaptor;
 
+import java.nio.file.FileSystem;
 import java.nio.file.Path;
 import java.util.List;
+import java.util.Optional;
 
 import static com.yahoo.yolean.Exceptions.uncheck;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -27,6 +33,7 @@ import static org.mockito.Mockito.when;
  * @author hakonhall
  */
 class CoresTest {
+    private final FileSystem fileSystem = TestFileSystem.create();
     private final ObjectMapper mapper = new ObjectMapper();
     private final ConfigServerApi configServerApi = mock(ConfigServerApi.class);
     private final Cores cores = new CoresImpl(configServerApi);
@@ -35,7 +42,8 @@ class CoresTest {
     private final CoreDumpMetadata metadata = new CoreDumpMetadata()
             .setKernelVersion("4.18.0-372.26.1.el8_6.x86_64")
             .setCpuMicrocodeVersion("0x1000065")
-            .setCoreDumpPath(Path.of("/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813"))
+            .setCoreDumpPath(fileSystem.getPath("/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813"))
+            .setDecryptionToken("987def")
             .setDockerImage(DockerImage.fromString("us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8"))
             .setBinPath("/usr/bin/java")
             .setVespaVersion("8.68.8")
@@ -75,6 +83,7 @@ class CoresTest {
                        "bin_path": "/usr/bin/java",
                        "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813",
                        "cpu_microcode_version": "0x1000065",
+                       "decryption_token": "987def",
                        "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8",
                        "kernel_version": "4.18.0-372.26.1.el8_6.x86_64",
                        "vespa_version": "8.68.8"
@@ -97,4 +106,39 @@ class CoresTest {
         var bodyJsonPojoCaptor = ArgumentCaptor.forClass(Object.class);
         verify(configServerApi).post(pathCaptor.capture(), bodyJsonPojoCaptor.capture(), any());
     }
+
+    @Test
+    void serialization() {
+        Path path = fileSystem.getPath("/foo.json");
+        ReportCoreDumpRequest request = new ReportCoreDumpRequest().fillFrom(metadata);
+        request.save(path);
+        assertEquals("""
+                     {
+                       "backtrace": [
+                         "Example",
+                         "of",
+                         "backtrace"
+                       ],
+                       "backtrace_all_threads": [
+                         "Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...",
+                         "Debugger attached successfully.",
+                         " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)",
+                         " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)"
+                       ],
+                       "bin_path": "/usr/bin/java",
+                       "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813",
+                       "cpu_microcode_version": "0x1000065",
+                       "decryption_token": "987def",
+                       "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8",
+                       "kernel_version": "4.18.0-372.26.1.el8_6.x86_64",
+                       "vespa_version": "8.68.8"
+                     }""",
+                     JsonTestHelper.normalize(new UnixPath(path).readUtf8File()));
+
+        Optional<ReportCoreDumpRequest> loaded = ReportCoreDumpRequest.load(path);
+        assertTrue(loaded.isPresent());
+        var meta = new CoreDumpMetadata();
+        loaded.get().populateMetadata(meta, fileSystem);
+        assertEquals(metadata, meta);
+    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
index 3906c13c4a4..9b73be6fbb1 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
@@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
 import com.yahoo.security.SealedSharedKey;
 import com.yahoo.security.SecretSharedKey;
 import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores;
 import com.yahoo.vespa.hosted.node.admin.container.metrics.DimensionMetrics;
 import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
@@ -52,6 +54,7 @@ public class CoredumpHandlerTest {
     private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps");
 
     private final CoreCollector coreCollector = mock(CoreCollector.class);
+    private final Cores cores = mock(Cores.class);
     private final CoredumpReporter coredumpReporter = mock(CoredumpReporter.class);
     private final Metrics metrics = new Metrics();
     private final ManualClock clock = new ManualClock();
@@ -59,10 +62,11 @@ public class CoredumpHandlerTest {
     private final Supplier<String> coredumpIdSupplier = mock(Supplier.class);
     @SuppressWarnings("unchecked")
     private final Supplier<SecretSharedKey> secretSharedKeySupplier = mock(Supplier.class);
-    private final CoredumpHandler coredumpHandler = new CoredumpHandler(coreCollector, coredumpReporter,
-            containerCrashPath.pathInContainer(), doneCoredumpsPath, metrics, clock, coredumpIdSupplier,
-            secretSharedKeySupplier);
-
+    private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
+    private final CoredumpHandler coredumpHandler =
+            new CoredumpHandler(coreCollector, cores, coredumpReporter, containerCrashPath.pathInContainer(),
+                                doneCoredumpsPath, metrics, clock, coredumpIdSupplier, secretSharedKeySupplier,
+                                flagSource);
 
     @Test
     void coredump_enqueue_test() throws IOException {