Merge pull request #19178 from vespa-engine/bjorncs/service-dump

Bjorncs/service dump

6 files changed, 45 insertions, 11 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 65e046b448d..4dd380324fa 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -35,6 +35,7 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
 import com.yahoo.vespa.serviceview.bindings.ApplicationView;
+import com.yahoo.yolean.concurrent.Sleeper;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -73,6 +74,7 @@ public class Controller extends AbstractComponent {
     private final TenantController tenantController;
     private final JobController jobController;
     private final Clock clock;
+    private final Sleeper sleeper;
     private final ZoneRegistry zoneRegistry;
     private final ServiceRegistry serviceRegistry;
     private final AuditLogger auditLogger;
@@ -97,18 +99,20 @@ public class Controller extends AbstractComponent {
                       MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore,
                       ControllerConfig controllerConfig) {
         this(curator, rotationsConfig, accessControl, com.yahoo.net.HostName::getLocalhost, flagSource,
-             mavenRepository, serviceRegistry, metric, secretStore, controllerConfig);
+             mavenRepository, serviceRegistry, metric, secretStore, controllerConfig, Sleeper.DEFAULT);
     }
 
     public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl,
                       Supplier<String> hostnameSupplier, FlagSource flagSource, MavenRepository mavenRepository,
-                      ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, ControllerConfig controllerConfig) {
+                      ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore,
+                      ControllerConfig controllerConfig, Sleeper sleeper) {
 
         this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null");
         this.curator = Objects.requireNonNull(curator, "Curator cannot be null");
         this.serviceRegistry = Objects.requireNonNull(serviceRegistry, "ServiceRegistry cannot be null");
         this.zoneRegistry = Objects.requireNonNull(serviceRegistry.zoneRegistry(), "ZoneRegistry cannot be null");
         this.clock = Objects.requireNonNull(serviceRegistry.clock(), "Clock cannot be null");
+        this.sleeper = Objects.requireNonNull(sleeper);
         this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null");
         this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null");
         this.metric = Objects.requireNonNull(metric, "Metric cannot be null");
@@ -158,6 +162,8 @@ public class Controller extends AbstractComponent {
 
     public Clock clock() { return clock; }
 
+    public Sleeper sleeper() { return sleeper; }
+
     public ZoneRegistry zoneRegistry() { return zoneRegistry; }
 
     public NameServiceForwarder nameServiceForwarder() { return nameServiceForwarder; }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java
index 6cc7446499f..1aeae76aad4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.proxy;
 
-import ai.vespa.util.http.hc4.retry.Sleeper;
 import com.google.inject.Inject;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.jdisc.http.HttpRequest.Method;
@@ -10,6 +9,7 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
 import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
+import com.yahoo.yolean.concurrent.Sleeper;
 import org.apache.http.Header;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.config.RequestConfig;
@@ -69,7 +69,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C
 
     @Inject
     public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, ServiceIdentityProvider sslContextProvider) {
-        this(zoneRegistry, sslContextProvider.getIdentitySslContext(), new Sleeper.Default(),
+        this(zoneRegistry, sslContextProvider.getIdentitySslContext(), Sleeper.DEFAULT,
              new ConnectionReuseStrategy(zoneRegistry));
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 3c7c5f6ac30..1a2acb82348 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -77,7 +77,6 @@ import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition;
 import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
 import com.yahoo.vespa.hosted.controller.application.ActivateResult;
-import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
@@ -87,6 +86,7 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList;
 import com.yahoo.vespa.hosted.controller.application.QuotaUsage;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps;
@@ -2086,7 +2086,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         if (report != null) {
             Cursor cursor = report.get();
             // Note: same behaviour for both value '0' and missing value.
-            if (cursor.field("failedAt").asLong() == 0 && cursor.field("completedAt").asLong() == 0) {
+            boolean force = request.getBooleanProperty("force");
+            if (!force && cursor.field("failedAt").asLong() == 0 && cursor.field("completedAt").asLong() == 0) {
                 throw new IllegalArgumentException("Service dump already in progress for " + cursor.field("configId").asString());
             }
         }
@@ -2119,9 +2120,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         if (expiresAt > 0) {
             dumpRequestCursor.setLong("expiresAt", expiresAt);
         }
+        Cursor dumpOptionsCursor = requestPayloadCursor.field("dumpOptions");
+        if (dumpOptionsCursor.children() > 0) {
+            SlimeUtils.copyObject(dumpOptionsCursor, dumpRequestCursor.setObject("dumpOptions"));
+        }
         var reportsUpdate = Map.of("serviceDump", new String(uncheck(() -> SlimeUtils.toJsonBytes(dumpRequest))));
         nodeRepository.updateReports(zone, hostname, reportsUpdate);
-        return new MessageResponse("Request created");
+        boolean wait = request.getBooleanProperty("wait");
+        if (!wait) return new MessageResponse("Request created");
+        return waitForServiceDumpResult(nodeRepository, zone, tenant, application, instance, hostname);
     }
 
     private HttpResponse getServiceDump(String tenant, String application, String instance, String environment,
@@ -2133,6 +2140,24 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         return new SlimeJsonResponse(report);
     }
 
+    private HttpResponse waitForServiceDumpResult(NodeRepository nodeRepository, ZoneId zone, String tenant,
+                                                  String application, String instance, String hostname) {
+        int pollInterval = 2;
+        Slime report;
+        while (true) {
+            report = getReport(nodeRepository, zone, tenant, application, instance, hostname).get();
+            Cursor cursor = report.get();
+            if (cursor.field("completedAt").asLong() > 0 || cursor.field("failedAt").asLong() > 0) {
+                break;
+            }
+            final Slime copyForLambda = report;
+            log.fine(() -> uncheck(() -> new String(SlimeUtils.toJsonBytes(copyForLambda))));
+            log.fine("Sleeping " + pollInterval + " seconds before checking report status again");
+            controller.sleeper().sleep(Duration.ofSeconds(pollInterval));
+        }
+        return new SlimeJsonResponse(report);
+    }
+
     private Optional<Slime> getReport(NodeRepository nodeRepository, ZoneId zone, String tenant,
                                       String application, String instance, String hostname) {
         Node node;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index bae0867736a..445a1c73297 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -57,6 +57,7 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.vespa.hosted.controller.versions.ControllerVersion;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
+import com.yahoo.yolean.concurrent.Sleeper;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -387,7 +388,8 @@ public final class ControllerTester {
                                                new MockMavenRepository(),
                                                serviceRegistry,
                                                new MetricsMock(), new SecretStoreMock(),
-                                               new ControllerConfig.Builder().build());
+                                               new ControllerConfig.Builder().build(),
+                                               Sleeper.NOOP);
         // Calculate initial versions
         controller.updateVersionStatus(VersionStatus.compute(controller));
         return controller;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java
index 1fce7ba5695..7108bb5d423 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java
@@ -7,6 +7,7 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
+import com.yahoo.yolean.concurrent.Sleeper;
 import org.apache.http.protocol.HttpContext;
 import org.apache.http.protocol.HttpCoreContext;
 import org.junit.Rule;
@@ -38,7 +39,7 @@ public class ConfigServerRestExecutorImplTest {
     public void proxy_with_retries() throws Exception {
         var connectionReuseStrategy = new CountingConnectionReuseStrategy(Set.of("127.0.0.1"));
         var proxy = new ConfigServerRestExecutorImpl(new ZoneRegistryMock(SystemName.cd), SSLContext.getDefault(),
-                                                     (duration) -> {}, connectionReuseStrategy);
+                                                     Sleeper.NOOP, connectionReuseStrategy);
 
         URI url = url();
         String path = url.getPath();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 72f0fc283e5..66c3f7bba16 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -621,14 +621,14 @@ public class ApplicationApiTest extends ControllerContainerTest {
         // POST to request a service dump
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/node/host-tenant1:application1:instance1-prod.us-central-1/service-dump", POST)
                         .userIdentity(HOSTED_VESPA_OPERATOR)
-                        .data("{\"configId\":\"default/container.1\",\"artifacts\":[\"jvm-dump\"]}"),
+                        .data("{\"configId\":\"default/container.1\",\"artifacts\":[\"jvm-dump\"],\"dumpOptions\":{\"duration\":30}}"),
                 "{\"message\":\"Request created\"}");
 
         // GET to get status of service dump
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/node/host-tenant1:application1:instance1-prod.us-central-1/service-dump", GET)
                         .userIdentity(HOSTED_VESPA_OPERATOR),
                 "{\"createdMillis\":" + tester.controller().clock().millis() + ",\"configId\":\"default/container.1\"" +
-                        ",\"artifacts\":[\"jvm-dump\"]}");
+                        ",\"artifacts\":[\"jvm-dump\"],\"dumpOptions\":{\"duration\":30}}");
 
         // POST a 'restart application' command
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST)