diff options
4 files changed, 87 insertions, 3 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 327175c19ed..135429be8f9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -153,6 +153,10 @@ enum PathGroup { "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/reindex", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/reindexing"), + serviceDump(Matcher.tenant, + Matcher.application, + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/node/{node}/service-dump"), + /** Paths used for development deployments. */ developmentDeployment(Matcher.tenant, Matcher.application, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index eae5ad5b685..ef08ded917c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -84,7 +84,7 @@ enum Policy { /** Read access to application information and settings. */ applicationRead(Privilege.grant(Action.read) - .on(PathGroup.application, PathGroup.applicationInfo, PathGroup.reindexing) + .on(PathGroup.application, PathGroup.applicationInfo, PathGroup.reindexing, PathGroup.serviceDump) .in(SystemName.all())), /** Read access to application information and settings. */ @@ -99,7 +99,7 @@ enum Policy { /** Full access to application information and settings. */ applicationOperations(Privilege.grant(Action.write()) - .on(PathGroup.applicationInfo, PathGroup.productionRestart, PathGroup.reindexing) + .on(PathGroup.applicationInfo, PathGroup.productionRestart, PathGroup.reindexing, PathGroup.serviceDump) .in(SystemName.all())), /** Access to create and delete developer and deploy keys under a tenant. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index e2f456b33f0..5d4e25148f9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -51,6 +51,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbi import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.aws.TenantRoles; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; @@ -60,6 +61,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -143,12 +145,12 @@ import java.util.OptionalLong; import java.util.Scanner; import java.util.StringJoiner; import java.util.logging.Level; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.CONFLICT; +import static com.yahoo.yolean.Exceptions.uncheck; import static java.util.Map.Entry.comparingByKey; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; @@ -265,6 +267,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/content/{*}")) return content(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/access/support")) return supportAccess(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/node/{node}/service-dump")) return getServiceDump(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("node"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/metrics")) return metrics(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("endpointId"))); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); @@ -315,6 +318,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspend")) return suspend(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/access/support")) return allowSupportAccess(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/node/{node}/service-dump")) return requestServiceDump(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("node"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -2048,6 +2052,71 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { controller.applications().reachableContentClustersByZone(deployments))); } + private HttpResponse requestServiceDump(String tenant, String application, String instance, String environment, + String region, String hostname, HttpRequest request) { + NodeRepository nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); + ZoneId zone = requireZone(environment, region); + + // Check that no other service dump is in progress + Slime report = getReport(nodeRepository, zone, tenant, application, instance, hostname).orElse(null); + if (report != null) { + Cursor cursor = report.get(); + // Note: same behaviour for both value '0' and missing value. + if (cursor.field("failedAt").asLong() == 0 && cursor.field("completedAt").asLong() == 0) { + throw new IllegalArgumentException("Service dump already in progress for " + cursor.field("configId").asString()); + } + } + Slime requestPayload; + try { + requestPayload = SlimeUtils.jsonToSlimeOrThrow(request.getData().readAllBytes()); + } catch (Exception e) { + throw new IllegalArgumentException("Missing or invalid JSON in request content", e); + } + Cursor requestPayloadCursor = requestPayload.get(); + String configId = requestPayloadCursor.field("configId").asString(); + long expiresAt = requestPayloadCursor.field("expiresAt").asLong(); + if (configId.isEmpty()) { + throw new IllegalArgumentException("Missing configId"); + } + Slime dumpRequest = new Slime(); + Cursor dumpRequestCursor = dumpRequest.setObject(); + dumpRequestCursor.setLong("createdMillis", controller.clock().millis()); + dumpRequestCursor.setString("configId", configId); + if (expiresAt > 0) { + dumpRequestCursor.setLong("expiresAt", expiresAt); + } + var reportsUpdate = Map.of("serviceDump", new String(uncheck(() -> SlimeUtils.toJsonBytes(dumpRequest)))); + nodeRepository.updateReports(zone, hostname, reportsUpdate); + return new MessageResponse("Request created"); + } + + private HttpResponse getServiceDump(String tenant, String application, String instance, String environment, + String region, String hostname, HttpRequest request) { + NodeRepository nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); + ZoneId zone = requireZone(environment, region); + Slime report = getReport(nodeRepository, zone, tenant, application, instance, hostname).orElse(null); + if (report == null) throw new NotExistsException("No service dump for node " + hostname); + return new SlimeJsonResponse(report); + } + + private Optional<Slime> getReport(NodeRepository nodeRepository, ZoneId zone, String tenant, + String application, String instance, String hostname) { + Node node; + try { + node = nodeRepository.getNode(zone, hostname); + } catch (IllegalArgumentException e) { + throw new NotExistsException(new Hostname(hostname)); + } + ApplicationId app = ApplicationId.from(tenant, application, instance); + ApplicationId owner = node.owner().orElseThrow(() -> new IllegalArgumentException("Node has no owner")); + if (!app.equals(owner)) { + throw new IllegalArgumentException("Node is owned by " + owner.toFullString()); + } + String json = node.reports().get("serviceDump"); + if (json == null) return Optional.empty(); + return Optional.of(SlimeUtils.jsonToSlimeOrThrow(json)); + } + private static SourceRevision toSourceRevision(Inspector object) { if (!object.field("repository").valid() || !object.field("branch").valid() || diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 8a182ce5eca..ed6fc776e8d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -611,6 +611,17 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), "{\"enabled\":true,\"clusters\":[{\"name\":\"cluster\",\"pending\":[{\"type\":\"type\",\"requiredGeneration\":100}],\"ready\":[{\"type\":\"type\",\"readyAtMillis\":345,\"startedAtMillis\":456,\"endedAtMillis\":567,\"state\":\"failed\",\"message\":\"(＃｀д´)ﾉ\",\"progress\":0.1}]}]}"); + // POST to request a service dump + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/node/host-tenant1:application1:instance1-prod.us-central-1/service-dump", POST) + .userIdentity(HOSTED_VESPA_OPERATOR) + .data("{\"configId\":\"default/container.1\"}"), + "{\"message\":\"Request created\"}"); + + // GET to get status of service dump + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/node/host-tenant1:application1:instance1-prod.us-central-1/service-dump", GET) + .userIdentity(HOSTED_VESPA_OPERATOR), + "{\"createdMillis\":" + tester.controller().clock().millis() + ",\"configId\":\"default/container.1\"}"); + // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) .userIdentity(USER_ID), |