diff options
8 files changed, 46 insertions, 22 deletions
diff --git a/container-core/src/main/sh/vespa-jvm-dumper b/container-core/src/main/sh/vespa-jvm-dumper index 6e829e029b1..bc886be39f1 100755 --- a/container-core/src/main/sh/vespa-jvm-dumper +++ b/container-core/src/main/sh/vespa-jvm-dumper @@ -92,16 +92,6 @@ fi jvm_pid=$($VESPA_HOME/libexec/vespa/find-pid "$service") echo "Pid for '${service}' is '${jvm_pid}'" -echo "Starting Java Flight Recorder recording" -jcmd ${jvm_pid} JFR.start name=vespa-jvm-dumper path-to-gc-roots=true settings=profile filename=${output_directory}/jvm-jfr-dump.jfr - -readonly sleep_seconds=15 -echo "Waiting ${sleep_seconds} before stopping Java Flight Recorder" -sleep ${sleep_seconds}s - -echo "Dumping Java Flight Recorder recording to file" -jcmd ${jvm_pid} JFR.dump name=vespa-jvm-dumper - echo "Creating heap dump" readonly heap_dump_file=${output_directory}/jvm-heap-dump.hprof if test -f "${heap_dump_file}"; then diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java index 53e80fcb2ed..951bdb842aa 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java @@ -36,7 +36,7 @@ public class ZtsClientMock implements ZtsClient { @Override public List<AthenzDomain> getTenantDomains(AthenzIdentity providerIdentity, AthenzIdentity userIdentity, String roleName) { - log.log(Level.INFO, String.format("getTenantDomains(providerIdentity='%s', userIdentity='%s', roleName='%s')", + log.log(Level.FINE, String.format("getTenantDomains(providerIdentity='%s', userIdentity='%s', roleName='%s')", providerIdentity.getFullName(), userIdentity.getFullName(), roleName)); return athenz.domains.values().stream() .filter(domain -> domain.tenantAdmins.contains(userIdentity) || domain.admins.contains(userIdentity)) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 65e046b448d..4dd380324fa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -35,6 +35,7 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import com.yahoo.vespa.serviceview.bindings.ApplicationView; +import com.yahoo.yolean.concurrent.Sleeper; import java.time.Clock; import java.time.Duration; @@ -73,6 +74,7 @@ public class Controller extends AbstractComponent { private final TenantController tenantController; private final JobController jobController; private final Clock clock; + private final Sleeper sleeper; private final ZoneRegistry zoneRegistry; private final ServiceRegistry serviceRegistry; private final AuditLogger auditLogger; @@ -97,18 +99,20 @@ public class Controller extends AbstractComponent { MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, ControllerConfig controllerConfig) { this(curator, rotationsConfig, accessControl, com.yahoo.net.HostName::getLocalhost, flagSource, - mavenRepository, serviceRegistry, metric, secretStore, controllerConfig); + mavenRepository, serviceRegistry, metric, secretStore, controllerConfig, Sleeper.DEFAULT); } public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, Supplier<String> hostnameSupplier, FlagSource flagSource, MavenRepository mavenRepository, - ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, ControllerConfig controllerConfig) { + ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, + ControllerConfig controllerConfig, Sleeper sleeper) { this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null"); this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); this.serviceRegistry = Objects.requireNonNull(serviceRegistry, "ServiceRegistry cannot be null"); this.zoneRegistry = Objects.requireNonNull(serviceRegistry.zoneRegistry(), "ZoneRegistry cannot be null"); this.clock = Objects.requireNonNull(serviceRegistry.clock(), "Clock cannot be null"); + this.sleeper = Objects.requireNonNull(sleeper); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null"); this.metric = Objects.requireNonNull(metric, "Metric cannot be null"); @@ -158,6 +162,8 @@ public class Controller extends AbstractComponent { public Clock clock() { return clock; } + public Sleeper sleeper() { return sleeper; } + public ZoneRegistry zoneRegistry() { return zoneRegistry; } public NameServiceForwarder nameServiceForwarder() { return nameServiceForwarder; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 6cc7446499f..1aeae76aad4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import ai.vespa.util.http.hc4.retry.Sleeper; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.HttpRequest.Method; @@ -10,6 +9,7 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.yolean.concurrent.Sleeper; import org.apache.http.Header; import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; @@ -69,7 +69,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C @Inject public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, ServiceIdentityProvider sslContextProvider) { - this(zoneRegistry, sslContextProvider.getIdentitySslContext(), new Sleeper.Default(), + this(zoneRegistry, sslContextProvider.getIdentitySslContext(), Sleeper.DEFAULT, new ConnectionReuseStrategy(zoneRegistry)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 3c7c5f6ac30..1a2acb82348 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -77,7 +77,6 @@ import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.application.ActivateResult; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -87,6 +86,7 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; @@ -2086,7 +2086,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (report != null) { Cursor cursor = report.get(); // Note: same behaviour for both value '0' and missing value. - if (cursor.field("failedAt").asLong() == 0 && cursor.field("completedAt").asLong() == 0) { + boolean force = request.getBooleanProperty("force"); + if (!force && cursor.field("failedAt").asLong() == 0 && cursor.field("completedAt").asLong() == 0) { throw new IllegalArgumentException("Service dump already in progress for " + cursor.field("configId").asString()); } } @@ -2119,9 +2120,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (expiresAt > 0) { dumpRequestCursor.setLong("expiresAt", expiresAt); } + Cursor dumpOptionsCursor = requestPayloadCursor.field("dumpOptions"); + if (dumpOptionsCursor.children() > 0) { + SlimeUtils.copyObject(dumpOptionsCursor, dumpRequestCursor.setObject("dumpOptions")); + } var reportsUpdate = Map.of("serviceDump", new String(uncheck(() -> SlimeUtils.toJsonBytes(dumpRequest)))); nodeRepository.updateReports(zone, hostname, reportsUpdate); - return new MessageResponse("Request created"); + boolean wait = request.getBooleanProperty("wait"); + if (!wait) return new MessageResponse("Request created"); + return waitForServiceDumpResult(nodeRepository, zone, tenant, application, instance, hostname); } private HttpResponse getServiceDump(String tenant, String application, String instance, String environment, @@ -2133,6 +2140,24 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(report); } + private HttpResponse waitForServiceDumpResult(NodeRepository nodeRepository, ZoneId zone, String tenant, + String application, String instance, String hostname) { + int pollInterval = 2; + Slime report; + while (true) { + report = getReport(nodeRepository, zone, tenant, application, instance, hostname).get(); + Cursor cursor = report.get(); + if (cursor.field("completedAt").asLong() > 0 || cursor.field("failedAt").asLong() > 0) { + break; + } + final Slime copyForLambda = report; + log.fine(() -> uncheck(() -> new String(SlimeUtils.toJsonBytes(copyForLambda)))); + log.fine("Sleeping " + pollInterval + " seconds before checking report status again"); + controller.sleeper().sleep(Duration.ofSeconds(pollInterval)); + } + return new SlimeJsonResponse(report); + } + private Optional<Slime> getReport(NodeRepository nodeRepository, ZoneId zone, String tenant, String application, String instance, String hostname) { Node node; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index bae0867736a..445a1c73297 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -57,6 +57,7 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import com.yahoo.yolean.concurrent.Sleeper; import java.time.Duration; import java.time.Instant; @@ -387,7 +388,8 @@ public final class ControllerTester { new MockMavenRepository(), serviceRegistry, new MetricsMock(), new SecretStoreMock(), - new ControllerConfig.Builder().build()); + new ControllerConfig.Builder().build(), + Sleeper.NOOP); // Calculate initial versions controller.updateVersionStatus(VersionStatus.compute(controller)); return controller; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java index 1fce7ba5695..7108bb5d423 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; +import com.yahoo.yolean.concurrent.Sleeper; import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpCoreContext; import org.junit.Rule; @@ -38,7 +39,7 @@ public class ConfigServerRestExecutorImplTest { public void proxy_with_retries() throws Exception { var connectionReuseStrategy = new CountingConnectionReuseStrategy(Set.of("127.0.0.1")); var proxy = new ConfigServerRestExecutorImpl(new ZoneRegistryMock(SystemName.cd), SSLContext.getDefault(), - (duration) -> {}, connectionReuseStrategy); + Sleeper.NOOP, connectionReuseStrategy); URI url = url(); String path = url.getPath(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 72f0fc283e5..66c3f7bba16 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -621,14 +621,14 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST to request a service dump tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/node/host-tenant1:application1:instance1-prod.us-central-1/service-dump", POST) .userIdentity(HOSTED_VESPA_OPERATOR) - .data("{\"configId\":\"default/container.1\",\"artifacts\":[\"jvm-dump\"]}"), + .data("{\"configId\":\"default/container.1\",\"artifacts\":[\"jvm-dump\"],\"dumpOptions\":{\"duration\":30}}"), "{\"message\":\"Request created\"}"); // GET to get status of service dump tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/node/host-tenant1:application1:instance1-prod.us-central-1/service-dump", GET) .userIdentity(HOSTED_VESPA_OPERATOR), "{\"createdMillis\":" + tester.controller().clock().millis() + ",\"configId\":\"default/container.1\"" + - ",\"artifacts\":[\"jvm-dump\"]}"); + ",\"artifacts\":[\"jvm-dump\"],\"dumpOptions\":{\"duration\":30}}"); // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) |