diff options
14 files changed, 147 insertions, 21 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/RunDataStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/RunDataStore.java index 9cf508d37c9..3558d18f721 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/RunDataStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/RunDataStore.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import java.io.InputStream; import java.util.Optional; /** @@ -29,4 +30,10 @@ public interface RunDataStore { /** Deletes all data associated with the given application. */ void delete(ApplicationId id); + /** Stores Vespa logs for the run. */ + void putLogs(RunId id, boolean tester, InputStream logs); + + /** Fetches Vespa logs for the run. */ + InputStream getLogs(RunId id, boolean tester); + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockRunDataStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockRunDataStore.java index 32591cfe4e2..cc1e084de8e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockRunDataStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockRunDataStore.java @@ -5,10 +5,14 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import java.io.ByteArrayInputStream; +import java.io.InputStream; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import static com.yahoo.yolean.Exceptions.uncheck; + /** * @author jonmv */ @@ -16,6 +20,8 @@ public class MockRunDataStore implements RunDataStore { private final Map<RunId, byte[]> logs = new ConcurrentHashMap<>(); private final Map<RunId, byte[]> reports = new ConcurrentHashMap<>(); + private final Map<RunId, byte[]> vespaLogs = new ConcurrentHashMap<>(); + private final Map<RunId, byte[]> testerLogs = new ConcurrentHashMap<>(); @Override public Optional<byte[]> get(RunId id) { @@ -49,4 +55,14 @@ public class MockRunDataStore implements RunDataStore { reports.keySet().removeIf(runId -> runId.application().equals(id)); } + @Override + public void putLogs(RunId id, boolean tester, InputStream logs) { + (tester ? testerLogs : vespaLogs).put(id, uncheck(logs::readAllBytes)); + } + + @Override + public InputStream getLogs(RunId id, boolean tester) { + return new ByteArrayInputStream((tester ? testerLogs : vespaLogs).get(id)); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 3d4f4833757..069aa816af5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -38,6 +38,9 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; @@ -181,20 +184,30 @@ public class JobController { if ( ! run.hasStep(copyVespaLogs)) return run; + storeVespaLogs(id); + + // TODO jonmv: remove all the below around start of 2023. ZoneId zone = id.type().zone(); Optional<Deployment> deployment = Optional.ofNullable(controller.applications().requireInstance(id.application()) .deployments().get(zone)); if (deployment.isEmpty() || deployment.get().at().isBefore(run.start())) return run; - Instant deployedAt = run.stepInfo(installInitialReal).or(() -> run.stepInfo(installReal)).flatMap(StepInfo::startTime).orElseThrow(); - Instant from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.minusSeconds(10); - List<LogEntry> log = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() - .getLogs(new DeploymentId(id.application(), zone), - Map.of("from", Long.toString(from.toEpochMilli()))), - from); + List<LogEntry> log; + Instant deployedAt; + Instant from; + if ( ! run.id().type().isProduction()) { + deployedAt = run.stepInfo(installInitialReal).or(() -> run.stepInfo(installReal)).flatMap(StepInfo::startTime).orElseThrow(); + from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.minusSeconds(10); + log = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() + .getLogs(new DeploymentId(id.application(), zone), + Map.of("from", Long.toString(from.toEpochMilli()))), + from); + } + else + log = List.of(); - if (run.hasStep(installTester)) { + if (id.type().isTest()) { deployedAt = run.stepInfo(installTester).flatMap(StepInfo::startTime).orElseThrow(); from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.minusSeconds(10); List<LogEntry> testerLog = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() @@ -216,6 +229,47 @@ public class JobController { }); } + public InputStream getVespaLogs(RunId id, long fromMillis, boolean tester) { + Run run = run(id); + return run.stepStatus(copyVespaLogs).map(succeeded::equals).orElse(false) + ? controller.serviceRegistry().runDataStore().getLogs(id, tester) + : getVespaLogsFromLogserver(run, fromMillis, tester); + } + + public static Optional<Instant> deploymentCompletedAt(Run run, boolean tester) { + return (tester ? run.stepInfo(installTester) + : run.stepInfo(installInitialReal).or(() -> run.stepInfo(installReal))) + .flatMap(StepInfo::startTime); + } + + public void storeVespaLogs(RunId id) { + Run run = run(id); + if ( ! id.type().isProduction()) { + try (InputStream logs = getVespaLogsFromLogserver(run, 0, false)) { + controller.serviceRegistry().runDataStore().putLogs(id, false, logs); + } + catch (IOException e) { + throw new UncheckedTimeoutException(e); + } + } + if (id.type().isTest()) { + try (InputStream logs = getVespaLogsFromLogserver(run, 0, true)) { + controller.serviceRegistry().runDataStore().putLogs(id, true, logs); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + private InputStream getVespaLogsFromLogserver(Run run, long fromMillis, boolean tester) { + long deploymentCompletedAtMillis = deploymentCompletedAt(run, tester).orElse(Instant.EPOCH).toEpochMilli(); + return controller.serviceRegistry().configServer().getLogs(new DeploymentId(tester ? run.id().tester().id() : run.id().application(), + run.id().type().zone()), + Map.of("from", Long.toString(Math.max(fromMillis, deploymentCompletedAtMillis)), + "to", Long.toString(run.end().orElse(controller.clock().instant()).toEpochMilli()))); + } + /** Fetches any new test log entries, and records the id of the last of these, for continuation. */ public void updateTestLog(RunId id) { locked(id, run -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index 5f9207953ca..f0ec39b8d1c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -63,6 +63,7 @@ public enum JobProfile { installTester, startTests, endTests, + copyVespaLogs, deactivateTester, report)), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 8d2fac84bc0..7a843b59c87 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -272,6 +272,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/diff/{number}")) return devApplicationPackageDiff(runIdFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}/logs")) return JobControllerApiHandlerHelper.vespaLogsResponse(controller.jobController(), runIdFromPath(path), asLong(request.getProperty("from"), 0), request.getBooleanProperty("tester")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return getReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 25953c16bf0..60f65070557 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -8,7 +8,6 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; @@ -16,16 +15,15 @@ import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.NotExistsException; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import com.yahoo.vespa.hosted.controller.deployment.JobController; @@ -39,11 +37,13 @@ import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.net.URI; import java.time.Instant; import java.time.format.TextStyle; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Locale; @@ -53,6 +53,7 @@ import java.util.stream.Stream; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.canary; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.broken; @@ -154,9 +155,27 @@ class JobControllerApiHandlerHelper { .map(Slime::get) .ifPresent(reportArrayCursor -> SlimeUtils.copyArray(reportArrayCursor, detailsObject.setArray("testReports"))); + boolean logsStored = run.stepStatus(copyVespaLogs).map(succeeded::equals).orElse(false); + if (run.hasStep(copyVespaLogs) && ! runId.type().isProduction() && JobController.deploymentCompletedAt(run, false).isPresent()) + detailsObject.setBool("vespaLogsActive", ! logsStored); + + if (runId.type().isTest() && JobController.deploymentCompletedAt(run, true).isPresent()) + detailsObject.setBool("testerLogsActive", ! logsStored); + return new SlimeJsonResponse(slime); } + /** Proxies a Vespa log request for a run to S3 once logs have been copied, or to logserver before this. */ + static HttpResponse vespaLogsResponse(JobController jobController, RunId runId, long fromMillis, boolean tester) { + return new HttpResponse(200) { + @Override public void render(OutputStream out) throws IOException { + try (InputStream logs = jobController.getVespaLogs(runId, fromMillis, tester)) { + logs.transferTo(out); + } + } + }; + } + private static void toSlime(Cursor summaryObject, ConvergenceSummary summary) { summaryObject.setLong("nodes", summary.nodes()); summaryObject.setLong("down", summary.down()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 233b460333e..168b9b374f3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -7,6 +7,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; @@ -144,6 +145,10 @@ public class JobControllerApiHandlerHelperTest { assertResponse(JobControllerApiHandlerHelper.runResponse(app.application(), tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), Optional.empty(), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json"); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), userApp.instanceId(), URI.create("https://some.url:43/root/")), "overview-user-instance.json"); assertResponse(JobControllerApiHandlerHelper.overviewResponse(tester.controller(), app.application().id(), URI.create("https://some.url:43/root/")), "deployment-overview-2.json"); + + tester.configServer().setLogStream(() -> "no more logs"); + assertResponse(JobControllerApiHandlerHelper.vespaLogsResponse(tester.jobs(), new RunId(app.instanceId(), stagingTest, 1), 0, false), "vespa.log"); + assertResponse(JobControllerApiHandlerHelper.vespaLogsResponse(tester.jobs(), new RunId(app.instanceId(), stagingTest, 1), 0, true), "vespa.log"); } @Test @@ -197,11 +202,17 @@ public class JobControllerApiHandlerHelperTest { Path path = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/").resolve(fileName); ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); - byte[] actualJson = SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlimeOrThrow(baos.toByteArray()).get(), false); - // Files.write(path, actualJson); - byte[] expected = Files.readAllBytes(path); - assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlimeOrThrow(expected).get(), false), UTF_8), - new String(actualJson, UTF_8)); + if (fileName.endsWith(".json")) { + byte[] actualJson = SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlimeOrThrow(baos.toByteArray()).get(), false); + // Files.write(path, actualJson); + byte[] expected = Files.readAllBytes(path); + assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlimeOrThrow(expected).get(), false), UTF_8), + new String(actualJson, UTF_8)); + } + else { + assertEquals(Files.readString(path), + baos.toString(UTF_8)); + } } catch (Exception e) { throw new RuntimeException(e); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index a7e2d1913cf..a02fb1fb375 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -967,6 +967,10 @@ "status": "succeeded" }, { + "name": "copyVespaLogs", + "status": "succeeded" + }, + { "name": "deactivateTester", "status": "succeeded" }, @@ -1016,6 +1020,10 @@ "status": "succeeded" }, { + "name": "copyVespaLogs", + "status": "succeeded" + }, + { "name": "deactivateTester", "status": "succeeded" }, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json index 9b391196d55..3190dee06fa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json @@ -82,5 +82,6 @@ "copyVespaLogs": { "status": "unfinished" } - } + }, + "vespaLogsActive": true } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json index 4ffac2bf738..cee7e0f4c92 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json @@ -39,5 +39,6 @@ "status": "succeeded", "startMillis": 0 } - } + }, + "vespaLogsActive": false } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index a2f62621f5b..0b8937c31a4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -232,5 +232,7 @@ "status": "succeeded", "startMillis": 14503000 } - } + }, + "vespaLogsActive": false, + "testerLogsActive": false } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json index a691762c40b..638a7e24ee7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json @@ -401,5 +401,7 @@ "status": "succeeded", "startMillis": 1600000000000 } - } + }, + "vespaLogsActive": false, + "testerLogsActive": false } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json index 4e8737d5f67..a8b38079b6f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json @@ -404,5 +404,7 @@ "failed": 0 } } - ] + ], + "vespaLogsActive": false, + "testerLogsActive": false } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/vespa.log b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/vespa.log new file mode 100644 index 00000000000..25916d9e6df --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/vespa.log @@ -0,0 +1 @@ +INFO - All good
\ No newline at end of file |