Update Vespa log on deployment job log requests iff job includes Vespa logs

9 files changed, 139 insertions, 6 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 6a1eed0132b..386b71607da 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -13,7 +13,6 @@ import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.io.IOUtils;
 import com.yahoo.security.KeyAlgorithm;
 import com.yahoo.security.KeyUtils;
 import com.yahoo.security.SignatureAlgorithm;
@@ -25,7 +24,6 @@ import com.yahoo.vespa.hosted.controller.api.ActivateResult;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
-import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
@@ -56,7 +54,6 @@ import java.security.cert.CertificateNotYetValidException;
 import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 0907f146f8a..c00f5d9d772 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -150,6 +150,9 @@ public class JobController {
     /** Fetches any new Vespa log entries, and records the timestamp of the last of these, for continuation. */
     public void updateVespaLog(RunId id) {
         locked(id, run -> {
+            if ( ! run.steps().containsKey(copyVespaLogs))
+                return run;
+
             ZoneId zone = id.type().zone(controller.system());
             Optional<Deployment> deployment = Optional.ofNullable(controller.applications().require(id.application())
                                                                             .deployments().get(zone));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index 293f4f9a8e3..157aa16abde 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -391,6 +391,8 @@ class JobControllerApiHandlerHelper {
         detailsObject.setBool("active", ! run.hasEnded());
         detailsObject.setString("status", nameOf(run.status()));
         jobController.updateTestLog(runId);
+        try { jobController.updateVespaLog(runId); }
+        catch (RuntimeException ignored) { } // May be perfectly fine, e.g., when logserver isn't up yet.
 
         RunLog runLog = (after == null ? jobController.details(runId) : jobController.details(runId, Long.parseLong(after)))
                 .orElseThrow(() -> new NotExistsException(String.format(
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index 614df953ca9..043ce712636 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -26,6 +26,7 @@ import java.util.Optional;
 
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devAwsUsEast2a;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1;
@@ -129,11 +130,16 @@ public class JobControllerApiHandlerHelperTest {
 
         ZoneId zone = JobType.devUsEast1.zone(tester.tester().controller().system());
         tester.jobs().deploy(appId, JobType.devUsEast1, Optional.empty(), applicationPackage);
+        tester.configServer().setLogStream("1554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n");
+        assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(appId, devUsEast1).get().id(), null), "dev-us-east-1-log-first-part.json");
+
+        tester.configServer().setLogStream("Nope, this won't be logged");
         tester.configServer().convergeServices(appId, zone);
         tester.setEndpoints(appId, zone);
         tester.runner().run();
 
         assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.tester().controller(), appId, URI.create("https://some.url:43/root")), "dev-overview.json");
+        assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(appId, devUsEast1).get().id(), "9"), "dev-us-east-1-log-second-part.json");
     }
 
     @Test
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json
index 82bdc4ca195..fe90ddd772e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json
@@ -10,7 +10,8 @@
     },
     "steps": {
       "deployReal": "succeeded",
-      "installReal": "succeeded"
+      "installReal": "succeeded",
+      "copyVespaLogs": "succeeded"
     },
     "tasks": {
       "deploy": "succeeded",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json
index 88bf5300ec2..bfaa62a602d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json
@@ -17,7 +17,8 @@
           },
           "steps": {
             "deployReal": "succeeded",
-            "installReal": "succeeded"
+            "installReal": "succeeded",
+            "copyVespaLogs": "succeeded"
           },
           "tasks": {
             "deploy": "succeeded",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json
new file mode 100644
index 00000000000..cce8fe196a7
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json
@@ -0,0 +1,62 @@
+{
+  "log": {
+    "deployReal": [
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "Deploying platform version 6.1 and application version unknown ..."
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "No services requiring restart."
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "Deployment successful."
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "foo"
+      }
+    ],
+    "installReal": [
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "Checking installation of 6.1 and unknown ..."
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "                         host-tenant:application:default-dev.us-east-1: unorchestrated  6.1                                                      "
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "Wanted config generation is 2"
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "                         host-tenant:application:default-dev.us-east-1:   container on port   43 has 1"
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "Installation not yet complete."
+      }
+    ],
+    "copyVespaLogs": [
+      {
+        "at": 1554970337935,
+        "type": "info",
+        "message": "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\nERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"
+      }
+    ]
+  },
+  "active": true,
+  "lastId": 9
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json
new file mode 100644
index 00000000000..5d1a1f0e370
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json
@@ -0,0 +1,60 @@
+{
+  "log": {
+    "installReal": [
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "Checking installation of 6.1 and unknown ..."
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "                         host-tenant:application:default-dev.us-east-1: unorchestrated  6.1                                                      "
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "Wanted config generation is 2"
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "All services on wanted config generation."
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "Attempting to find deployment endpoints ..."
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "Found endpoints:"
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "- dev.us-east-1"
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": " |-- https://default--application--tenant.us-east-1.dev.vespa:43 (cluster 'default')"
+      },
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "Installation succeeded!"
+      }
+    ],
+    "copyVespaLogs": [
+      {
+        "at": 0,
+        "type": "debug",
+        "message": "Copying Vespa log from nodes of tenant.application in dev.us-east-1 ..."
+      }
+    ]
+  },
+  "active": false,
+  "lastId": 19
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json
index 222cf88ed87..8c92288f88c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json
@@ -140,7 +140,8 @@
           },
           "steps": {
             "deployReal": "unfinished",
-            "installReal": "unfinished"
+            "installReal": "unfinished",
+            "copyVespaLogs": "unfinished"
           },
           "tasks": {},
           "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1/run/1"