Merge pull request #11836 from vespa-engine/hmusum/add-methods-for-tester-requests

Add bindings for requests that should be sent to tester containers

8 files changed, 231 insertions, 11 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 87c971a6cd6..d2f26738301 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -42,6 +42,7 @@ import com.yahoo.vespa.config.server.deploy.InfraDeployerProvider;
 import com.yahoo.vespa.config.server.http.InternalServerException;
 import com.yahoo.vespa.config.server.http.LogRetriever;
 import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
+import com.yahoo.vespa.config.server.http.TesterClient;
 import com.yahoo.vespa.config.server.http.v2.MetricsResponse;
 import com.yahoo.vespa.config.server.http.v2.PrepareResult;
 import com.yahoo.vespa.config.server.metrics.ApplicationMetricsRetriever;
@@ -107,6 +108,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private final FileDistributionStatus fileDistributionStatus;
     private final Orchestrator orchestrator;
     private final LogRetriever logRetriever;
+    private final TesterClient testerClient;
 
     @Inject
     public ApplicationRepository(TenantRepository tenantRepository,
@@ -115,7 +117,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                  ConfigConvergenceChecker configConvergenceChecker,
                                  HttpProxy httpProxy,
                                  ConfigserverConfig configserverConfig,
-                                 Orchestrator orchestrator) {
+                                 Orchestrator orchestrator,
+                                 TesterClient testerClient) {
         this(tenantRepository,
              hostProvisionerProvider.getHostProvisioner(),
              infraDeployerProvider.getInfraDeployer(),
@@ -125,7 +128,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
              orchestrator,
              new LogRetriever(),
              new FileDistributionStatus(),
-             Clock.systemUTC());
+             Clock.systemUTC(),
+             testerClient);
     }
 
     // For testing
@@ -138,7 +142,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
              orchestrator,
              new ConfigserverConfig(new ConfigserverConfig.Builder()),
              new LogRetriever(),
-             clock);
+             clock,
+             new TesterClient());
     }
 
     // For testing
@@ -147,7 +152,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                  Orchestrator orchestrator,
                                  ConfigserverConfig configserverConfig,
                                  LogRetriever logRetriever,
-                                 Clock clock) {
+                                 Clock clock,
+                                 TesterClient testerClient) {
         this(tenantRepository,
              Optional.of(hostProvisioner),
              Optional.empty(),
@@ -157,7 +163,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
              orchestrator,
              logRetriever,
              new FileDistributionStatus(),
-             clock);
+             clock,
+             testerClient);
     }
 
     private ApplicationRepository(TenantRepository tenantRepository,
@@ -169,7 +176,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                   Orchestrator orchestrator,
                                   LogRetriever logRetriever,
                                   FileDistributionStatus fileDistributionStatus,
-                                  Clock clock) {
+                                  Clock clock,
+                                  TesterClient testerClient) {
         this.tenantRepository = tenantRepository;
         this.hostProvisioner = hostProvisioner;
         this.infraDeployer = infraDeployer;
@@ -180,6 +188,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         this.logRetriever = logRetriever;
         this.fileDistributionStatus = fileDistributionStatus;
         this.clock = clock;
+        this.testerClient = testerClient;
     }
 
     // ---------------- Deploying ----------------------------------------------------------------
@@ -523,6 +532,41 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return logRetriever.getLogs(logServerURI);
     }
 
+
+    // ---------------- Methods to do call against tester containers in hosted ------------------------------
+
+    public HttpResponse getTesterStatus(ApplicationId applicationId) {
+        return testerClient.getStatus(getTesterHostname(applicationId), getTesterPort(applicationId));
+    }
+
+    public HttpResponse getTesterLog(ApplicationId applicationId, Long after) {
+        return testerClient.getLog(getTesterHostname(applicationId), getTesterPort(applicationId), after);
+    }
+
+    // TODO: Not implemented in TesterClient yet
+    public HttpResponse startTests(ApplicationId applicationId, String suite, String config) {
+        return testerClient.startTests(getTesterHostname(applicationId), suite, config);
+    }
+
+    private String getTesterHostname(ApplicationId applicationId) {
+        return getTesterServiceInfo(applicationId).getHostName();
+    }
+
+    private int getTesterPort(ApplicationId applicationId) {
+        ServiceInfo serviceInfo = getTesterServiceInfo(applicationId);
+        return serviceInfo.getPorts().stream().filter(portInfo -> portInfo.getTags().contains("http")).findFirst().get().getPort();
+    }
+
+    private ServiceInfo getTesterServiceInfo(ApplicationId applicationId) {
+        Application application = getApplication(applicationId);
+        return application.getModel().getHosts().stream()
+                .findFirst().orElseThrow(() -> new InternalServerException("Could not find any host for tester app " + applicationId.toFullString()))
+                .getServices().stream()
+                .filter(service -> CONTAINER.serviceName.equals(service.getServiceType()))
+                .findFirst()
+                .orElseThrow(() -> new InternalServerException("Could not find any tester container for tester app " + applicationId.toFullString()));
+    }
+
     // ---------------- Session operations ----------------------------------------------------------------
 
     /**
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java
index 4c310dd8a0d..9dc26f3a601 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java
@@ -11,7 +11,8 @@ import java.util.Optional;
 /**
  * Proxies response back to client, keeps Content-Type header if it is present
  *
- * @author Ola Aunrønning
+ * @author olaa
+ * @author freva
  */
 class ProxyResponse extends HttpResponse {
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java
new file mode 100644
index 00000000000..92c04bfb8e7
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java
@@ -0,0 +1,75 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.http;
+
+import ai.vespa.util.http.VespaHttpClientBuilder;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.log.LogLevel;
+import com.yahoo.yolean.Exceptions;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.utils.URIBuilder;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.logging.Logger;
+
+/**
+ * @author musum
+ */
+public class TesterClient {
+
+    private final HttpClient httpClient = VespaHttpClientBuilder.create().build();
+    private static final Logger logger = Logger.getLogger(TesterClient.class.getName());
+
+    public HttpResponse getStatus(String testerHostname, int port) {
+        URI testerUri;
+        try {
+            testerUri = new URIBuilder()
+                    .setScheme("https")
+                    .setHost(testerHostname)
+                    .setPort(port)
+                    .setPath("/tester/v1/status")
+                    .build();
+        } catch (URISyntaxException e) {
+            throw new IllegalArgumentException(e);
+        }
+
+        return execute(new HttpGet(testerUri), "Failed to get tester status");
+    }
+
+    public HttpResponse getLog(String testerHostname, int port, Long after) {
+        URI testerUri;
+        try {
+            testerUri = new URIBuilder()
+                    .setScheme("https")
+                    .setHost(testerHostname)
+                    .setPort(port)
+                    .setPath("/tester/v1/log")
+                    .addParameter("after", String.valueOf(after))
+                    .build();
+        } catch (URISyntaxException e) {
+            throw new IllegalArgumentException(e);
+        }
+
+        return execute(new HttpGet(testerUri), "Failed to get tester logs");
+    }
+
+    public HttpResponse startTests(String testerHostname, String suite, String config) {
+        throw new UnsupportedOperationException("Not implemented yet");
+    }
+
+
+    private HttpResponse execute(HttpUriRequest request, String messageIfRequestFails) {
+        // TODO: Change log level to DEBUG
+        logger.log(LogLevel.INFO, "Sending request to tester container " + request.getURI().toString());
+        try {
+            return new ProxyResponse(httpClient.execute(request));
+        } catch (IOException e) {
+            logger.warning(messageIfRequestFails + ": " + Exceptions.toMessageString(e));
+            return HttpErrorResponse.internalServerError(Exceptions.toMessageString(e));
+        }
+    }
+
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index babfdfa575d..d9a5aa7055d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -46,6 +46,7 @@ public class ApplicationHandler extends HttpHandler {
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/logs",
+            "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*",
             "http://*/application/v2/tenant/*/application/*")
             .map(UriPattern::new)
@@ -132,6 +133,23 @@ public class ApplicationHandler extends HttpHandler {
             return new ApplicationSuspendedResponse(applicationRepository.isSuspended(applicationId));
         }
 
+        if (isTesterRequest(request)) {
+            String testerCommand = getTesterCommandFromRequest(request);
+            switch (testerCommand) {
+                case "status":
+                    return applicationRepository.getTesterStatus(applicationId);
+                case "logs":
+                    Long after = Long.valueOf(request.getProperty("after"));
+                    return applicationRepository.getTesterLog(applicationId, after);
+                case "startTests":
+                    String suite = "foo"; // TODO
+                    String config = "bar"; // TODO
+                    return applicationRepository.startTests(applicationId, suite, config);
+                default:
+                    throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString());
+            }
+        }
+
         return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId));
     }
 
@@ -210,11 +228,21 @@ public class ApplicationHandler extends HttpHandler {
                 request.getUri().getPath().contains("/filedistributionstatus");
     }
 
+    private static boolean isTesterRequest(HttpRequest request) {
+        return getBindingMatch(request).groupCount() == 8 &&
+               request.getUri().getPath().contains("/tester");
+    }
+
     private static String getHostNameFromRequest(HttpRequest req) {
         BindingMatch<?> bm = getBindingMatch(req);
         return bm.group(7);
     }
 
+    private static String getTesterCommandFromRequest(HttpRequest req) {
+        BindingMatch<?> bm = getBindingMatch(req);
+        return bm.group(7);
+    }
+
     private static String getPathSuffix(HttpRequest req) {
         BindingMatch<?> bm = getBindingMatch(req);
         return bm.group(8);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index cd507178733..fb745bbb76b 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -323,7 +323,8 @@ public class ApplicationRepositoryTest {
                                          orchestrator,
                                          new ConfigserverConfig(new ConfigserverConfig.Builder()),
                                          new MockLogRetriever(),
-                                         clock);
+                                         clock,
+                                         new MockTesterClient());
     }
 
     private PrepareResult prepareAndActivateApp(File application) throws IOException {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java
new file mode 100644
index 00000000000..7cb878a5f2c
--- /dev/null
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java
@@ -0,0 +1,52 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.vespa.config.server.http.TesterClient;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * @author hmusum
+ */
+public class MockTesterClient extends TesterClient {
+
+    @Override
+    public HttpResponse getStatus(String testerHostname, int port) {
+        return new MockStatusResponse();
+    }
+
+    @Override
+    public HttpResponse getLog(String testerHostname, int port, Long after) {
+        return new MockLogResponse();
+    }
+
+    private static class MockStatusResponse extends HttpResponse {
+
+        private MockStatusResponse() {
+            super(200);
+        }
+
+        @Override
+        public void render(OutputStream outputStream) throws IOException {
+            outputStream.write("OK".getBytes(StandardCharsets.UTF_8));
+        }
+
+    }
+
+    private static class MockLogResponse extends HttpResponse {
+
+        private MockLogResponse() {
+            super(200);
+        }
+
+        @Override
+        public void render(OutputStream outputStream) throws IOException {
+            outputStream.write("log".getBytes(StandardCharsets.UTF_8));
+        }
+
+    }
+
+}
\ No newline at end of file
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
index 54a80adf676..9a15d6528ee 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
@@ -29,6 +29,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.MockTesterClient;
 import com.yahoo.vespa.config.server.TestComponentRegistry;
 import com.yahoo.vespa.config.server.TimeoutBudget;
 import com.yahoo.vespa.config.server.application.OrchestratorMock;
@@ -135,7 +136,8 @@ public class DeployTester {
                                                           new OrchestratorMock(),
                                                           configserverConfig,
                                                           new LogRetriever(),
-                                                          clock);
+                                                          clock,
+                                                          new MockTesterClient());
     }
 
     public Tenant tenant() {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index c2e78f2dd63..9e8d483cd86 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -12,6 +12,7 @@ import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.jdisc.Response;
 import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.MockLogRetriever;
+import com.yahoo.vespa.config.server.MockTesterClient;
 import com.yahoo.vespa.config.server.TestComponentRegistry;
 import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker;
 import com.yahoo.vespa.config.server.application.HttpProxy;
@@ -77,7 +78,8 @@ public class ApplicationHandlerTest {
                                                           orchestrator,
                                                           new ConfigserverConfig(new ConfigserverConfig.Builder()),
                                                           new MockLogRetriever(),
-                                                          Clock.systemUTC());
+                                                          Clock.systemUTC(),
+                                                          new MockTesterClient());
         listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(),
                                                               tenantRepository,
                                                               Zone.defaultZone());
@@ -174,7 +176,8 @@ public class ApplicationHandlerTest {
                                                                                 new ConfigConvergenceChecker(stateApiFactory),
                                                                                 mockHttpProxy,
                                                                                 new ConfigserverConfig(new ConfigserverConfig.Builder()),
-                                                                                new OrchestratorMock());
+                                                                                new OrchestratorMock(),
+                                                                                new MockTesterClient());
         ApplicationHandler mockHandler = createApplicationHandler(applicationRepository);
         when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName),eq("clustercontroller-status/v1/clusterName1")))
                 .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>"));
@@ -221,6 +224,20 @@ public class ApplicationHandlerTest {
         assertEquals("log line", baos.toString());
     }
 
+    @Test
+    public void testTesterStatus() throws IOException {
+        applicationRepository.deploy(testApp, prepareParams(applicationId));
+        String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/status";
+        ApplicationHandler mockHandler = createApplicationHandler();
+
+        HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET));
+        assertEquals(200, response.getStatus());
+
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        response.render(baos);
+        assertEquals("OK", baos.toString());
+    }
+
     private void assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException {
         String url = "http://myhost:14000/application/v2/tenant/" + mytenantName + "/application/default";
         deleteAndAssertResponse(url, Response.Status.METHOD_NOT_ALLOWED, HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, "{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Method '" + method + "' is not supported\"}",