From 2bb43f48f6f7a26d89b6015bdde06ff9c605dc01 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 17 Jan 2020 12:55:46 +0100 Subject: Add bindings for requests that should be sent to tester containers Add bindings to be able to send requests to /tester/v1 API on tester containers (not fully implemented yet) --- .../vespa/config/server/ApplicationRepository.java | 56 ++++++++++++++-- .../vespa/config/server/http/ProxyResponse.java | 3 +- .../vespa/config/server/http/TesterClient.java | 75 ++++++++++++++++++++++ .../config/server/http/v2/ApplicationHandler.java | 28 ++++++++ .../config/server/ApplicationRepositoryTest.java | 3 +- .../vespa/config/server/MockTesterClient.java | 52 +++++++++++++++ .../vespa/config/server/deploy/DeployTester.java | 4 +- .../server/http/v2/ApplicationHandlerTest.java | 21 +++++- 8 files changed, 231 insertions(+), 11 deletions(-) create mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java create mode 100644 configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java (limited to 'configserver') diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 87c971a6cd6..d2f26738301 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -42,6 +42,7 @@ import com.yahoo.vespa.config.server.deploy.InfraDeployerProvider; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; +import com.yahoo.vespa.config.server.http.TesterClient; import com.yahoo.vespa.config.server.http.v2.MetricsResponse; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.metrics.ApplicationMetricsRetriever; @@ -107,6 +108,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final FileDistributionStatus fileDistributionStatus; private final Orchestrator orchestrator; private final LogRetriever logRetriever; + private final TesterClient testerClient; @Inject public ApplicationRepository(TenantRepository tenantRepository, @@ -115,7 +117,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye ConfigConvergenceChecker configConvergenceChecker, HttpProxy httpProxy, ConfigserverConfig configserverConfig, - Orchestrator orchestrator) { + Orchestrator orchestrator, + TesterClient testerClient) { this(tenantRepository, hostProvisionerProvider.getHostProvisioner(), infraDeployerProvider.getInfraDeployer(), @@ -125,7 +128,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye orchestrator, new LogRetriever(), new FileDistributionStatus(), - Clock.systemUTC()); + Clock.systemUTC(), + testerClient); } // For testing @@ -138,7 +142,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye orchestrator, new ConfigserverConfig(new ConfigserverConfig.Builder()), new LogRetriever(), - clock); + clock, + new TesterClient()); } // For testing @@ -147,7 +152,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Orchestrator orchestrator, ConfigserverConfig configserverConfig, LogRetriever logRetriever, - Clock clock) { + Clock clock, + TesterClient testerClient) { this(tenantRepository, Optional.of(hostProvisioner), Optional.empty(), @@ -157,7 +163,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye orchestrator, logRetriever, new FileDistributionStatus(), - clock); + clock, + testerClient); } private ApplicationRepository(TenantRepository tenantRepository, @@ -169,7 +176,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Orchestrator orchestrator, LogRetriever logRetriever, FileDistributionStatus fileDistributionStatus, - Clock clock) { + Clock clock, + TesterClient testerClient) { this.tenantRepository = tenantRepository; this.hostProvisioner = hostProvisioner; this.infraDeployer = infraDeployer; @@ -180,6 +188,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.logRetriever = logRetriever; this.fileDistributionStatus = fileDistributionStatus; this.clock = clock; + this.testerClient = testerClient; } // ---------------- Deploying ---------------------------------------------------------------- @@ -523,6 +532,41 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return logRetriever.getLogs(logServerURI); } + + // ---------------- Methods to do call against tester containers in hosted ------------------------------ + + public HttpResponse getTesterStatus(ApplicationId applicationId) { + return testerClient.getStatus(getTesterHostname(applicationId), getTesterPort(applicationId)); + } + + public HttpResponse getTesterLog(ApplicationId applicationId, Long after) { + return testerClient.getLog(getTesterHostname(applicationId), getTesterPort(applicationId), after); + } + + // TODO: Not implemented in TesterClient yet + public HttpResponse startTests(ApplicationId applicationId, String suite, String config) { + return testerClient.startTests(getTesterHostname(applicationId), suite, config); + } + + private String getTesterHostname(ApplicationId applicationId) { + return getTesterServiceInfo(applicationId).getHostName(); + } + + private int getTesterPort(ApplicationId applicationId) { + ServiceInfo serviceInfo = getTesterServiceInfo(applicationId); + return serviceInfo.getPorts().stream().filter(portInfo -> portInfo.getTags().contains("http")).findFirst().get().getPort(); + } + + private ServiceInfo getTesterServiceInfo(ApplicationId applicationId) { + Application application = getApplication(applicationId); + return application.getModel().getHosts().stream() + .findFirst().orElseThrow(() -> new InternalServerException("Could not find any host for tester app " + applicationId.toFullString())) + .getServices().stream() + .filter(service -> CONTAINER.serviceName.equals(service.getServiceType())) + .findFirst() + .orElseThrow(() -> new InternalServerException("Could not find any tester container for tester app " + applicationId.toFullString())); + } + // ---------------- Session operations ---------------------------------------------------------------- /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java index 4c310dd8a0d..9dc26f3a601 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java @@ -11,7 +11,8 @@ import java.util.Optional; /** * Proxies response back to client, keeps Content-Type header if it is present * - * @author Ola Aunrønning + * @author olaa + * @author freva */ class ProxyResponse extends HttpResponse { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java new file mode 100644 index 00000000000..b3d59297503 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java @@ -0,0 +1,75 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import ai.vespa.util.http.VespaHttpClientBuilder; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.log.LogLevel; +import com.yahoo.yolean.Exceptions; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.utils.URIBuilder; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.logging.Logger; + +/** + * @author musum + */ +public class TesterClient { + + private final HttpClient httpClient = VespaHttpClientBuilder.create().build(); + private static final Logger logger = Logger.getLogger(TesterClient.class.getName()); + + public HttpResponse getStatus(String testerHostname, int port) { + URI testerUri; + try { + testerUri = new URIBuilder() + .setScheme("http") + .setHost(testerHostname) + .setPort(port) + .setPath("/tester/v1/status") + .build(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + + return execute(new HttpGet(testerUri), "Failed to get tester status"); + } + + public HttpResponse getLog(String testerHostname, int port, Long after) { + URI testerUri; + try { + testerUri = new URIBuilder() + .setScheme("http") + .setHost(testerHostname) + .setPort(port) + .setPath("/tester/v1/log") + .addParameter("after", String.valueOf(after)) + .build(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + + return execute(new HttpGet(testerUri), "Failed to get tester logs"); + } + + public HttpResponse startTests(String testerHostname, String suite, String config) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + + private HttpResponse execute(HttpUriRequest request, String messageIfRequestFails) { + // TODO: Change log level to DEBUG + logger.log(LogLevel.INFO, "Sending request to tester container " + request.getURI().toString()); + try { + return new ProxyResponse(httpClient.execute(request)); + } catch (IOException e) { + logger.warning(messageIfRequestFails + ": " + Exceptions.toMessageString(e)); + return HttpErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index babfdfa575d..d9a5aa7055d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -46,6 +46,7 @@ public class ApplicationHandler extends HttpHandler { "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/logs", + "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*", "http://*/application/v2/tenant/*/application/*") .map(UriPattern::new) @@ -132,6 +133,23 @@ public class ApplicationHandler extends HttpHandler { return new ApplicationSuspendedResponse(applicationRepository.isSuspended(applicationId)); } + if (isTesterRequest(request)) { + String testerCommand = getTesterCommandFromRequest(request); + switch (testerCommand) { + case "status": + return applicationRepository.getTesterStatus(applicationId); + case "logs": + Long after = Long.valueOf(request.getProperty("after")); + return applicationRepository.getTesterLog(applicationId, after); + case "startTests": + String suite = "foo"; // TODO + String config = "bar"; // TODO + return applicationRepository.startTests(applicationId, suite, config); + default: + throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString()); + } + } + return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId)); } @@ -210,11 +228,21 @@ public class ApplicationHandler extends HttpHandler { request.getUri().getPath().contains("/filedistributionstatus"); } + private static boolean isTesterRequest(HttpRequest request) { + return getBindingMatch(request).groupCount() == 8 && + request.getUri().getPath().contains("/tester"); + } + private static String getHostNameFromRequest(HttpRequest req) { BindingMatch bm = getBindingMatch(req); return bm.group(7); } + private static String getTesterCommandFromRequest(HttpRequest req) { + BindingMatch bm = getBindingMatch(req); + return bm.group(7); + } + private static String getPathSuffix(HttpRequest req) { BindingMatch bm = getBindingMatch(req); return bm.group(8); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index cd507178733..fb745bbb76b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -323,7 +323,8 @@ public class ApplicationRepositoryTest { orchestrator, new ConfigserverConfig(new ConfigserverConfig.Builder()), new MockLogRetriever(), - clock); + clock, + new MockTesterClient()); } private PrepareResult prepareAndActivateApp(File application) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java new file mode 100644 index 00000000000..7cb878a5f2c --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java @@ -0,0 +1,52 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.config.server.http.TesterClient; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +/** + * @author hmusum + */ +public class MockTesterClient extends TesterClient { + + @Override + public HttpResponse getStatus(String testerHostname, int port) { + return new MockStatusResponse(); + } + + @Override + public HttpResponse getLog(String testerHostname, int port, Long after) { + return new MockLogResponse(); + } + + private static class MockStatusResponse extends HttpResponse { + + private MockStatusResponse() { + super(200); + } + + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("OK".getBytes(StandardCharsets.UTF_8)); + } + + } + + private static class MockLogResponse extends HttpResponse { + + private MockLogResponse() { + super(200); + } + + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("log".getBytes(StandardCharsets.UTF_8)); + } + + } + +} \ No newline at end of file diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 54a80adf676..9a15d6528ee 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -29,6 +29,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.Zone; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.MockTesterClient; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.OrchestratorMock; @@ -135,7 +136,8 @@ public class DeployTester { new OrchestratorMock(), configserverConfig, new LogRetriever(), - clock); + clock, + new MockTesterClient()); } public Tenant tenant() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index c2e78f2dd63..9e8d483cd86 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -12,6 +12,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockLogRetriever; +import com.yahoo.vespa.config.server.MockTesterClient; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.application.HttpProxy; @@ -77,7 +78,8 @@ public class ApplicationHandlerTest { orchestrator, new ConfigserverConfig(new ConfigserverConfig.Builder()), new MockLogRetriever(), - Clock.systemUTC()); + Clock.systemUTC(), + new MockTesterClient()); listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), tenantRepository, Zone.defaultZone()); @@ -174,7 +176,8 @@ public class ApplicationHandlerTest { new ConfigConvergenceChecker(stateApiFactory), mockHttpProxy, new ConfigserverConfig(new ConfigserverConfig.Builder()), - new OrchestratorMock()); + new OrchestratorMock(), + new MockTesterClient()); ApplicationHandler mockHandler = createApplicationHandler(applicationRepository); when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName),eq("clustercontroller-status/v1/clusterName1"))) .thenReturn(new StaticResponse(200, "text/html", "...")); @@ -221,6 +224,20 @@ public class ApplicationHandlerTest { assertEquals("log line", baos.toString()); } + @Test + public void testTesterStatus() throws IOException { + applicationRepository.deploy(testApp, prepareParams(applicationId)); + String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/status"; + ApplicationHandler mockHandler = createApplicationHandler(); + + HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + assertEquals(200, response.getStatus()); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.render(baos); + assertEquals("OK", baos.toString()); + } + private void assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException { String url = "http://myhost:14000/application/v2/tenant/" + mytenantName + "/application/default"; deleteAndAssertResponse(url, Response.Status.METHOD_NOT_ALLOWED, HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, "{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Method '" + method + "' is not supported\"}", -- cgit v1.2.3 From 5cb0660839830be165457e5779f271a2d963687c Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Sat, 18 Jan 2020 11:36:54 +0100 Subject: Use https --- .../main/java/com/yahoo/vespa/config/server/http/TesterClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'configserver') diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java index b3d59297503..92c04bfb8e7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java @@ -27,7 +27,7 @@ public class TesterClient { URI testerUri; try { testerUri = new URIBuilder() - .setScheme("http") + .setScheme("https") .setHost(testerHostname) .setPort(port) .setPath("/tester/v1/status") @@ -43,7 +43,7 @@ public class TesterClient { URI testerUri; try { testerUri = new URIBuilder() - .setScheme("http") + .setScheme("https") .setHost(testerHostname) .setPort(port) .setPath("/tester/v1/log") -- cgit v1.2.3