From e2dace4bc4f72e86a671664cb1b32eea48361cfb Mon Sep 17 00:00:00 2001 From: Morten Tokle Date: Mon, 22 Jun 2020 08:55:36 +0200 Subject: Execute Junit tests in container --- .../java/ai/vespa/hosted/cd/VespaTestRuntime.java | 57 ----------- .../ai/vespa/hosted/cd/http/HttpDeployment.java | 37 ------- .../java/ai/vespa/hosted/cd/http/HttpEndpoint.java | 68 ------------- .../ai/vespa/hosted/cd/impl/VespaTestRuntime.java | 113 +++++++++++++++++++++ .../vespa/hosted/cd/impl/http/HttpDeployment.java | 37 +++++++ .../ai/vespa/hosted/cd/impl/http/HttpEndpoint.java | 68 +++++++++++++ .../yahoo/vespa/hosted/cd/impl/package-info.java | 9 ++ .../services/ai.vespa.hosted.cd.TestRuntime | 2 +- .../configdefinitions/cloud-tenant-cd.def | 9 ++ 9 files changed, 237 insertions(+), 163 deletions(-) delete mode 100644 cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/VespaTestRuntime.java delete mode 100644 cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java delete mode 100644 cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java create mode 100644 cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/impl/VespaTestRuntime.java create mode 100644 cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/impl/http/HttpDeployment.java create mode 100644 cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/impl/http/HttpEndpoint.java create mode 100644 cloud-tenant-cd/src/main/java/com/yahoo/vespa/hosted/cd/impl/package-info.java create mode 100644 cloud-tenant-cd/src/main/resources/configdefinitions/cloud-tenant-cd.def (limited to 'cloud-tenant-cd/src') diff --git a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/VespaTestRuntime.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/VespaTestRuntime.java deleted file mode 100644 index 75aaaec78ba..00000000000 --- a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/VespaTestRuntime.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd; - -import ai.vespa.cloud.Zone; -import ai.vespa.hosted.api.ControllerHttpClient; -import ai.vespa.hosted.api.Properties; -import ai.vespa.hosted.api.TestConfig; -import ai.vespa.hosted.cd.http.HttpDeployment; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.zone.ZoneId; - -import java.nio.file.Files; -import java.nio.file.Paths; - -/** - * @author mortent - */ -public class VespaTestRuntime implements TestRuntime { - private final TestConfig config; - private final Deployment deploymentToTest; - - public VespaTestRuntime() { - String configPath = System.getProperty("vespa.test.config"); - TestConfig config = configPath != null ? fromFile(configPath) : fromController(); - this.config = config; - this.deploymentToTest = new HttpDeployment(config.deployments().get(config.zone()), new ai.vespa.hosted.auth.EndpointAuthenticator(config.system())); - } - - @Override - public Zone zone() { - return new Zone( - ai.vespa.cloud.Environment.valueOf(config.zone().environment().name()), - config.zone().region().value()); } - - /** Returns the deployment this is testing. */ - @Override - public Deployment deploymentToTest() { return deploymentToTest; } - - private static TestConfig fromFile(String path) { - try { - return TestConfig.fromJson(Files.readAllBytes(Paths.get(path))); - } - catch (Exception e) { - throw new IllegalArgumentException("Failed reading config from '" + path + "'!", e); - } - } - - private static TestConfig fromController() { - ControllerHttpClient controller = new ai.vespa.hosted.auth.ApiAuthenticator().controller(); - ApplicationId id = Properties.application(); - Environment environment = Properties.environment().orElse(Environment.dev); - ZoneId zone = Properties.region().map(region -> ZoneId.from(environment, region)) - .orElseGet(() -> controller.defaultZone(environment)); - return controller.testConfig(id, zone); - } -} diff --git a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java deleted file mode 100644 index 80d5416ab34..00000000000 --- a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd.http; - -import ai.vespa.hosted.api.EndpointAuthenticator; -import ai.vespa.hosted.cd.Deployment; -import ai.vespa.hosted.cd.Endpoint; - -import java.net.URI; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.stream.Collectors; - -/** - * A remote deployment of a Vespa application, reachable over HTTP. Contains {@link HttpEndpoint}s. - * - * @author jonmv - */ -public class HttpDeployment implements Deployment { - - private final Map endpoints; - - /** Creates a representation of the given deployment endpoints, using the authenticator for data plane access. */ - public HttpDeployment(Map endpoints, EndpointAuthenticator authenticator) { - this.endpoints = endpoints.entrySet().stream() - .collect(Collectors.toUnmodifiableMap(entry -> entry.getKey(), - entry -> new HttpEndpoint(entry.getValue(), authenticator))); - } - - @Override - public Endpoint endpoint(String id) { - if ( ! endpoints.containsKey(id)) - throw new NoSuchElementException("No cluster with id '" + id + "'"); - - return endpoints.get(id); - } - -} diff --git a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java deleted file mode 100644 index a803fc3e0e2..00000000000 --- a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd.http; - -import ai.vespa.hosted.api.EndpointAuthenticator; -import ai.vespa.hosted.cd.Endpoint; - -import javax.net.ssl.SSLParameters; -import java.io.IOException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; -import java.util.Map; -import java.util.stream.Collectors; - -import static java.net.URLEncoder.encode; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; - -/** - * A remote endpoint in a {@link HttpDeployment} of a Vespa application, reachable over HTTP. - * - * @author jonmv - */ -public class HttpEndpoint implements Endpoint { - - private final URI endpoint; - private final HttpClient client; - private final EndpointAuthenticator authenticator; - - public HttpEndpoint(URI endpoint, EndpointAuthenticator authenticator) { - this.endpoint = requireNonNull(endpoint); - this.authenticator = requireNonNull(authenticator); - SSLParameters sslParameters = new SSLParameters(); - sslParameters.setProtocols(new String[] {"TLSv1.2" }); - this.client = HttpClient.newBuilder() - .sslContext(authenticator.sslContext()) - .connectTimeout(Duration.ofSeconds(5)) - .version(HttpClient.Version.HTTP_1_1) - .sslParameters(sslParameters) - .build(); - } - - @Override - public URI uri() { - return endpoint; - } - - @Override - public HttpResponse send(HttpRequest.Builder request, HttpResponse.BodyHandler handler) { - try { - return client.send(authenticator.authenticated(request).build(), handler); - } - catch (IOException | InterruptedException e) { - throw new RuntimeException(request.build() + " failed: " + e.getMessage(), e); - } - } - - @Override - public HttpRequest.Builder request(String path, Map properties) { - return HttpRequest.newBuilder(endpoint.resolve(path + - properties.entrySet().stream() - .map(entry -> encode(entry.getKey(), UTF_8) + "=" + encode(entry.getValue(), UTF_8)) - .collect(Collectors.joining("&", path.contains("?") ? "&" : "?", "")))); - } - -} diff --git a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/impl/VespaTestRuntime.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/impl/VespaTestRuntime.java new file mode 100644 index 00000000000..d2367d588f6 --- /dev/null +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/impl/VespaTestRuntime.java @@ -0,0 +1,113 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.hosted.cd.impl; + +import ai.vespa.cloud.Zone; +import ai.vespa.hosted.api.ControllerHttpClient; +import ai.vespa.hosted.api.Properties; +import ai.vespa.hosted.api.TestConfig; +import ai.vespa.hosted.cd.Deployment; +import ai.vespa.hosted.cd.TestRuntime; +import ai.vespa.hosted.cd.impl.http.HttpDeployment; +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.cd.impl.CloudTenantCdConfig; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author mortent + */ +public class VespaTestRuntime extends AbstractComponent implements TestRuntime { + private final TestConfig config; + private final Deployment deploymentToTest; + + /* + * Used when executing tests locally + */ + public VespaTestRuntime() { + this(configFromPropertyOrController()); + } + + /* + * Used when executing tests from using Vespa test framework in container + */ + @Inject + public VespaTestRuntime(CloudTenantCdConfig c) { + + this(fromVespaConfig(c)); + } + private VespaTestRuntime(TestConfig config) { + this.config = config; + this.deploymentToTest = new HttpDeployment(config.deployments().get(config.zone()), new ai.vespa.hosted.auth.EndpointAuthenticator(config.system())); + } + + @Override + public Zone zone() { + return new Zone( + ai.vespa.cloud.Environment.valueOf(config.zone().environment().name()), + config.zone().region().value()); } + + /** Returns the deployment this is testing. */ + @Override + public Deployment deploymentToTest() { return deploymentToTest; } + + private static TestConfig configFromPropertyOrController() { + String configPath = System.getProperty("vespa.test.config"); + return configPath != null ? fromFile(configPath) : fromController(); + } + + private static TestConfig fromFile(String path) { + try { + return TestConfig.fromJson(Files.readAllBytes(Paths.get(path))); + } + catch (Exception e) { + throw new IllegalArgumentException("Failed reading config from '" + path + "'!", e); + } + } + + private static TestConfig fromController() { + ControllerHttpClient controller = new ai.vespa.hosted.auth.ApiAuthenticator().controller(); + ApplicationId id = Properties.application(); + Environment environment = Properties.environment().orElse(Environment.dev); + ZoneId zone = Properties.region().map(region -> ZoneId.from(environment, region)) + .orElseGet(() -> controller.defaultZone(environment)); + return controller.testConfig(id, zone); + } + + private static TestConfig fromVespaConfig(CloudTenantCdConfig config) { + Map> deployments = new HashMap<>(); + Map> contentClusters = new HashMap<>(); + for (Map.Entry entry : config.zones().entrySet()) { + ZoneId zoneId = ZoneId.from(entry.getKey()); + + Map zoneDeployments = entry.getValue().deployments().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> URI.create(e.getValue()))); + + deployments.put(zoneId, zoneDeployments); + + contentClusters.put(zoneId, entry.getValue().contentClusters()); + } + return new TestConfig( + ApplicationId.fromFullString(config.application()), + ZoneId.from(config.zone()), + SystemName.from(config.systemName()), + config.isCi(), + deployments, + contentClusters); + } + + @Override + public void deconstruct() { + super.deconstruct(); + } +} diff --git a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/impl/http/HttpDeployment.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/impl/http/HttpDeployment.java new file mode 100644 index 00000000000..65210455b85 --- /dev/null +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/impl/http/HttpDeployment.java @@ -0,0 +1,37 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.hosted.cd.impl.http; + +import ai.vespa.hosted.api.EndpointAuthenticator; +import ai.vespa.hosted.cd.Deployment; +import ai.vespa.hosted.cd.Endpoint; + +import java.net.URI; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; + +/** + * A remote deployment of a Vespa application, reachable over HTTP. Contains {@link HttpEndpoint}s. + * + * @author jonmv + */ +public class HttpDeployment implements Deployment { + + private final Map endpoints; + + /** Creates a representation of the given deployment endpoints, using the authenticator for data plane access. */ + public HttpDeployment(Map endpoints, EndpointAuthenticator authenticator) { + this.endpoints = endpoints.entrySet().stream() + .collect(Collectors.toUnmodifiableMap(entry -> entry.getKey(), + entry -> new HttpEndpoint(entry.getValue(), authenticator))); + } + + @Override + public Endpoint endpoint(String id) { + if ( ! endpoints.containsKey(id)) + throw new NoSuchElementException("No cluster with id '" + id + "'"); + + return endpoints.get(id); + } + +} diff --git a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/impl/http/HttpEndpoint.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/impl/http/HttpEndpoint.java new file mode 100644 index 00000000000..f48973b7382 --- /dev/null +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/impl/http/HttpEndpoint.java @@ -0,0 +1,68 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.hosted.cd.impl.http; + +import ai.vespa.hosted.api.EndpointAuthenticator; +import ai.vespa.hosted.cd.Endpoint; + +import javax.net.ssl.SSLParameters; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.net.URLEncoder.encode; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; + +/** + * A remote endpoint in a {@link HttpDeployment} of a Vespa application, reachable over HTTP. + * + * @author jonmv + */ +public class HttpEndpoint implements Endpoint { + + private final URI endpoint; + private final HttpClient client; + private final EndpointAuthenticator authenticator; + + public HttpEndpoint(URI endpoint, EndpointAuthenticator authenticator) { + this.endpoint = requireNonNull(endpoint); + this.authenticator = requireNonNull(authenticator); + SSLParameters sslParameters = new SSLParameters(); + sslParameters.setProtocols(new String[] {"TLSv1.2" }); + this.client = HttpClient.newBuilder() + .sslContext(authenticator.sslContext()) + .connectTimeout(Duration.ofSeconds(5)) + .version(HttpClient.Version.HTTP_1_1) + .sslParameters(sslParameters) + .build(); + } + + @Override + public URI uri() { + return endpoint; + } + + @Override + public HttpResponse send(HttpRequest.Builder request, HttpResponse.BodyHandler handler) { + try { + return client.send(authenticator.authenticated(request).build(), handler); + } + catch (IOException | InterruptedException e) { + throw new RuntimeException(request.build() + " failed: " + e.getMessage(), e); + } + } + + @Override + public HttpRequest.Builder request(String path, Map properties) { + return HttpRequest.newBuilder(endpoint.resolve(path + + properties.entrySet().stream() + .map(entry -> encode(entry.getKey(), UTF_8) + "=" + encode(entry.getValue(), UTF_8)) + .collect(Collectors.joining("&", path.contains("?") ? "&" : "?", "")))); + } + +} diff --git a/cloud-tenant-cd/src/main/java/com/yahoo/vespa/hosted/cd/impl/package-info.java b/cloud-tenant-cd/src/main/java/com/yahoo/vespa/hosted/cd/impl/package-info.java new file mode 100644 index 00000000000..b5d6f0450ec --- /dev/null +++ b/cloud-tenant-cd/src/main/java/com/yahoo/vespa/hosted/cd/impl/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * @author mortent + */ +@ExportPackage +package com.yahoo.vespa.hosted.cd.impl; + +import com.yahoo.osgi.annotation.ExportPackage; \ No newline at end of file diff --git a/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime b/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime index 695fe363e4e..35cb2ed7c25 100644 --- a/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime +++ b/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime @@ -1,2 +1,2 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -ai.vespa.hosted.cd.VespaTestRuntime \ No newline at end of file +ai.vespa.hosted.cd.impl.VespaTestRuntime \ No newline at end of file diff --git a/cloud-tenant-cd/src/main/resources/configdefinitions/cloud-tenant-cd.def b/cloud-tenant-cd/src/main/resources/configdefinitions/cloud-tenant-cd.def new file mode 100644 index 00000000000..bac21f386be --- /dev/null +++ b/cloud-tenant-cd/src/main/resources/configdefinitions/cloud-tenant-cd.def @@ -0,0 +1,9 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.hosted.cd.impl + +application string +zone string +systemName string +isCi bool +zones{}.deployments{} string +zones{}.contentClusters[] string -- cgit v1.2.3