diff options
author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2019-06-11 12:39:47 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-06-11 12:39:47 +0200 |
commit | 89e40d3684d48b94e82a7199d17b0e32dd07faab (patch) | |
tree | d20ab6f7938a47e85a00893bbb39a5853514555a /tenant-cd | |
parent | b285157131ccf7a9a7e5b10e73d1ded8df1eacfc (diff) | |
parent | ce08379e47b9e02836026d111e1a27681b21c715 (diff) |
Merge pull request #9731 from vespa-engine/jvenstad/tenant-cd
Jvenstad/tenant cd
Diffstat (limited to 'tenant-cd')
27 files changed, 986 insertions, 1 deletions
diff --git a/tenant-cd/pom.xml b/tenant-cd/pom.xml index 8907e56762c..7cc2c9a2d5b 100644 --- a/tenant-cd/pom.xml +++ b/tenant-cd/pom.xml @@ -5,6 +5,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> + <groupId>ai.vespa.hosted</groupId> <artifactId>tenant-cd</artifactId> <name>Hosted Vespa tenant CD</name> <description>Test library for hosted Vespa applications.</description> @@ -20,6 +21,36 @@ <dependencies> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-auth</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-api</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.12</version> diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java new file mode 100644 index 00000000000..277632b74c7 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java @@ -0,0 +1,19 @@ +package ai.vespa.hosted.cd; + +/** + * A deployment of a Vespa application, which contains endpoints for document and metrics retrieval. + * + * @author jonmv + */ +public interface Deployment { + + /** Returns an Endpoint in the cluster with the "default" id. */ + Endpoint endpoint(); + + /** Returns an Endpoint in the cluster with the given id. */ + Endpoint endpoint(String id); + + /** Returns a {@link TestDeployment} representation of this, or throws if this is a production deployment. */ + TestDeployment asTestDeployment(); + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Digest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Digest.java new file mode 100644 index 00000000000..dee13fdca13 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Digest.java @@ -0,0 +1,28 @@ +package ai.vespa.hosted.cd; + +import java.util.Set; + +/** + * An immutable report of the outcome of a {@link Feed} sent to a {@link TestEndpoint}. + * + * @author jonmv + */ +public class Digest { + + public Set<DocumentId> created() { + return null; + } + + public Set<DocumentId> updated() { + return null; + } + + public Set<DocumentId> deleted() { + return null; + } + + public Set<DocumentId> failed() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Document.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Document.java new file mode 100644 index 00000000000..91adeded65c --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Document.java @@ -0,0 +1,16 @@ +package ai.vespa.hosted.cd; + +/** + * A schema-less representation of a generic Vespa document. + * + * @author jonmv + */ +public class Document { + + + /** Returns a copy of this document, updated with the data in the given document. */ + public Document updatedBy(Document update) { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/DocumentId.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/DocumentId.java new file mode 100644 index 00000000000..9aa8e80c977 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/DocumentId.java @@ -0,0 +1,71 @@ +package ai.vespa.hosted.cd; + +import java.util.Arrays; +import java.util.List; + +/** + * Unique, immutable ID of a Vespa document, which contains information pertinent to its storage. + * + * @author jonmv + */ +public class DocumentId { + + private final String namespace; + private final String documentType; + private final String group; + private final Long number; + private final String userDefined; + + private DocumentId(String namespace, String documentType, String group, Long number, String userDefined) { + this.namespace = namespace; + this.documentType = documentType; + this.group = group; + this.number = number; + this.userDefined = userDefined; + } + + public static DocumentId of(String namespace, String documentType, String id) { + return new DocumentId(requireNonEmpty(namespace), requireNonEmpty(documentType), null, null, requireNonEmpty(id)); + } + + public static DocumentId of(String namespace, String documentType, String group, String id) { + return new DocumentId(requireNonEmpty(namespace), requireNonEmpty(documentType), requireNonEmpty(group), null, requireNonEmpty(id)); + } + + public static DocumentId of(String namespace, String documentType, long number, String id) { + return new DocumentId(requireNonEmpty(namespace), requireNonEmpty(documentType), null, number, requireNonEmpty(id)); + } + + public static DocumentId ofValue(String value) { + List<String> parts = Arrays.asList(value.split(":")); + String id = String.join(":", parts.subList(4, parts.size())); + if ( parts.size() < 5 + || ! parts.get(0).equals("id") + || id.isEmpty() + || ! parts.get(3).matches("((n=\\d+)|(g=\\w+))?")) + throw new IllegalArgumentException("Document id must be on the form" + + " 'id:<namespace>:<document type>:n=<integer>|g=<name>|<empty>:<user defined id>'," + + " but was '" + value + "'."); + + if (parts.get(3).matches("n=\\d+")) + return of(parts.get(1), parts.get(2), Long.parseLong(parts.get(3).substring(2)), id); + if (parts.get(3).matches("g=\\w+")) + return of(parts.get(1), parts.get(2), parts.get(3).substring(2), id); + return of(parts.get(1), parts.get(2), id); + } + + public String asValue() { + return "id:" + namespace + ":" + documentType + ":" + grouper() + ":" + userDefined; + } + + private String grouper() { + return group != null ? group : number != null ? number.toString() : ""; + } + + private static String requireNonEmpty(String string) { + if (string.isEmpty()) + throw new IllegalArgumentException("The empty string is not allowed."); + return string; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/EmptyGroup.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/EmptyGroup.java new file mode 100644 index 00000000000..8deca3cfb11 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/EmptyGroup.java @@ -0,0 +1,9 @@ +package ai.vespa.hosted.cd; + +/** + * The Surefire configuration element <excludedGroups> requires a non-empty argument to reset another. + * This class serves that purpose. Without it, no tests run in the various integration test profiles. + * + * @author jonmv + */ +public interface EmptyGroup { } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java new file mode 100644 index 00000000000..348fc329682 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java @@ -0,0 +1,21 @@ +package ai.vespa.hosted.cd; + +import ai.vespa.hosted.cd.metric.Metrics; + +/** + * An endpoint in a Vespa application {@link Deployment}, which allows document and metrics retrieval. + * + * The endpoint translates {@link Query}s to {@link Search}s, and {@link Selection}s to {@link Visit}s. + * It also supplies {@link Metrics}. + * + * @author jonmv + */ +public interface Endpoint { + + Search search(Query query); + + Visit visit(Selection selection); + + Metrics metrics(); + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Feed.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Feed.java new file mode 100644 index 00000000000..e9a0a0aeff0 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Feed.java @@ -0,0 +1,25 @@ +package ai.vespa.hosted.cd; + +import java.util.Map; +import java.util.Set; + +/** + * An immutable set of document feed / update / delete operations, which can be sent to a Vespa {@link TestEndpoint}. + * + * @author jonmv + */ +public class Feed { + + Map<DocumentId, Document> creations() { + return null; + } + + Map<DocumentId, Document> updates() { + return null; + } + + Set<DocumentId> deletions() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/FunctionalTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/FunctionalTest.java new file mode 100644 index 00000000000..e6beb313d28 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/FunctionalTest.java @@ -0,0 +1,31 @@ +package ai.vespa.hosted.cd; + +/** + * Tests that compare the behaviour of a Vespa application deployment against a fixed specification. + * + * These tests are run whenever a change is pushed to a Vespa application, and whenever the Vespa platform + * is upgraded, and before any deployments to production zones. When these tests fails, the tested change to + * the Vespa application is not rolled out. + * + * A typical functional test is to feed some documents, optionally verifying that the documents have been processed + * as expected, and then to see that queries give the expected results. Another common use is to verify integration + * with external services. + * + * @author jonmv + */ +public interface FunctionalTest { + + // Want to feed some documents. + // Want to verify document processing and routing is as expected. + // Want to check recall on those documents. + // Want to verify queries give expected documents. + // Want to verify searchers. + // Want to verify updates. + // Want to verify deletion. + // May want to verify reprocessing. + // Must likely delete documents between tests. + // Must be able to feed documents, setting route. + // Must be able to search. + // Must be able to visit. + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java index a756b665c1a..6cf5fb07f58 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java @@ -1,6 +1,23 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; -public class ProductionTest { +/** + * Tests that verify the health of production deployments of Vespa applications. + * + * These tests are typically run some time after deployment to a production zone, to ensure + * the deployment is still healthy and working as expected. When these tests fail, deployment + * of the tested change is halted until it succeeds, or is superseded by a remedying change. + * + * A typical production test is to verify that a set of metrics, measured by the Vespa + * deployment itself, are within specified parameters, or that some higher-level measure + * of quality, such as engagement among end users of the application, is as expected. + * + * @author jonmv + */ +public interface ProductionTest { + + // Want to verify metrics (Vespa). + // Want to verify external metrics (YAMAS, other). + // May want to verify search gives expected results. } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Query.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Query.java new file mode 100644 index 00000000000..d421dc14322 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Query.java @@ -0,0 +1,60 @@ +package ai.vespa.hosted.cd; + +import java.util.Map; +import java.util.stream.Stream; + +import static java.util.Map.copyOf; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toUnmodifiableMap; + +/** + * An immutable query to send to a Vespa {@link Endpoint}, to receive a {@link Search}. + * + * @author jonmv + */ +public class Query { + + private final String rawQuery; + private final Map<String, String> parameters; + + private Query(String rawQuery, Map<String, String> parameters) { + this.rawQuery = rawQuery; + this.parameters = parameters; + } + + /** Creates a query with the given raw query part. */ + public static Query ofRaw(String rawQuery) { + if (rawQuery.isBlank()) + throw new IllegalArgumentException("Query can not be blank."); + + return new Query(rawQuery, + Stream.of(rawQuery.split("&")) + .map(pair -> pair.split("=")) + .collect(toUnmodifiableMap(pair -> pair[0], pair -> pair[1]))); + } + + /** Creates a query with the given name-value pairs. */ + public static Query ofParameters(Map<String, String> parameters) { + if (parameters.isEmpty()) + throw new IllegalArgumentException("Parameters can not be empty."); + + return new Query(parameters.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(joining("&")), + copyOf(parameters)); + } + + /** Returns a copy of this with the given name-value pair added, potentially overriding any current value. */ + public Query withParameter(String name, String value) { + return ofParameters(Stream.concat(parameters.entrySet().stream().filter(entry -> ! entry.getKey().equals(name)), + Stream.of(Map.entry(name, value))) + .collect(toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue))); + } + + /** Returns the raw string representation of this query. */ + public String rawQuery() { return rawQuery; } + + /** Returns the parameters of this query. */ + public Map<String, String> parameters() { return parameters; } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Search.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Search.java new file mode 100644 index 00000000000..a6c1d188591 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Search.java @@ -0,0 +1,24 @@ +package ai.vespa.hosted.cd; + +import java.util.Map; + +/** + * The immutable result of sending a {@link Query} to a Vespa {@link Endpoint}. + * + * @author jonmv + */ +public class Search { + + // hits + // coverage + // searched + // full? + // results? + // resultsFull? + + /** Returns the documents that were returned as the result, with iteration order as returned. */ + Map<DocumentId, Document> documents() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Selection.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Selection.java new file mode 100644 index 00000000000..158ae279cb6 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Selection.java @@ -0,0 +1,58 @@ +package ai.vespa.hosted.cd; + +/** + * A document selection expression, type and cluster, which can be used to visit an {@link Endpoint}. + * + * @author jonmv + */ +public class Selection { + + private final String selection; + private final String namespace; + private final String type; + private final String group; + private final String cluster; + private final int concurrency; + + private Selection(String selection, String namespace, String type, String group, String cluster, int concurrency) { + this.selection = selection; + this.namespace = namespace; + this.type = type; + this.group = group; + this.cluster = cluster; + this.concurrency = concurrency; + } + + /** Returns a new selection which will visit documents in the given cluster. */ + public static Selection in(String cluster) { + if (cluster.isBlank()) throw new IllegalArgumentException("Cluster name can not be blank."); + return new Selection(null, null, null, cluster, null, 1); + } + + /** Returns a new selection which will visit documents in the given namespace and of the given type. */ + public static Selection of(String namespace, String type) { + if (namespace.isBlank()) throw new IllegalArgumentException("Namespace can not be blank."); + if (type.isBlank()) throw new IllegalArgumentException("Document type can not be blank."); + return new Selection(null, namespace, type, null, null, 1); + } + + /** Returns a copy of this with the given selection criterion set. */ + public Selection matching(String selection) { + if (selection.isBlank()) throw new IllegalArgumentException("Selection can not be blank."); + return new Selection(selection, namespace, type, cluster, group, concurrency); + } + + /** Returns a copy of this selection, with the group set to the specified value. Requires namespace and type to be set. */ + public Selection limitedTo(String group) { + if (namespace == null || type == null) throw new IllegalArgumentException("Namespace and type must be specified to set group."); + if (group.isBlank()) throw new IllegalArgumentException("Group name can not be blank."); + return new Selection(selection, namespace, type, cluster, group, concurrency); + } + + /** Returns a copy of this, with concurrency set to the given positive value. */ + public Selection concurrently(int concurrency) { + if (concurrency < 1) throw new IllegalArgumentException("Concurrency must be a positive integer."); + return new Selection(selection, namespace, type, cluster, group, concurrency); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java index 789b9deadb0..ee2ee0add4c 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java @@ -1,6 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; +/** + * @deprecated Use {@link UpgradeTest}. + */ +@Deprecated public class StagingTest { } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java index 889acb8b9c4..6a8d1b4cbe4 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java @@ -1,6 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; +/** + * @deprecated use {@link FunctionalTest}. + */ +@Deprecated public class SystemTest { } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestConfig.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestConfig.java new file mode 100644 index 00000000000..36c14a38b37 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestConfig.java @@ -0,0 +1,101 @@ +package ai.vespa.hosted.cd; + +import ai.vespa.hosted.api.ControllerHttpClient; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.slime.Slime; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The place to obtain environment-dependent configuration for the current test run. + * + * If the system property 'vespa.test.config' is set, this class attempts to parse config + * from a JSON file at that location -- otherwise, attempts to access the config will return null. + * + * @author jvenstad + */ +public class TestConfig { + + private static TestConfig theConfig; + + private final ApplicationId application; + private final ZoneId zone; + private final SystemName system; + private final Map<ZoneId, Deployment> deployments; + + private TestConfig(ApplicationId application, ZoneId zone, SystemName system, Map<ZoneId, Deployment> deployments) { + this.application = application; + this.zone = zone; + this.system = system; + this.deployments = Map.copyOf(deployments); + } + + /** Returns the config for this test, or null if it has not been provided. */ + public static synchronized TestConfig get() { + if (theConfig == null) { + String configPath = System.getProperty("vespa.test.config"); + theConfig = configPath != null ? fromFile(configPath) : fromController(); + } + return theConfig; + } + + /** Returns the full id of the application to be tested. */ + public ApplicationId application() { return application; } + + /** Returns the zone of the deployment to test. */ + public ZoneId zone() { return zone; } + + /** Returns an immutable view of all configured endpoints for each zone of the application to test. */ + public Map<ZoneId, Deployment> allDeployments() { return deployments; } + + /** Returns the deployment to test in this test runtime. */ + public Deployment deploymentToTest() { return deployments.get(zone); } + + /** Returns the system this is run against. */ + public SystemName system() { return system; } + + static TestConfig fromFile(String path) { + if (path == null) + return null; + + try { + return fromJson(Files.readAllBytes(Paths.get(path))); + } + catch (Exception e) { + throw new IllegalArgumentException("Failed reading config from '" + path + "'!", e); + } + } + + static TestConfig fromController() { + return null; + } + + static TestConfig fromJson(byte[] jsonBytes) { + Inspector config = new JsonDecoder().decode(new Slime(), jsonBytes).get(); + ApplicationId application = ApplicationId.fromSerializedForm(config.field("application").asString()); + ZoneId zone = ZoneId.from(config.field("zone").asString()); + SystemName system = SystemName.from(config.field("system").asString()); + Map<ZoneId, Deployment> endpoints = new HashMap<>(); + config.field("endpoints").traverse((ObjectTraverser) (zoneId, endpointArray) -> { + List<URI> uris = new ArrayList<>(); + endpointArray.traverse((ArrayTraverser) (__, uri) -> uris.add(URI.create(uri.asString()))); + endpoints.put(ZoneId.from(zoneId), null); // TODO jvenstad + }); + return new TestConfig(application, zone, system, endpoints); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestDeployment.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestDeployment.java new file mode 100644 index 00000000000..3360c12e374 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestDeployment.java @@ -0,0 +1,14 @@ +package ai.vespa.hosted.cd; + +/** + * A deployment of a Vespa application, which also contains endpoints for document manipulation. + * + * @author jonmv + */ +public interface TestDeployment extends Deployment { + + TestEndpoint endpoint(); + + TestEndpoint endpoint(String id); + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestEndpoint.java new file mode 100644 index 00000000000..f6f8a722f19 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestEndpoint.java @@ -0,0 +1,13 @@ +package ai.vespa.hosted.cd; + +/** + * An endpoint in a Vespa application {@link TestDeployment}, which also translates {@link Feed}s to {@link Digest}s. + * + * @author jonmv + */ +public interface TestEndpoint extends Endpoint { + + /** Sends the given Feed to this TestEndpoint, blocking until it is digested, and returns a feed report. */ + Digest digest(Feed feed); + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/UpgradeTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/UpgradeTest.java new file mode 100644 index 00000000000..32083fbd5f6 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/UpgradeTest.java @@ -0,0 +1,23 @@ +package ai.vespa.hosted.cd; + +/** + * Tests that assert continuity of behaviour for Vespa application deployments, through upgrades. + * + * These tests are run whenever a change is pushed to a Vespa application, and whenever the Vespa platform + * is upgraded, and before any deployments to production zones. When these tests fails, the tested change to + * the Vespa application is not rolled out. + * + * A typical upgrade test is to do some operations against a test deployment prior to upgrade, like feed and + * search for some documents, perhaps recording some metrics from the deployment, and then to upgrade it, + * repeat the exercise, and compare the results from pre and post upgrade. + * + * TODO Split in platform upgrades and application upgrades? + * + * @author jonmv + */ +public interface UpgradeTest { + + // Want to verify documents are not damaged by upgrade. + // May want to verify metrics during upgrade. + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Visit.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Visit.java new file mode 100644 index 00000000000..3bb2f59de97 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Visit.java @@ -0,0 +1,17 @@ +package ai.vespa.hosted.cd; + +import java.util.Map; + +/** + * A stateful visit operation against a {@link Endpoint}. + * + * @author jonmv + */ +public class Visit { + + // Delegate to a blocking iterator, which can be used for iteration as visit is ongoing. + public Map<DocumentId, Document> documents() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/VisitEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/VisitEndpoint.java new file mode 100644 index 00000000000..618a004a571 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/VisitEndpoint.java @@ -0,0 +1,10 @@ +package ai.vespa.hosted.cd; + +/** + * A remote endpoint in a Vespa application {@link Deployment}, which translates {@link Selection}s to {@link Visit}s. + * + * @author jonmv + */ +public interface VisitEndpoint { + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java new file mode 100644 index 00000000000..e0d3787a21c --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java @@ -0,0 +1,85 @@ +package ai.vespa.hosted.cd.http; + +import ai.vespa.hosted.auth.Authenticator; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import ai.vespa.hosted.cd.Digest; +import ai.vespa.hosted.cd.Feed; +import ai.vespa.hosted.cd.Query; +import ai.vespa.hosted.cd.Search; +import ai.vespa.hosted.cd.Selection; +import ai.vespa.hosted.cd.TestEndpoint; +import ai.vespa.hosted.cd.Visit; +import ai.vespa.hosted.cd.metric.Metrics; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; + +import static java.util.Objects.requireNonNull; + +public class HttpEndpoint implements TestEndpoint { + + static final String metricsPath = "/state/v1/metrics"; + static final String documentApiPath = "/document/v1"; + static final String searchApiPath = "/search"; + + private final URI endpoint; + private final HttpClient client; + private final Authenticator authenticator; + + public HttpEndpoint(URI endpoint) { + this.endpoint = requireNonNull(endpoint); + this.authenticator = new Authenticator(); + this.client = HttpClient.newBuilder() + .sslContext(authenticator.sslContext()) + .connectTimeout(Duration.ofSeconds(5)) + .version(HttpClient.Version.HTTP_1_1) + .build(); + } + + @Override + public Digest digest(Feed feed) { + return null; + } + + @Override + public Search search(Query query) { + try { + URI target = endpoint.resolve(searchApiPath).resolve("?" + query.rawQuery()); + HttpRequest request = HttpRequest.newBuilder() + .timeout(Duration.ofSeconds(5)) + .uri(target) + .build(); + HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() / 100 != 2) // TODO consider allowing 504 if specified. + throw new RuntimeException("Non-OK status code " + response.statusCode() + " at " + target + + ", with response \n" + new String(response.body())); + + return toSearch(response.body()); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + static Search toSearch(byte[] body) { + Inspector rootObject = new JsonDecoder().decode(new Slime(), body).get(); + // TODO jvenstad + return new Search(); + } + + @Override + public Visit visit(Selection selection) { + return null; + } + + @Override + public Metrics metrics() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java new file mode 100644 index 00000000000..cb3c8e77a9a --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java @@ -0,0 +1,87 @@ +package ai.vespa.hosted.cd.metric; + +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.StringJoiner; + +import static java.util.Map.copyOf; +import static java.util.stream.Collectors.toUnmodifiableMap; + +/** + * A set of statistics for a metric, for points over a space with named dimensions of arbitrary type. + * + * @author jonmv + */ +public class Metric { + + private final Map<Map<String, ?>, Statistic> statistics; + + private Metric(Map<Map<String, ?>, Statistic> statistics) { + this.statistics = statistics; + } + + /** Creates a new Metric with a copy of the given data. */ + public static Metric of(Map<Map<String, ?>, Statistic> data) { + if (data.isEmpty()) + throw new IllegalArgumentException("No data given."); + + Map<Map<String, ?>, Statistic> copies = new HashMap<>(); + Set<String> dimensions = data.keySet().iterator().next().keySet(); + data.forEach((point, statistic) -> { + if ( ! point.keySet().equals(dimensions)) + throw new IllegalArgumentException("Given data has inconsistent dimensions: '" + dimensions + "' vs '" + point.keySet() + "'."); + + copies.put(copyOf(point), statistic); + }); + + return new Metric(copyOf(copies)); + } + + /** Returns a Metric view of the subset of points in the given hyperplane; its dimensions must be a subset of those of this Metric. */ + public Metric at(Map<String, ?> hyperplane) { + return new Metric(statistics.keySet().stream() + .filter(point -> point.entrySet().containsAll(hyperplane.entrySet())) + .collect(toUnmodifiableMap(point -> point, statistics::get))); + } + + /** Returns a version of this where statistics along the given hyperspace are aggregated. This does not preserve last, 95 and 99 percentile values. */ + public Metric collapse(Set<String> hyperspace) { + return new Metric(statistics.keySet().stream() + .collect(toUnmodifiableMap(point -> point.keySet().stream() + .filter(dimension -> ! hyperspace.contains(dimension)) + .collect(toUnmodifiableMap(dimension -> dimension, point::get)), + statistics::get, + Statistic::mergedWith))); + } + + /** Returns a collapsed version of this, with all statistics aggregated. This does not preserve last, 95 and 99 percentile values. */ + public Metric collapse() { + return collapse(statistics.keySet().iterator().next().keySet()); + } + + /** If this Metric contains a single point, returns the Statistic of that point; otherwise, throws an exception. */ + public Statistic statistic() { + if (statistics.size() == 1) + return statistics.values().iterator().next(); + + if (statistics.isEmpty()) + throw new NoSuchElementException("This Metric has no data."); + + throw new IllegalStateException("This Metric has more than one point of data."); + } + + /** Returns the underlying, unmodifiable Map. */ + public Map<Map<String, ?>, Statistic> asMap() { + return statistics; + } + + @Override + public String toString() { + return new StringJoiner(", ", Metric.class.getSimpleName() + "[", "]") + .add("statistics=" + statistics) + .toString(); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java new file mode 100644 index 00000000000..3aa5a126745 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java @@ -0,0 +1,73 @@ +package ai.vespa.hosted.cd.metric; + +import ai.vespa.hosted.cd.Endpoint; + +import java.time.Instant; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.StringJoiner; + +import static java.util.Map.copyOf; + +/** + * Metrics from a Vespa application {@link Endpoint}, indexed by their names, and optionally by a set of custom dimensions. + * + * Metrics are collected from the <a href="https://docs.vespa.ai/documentation/reference/metrics-health-format.html">metrics</a> + * API of a Vespa endpoint, and contain the current health status of the endpoint, values for all configured metrics in + * that endpoint, and the time interval from which these metrics were sampled. + * + * Each metric is indexed by a name, and, optionally, along a custom set of dimensions, given by a {@code Map<String, String>}. + * + * @author jonmv + */ +public class Metrics { + + private final Instant start, end; + private final Map<String, Metric> metrics; + + private Metrics(Instant start, Instant end, Map<String, Metric> metrics) { + this.start = start; + this.end = end; + this.metrics = metrics; + } + + public static Metrics of(Instant start, Instant end, Map<String, Metric> metrics) { + if ( ! start.isBefore(end)) + throw new IllegalArgumentException("Given time interval must be positive: '" + start + "' to '" + end + "'."); + + return new Metrics(start, end, copyOf(metrics)); + } + + /** Returns the start of the time window from which these metrics were sampled, or throws if the status is {@code Status.down}. */ + public Instant start() { + return start; + } + + /** Returns the end of the time window from which these metrics were sampled, or throws if the status is {@code Status.down}. */ + public Instant end() { + return end; + } + + /** Returns the metric with the given name, or throws a NoSuchElementException if no such Metric is known. */ + public Metric get(String name) { + if ( ! metrics.containsKey(name)) + throw new NoSuchElementException("No metric with name '" + name + "'."); + + return metrics.get(name); + } + + /** Returns the underlying, unmodifiable Map. */ + public Map<String, Metric> asMap() { + return metrics; + } + + @Override + public String toString() { + return new StringJoiner(", ", Metrics.class.getSimpleName() + "[", "]") + .add("start=" + start) + .add("end=" + end) + .add("metrics=" + metrics) + .toString(); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java new file mode 100644 index 00000000000..ea771ca5dd9 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java @@ -0,0 +1,44 @@ +package ai.vespa.hosted.cd.metric; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.IntStream; + +import static java.util.stream.Collectors.toUnmodifiableMap; + +/** + * Used to easily generate points for a pre-defined space. + * + * @author jonmv + */ +public class Space { + + private final List<String> dimensions; + + private Space(List<String> dimensions) { + this.dimensions = dimensions; + } + + /** Creates a new space with the given named dimensions, in order. */ + public static Space of(List<String> dimensions) { + if (Set.copyOf(dimensions).size() != dimensions.size()) + throw new IllegalArgumentException("Duplicated dimension names in '" + dimensions + "'."); + + return new Space(List.copyOf(dimensions)); + } + + /** Returns a point in this space, with the given values along each dimensions, in order. */ + public Map<String, ?> at(List<?> values) { + if (dimensions.size() != values.size()) + throw new IllegalArgumentException("This space has " + dimensions.size() + " dimensions, but " + values.size() + " were given."); + + return IntStream.range(0, dimensions.size()).boxed().collect(toUnmodifiableMap(dimensions::get, values::get)); + } + + /** Returns a point in this space, with the given values along each dimensions, in order. */ + public Map<String, ?> at(Object... values) { + return at(List.of(values)); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java new file mode 100644 index 00000000000..fc52900bdac --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java @@ -0,0 +1,68 @@ +package ai.vespa.hosted.cd.metric; + +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.StringJoiner; + +import static java.util.Map.copyOf; + +/** + * Known statistic about a metric, at a certain point. + * + * @author jonmv + */ +public class Statistic { + + private final Map<Type, Double> data; + + /** Creates a new Statistic with a copy of the given data. */ + private Statistic(Map<Type, Double> data) { + this.data = data; + } + + public static Statistic of(Map<Type, Double> data) { + return new Statistic(copyOf(data)); + } + + /** Returns the value of the given type, or throws a NoSuchElementException if this isn't known. */ + public double get(Type key) { + if ( ! data.containsKey(key)) + throw new NoSuchElementException("No value with key '" + key + "' is known."); + + return data.get(key); + } + + /** Returns the underlying, unmodifiable Map. */ + public Map<Type, Double> asMap() { + return data; + } + + Statistic mergedWith(Statistic other) { + if (data.keySet().equals(other.data.keySet())) + throw new IllegalArgumentException("Incompatible key sets '" + data.keySet() + "' and '" + other.data.keySet() + "'."); + + Map<Type, Double> merged = new HashMap<>(); + double n1 = get(Type.count), n2 = other.get(Type.count); + for (Type type : data.keySet()) switch (type) { + case count: merged.put(type, n1 + n2); break; + case rate: merged.put(type, get(Type.rate) + other.get(Type.rate)); break; + case max: merged.put(type, Math.max(get(Type.max), other.get(Type.max))); break; + case min: merged.put(type, Math.min(get(Type.min), other.get(Type.min))); break; + case average: merged.put(type, (n1 * get(Type.average) + n2 * other.get(Type.average)) / (n1 + n2)); break; + case last: + case percentile95: + case percentile99: break; + default: throw new IllegalArgumentException("Unexpected type '" + type + "'."); + } + return of(merged); + } + + @Override + public String toString() { + return new StringJoiner(", ", Statistic.class.getSimpleName() + "[", "]") + .add("data=" + data) + .toString(); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java new file mode 100644 index 00000000000..d48b4566f6d --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java @@ -0,0 +1,32 @@ +package ai.vespa.hosted.cd.metric; + +/** + * Known statistic types. + */ +public enum Type { + + /** 95th percentile measurement. */ + percentile95, + + /** 99th percentile measurement. */ + percentile99, + + /** Average over all measurements. */ + average, + + /** Number of measurements. */ + count, + + /** Last measurement. */ + last, + + /** Maximum measurement. */ + max, + + /** Minimum measurement. */ + min, + + /** Number of measurements per second. */ + rate; + +} |