Merge pull request #9731 from vespa-engine/jvenstad/tenant-cd

Jvenstad/tenant cd

26 files changed, 955 insertions, 1 deletions

diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java
new file mode 100644
index 00000000000..277632b74c7
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java
@@ -0,0 +1,19 @@
+package ai.vespa.hosted.cd;
+
+/**
+ * A deployment of a Vespa application, which contains endpoints for document and metrics retrieval.
+ *
+ * @author jonmv
+ */
+public interface Deployment {
+
+    /** Returns an Endpoint in the cluster with the "default" id. */
+    Endpoint endpoint();
+
+    /** Returns an Endpoint in the cluster with the given id. */
+    Endpoint endpoint(String id);
+
+    /** Returns a {@link TestDeployment} representation of this, or throws if this is a production deployment. */
+    TestDeployment asTestDeployment();
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Digest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Digest.java
new file mode 100644
index 00000000000..dee13fdca13
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Digest.java
@@ -0,0 +1,28 @@
+package ai.vespa.hosted.cd;
+
+import java.util.Set;
+
+/**
+ * An immutable report of the outcome of a {@link Feed} sent to a {@link TestEndpoint}.
+ *
+ * @author jonmv
+ */
+public class Digest {
+
+    public Set<DocumentId> created() {
+        return null;
+    }
+
+    public Set<DocumentId> updated() {
+        return null;
+    }
+
+    public Set<DocumentId> deleted() {
+        return null;
+    }
+
+    public Set<DocumentId> failed() {
+        return null;
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Document.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Document.java
new file mode 100644
index 00000000000..91adeded65c
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Document.java
@@ -0,0 +1,16 @@
+package ai.vespa.hosted.cd;
+
+/**
+ * A schema-less representation of a generic Vespa document.
+ *
+ * @author jonmv
+ */
+public class Document {
+
+
+    /** Returns a copy of this document, updated with the data in the given document. */
+    public Document updatedBy(Document update) {
+        return null;
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/DocumentId.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/DocumentId.java
new file mode 100644
index 00000000000..9aa8e80c977
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/DocumentId.java
@@ -0,0 +1,71 @@
+package ai.vespa.hosted.cd;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Unique, immutable ID of a Vespa document, which contains information pertinent to its storage.
+ *
+ * @author jonmv
+ */
+public class DocumentId {
+
+    private final String namespace;
+    private final String documentType;
+    private final String group;
+    private final Long number;
+    private final String userDefined;
+
+    private DocumentId(String namespace, String documentType, String group, Long number, String userDefined) {
+        this.namespace = namespace;
+        this.documentType = documentType;
+        this.group = group;
+        this.number = number;
+        this.userDefined = userDefined;
+    }
+
+    public static DocumentId of(String namespace, String documentType, String id) {
+        return new DocumentId(requireNonEmpty(namespace), requireNonEmpty(documentType), null, null, requireNonEmpty(id));
+    }
+
+    public static DocumentId of(String namespace, String documentType, String group, String id) {
+        return new DocumentId(requireNonEmpty(namespace), requireNonEmpty(documentType), requireNonEmpty(group), null, requireNonEmpty(id));
+    }
+
+    public static DocumentId of(String namespace, String documentType, long number, String id) {
+        return new DocumentId(requireNonEmpty(namespace), requireNonEmpty(documentType), null, number, requireNonEmpty(id));
+    }
+
+    public static DocumentId ofValue(String value) {
+        List<String> parts = Arrays.asList(value.split(":"));
+        String id = String.join(":", parts.subList(4, parts.size()));
+        if (     parts.size() < 5
+            || ! parts.get(0).equals("id")
+            ||   id.isEmpty()
+            || ! parts.get(3).matches("((n=\\d+)|(g=\\w+))?"))
+            throw new IllegalArgumentException("Document id must be on the form" +
+                                               " 'id:<namespace>:<document type>:n=<integer>|g=<name>|<empty>:<user defined id>'," +
+                                               " but was '" + value + "'.");
+
+        if (parts.get(3).matches("n=\\d+"))
+            return of(parts.get(1), parts.get(2), Long.parseLong(parts.get(3).substring(2)), id);
+        if (parts.get(3).matches("g=\\w+"))
+            return of(parts.get(1), parts.get(2), parts.get(3).substring(2), id);
+        return of(parts.get(1), parts.get(2), id);
+    }
+
+    public String asValue() {
+        return "id:" + namespace + ":" + documentType + ":" + grouper() + ":" + userDefined;
+    }
+
+    private String grouper() {
+        return group != null ? group : number != null ? number.toString() : "";
+    }
+
+    private static String requireNonEmpty(String string) {
+        if (string.isEmpty())
+            throw new IllegalArgumentException("The empty string is not allowed.");
+        return string;
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/EmptyGroup.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/EmptyGroup.java
new file mode 100644
index 00000000000..8deca3cfb11
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/EmptyGroup.java
@@ -0,0 +1,9 @@
+package ai.vespa.hosted.cd;
+
+/**
+ * The Surefire configuration element &lt;excludedGroups&gt; requires a non-empty argument to reset another.
+ * This class serves that purpose. Without it, no tests run in the various integration test profiles.
+ *
+ * @author jonmv
+ */
+public interface EmptyGroup { }
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java
new file mode 100644
index 00000000000..348fc329682
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java
@@ -0,0 +1,21 @@
+package ai.vespa.hosted.cd;
+
+import ai.vespa.hosted.cd.metric.Metrics;
+
+/**
+ * An endpoint in a Vespa application {@link Deployment}, which allows document and metrics retrieval.
+ *
+ * The endpoint translates {@link Query}s to {@link Search}s, and {@link Selection}s to {@link Visit}s.
+ * It also supplies {@link Metrics}.
+ *
+ * @author jonmv
+ */
+public interface Endpoint {
+
+    Search search(Query query);
+
+    Visit visit(Selection selection);
+
+    Metrics metrics();
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Feed.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Feed.java
new file mode 100644
index 00000000000..e9a0a0aeff0
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Feed.java
@@ -0,0 +1,25 @@
+package ai.vespa.hosted.cd;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * An immutable set of document feed / update / delete operations, which can be sent to a Vespa {@link TestEndpoint}.
+ *
+ * @author jonmv
+ */
+public class Feed {
+
+    Map<DocumentId, Document> creations() {
+        return null;
+    }
+
+    Map<DocumentId, Document> updates() {
+        return null;
+    }
+
+    Set<DocumentId> deletions() {
+        return null;
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/FunctionalTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/FunctionalTest.java
new file mode 100644
index 00000000000..e6beb313d28
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/FunctionalTest.java
@@ -0,0 +1,31 @@
+package ai.vespa.hosted.cd;
+
+/**
+ * Tests that compare the behaviour of a Vespa application deployment against a fixed specification.
+ *
+ * These tests are run whenever a change is pushed to a Vespa application, and whenever the Vespa platform
+ * is upgraded, and before any deployments to production zones. When these tests fails, the tested change to
+ * the Vespa application is not rolled out.
+ *
+ * A typical functional test is to feed some documents, optionally verifying that the documents have been processed
+ * as expected, and then to see that queries give the expected results. Another common use is to verify integration
+ * with external services.
+ *
+ * @author jonmv
+ */
+public interface FunctionalTest {
+
+    // Want to feed some documents.
+    // Want to verify document processing and routing is as expected.
+    // Want to check recall on those documents.
+    // Want to verify queries give expected documents.
+    // Want to verify searchers.
+    // Want to verify updates.
+    // Want to verify deletion.
+    // May want to verify reprocessing.
+    // Must likely delete documents between tests.
+    // Must be able to feed documents, setting route.
+    // Must be able to search.
+    // Must be able to visit.
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java
index a756b665c1a..6cf5fb07f58 100644
--- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java
@@ -1,6 +1,23 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package ai.vespa.hosted.cd;
 
-public class ProductionTest {
+/**
+ * Tests that verify the health of production deployments of Vespa applications.
+ *
+ * These tests are typically run some time after deployment to a production zone, to ensure
+ * the deployment is still healthy and working as expected. When these tests fail, deployment
+ * of the tested change is halted until it succeeds, or is superseded by a remedying change.
+ *
+ * A typical production test is to verify that a set of metrics, measured by the Vespa
+ * deployment itself, are within specified parameters, or that some higher-level measure
+ * of quality, such as engagement among end users of the application, is as expected.
+ *
+ * @author jonmv
+ */
+public interface ProductionTest {
+
+    // Want to verify metrics (Vespa).
+    // Want to verify external metrics (YAMAS, other).
+    // May want to verify search gives expected results.
 
 }
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Query.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Query.java
new file mode 100644
index 00000000000..d421dc14322
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Query.java
@@ -0,0 +1,60 @@
+package ai.vespa.hosted.cd;
+
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static java.util.Map.copyOf;
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toUnmodifiableMap;
+
+/**
+ * An immutable query to send to a Vespa {@link Endpoint}, to receive a {@link Search}.
+ *
+ * @author jonmv
+ */
+public class Query {
+
+    private final String rawQuery;
+    private final Map<String, String> parameters;
+
+    private Query(String rawQuery, Map<String, String> parameters) {
+        this.rawQuery = rawQuery;
+        this.parameters = parameters;
+    }
+
+    /** Creates a query with the given raw query part. */
+    public static Query ofRaw(String rawQuery) {
+        if (rawQuery.isBlank())
+            throw new IllegalArgumentException("Query can not be blank.");
+
+        return new Query(rawQuery,
+                         Stream.of(rawQuery.split("&"))
+                               .map(pair -> pair.split("="))
+                               .collect(toUnmodifiableMap(pair -> pair[0], pair -> pair[1])));
+    }
+
+    /** Creates a query with the given name-value pairs. */
+    public static Query ofParameters(Map<String, String> parameters) {
+        if (parameters.isEmpty())
+            throw new IllegalArgumentException("Parameters can not be empty.");
+
+        return new Query(parameters.entrySet().stream()
+                                   .map(entry -> entry.getKey() + "=" + entry.getValue())
+                                   .collect(joining("&")),
+                         copyOf(parameters));
+    }
+
+    /** Returns a copy of this with the given name-value pair added, potentially overriding any current value. */
+    public Query withParameter(String name, String value) {
+        return ofParameters(Stream.concat(parameters.entrySet().stream().filter(entry -> ! entry.getKey().equals(name)),
+                                          Stream.of(Map.entry(name, value)))
+                                  .collect(toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)));
+    }
+
+    /** Returns the raw string representation of this query. */
+    public String rawQuery() { return rawQuery; }
+
+    /** Returns the parameters of this query. */
+    public Map<String, String> parameters() { return parameters; }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Search.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Search.java
new file mode 100644
index 00000000000..a6c1d188591
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Search.java
@@ -0,0 +1,24 @@
+package ai.vespa.hosted.cd;
+
+import java.util.Map;
+
+/**
+ * The immutable result of sending a {@link Query} to a Vespa {@link Endpoint}.
+ *
+ * @author jonmv
+ */
+public class Search {
+
+    // hits
+    // coverage
+    // searched
+    // full?
+    // results?
+    // resultsFull?
+
+    /** Returns the documents that were returned as the result, with iteration order as returned. */
+    Map<DocumentId, Document> documents() {
+        return null;
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Selection.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Selection.java
new file mode 100644
index 00000000000..158ae279cb6
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Selection.java
@@ -0,0 +1,58 @@
+package ai.vespa.hosted.cd;
+
+/**
+ * A document selection expression, type and cluster, which can be used to visit an {@link Endpoint}.
+ *
+ * @author jonmv
+ */
+public class Selection {
+
+    private final String selection;
+    private final String namespace;
+    private final String type;
+    private final String group;
+    private final String cluster;
+    private final int concurrency;
+
+    private Selection(String selection, String namespace, String type, String group, String cluster, int concurrency) {
+        this.selection = selection;
+        this.namespace = namespace;
+        this.type = type;
+        this.group = group;
+        this.cluster = cluster;
+        this.concurrency = concurrency;
+    }
+
+    /** Returns a new selection which will visit documents in the given cluster. */
+    public static Selection in(String cluster) {
+        if (cluster.isBlank()) throw new IllegalArgumentException("Cluster name can not be blank.");
+        return new Selection(null, null, null, cluster, null, 1);
+    }
+
+    /** Returns a new selection which will visit documents in the given namespace and of the given type. */
+    public static Selection of(String namespace, String type) {
+        if (namespace.isBlank()) throw new IllegalArgumentException("Namespace can not be blank.");
+        if (type.isBlank()) throw new IllegalArgumentException("Document type can not be blank.");
+        return new Selection(null, namespace, type, null, null, 1);
+    }
+
+    /** Returns a copy of this with the given selection criterion set. */
+    public Selection matching(String selection) {
+        if (selection.isBlank()) throw new IllegalArgumentException("Selection can not be blank.");
+        return new Selection(selection, namespace, type, cluster, group, concurrency);
+    }
+
+    /** Returns a copy of this selection, with the group set to the specified value. Requires namespace and type to be set. */
+    public Selection limitedTo(String group) {
+        if (namespace == null || type == null) throw new IllegalArgumentException("Namespace and type must be specified to set group.");
+        if (group.isBlank()) throw new IllegalArgumentException("Group name can not be blank.");
+        return new Selection(selection, namespace, type, cluster, group, concurrency);
+    }
+
+    /** Returns a copy of this, with concurrency set to the given positive value. */
+    public Selection concurrently(int concurrency) {
+        if (concurrency < 1) throw new IllegalArgumentException("Concurrency must be a positive integer.");
+        return new Selection(selection, namespace, type, cluster, group, concurrency);
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java
index 789b9deadb0..ee2ee0add4c 100644
--- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java
@@ -1,6 +1,10 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package ai.vespa.hosted.cd;
 
+/**
+ * @deprecated Use {@link UpgradeTest}.
+ */
+@Deprecated
 public class StagingTest {
 
 }
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java
index 889acb8b9c4..6a8d1b4cbe4 100644
--- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java
@@ -1,6 +1,10 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package ai.vespa.hosted.cd;
 
+/**
+ * @deprecated use {@link FunctionalTest}.
+ */
+@Deprecated
 public class SystemTest {
 
 }
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestConfig.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestConfig.java
new file mode 100644
index 00000000000..36c14a38b37
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestConfig.java
@@ -0,0 +1,101 @@
+package ai.vespa.hosted.cd;
+
+import ai.vespa.hosted.api.ControllerHttpClient;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.JsonDecoder;
+import com.yahoo.slime.ObjectTraverser;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.slime.Slime;
+
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * The place to obtain environment-dependent configuration for the current test run.
+ *
+ * If the system property 'vespa.test.config' is set, this class attempts to parse config
+ * from a JSON file at that location -- otherwise, attempts to access the config will return null.
+ *
+ * @author jvenstad
+ */
+public class TestConfig {
+
+    private static TestConfig theConfig;
+
+    private final ApplicationId application;
+    private final ZoneId zone;
+    private final SystemName system;
+    private final Map<ZoneId, Deployment> deployments;
+
+    private TestConfig(ApplicationId application, ZoneId zone, SystemName system, Map<ZoneId, Deployment> deployments) {
+        this.application = application;
+        this.zone = zone;
+        this.system = system;
+        this.deployments = Map.copyOf(deployments);
+    }
+
+    /** Returns the config for this test, or null if it has not been provided. */
+    public static synchronized TestConfig get() {
+        if (theConfig == null) {
+            String configPath = System.getProperty("vespa.test.config");
+            theConfig = configPath != null ? fromFile(configPath) : fromController();
+        }
+        return theConfig;
+    }
+
+    /** Returns the full id of the application to be tested. */
+    public ApplicationId application() { return application; }
+
+    /** Returns the zone of the deployment to test. */
+    public ZoneId zone() { return zone; }
+
+    /** Returns an immutable view of all configured endpoints for each zone of the application to test. */
+    public Map<ZoneId, Deployment> allDeployments() { return deployments; }
+
+    /** Returns the deployment to test in this test runtime. */
+    public Deployment deploymentToTest() { return deployments.get(zone); }
+
+    /** Returns the system this is run against. */
+    public SystemName system() { return system; }
+
+    static TestConfig fromFile(String path) {
+        if (path == null)
+            return null;
+
+        try {
+            return fromJson(Files.readAllBytes(Paths.get(path)));
+        }
+        catch (Exception e) {
+            throw new IllegalArgumentException("Failed reading config from '" + path + "'!", e);
+        }
+    }
+
+    static TestConfig fromController() {
+        return null;
+    }
+
+    static TestConfig fromJson(byte[] jsonBytes) {
+        Inspector config = new JsonDecoder().decode(new Slime(), jsonBytes).get();
+        ApplicationId application = ApplicationId.fromSerializedForm(config.field("application").asString());
+        ZoneId zone = ZoneId.from(config.field("zone").asString());
+        SystemName system = SystemName.from(config.field("system").asString());
+        Map<ZoneId, Deployment> endpoints = new HashMap<>();
+        config.field("endpoints").traverse((ObjectTraverser) (zoneId, endpointArray) -> {
+            List<URI> uris = new ArrayList<>();
+            endpointArray.traverse((ArrayTraverser) (__, uri) -> uris.add(URI.create(uri.asString())));
+            endpoints.put(ZoneId.from(zoneId), null); // TODO jvenstad
+        });
+        return new TestConfig(application, zone, system, endpoints);
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestDeployment.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestDeployment.java
new file mode 100644
index 00000000000..3360c12e374
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestDeployment.java
@@ -0,0 +1,14 @@
+package ai.vespa.hosted.cd;
+
+/**
+ * A deployment of a Vespa application, which also contains endpoints for document manipulation.
+ *
+ * @author jonmv
+ */
+public interface TestDeployment extends Deployment {
+
+    TestEndpoint endpoint();
+
+    TestEndpoint endpoint(String id);
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestEndpoint.java
new file mode 100644
index 00000000000..f6f8a722f19
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestEndpoint.java
@@ -0,0 +1,13 @@
+package ai.vespa.hosted.cd;
+
+/**
+ * An endpoint in a Vespa application {@link TestDeployment}, which also translates {@link Feed}s to {@link Digest}s.
+ *
+ * @author jonmv
+ */
+public interface TestEndpoint extends Endpoint {
+
+    /** Sends the given Feed to this TestEndpoint, blocking until it is digested, and returns a feed report. */
+    Digest digest(Feed feed);
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/UpgradeTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/UpgradeTest.java
new file mode 100644
index 00000000000..32083fbd5f6
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/UpgradeTest.java
@@ -0,0 +1,23 @@
+package ai.vespa.hosted.cd;
+
+/**
+ * Tests that assert continuity of behaviour for Vespa application deployments, through upgrades.
+ *
+ * These tests are run whenever a change is pushed to a Vespa application, and whenever the Vespa platform
+ * is upgraded, and before any deployments to production zones. When these tests fails, the tested change to
+ * the Vespa application is not rolled out.
+ *
+ * A typical upgrade test is to do some operations against a test deployment prior to upgrade, like feed and
+ * search for some documents, perhaps recording some metrics from the deployment, and then to upgrade it,
+ * repeat the exercise, and compare the results from pre and post upgrade.
+ *
+ * TODO Split in platform upgrades and application upgrades?
+ *
+ * @author jonmv
+ */
+public interface UpgradeTest {
+
+    // Want to verify documents are not damaged by upgrade.
+    // May want to verify metrics during upgrade.
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Visit.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Visit.java
new file mode 100644
index 00000000000..3bb2f59de97
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Visit.java
@@ -0,0 +1,17 @@
+package ai.vespa.hosted.cd;
+
+import java.util.Map;
+
+/**
+ * A stateful visit operation against a {@link Endpoint}.
+ *
+ * @author jonmv
+ */
+public class Visit {
+
+    // Delegate to a blocking iterator, which can be used for iteration as visit is ongoing.
+    public Map<DocumentId, Document> documents() {
+        return null;
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/VisitEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/VisitEndpoint.java
new file mode 100644
index 00000000000..618a004a571
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/VisitEndpoint.java
@@ -0,0 +1,10 @@
+package ai.vespa.hosted.cd;
+
+/**
+ * A remote endpoint in a Vespa application {@link Deployment}, which translates {@link Selection}s to {@link Visit}s.
+ *
+ * @author jonmv
+ */
+public interface VisitEndpoint {
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java
new file mode 100644
index 00000000000..e0d3787a21c
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java
@@ -0,0 +1,85 @@
+package ai.vespa.hosted.cd.http;
+
+import ai.vespa.hosted.auth.Authenticator;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.JsonDecoder;
+import com.yahoo.slime.Slime;
+import ai.vespa.hosted.cd.Digest;
+import ai.vespa.hosted.cd.Feed;
+import ai.vespa.hosted.cd.Query;
+import ai.vespa.hosted.cd.Search;
+import ai.vespa.hosted.cd.Selection;
+import ai.vespa.hosted.cd.TestEndpoint;
+import ai.vespa.hosted.cd.Visit;
+import ai.vespa.hosted.cd.metric.Metrics;
+
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.time.Duration;
+
+import static java.util.Objects.requireNonNull;
+
+public class HttpEndpoint implements TestEndpoint {
+
+    static final String metricsPath = "/state/v1/metrics";
+    static final String documentApiPath = "/document/v1";
+    static final String searchApiPath = "/search";
+
+    private final URI endpoint;
+    private final HttpClient client;
+    private final Authenticator authenticator;
+
+    public HttpEndpoint(URI endpoint) {
+        this.endpoint = requireNonNull(endpoint);
+        this.authenticator = new Authenticator();
+        this.client = HttpClient.newBuilder()
+                                .sslContext(authenticator.sslContext())
+                                .connectTimeout(Duration.ofSeconds(5))
+                                .version(HttpClient.Version.HTTP_1_1)
+                                .build();
+    }
+
+    @Override
+    public Digest digest(Feed feed) {
+        return null;
+    }
+
+    @Override
+    public Search search(Query query) {
+        try {
+            URI target = endpoint.resolve(searchApiPath).resolve("?" + query.rawQuery());
+            HttpRequest request = HttpRequest.newBuilder()
+                                             .timeout(Duration.ofSeconds(5))
+                                             .uri(target)
+                                             .build();
+            HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
+            if (response.statusCode() / 100 != 2) // TODO consider allowing 504 if specified.
+                throw new RuntimeException("Non-OK status code " + response.statusCode() + " at " + target +
+                                           ", with response \n" + new String(response.body()));
+
+            return toSearch(response.body());
+        }
+        catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    static Search toSearch(byte[] body) {
+        Inspector rootObject = new JsonDecoder().decode(new Slime(), body).get();
+        // TODO jvenstad
+        return new Search();
+    }
+
+    @Override
+    public Visit visit(Selection selection) {
+        return null;
+    }
+
+    @Override
+    public Metrics metrics() {
+        return null;
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java
new file mode 100644
index 00000000000..cb3c8e77a9a
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java
@@ -0,0 +1,87 @@
+package ai.vespa.hosted.cd.metric;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.StringJoiner;
+
+import static java.util.Map.copyOf;
+import static java.util.stream.Collectors.toUnmodifiableMap;
+
+/**
+ * A set of statistics for a metric, for points over a space with named dimensions of arbitrary type.
+ *
+ * @author jonmv
+ */
+public class Metric {
+
+    private final Map<Map<String, ?>, Statistic> statistics;
+
+    private Metric(Map<Map<String, ?>, Statistic> statistics) {
+        this.statistics = statistics;
+    }
+
+    /** Creates a new Metric with a copy of the given data. */
+    public static Metric of(Map<Map<String, ?>, Statistic> data) {
+        if (data.isEmpty())
+            throw new IllegalArgumentException("No data given.");
+
+        Map<Map<String, ?>, Statistic> copies = new HashMap<>();
+        Set<String> dimensions = data.keySet().iterator().next().keySet();
+        data.forEach((point, statistic) -> {
+            if ( ! point.keySet().equals(dimensions))
+                throw new IllegalArgumentException("Given data has inconsistent dimensions: '" + dimensions + "' vs '" + point.keySet() + "'.");
+
+            copies.put(copyOf(point), statistic);
+        });
+
+        return new Metric(copyOf(copies));
+    }
+
+    /** Returns a Metric view of the subset of points in the given hyperplane; its dimensions must be a subset of those of this Metric. */
+    public Metric at(Map<String, ?> hyperplane) {
+        return new Metric(statistics.keySet().stream()
+                                    .filter(point -> point.entrySet().containsAll(hyperplane.entrySet()))
+                                    .collect(toUnmodifiableMap(point -> point, statistics::get)));
+    }
+
+    /** Returns a version of this where statistics along the given hyperspace are aggregated. This does not preserve last, 95 and 99 percentile values. */
+    public Metric collapse(Set<String> hyperspace) {
+        return new Metric(statistics.keySet().stream()
+                                    .collect(toUnmodifiableMap(point -> point.keySet().stream()
+                                                                             .filter(dimension -> ! hyperspace.contains(dimension))
+                                                                             .collect(toUnmodifiableMap(dimension -> dimension, point::get)),
+                                                               statistics::get,
+                                                               Statistic::mergedWith)));
+    }
+
+    /** Returns a collapsed version of this, with all statistics aggregated. This does not preserve last, 95 and 99 percentile values. */
+    public Metric collapse() {
+        return collapse(statistics.keySet().iterator().next().keySet());
+    }
+
+    /** If this Metric contains a single point, returns the Statistic of that point; otherwise, throws an exception. */
+    public Statistic statistic() {
+        if (statistics.size() == 1)
+            return statistics.values().iterator().next();
+
+        if (statistics.isEmpty())
+            throw new NoSuchElementException("This Metric has no data.");
+
+        throw new IllegalStateException("This Metric has more than one point of data.");
+    }
+
+    /** Returns the underlying, unmodifiable Map. */
+    public Map<Map<String, ?>, Statistic> asMap() {
+        return statistics;
+    }
+
+    @Override
+    public String toString() {
+        return new StringJoiner(", ", Metric.class.getSimpleName() + "[", "]")
+                .add("statistics=" + statistics)
+                .toString();
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java
new file mode 100644
index 00000000000..3aa5a126745
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java
@@ -0,0 +1,73 @@
+package ai.vespa.hosted.cd.metric;
+
+import ai.vespa.hosted.cd.Endpoint;
+
+import java.time.Instant;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.StringJoiner;
+
+import static java.util.Map.copyOf;
+
+/**
+ * Metrics from a Vespa application {@link Endpoint}, indexed by their names, and optionally by a set of custom dimensions.
+ *
+ * Metrics are collected from the <a href="https://docs.vespa.ai/documentation/reference/metrics-health-format.html">metrics</a>
+ * API of a Vespa endpoint, and contain the current health status of the endpoint, values for all configured metrics in
+ * that endpoint, and the time interval from which these metrics were sampled.
+ *
+ * Each metric is indexed by a name, and, optionally, along a custom set of dimensions, given by a {@code Map<String, String>}.
+ *
+ * @author jonmv
+ */
+public class Metrics {
+
+    private final Instant start, end;
+    private final Map<String, Metric> metrics;
+
+    private Metrics(Instant start, Instant end, Map<String, Metric> metrics) {
+        this.start = start;
+        this.end = end;
+        this.metrics = metrics;
+    }
+
+    public static Metrics of(Instant start, Instant end, Map<String, Metric> metrics) {
+        if ( ! start.isBefore(end))
+            throw new IllegalArgumentException("Given time interval must be positive: '" + start + "' to '" + end + "'.");
+
+        return new Metrics(start, end, copyOf(metrics));
+    }
+
+    /** Returns the start of the time window from which these metrics were sampled, or throws if the status is {@code Status.down}. */
+    public Instant start() {
+        return start;
+    }
+
+    /** Returns the end of the time window from which these metrics were sampled, or throws if the status is {@code Status.down}. */
+    public Instant end() {
+        return end;
+    }
+
+    /** Returns the metric with the given name, or throws a NoSuchElementException if no such Metric is known. */
+    public Metric get(String name) {
+        if ( ! metrics.containsKey(name))
+            throw new NoSuchElementException("No metric with name '" + name + "'.");
+
+        return metrics.get(name);
+    }
+
+    /** Returns the underlying, unmodifiable Map. */
+    public Map<String, Metric> asMap() {
+        return metrics;
+    }
+
+    @Override
+    public String toString() {
+        return new StringJoiner(", ", Metrics.class.getSimpleName() + "[", "]")
+                .add("start=" + start)
+                .add("end=" + end)
+                .add("metrics=" + metrics)
+                .toString();
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java
new file mode 100644
index 00000000000..ea771ca5dd9
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java
@@ -0,0 +1,44 @@
+package ai.vespa.hosted.cd.metric;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.IntStream;
+
+import static java.util.stream.Collectors.toUnmodifiableMap;
+
+/**
+ * Used to easily generate points for a pre-defined space.
+ *
+ * @author jonmv
+ */
+public class Space {
+
+    private final List<String> dimensions;
+
+    private Space(List<String> dimensions) {
+        this.dimensions = dimensions;
+    }
+
+    /** Creates a new space with the given named dimensions, in order. */
+    public static Space of(List<String> dimensions) {
+        if (Set.copyOf(dimensions).size() != dimensions.size())
+            throw new IllegalArgumentException("Duplicated dimension names in '" + dimensions + "'.");
+
+        return new Space(List.copyOf(dimensions));
+    }
+
+    /** Returns a point in this space, with the given values along each dimensions, in order. */
+    public Map<String, ?> at(List<?> values) {
+        if (dimensions.size() != values.size())
+            throw new IllegalArgumentException("This space has " + dimensions.size() + " dimensions, but " + values.size() + " were given.");
+
+        return IntStream.range(0, dimensions.size()).boxed().collect(toUnmodifiableMap(dimensions::get, values::get));
+    }
+
+    /** Returns a point in this space, with the given values along each dimensions, in order. */
+    public Map<String, ?> at(Object... values) {
+        return at(List.of(values));
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java
new file mode 100644
index 00000000000..fc52900bdac
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java
@@ -0,0 +1,68 @@
+package ai.vespa.hosted.cd.metric;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.StringJoiner;
+
+import static java.util.Map.copyOf;
+
+/**
+ * Known statistic about a metric, at a certain point.
+ *
+ * @author jonmv
+ */
+public class Statistic {
+
+    private final Map<Type, Double> data;
+
+    /** Creates a new Statistic with a copy of the given data. */
+    private Statistic(Map<Type, Double> data) {
+        this.data = data;
+    }
+
+    public static Statistic of(Map<Type, Double> data) {
+        return new Statistic(copyOf(data));
+    }
+
+    /** Returns the value of the given type, or throws a NoSuchElementException if this isn't known. */
+    public double get(Type key) {
+        if ( ! data.containsKey(key))
+            throw new NoSuchElementException("No value with key '" + key + "' is known.");
+
+        return data.get(key);
+    }
+
+    /** Returns the underlying, unmodifiable Map. */
+    public Map<Type, Double> asMap() {
+        return data;
+    }
+
+    Statistic mergedWith(Statistic other) {
+        if (data.keySet().equals(other.data.keySet()))
+            throw new IllegalArgumentException("Incompatible key sets '" + data.keySet() + "' and '" + other.data.keySet() + "'.");
+
+        Map<Type, Double> merged = new HashMap<>();
+        double n1 = get(Type.count), n2 = other.get(Type.count);
+        for (Type type : data.keySet()) switch (type) {
+            case count: merged.put(type, n1 + n2); break;
+            case rate: merged.put(type, get(Type.rate) + other.get(Type.rate)); break;
+            case max: merged.put(type, Math.max(get(Type.max), other.get(Type.max))); break;
+            case min: merged.put(type, Math.min(get(Type.min), other.get(Type.min))); break;
+            case average: merged.put(type, (n1 * get(Type.average) + n2 * other.get(Type.average)) / (n1 + n2)); break;
+            case last:
+            case percentile95:
+            case percentile99: break;
+            default: throw new IllegalArgumentException("Unexpected type '" + type + "'.");
+        }
+        return of(merged);
+    }
+
+    @Override
+    public String toString() {
+        return new StringJoiner(", ", Statistic.class.getSimpleName() + "[", "]")
+                .add("data=" + data)
+                .toString();
+    }
+
+}
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java
new file mode 100644
index 00000000000..d48b4566f6d
--- /dev/null
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java
@@ -0,0 +1,32 @@
+package ai.vespa.hosted.cd.metric;
+
+/**
+ * Known statistic types.
+ */
+public enum Type {
+
+    /** 95th percentile measurement. */
+    percentile95,
+
+    /** 99th percentile measurement. */
+    percentile99,
+
+    /** Average over all measurements. */
+    average,
+
+    /** Number of measurements. */
+    count,
+
+    /** Last measurement. */
+    last,
+
+    /** Maximum measurement. */
+    max,
+
+    /** Minimum measurement. */
+    min,
+
+    /** Number of measurements per second. */
+    rate;
+
+}