Merge pull request #10881 from vespa-engine/jvenstad/expand-test-config-in-application-v4

List content clusters also for application/v4 test-config responses

10 files changed, 84 insertions, 24 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 34ec38e8c48..bfe7fc1ee2e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationId;
@@ -212,6 +213,14 @@ public class ApplicationController {
 
     public ApplicationStore applicationStore() {  return applicationStore; }
 
+    /** Returns all content clusters in all current deployments of the given application. */
+    public Map<ZoneId, List<String>> listClusters(ApplicationId id, Iterable<ZoneId> zones) {
+        ImmutableMap.Builder<ZoneId, List<String>> clusters = ImmutableMap.builder();
+        for (ZoneId zone : zones)
+            clusters.put(zone, ImmutableList.copyOf(configServer.getContentClusters(new DeploymentId(id, zone))));
+        return clusters.build();
+    }
+
     /** Returns the oldest Vespa version installed on any active or reserved production node for the given application. */
     public Version oldestInstalledPlatform(TenantAndApplicationId id) {
         return requireApplication(id).instances().values().stream()
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 1828a189cad..42e270edd5e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -468,7 +468,7 @@ public class InternalStepRunner implements StepRunner {
                                                       testConfigSerializer.configJson(id.application(),
                                                                                       id.type(),
                                                                                       endpoints,
-                                                                                      listClusters(id.application(), zones)));
+                                                                                      controller.applications().listClusters(id.application(), zones)));
         return Optional.of(running);
     }
 
@@ -690,14 +690,6 @@ public class InternalStepRunner implements StepRunner {
         throw new IllegalStateException("No step deploys to the zone this run is for!");
     }
 
-    /** Returns all content clusters in all current deployments of the given real application. */
-    private Map<ZoneId, List<String>> listClusters(ApplicationId id, Iterable<ZoneId> zones) {
-        ImmutableMap.Builder<ZoneId, List<String>> clusters = ImmutableMap.builder();
-        for (ZoneId zone : zones)
-            clusters.put(zone, ImmutableList.copyOf(controller.serviceRegistry().configServer().getContentClusters(new DeploymentId(id, zone))));
-        return clusters.build();
-    }
-
     /** Returns the generated services.xml content for the tester application. */
     static byte[] servicesXml(AthenzDomain domain, boolean useAthenzCredentials, boolean useTesterCertificate,
                               NodeResources resources) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 8b1025cd9b7..b76d0ae1094 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -133,6 +133,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
     private final Controller controller;
     private final AccessControlRequests accessControlRequests;
+    private final TestConfigSerializer testConfigSerializer;
 
     @Inject
     public ApplicationApiHandler(LoggingRequestHandler.Context parentCtx,
@@ -141,6 +142,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         super(parentCtx);
         this.controller = controller;
         this.accessControlRequests = accessControlRequests;
+        this.testConfigSerializer = new TestConfigSerializer(controller.system());
     }
 
     @Override
@@ -1315,11 +1317,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     }
 
     private HttpResponse testConfig(ApplicationId id, JobType type) {
-        var endpoints = controller.applications().clusterEndpoints(id, controller.jobController().testedZoneAndProductionZones(id, type));
-        return new SlimeJsonResponse(new TestConfigSerializer(controller.system()).configSlime(id,
-                                                                                               type,
-                                                                                               endpoints,
-                                                                                               Collections.emptyMap()));
+        Set<ZoneId> zones = controller.jobController().testedZoneAndProductionZones(id, type);
+        return new SlimeJsonResponse(testConfigSerializer.configSlime(id,
+                                                                      type,
+                                                                      controller.applications().clusterEndpoints(id, zones),
+                                                                      controller.applications().listClusters(id, zones)));
     }
 
     private static DeploymentJobs.JobReport toJobReport(String tenantName, String applicationName, Inspector report) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index a415f09f54c..7cacd91a5c4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -549,6 +549,11 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .oktaAccessToken(OKTA_AT),
                               new File("delete-with-active-deployments.json"), 400);
 
+        // GET test-config for local tests against a prod deployment
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/test-config", GET)
+                                      .userIdentity(USER_ID),
+                              new File("test-config.json"));
+
         // DELETE (deactivate) a deployment - dev
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1", DELETE)
                                       .userIdentity(USER_ID),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json
new file mode 100644
index 00000000000..2338543b019
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json
@@ -0,0 +1,20 @@
+{
+  "application": "tenant1:application1:instance1",
+  "zone": "prod.us-central-1",
+  "system": "main",
+  "endpoints": {
+    "prod.us-central-1": [
+      "http://old-endpoint.vespa.yahooapis.com:4080"
+    ]
+  },
+  "zoneEndpoints": {
+    "prod.us-central-1": {
+      "default": "http://old-endpoint.vespa.yahooapis.com:4080"
+    }
+  },
+  "clusters": {
+    "prod.us-central-1": [
+      "music"
+    ]
+  }
+}
diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java
index 61893a30e7e..0ca1b3e5603 100644
--- a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java
+++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java
@@ -38,8 +38,8 @@ public class Properties {
         return Paths.get(requireNonBlankProperty("privateKeyFile"));
     }
 
-    public static Path certificateFile() {
-        return Paths.get(requireNonBlankProperty("certificateFile"));
+    public static Optional<Path> certificateFile() {
+        return getNonBlankProperty("certificateFile").map(Paths::get);
     }
 
     /** Returns the system property with the given name if it is set, or empty. */
diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java b/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java
index b8698eab15f..c1104c649f2 100644
--- a/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java
+++ b/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java
@@ -3,13 +3,16 @@ package ai.vespa.hosted.api;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.JsonDecoder;
 import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.slime.Slime;
 
 import java.net.URI;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
@@ -26,8 +29,10 @@ public class TestConfig {
     private final ZoneId zone;
     private final SystemName system;
     private final Map<ZoneId, Map<String, URI>> deployments;
+    private final Map<ZoneId, List<String>> contentClusters;
 
-    public TestConfig(ApplicationId application, ZoneId zone, SystemName system, Map<ZoneId, Map<String, URI>> deployments) {
+    public TestConfig(ApplicationId application, ZoneId zone, SystemName system, Map<ZoneId, Map<String, URI>> deployments,
+                      Map<ZoneId, List<String>> contentClusters) {
         if ( ! deployments.containsKey(zone))
             throw new IllegalArgumentException("Config must contain a deployment for its zone, but only does for " + deployments.keySet());
         this.application = requireNonNull(application);
@@ -36,12 +41,15 @@ public class TestConfig {
         this.deployments = deployments.entrySet().stream()
                                       .collect(Collectors.toUnmodifiableMap(entry -> entry.getKey(),
                                                                             entry -> Map.copyOf(entry.getValue())));
+        this.contentClusters = contentClusters.entrySet().stream()
+                                              .collect(Collectors.toUnmodifiableMap(entry -> entry.getKey(),
+                                                                                    entry -> List.copyOf(entry.getValue())));
     }
 
     /**
      * Parses the given test config JSON and returns a new config instance.
      *
-     * If the given JSON has a "clusters" element, a config object with default values
+     * If the given JSON has a "localEndpoints" element, a config object with default values
      * is returned, using {@link #fromEndpointsOnly}. Otherwise, all config attributes are parsed.
      */
     public static TestConfig fromJson(byte[] jsonBytes) {
@@ -56,7 +64,13 @@ public class TestConfig {
         config.field("zoneEndpoints").traverse((ObjectTraverser) (zoneId, clustersObject) -> {
             deployments.put(ZoneId.from(zoneId), toClusterMap(clustersObject));
         });
-        return new TestConfig(application, zone, system, deployments);
+        Map<ZoneId, List<String>> contentClusters = new HashMap<>();
+        config.field("clusters").traverse(((ObjectTraverser) (zoneId, clustersArray) -> {
+            List<String> clusters = new ArrayList<>();
+            clustersArray.traverse((ArrayTraverser) (__, cluster) -> clusters.add(cluster.asString()));
+            contentClusters.put(ZoneId.from(zoneId), clusters);
+        }));
+        return new TestConfig(application, zone, system, deployments, contentClusters);
     }
 
     static Map<String, URI> toClusterMap(Inspector clustersObject) {
@@ -73,7 +87,8 @@ public class TestConfig {
         return new TestConfig(ApplicationId.defaultId(),
                               ZoneId.defaultId(),
                               SystemName.defaultSystem(),
-                              Map.of(ZoneId.defaultId(), endpoints));
+                              Map.of(ZoneId.defaultId(), endpoints),
+                              Map.of());
     }
 
     /** Returns the full id of the application to test. */
@@ -85,6 +100,9 @@ public class TestConfig {
     /** Returns an immutable view of deployments, per zone, of the application to test. */
     public Map<ZoneId, Map<String, URI>> deployments() { return deployments; }
 
+    /** Returns an immutable view of content clusters, per zone, of the application to test. */
+    public Map<ZoneId, List<String>> contentClusters() { return contentClusters; }
+
     /** Returns the hosted Vespa system this is run against. */
     public SystemName system() { return system; }
 
diff --git a/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java b/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java
index bad838f0579..5ed008cc2ec 100644
--- a/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java
+++ b/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java
@@ -9,6 +9,7 @@ import java.io.IOException;
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.List;
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
@@ -32,6 +33,9 @@ public class TestConfigTest {
                             ZoneId.from("prod", "aws-us-east-1a"),
                             Map.of("default", URI.create("https://prod.endpoint:443/"))),
                      config.deployments());
+        assertEquals(Map.of(ZoneId.from("prod", "aws-us-east-1c"),
+                            List.of("documents")),
+                     config.contentClusters());
     }
 
     @Test
diff --git a/hosted-api/src/test/resources/test-config.json b/hosted-api/src/test/resources/test-config.json
index 9d36f9496a0..bd337e1c28a 100644
--- a/hosted-api/src/test/resources/test-config.json
+++ b/hosted-api/src/test/resources/test-config.json
@@ -9,5 +9,10 @@
     "prod.aws-us-east-1a": {
       "default": "https://prod.endpoint:443/"
     }
+  },
+  "clusters": {
+    "prod.aws-us-east-1c": [
+      "documents"
+    ]
   }
 }
diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java
index 2b5bbb188dc..9de06e7f4da 100644
--- a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java
+++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java
@@ -5,12 +5,17 @@ import ai.vespa.hosted.api.Properties;
 
 public class ApiAuthenticator implements ai.vespa.hosted.api.ApiAuthenticator {
 
-    /** Returns an authenticating controller client, using private key signatures for authentication. */
+    /** Returns a controller client using mTLS if a key and certificate pair is provided, or signed requests otherwise. */
     @Override
     public ControllerHttpClient controller() {
-        return ControllerHttpClient.withSignatureKey(Properties.endpoint(),
-                                                     Properties.privateKeyFile(),
-                                                     Properties.application());
+        return Properties.certificateFile()
+                         .map(certificateFile -> ControllerHttpClient.withKeyAndCertificate(Properties.endpoint(),
+                                                                                            Properties.privateKeyFile(),
+                                                                                            certificateFile))
+                         .orElseGet(() ->
+                                            ControllerHttpClient.withSignatureKey(Properties.endpoint(),
+                                                                                  Properties.privateKeyFile(),
+                                                                                  Properties.application()));
     }
 
 }