Check private endpoints from same-zone config server instead

11 files changed, 231 insertions, 92 deletions

diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java b/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java
index 94988293d07..d9ced0177e5 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java
@@ -17,34 +17,39 @@ import java.util.Optional;
 /**
  * @author jonmv
  */
-public class EndpointsChecker {
+public interface EndpointsChecker {
 
-    public record Endpoint(ClusterSpec.Id clusterName,
-                           HttpURL url,
-                           Optional<InetAddress> ipAddress,
-                           Optional<DomainName> canonicalName,
-                           boolean isPublic) { }
+    record Endpoint(ClusterSpec.Id clusterName,
+                    HttpURL url,
+                    Optional<InetAddress> ipAddress,
+                    Optional<DomainName> canonicalName,
+                    boolean isPublic) { }
 
-    public enum Status { available, endpointsUnavailable, containersUnhealthy }
+    /** Status sorted by increasing readiness. */
+    enum Status { endpointsUnavailable, containersUnhealthy, available }
 
-    public record Availability(Status status, String message) { }
+    record Availability(Status status, String message) { }
 
-    public interface HostNameResolver { Optional<InetAddress> resolve(DomainName hostName); }
+    interface HostNameResolver { Optional<InetAddress> resolve(DomainName hostName); }
 
-    public interface CNameResolver { Optional<DomainName> resolve(DomainName hostName); }
+    interface CNameResolver { Optional<DomainName> resolve(DomainName hostName); }
 
-    public interface ContainerHealthChecker { boolean healthy(Endpoint endpoint); }
+    interface ContainerHealthChecker { boolean healthy(Endpoint endpoint); }
 
-    private EndpointsChecker() { }
+    static EndpointsChecker of(ContainerHealthChecker containerHealthChecker) {
+        return zoneEndpoints -> endpointsAvailable(zoneEndpoints, EndpointsChecker::resolveHostName, EndpointsChecker::resolveCname, containerHealthChecker);
+    }
 
-    public static Availability endpointsAvailable(List<Endpoint> zoneEndpoints, ContainerHealthChecker containerHealthChecker) {
-        return endpointsAvailable(zoneEndpoints, EndpointsChecker::resolveHostName, EndpointsChecker::resolveCname, containerHealthChecker);
+    static EndpointsChecker mock(HostNameResolver hostNameResolver, CNameResolver cNameResolver, ContainerHealthChecker containerHealthChecker) {
+        return zoneEndpoints -> endpointsAvailable(zoneEndpoints, hostNameResolver, cNameResolver, containerHealthChecker);
     }
 
-    public static Availability endpointsAvailable(List<Endpoint> zoneEndpoints,
-                                                  HostNameResolver hostNameResolver,
-                                                  CNameResolver cNameResolver,
-                                                  ContainerHealthChecker containerHealthChecker) {
+    Availability endpointsAvailable(List<Endpoint> zoneEndpoints);
+
+    private static Availability endpointsAvailable(List<Endpoint> zoneEndpoints,
+                                                   HostNameResolver hostNameResolver,
+                                                   CNameResolver cNameResolver,
+                                                   ContainerHealthChecker containerHealthChecker) {
         if (zoneEndpoints.isEmpty())
             return new Availability(Status.endpointsUnavailable, "Endpoints not yet ready.");
 
@@ -92,7 +97,7 @@ public class EndpointsChecker {
     }
 
     /** Returns the IP address of the given host name, if any. */
-    static Optional<InetAddress> resolveHostName(DomainName hostname) {
+    private static Optional<InetAddress> resolveHostName(DomainName hostname) {
         try {
             return Optional.of(InetAddress.getByName(hostname.value()));
         }
@@ -102,7 +107,7 @@ public class EndpointsChecker {
     }
 
     /** Returns the host name of the given CNAME, if any. */
-    static Optional<DomainName> resolveCname(DomainName endpoint) {
+    private static Optional<DomainName> resolveCname(DomainName endpoint) {
         try {
             InitialDirContext ctx = new InitialDirContext();
             try {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index ca06fe202d9..edcffcca878 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -4,9 +4,9 @@ package com.yahoo.vespa.config.server;
 import ai.vespa.http.DomainName;
 import ai.vespa.http.HttpURL;
 import ai.vespa.http.HttpURL.Query;
-import com.yahoo.component.annotation.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.Version;
+import com.yahoo.component.annotation.Inject;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.ApplicationFile;
 import com.yahoo.config.application.api.ApplicationMetaData;
@@ -17,6 +17,9 @@ import com.yahoo.config.provision.ActivationContext;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationTransaction;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.EndpointsChecker;
+import com.yahoo.config.provision.EndpointsChecker.Availability;
+import com.yahoo.config.provision.EndpointsChecker.Endpoint;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.InfraDeployer;
@@ -57,6 +60,8 @@ import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger;
 import com.yahoo.vespa.config.server.deploy.Deployment;
 import com.yahoo.vespa.config.server.deploy.InfraDeployerProvider;
 import com.yahoo.vespa.config.server.filedistribution.FileDirectory;
+import com.yahoo.vespa.config.server.http.HttpFetcher;
+import com.yahoo.vespa.config.server.http.HttpFetcher.Params;
 import com.yahoo.vespa.config.server.http.InternalServerException;
 import com.yahoo.vespa.config.server.http.LogRetriever;
 import com.yahoo.vespa.config.server.http.SecretStoreValidator;
@@ -87,6 +92,8 @@ import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.orchestrator.Orchestrator;
+import org.apache.hc.client5.http.ssl.DefaultHostnameVerifier;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -112,6 +119,7 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
+import static ai.vespa.http.HttpURL.Path.parse;
 import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER;
 import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER;
 import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse;
@@ -141,6 +149,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private final Optional<InfraDeployer> infraDeployer;
     private final ConfigConvergenceChecker convergeChecker;
     private final HttpProxy httpProxy;
+    private final EndpointsChecker endpointsChecker;
     private final Clock clock;
     private final ConfigserverConfig configserverConfig;
     private final FileDistributionStatus fileDistributionStatus = new FileDistributionStatus();
@@ -169,6 +178,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
              infraDeployerProvider.getInfraDeployer(),
              configConvergenceChecker,
              httpProxy,
+             createEndpointsChecker(),
              configserverConfig,
              orchestrator,
              new LogRetriever(),
@@ -185,6 +195,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                   Optional<InfraDeployer> infraDeployer,
                                   ConfigConvergenceChecker configConvergenceChecker,
                                   HttpProxy httpProxy,
+                                  EndpointsChecker endpointsChecker,
                                   ConfigserverConfig configserverConfig,
                                   Orchestrator orchestrator,
                                   LogRetriever logRetriever,
@@ -199,6 +210,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         this.infraDeployer = Objects.requireNonNull(infraDeployer);
         this.convergeChecker = Objects.requireNonNull(configConvergenceChecker);
         this.httpProxy = Objects.requireNonNull(httpProxy);
+        this.endpointsChecker = Objects.requireNonNull(endpointsChecker);
         this.configserverConfig = Objects.requireNonNull(configserverConfig);
         this.orchestrator = Objects.requireNonNull(orchestrator);
         this.logRetriever = Objects.requireNonNull(logRetriever);
@@ -215,6 +227,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         private TenantRepository tenantRepository;
         private Optional<Provisioner> hostProvisioner;
         private HttpProxy httpProxy = new HttpProxy(new SimpleHttpFetcher(Duration.ofSeconds(30)));
+        private EndpointsChecker endpointsChecker = __ -> { throw new UnsupportedOperationException(); };
         private Clock clock = Clock.systemUTC();
         private ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder().build();
         private Orchestrator orchestrator;
@@ -292,12 +305,18 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
             return this;
         }
 
+        public Builder withEndpointsChecker(EndpointsChecker endpointsChecker) {
+            this.endpointsChecker = endpointsChecker;
+            return this;
+        }
+
         public ApplicationRepository build() {
             return new ApplicationRepository(tenantRepository,
                                              hostProvisioner,
                                              InfraDeployerProvider.empty().getInfraDeployer(),
                                              configConvergenceChecker,
                                              httpProxy,
+                                             endpointsChecker,
                                              configserverConfig,
                                              orchestrator,
                                              logRetriever,
@@ -730,6 +749,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
 
     public ConfigConvergenceChecker configConvergenceChecker() { return convergeChecker; }
 
+    public Availability verifyEndpoints(List<Endpoint> endpoints) {
+        return endpointsChecker.endpointsAvailable(endpoints);
+    }
+
     // ---------------- Logs ----------------------------------------------------------------
 
     public HttpResponse getLogs(ApplicationId applicationId, Optional<DomainName> hostname, String apiParams) {
@@ -1211,4 +1234,24 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
 
     }
 
+    private static EndpointsChecker createEndpointsChecker() {
+        HttpFetcher fetcher = new SimpleHttpFetcher(Duration.ofSeconds(10), new DefaultHostnameVerifier()::verify);
+        return EndpointsChecker.of(endpoint -> {
+            int remainingFailures = 3;
+            int remainingSuccesses = 100;
+            while (remainingSuccesses > 0 && remainingFailures > 0) {
+                try {
+                    HttpResponse response = fetcher.get(new Params(3000),
+                                                        endpoint.url().withPath(parse("/status.html")).asURI());
+                    if (response.getStatus() == 200) remainingSuccesses--;
+                    else remainingFailures--;
+                }
+                catch (Exception e) {
+                    remainingFailures--;
+                }
+            }
+            return remainingSuccesses == 0;
+        });
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 68d0b81dc2b..9619ad69b3c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -10,6 +10,9 @@ import com.yahoo.config.application.api.ApplicationFile;
 import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.EndpointsChecker.Availability;
+import com.yahoo.config.provision.EndpointsChecker.Endpoint;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.Zone;
@@ -20,7 +23,10 @@ import com.yahoo.jdisc.Response;
 import com.yahoo.restapi.ErrorResponse;
 import com.yahoo.restapi.MessageResponse;
 import com.yahoo.restapi.Path;
+import com.yahoo.restapi.SlimeJsonResponse;
+import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
 import com.yahoo.slime.SlimeUtils;
 import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.config.server.ApplicationRepository;
@@ -43,9 +49,12 @@ import com.yahoo.vespa.config.server.tenant.Tenant;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.net.InetAddress;
 import java.net.URI;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -104,6 +113,7 @@ public class ApplicationHandler extends HttpHandler {
     public HttpResponse handlePOST(HttpRequest request) {
         Path path = new Path(request.getUri());
 
+        if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/verify-endpoints")) return verifyEndpoints(request);
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindex")) return triggerReindexing(applicationId(path), request);
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return enableReindexing(applicationId(path));
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/restart")) return restart(applicationId(path), request);
@@ -322,6 +332,32 @@ public class ApplicationHandler extends HttpHandler {
         return new MessageResponse("Success");
     }
 
+    private HttpResponse verifyEndpoints(HttpRequest request) {
+        byte[] data = uncheck(() -> request.getData().readAllBytes());
+        List<Endpoint> endpoints = new ArrayList<>();
+        SlimeUtils.jsonToSlime(data).get()
+                  .field("endpoints")
+                  .traverse((ArrayTraverser) (__, endpointObject) -> {
+                      endpoints.add(new Endpoint(ClusterSpec.Id.from(endpointObject.field("clusterName").asString()),
+                                                 HttpURL.from(URI.create(endpointObject.field("url").asString())),
+                                                 SlimeUtils.optionalString(endpointObject.field("ipAddress")).map(uncheck(InetAddress::getByName)),
+                                                 SlimeUtils.optionalString(endpointObject.field("canonicalName")).map(DomainName::of),
+                                                 endpointObject.field("public").asBool()));
+                  });
+        if (endpoints.isEmpty()) throw new IllegalArgumentException("No endpoints in request " + request);
+
+        Availability availability = applicationRepository.verifyEndpoints(endpoints);
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        root.setString("status", switch (availability.status()) {
+            case available -> "available";
+            case endpointsUnavailable -> "endpointsUnavailable";
+            case containersUnhealthy -> "containersUnhealthy";
+        });
+        root.setString("message", availability.message());
+        return new SlimeJsonResponse(slime);
+    }
+
     private HttpResponse testerStartTests(ApplicationId applicationId, String suite, HttpRequest request) {
         byte[] data;
         try {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index 3925899e1cd..c270b4559f9 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.http.v2;
 
+import ai.vespa.http.DomainName;
 import ai.vespa.http.HttpURL;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.Version;
@@ -9,6 +10,10 @@ import com.yahoo.config.model.api.PortInfo;
 import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.EndpointsChecker;
+import com.yahoo.config.provision.EndpointsChecker.Availability;
+import com.yahoo.config.provision.EndpointsChecker.Endpoint;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
@@ -53,6 +58,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
+import java.net.InetAddress;
 import java.net.URI;
 import java.net.URLEncoder;
 import java.time.Duration;
@@ -60,6 +66,7 @@ import java.time.Instant;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
@@ -77,10 +84,12 @@ import static com.yahoo.vespa.config.server.http.HandlerTest.assertHttpStatusCod
 import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString;
 import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServiceListResponse;
 import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServiceResponse.createResponse;
+import static com.yahoo.yolean.Exceptions.uncheck;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -107,6 +116,8 @@ public class ApplicationHandlerTest {
     private MockProvisioner provisioner;
     private OrchestratorMock orchestrator;
     private ManualClock clock;
+    private List<Endpoint> expectedEndpoints;
+    private Availability availability;
 
     @Rule
     public TemporaryFolder temporaryFolder = new TemporaryFolder();
@@ -139,6 +150,7 @@ public class ApplicationHandlerTest {
                 .withLogRetriever(logRetriever)
                 .withConfigserverConfig(configserverConfig)
                 .withSecretStoreValidator(secretStoreValidator)
+                .withEndpointsChecker(endpoints -> { assertEquals(expectedEndpoints, endpoints); return availability; })
                 .build();
     }
 
@@ -504,6 +516,35 @@ public class ApplicationHandlerTest {
     }
 
     @Test
+    public void testVerifyEndpoints() {
+        expectedEndpoints = List.of(new Endpoint(ClusterSpec.Id.from("bluster"),
+                                                 HttpURL.from(URI.create("https://bluster.tld:1234")),
+                                                 Optional.of(uncheck(() -> InetAddress.getByName("4.3.2.1"))),
+                                                 Optional.of(DomainName.of("fluster.tld")),
+                                                 false));
+        availability = new Availability(EndpointsChecker.Status.available, "Endpoints are ready");
+        ApplicationHandler handler = createApplicationHandler();
+        HttpRequest request = createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/verify-endpoints",
+                                                POST,
+                                                new ByteArrayInputStream("""
+                                                                         {
+                                                                           "endpoints": [
+                                                                             {
+                                                                               "clusterName": "bluster",
+                                                                               "url": "https://bluster.tld:1234",
+                                                                               "ipAddress": "4.3.2.1",
+                                                                               "canonicalName": "fluster.tld",
+                                                                               "public": false
+                                                                             }
+                                                                           ]
+                                                                         }""".getBytes(UTF_8)));
+        HttpResponse response = handler.handle(request);
+        assertEquals(200, response.getStatus());
+        assertEquals("{\"status\":\"available\",\"message\":\"Endpoints are ready\"}",
+                     new ByteArrayOutputStream() {{ uncheck(() -> response.render(this)); }}.toString(UTF_8));
+    }
+
+    @Test
     public void testClusterReindexingStateSerialization() {
         Stream.of(ClusterReindexing.State.values()).forEach(ClusterReindexing.State::toString);
     }
@@ -592,12 +633,12 @@ public class ApplicationHandlerTest {
                                                    hostAndPort,
                                                    uri);
             assertResponse("{\n" +
-                                   "  \"url\": \"" + uri.toString() + "\",\n" +
-                                   "  \"host\": \"" + hostAndPort + "\",\n" +
-                                   "  \"wantedGeneration\": 3,\n" +
-                                   "  \"converged\": true,\n" +
-                                   "  \"currentGeneration\": 3\n" +
-                                   "}",
+                           "  \"url\": \"" + uri.toString() + "\",\n" +
+                           "  \"host\": \"" + hostAndPort + "\",\n" +
+                           "  \"wantedGeneration\": 3,\n" +
+                           "  \"converged\": true,\n" +
+                           "  \"currentGeneration\": 3\n" +
+                           "}",
                            200,
                            response);
         }
@@ -609,11 +650,11 @@ public class ApplicationHandlerTest {
                                                    uri);
 
             assertResponse("{\n" +
-                                   "  \"url\": \"" + uri.toString() + "\",\n" +
-                                   "  \"host\": \"" + hostAndPort + "\",\n" +
-                                   "  \"wantedGeneration\": 3,\n" +
-                                   "  \"problem\": \"Host:port (service) no longer part of application, refetch list of services.\"\n" +
-                                   "}",
+                           "  \"url\": \"" + uri.toString() + "\",\n" +
+                           "  \"host\": \"" + hostAndPort + "\",\n" +
+                           "  \"wantedGeneration\": 3,\n" +
+                           "  \"problem\": \"Host:port (service) no longer part of application, refetch list of services.\"\n" +
+                           "}",
                            410,
                            response);
         }
@@ -635,20 +676,20 @@ public class ApplicationHandlerTest {
                                                                         3L),
                                                 requestUrl);
             assertResponse("{\n" +
-                                   "  \"services\": [\n" +
-                                   "    {\n" +
-                                   "      \"host\": \"" + hostname + "\",\n" +
-                                   "      \"port\": " + port + ",\n" +
-                                   "      \"type\": \"container\",\n" +
-                                   "      \"url\": \"" + serviceUrl.toString() + "\",\n" +
-                                   "      \"currentGeneration\":" + 3 + "\n" +
-                                   "    }\n" +
-                                   "  ],\n" +
-                                   "  \"url\": \"" + requestUrl.toString() + "\",\n" +
-                                   "  \"currentGeneration\": 3,\n" +
-                                   "  \"wantedGeneration\": 3,\n" +
-                                   "  \"converged\": true\n" +
-                                   "}",
+                           "  \"services\": [\n" +
+                           "    {\n" +
+                           "      \"host\": \"" + hostname + "\",\n" +
+                           "      \"port\": " + port + ",\n" +
+                           "      \"type\": \"container\",\n" +
+                           "      \"url\": \"" + serviceUrl.toString() + "\",\n" +
+                           "      \"currentGeneration\":" + 3 + "\n" +
+                           "    }\n" +
+                           "  ],\n" +
+                           "  \"url\": \"" + requestUrl.toString() + "\",\n" +
+                           "  \"currentGeneration\": 3,\n" +
+                           "  \"wantedGeneration\": 3,\n" +
+                           "  \"converged\": true\n" +
+                           "}",
                            200,
                            response);
         }
@@ -669,27 +710,27 @@ public class ApplicationHandlerTest {
                                                                         3L),
                                                 requestUrl);
             assertResponse("{\n" +
-                                   "  \"services\": [\n" +
-                                   "    {\n" +
-                                   "      \"host\": \"" + hostname + "\",\n" +
-                                   "      \"port\": " + port + ",\n" +
-                                   "      \"type\": \"container\",\n" +
-                                   "      \"url\": \"" + serviceUrl.toString() + "\",\n" +
-                                   "      \"currentGeneration\":" + 4 + "\n" +
-                                   "    },\n" +
-                                   "    {\n" +
-                                   "      \"host\": \"" + hostname2 + "\",\n" +
-                                   "      \"port\": " + port2 + ",\n" +
-                                   "      \"type\": \"container\",\n" +
-                                   "      \"url\": \"" + serviceUrl2.toString() + "\",\n" +
-                                   "      \"currentGeneration\":" + 3 + "\n" +
-                                   "    }\n" +
-                                   "  ],\n" +
-                                   "  \"url\": \"" + requestUrl.toString() + "\",\n" +
-                                   "  \"currentGeneration\": 3,\n" +
-                                   "  \"wantedGeneration\": 4,\n" +
-                                   "  \"converged\": false\n" +
-                                   "}",
+                           "  \"services\": [\n" +
+                           "    {\n" +
+                           "      \"host\": \"" + hostname + "\",\n" +
+                           "      \"port\": " + port + ",\n" +
+                           "      \"type\": \"container\",\n" +
+                           "      \"url\": \"" + serviceUrl.toString() + "\",\n" +
+                           "      \"currentGeneration\":" + 4 + "\n" +
+                           "    },\n" +
+                           "    {\n" +
+                           "      \"host\": \"" + hostname2 + "\",\n" +
+                           "      \"port\": " + port2 + ",\n" +
+                           "      \"type\": \"container\",\n" +
+                           "      \"url\": \"" + serviceUrl2.toString() + "\",\n" +
+                           "      \"currentGeneration\":" + 3 + "\n" +
+                           "    }\n" +
+                           "  ],\n" +
+                           "  \"url\": \"" + requestUrl.toString() + "\",\n" +
+                           "  \"currentGeneration\": 3,\n" +
+                           "  \"wantedGeneration\": 4,\n" +
+                           "  \"converged\": false\n" +
+                           "}",
                            200,
                            response);
         }
@@ -707,11 +748,11 @@ public class ApplicationHandlerTest {
                                                uri);
 
         assertResponse("{\n" +
-                               "  \"url\": \"" + uri.toString() + "\",\n" +
-                               "  \"host\": \"" + hostAndPort + "\",\n" +
-                               "  \"wantedGeneration\": 3,\n" +
-                               "  \"error\": \"some error message\"" +
-                               "}",
+                       "  \"url\": \"" + uri + "\",\n" +
+                       "  \"host\": \"" + hostAndPort + "\",\n" +
+                       "  \"wantedGeneration\": 3,\n" +
+                       "  \"error\": \"some error message\"" +
+                       "}",
                        404,
                        response);
     }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
index cc05bc01d99..93ac16c606d 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
@@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver;
 import ai.vespa.http.HttpURL.Query;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.EndpointsChecker.Availability;
+import com.yahoo.config.provision.EndpointsChecker.Endpoint;
 import com.yahoo.config.provision.zone.ZoneId;
 import ai.vespa.http.DomainName;
 import ai.vespa.http.HttpURL.Path;
@@ -145,6 +147,8 @@ public interface ConfigServer {
 
     Optional<TestReport> getTestReport(DeploymentId deployment);
 
+    Availability verifyEndpoints(DeploymentId deploymentId, List<Endpoint> zoneEndpoints);
+
     /** Get maximum resources consumed */
     QuotaUsage getQuotaUsage(DeploymentId deploymentId);
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
index 9ad06a3311f..4095e4b03fd 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
@@ -29,7 +29,7 @@ public interface TesterCloud {
     /** Returns whether the test container is ready to serve */
     boolean testerReady(DeploymentId deploymentId);
 
-    Availability verifyEndpoints(List<Endpoint> endpoints);
+    Availability verifyEndpoints(DeploymentId deploymentId, List<Endpoint> endpoints);
 
     /** Returns the test report as JSON if available */
     Optional<TestReport> getTestReport(DeploymentId deploymentId);
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java
index e9a7c8bad33..e29e8086c80 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java
@@ -26,6 +26,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Teste
 public class MockTesterCloud implements TesterCloud {
 
     private final NameService nameService;
+    private final EndpointsChecker endpointsChecker = EndpointsChecker.mock(this::resolveHostName, this::resolveCname, __ -> true);
 
     private List<LogEntry> log = new ArrayList<>();
     private Status status = NOT_STARTED;
@@ -56,8 +57,8 @@ public class MockTesterCloud implements TesterCloud {
     }
 
     @Override
-    public Availability verifyEndpoints(List<Endpoint> endpoints) {
-        return EndpointsChecker.endpointsAvailable(endpoints, this::resolveHostName, this::resolveCname, __ -> true);
+    public Availability verifyEndpoints(DeploymentId deploymentId, List<Endpoint> endpoints) {
+        return endpointsChecker.endpointsAvailable(endpoints);
     }
 
     private Optional<InetAddress> resolveHostName(DomainName hostname) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index d7118c4b3c2..c0989f61e36 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -498,6 +498,7 @@ public class InternalStepRunner implements StepRunner {
         DeploymentRoutingContext context = controller.routing().of(deployment);
         boolean resolveEndpoints = context.routingMethod() == RoutingMethod.exclusive;
         return controller.serviceRegistry().testerCloud().verifyEndpoints(
+                deployment,
                 endpoints.getOrDefault(zone, List.of())
                          .stream()
                          .map(endpoint -> {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index e2b421afe61..5704af75cb9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -12,6 +12,8 @@ import com.yahoo.config.provision.CloudAccount;
 import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.EndpointsChecker.Availability;
+import com.yahoo.config.provision.EndpointsChecker.Endpoint;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeResources;
@@ -42,8 +44,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.QuotaUsage
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter;
 import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore;
+import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 
@@ -82,6 +86,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
  */
 public class ConfigServerMock extends AbstractComponent implements ConfigServer {
 
+    private final MockTesterCloud mockTesterCloud;
     private final Map<DeploymentId, Application> applications = new LinkedHashMap<>();
     private final Set<ZoneId> inactiveZones = new HashSet<>();
     private final Map<DeploymentId, EndpointStatus> endpoints = new HashMap<>();
@@ -105,9 +110,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     private Consumer<ApplicationId> prepareException = null;
     private Supplier<String> log = () -> "INFO - All good";
 
-    @Inject
-    public ConfigServerMock(ZoneRegistryMock zoneRegistry) {
+    public ConfigServerMock(ZoneRegistryMock zoneRegistry, NameService nameService) {
         bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.notController());
+        this.mockTesterCloud = new MockTesterCloud(nameService);
     }
 
     /** Assigns a reserved tenant node to the given deployment, with initial versions. */
@@ -370,8 +375,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     public Optional<TestReport> getTestReport(DeploymentId deployment) {
         return Optional.ofNullable(testReport.get(deployment));
     }
-    public void setTestReport(DeploymentId deploymentId, TestReport report) {
-        testReport.put(deploymentId, report);
+
+    @Override
+    public Availability verifyEndpoints(DeploymentId deploymentId, List<Endpoint> zoneEndpoints) {
+        return mockTesterCloud.verifyEndpoints(deploymentId, zoneEndpoints); // Wraps the same name service mock, which is updated by test harness.
     }
 
     /** Add any of given loadBalancers that do not already exist to the load balancers in zone */
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
index 382a697c4cd..0ba8866c990 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
@@ -100,8 +100,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
 
     public ServiceRegistryMock(SystemName system) {
         this.zoneRegistryMock = new ZoneRegistryMock(system);
-        this.configServerMock = new ConfigServerMock(zoneRegistryMock);
-        this.mockTesterCloud = new MockTesterCloud(nameService());
+        this.configServerMock = new ConfigServerMock(zoneRegistryMock, memoryNameService);
+        this.mockTesterCloud = new MockTesterCloud(memoryNameService);
         this.clock.setInstant(Instant.ofEpochSecond(1600000000));
         this.controllerVersion = new ControllerVersion(Version.fromString("6.1.0"), "badb01", clock.instant());
     }
diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8.java b/vespajlib/src/main/java/com/yahoo/text/Utf8.java
index 2a42cb5cdee..3a7ecaa727a 100644
--- a/vespajlib/src/main/java/com/yahoo/text/Utf8.java
+++ b/vespajlib/src/main/java/com/yahoo/text/Utf8.java
@@ -10,7 +10,8 @@ import java.nio.ReadOnlyBufferException;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetEncoder;
 import java.nio.charset.CodingErrorAction;
-import java.nio.charset.StandardCharsets;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 /**
  * Utility class with functions for handling UTF-8
@@ -23,16 +24,16 @@ public final class Utf8 {
 
     private static final byte [] TRUE = {(byte) 't', (byte) 'r', (byte) 'u', (byte) 'e'};
     private static final byte [] FALSE = {(byte) 'f', (byte) 'a', (byte) 'l', (byte) 's', (byte) 'e'};
-    private static final byte[] LONG_MIN_VALUE_BYTES = String.valueOf(Long.MIN_VALUE).getBytes(StandardCharsets.UTF_8);
+    private static final byte[] LONG_MIN_VALUE_BYTES = String.valueOf(Long.MIN_VALUE).getBytes(UTF_8);
 
     /** Returns the Charset instance for UTF-8 */
     public static Charset getCharset() {
-        return StandardCharsets.UTF_8;
+        return UTF_8;
     }
 
     /** To be used instead of String.String(byte[] bytes) */
     public static String toStringStd(byte[] data) {
-        return new String(data, StandardCharsets.UTF_8);
+        return new String(data, UTF_8);
     }
 
     /**
@@ -60,7 +61,7 @@ public final class Utf8 {
      * @return a decoded String
      */
     public static String toString(ByteBuffer data) {
-        CharBuffer c = StandardCharsets.UTF_8.decode(data);
+        CharBuffer c = UTF_8.decode(data);
         return c.toString();
     }
 
@@ -68,7 +69,7 @@ public final class Utf8 {
      * Uses String.getBytes directly.
      */
     public static byte[] toBytesStd(String str) {
-        return str.getBytes(StandardCharsets.UTF_8);
+        return str.getBytes(UTF_8);
     }
 
     /**
@@ -112,7 +113,7 @@ public final class Utf8 {
      */
     public static byte[] toBytes(String string) {
         // This is just wrapper for String::getBytes. Pre-Java 9 this had a more efficient approach for ASCII-only strings.
-        return string.getBytes(StandardCharsets.UTF_8);
+        return string.getBytes(UTF_8);
     }
     /**
      * Decode a UTF-8 string.
@@ -122,7 +123,7 @@ public final class Utf8 {
      */
     public static String toString(byte[] utf8) {
         // This is just wrapper for String::new. Pre-Java 9 this had a more efficient approach for ASCII-onlu strings.
-        return new String(utf8, StandardCharsets.UTF_8);
+        return new String(utf8, UTF_8);
     }
 
     /**
@@ -138,7 +139,7 @@ public final class Utf8 {
      */
     public static byte[] toBytes(String str, int offset, int length) {
         CharBuffer c = CharBuffer.wrap(str, offset, offset + length);
-        ByteBuffer b = StandardCharsets.UTF_8.encode(c);
+        ByteBuffer b = UTF_8.encode(c);
         byte[] result = new byte[b.remaining()];
         b.get(result);
         return result;
@@ -161,7 +162,7 @@ public final class Utf8 {
      */
     public static int toBytes(String str, int srcOffset, int srcLen, byte[] dst, int dstOffset) {
         CharBuffer c = CharBuffer.wrap(str, srcOffset, srcOffset + srcLen);
-        ByteBuffer b = StandardCharsets.UTF_8.encode(c);
+        ByteBuffer b = UTF_8.encode(c);
         int encoded = b.remaining();
         b.get(dst, dstOffset, encoded);
         return encoded;
@@ -206,7 +207,7 @@ public final class Utf8 {
      * @see Utf8#toBytes(String, int, int, ByteBuffer, CharsetEncoder)
      */
     public static CharsetEncoder getNewEncoder() {
-        return StandardCharsets.UTF_8.newEncoder().onMalformedInput(CodingErrorAction.REPLACE)
+        return UTF_8.newEncoder().onMalformedInput(CodingErrorAction.REPLACE)
                 .onUnmappableCharacter(CodingErrorAction.REPLACE);
     }