Check private endpoints from same-zone config server instead

3 files changed, 172 insertions, 52 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index ca06fe202d9..edcffcca878 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -4,9 +4,9 @@ package com.yahoo.vespa.config.server;
 import ai.vespa.http.DomainName;
 import ai.vespa.http.HttpURL;
 import ai.vespa.http.HttpURL.Query;
-import com.yahoo.component.annotation.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.Version;
+import com.yahoo.component.annotation.Inject;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.ApplicationFile;
 import com.yahoo.config.application.api.ApplicationMetaData;
@@ -17,6 +17,9 @@ import com.yahoo.config.provision.ActivationContext;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationTransaction;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.EndpointsChecker;
+import com.yahoo.config.provision.EndpointsChecker.Availability;
+import com.yahoo.config.provision.EndpointsChecker.Endpoint;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.InfraDeployer;
@@ -57,6 +60,8 @@ import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger;
 import com.yahoo.vespa.config.server.deploy.Deployment;
 import com.yahoo.vespa.config.server.deploy.InfraDeployerProvider;
 import com.yahoo.vespa.config.server.filedistribution.FileDirectory;
+import com.yahoo.vespa.config.server.http.HttpFetcher;
+import com.yahoo.vespa.config.server.http.HttpFetcher.Params;
 import com.yahoo.vespa.config.server.http.InternalServerException;
 import com.yahoo.vespa.config.server.http.LogRetriever;
 import com.yahoo.vespa.config.server.http.SecretStoreValidator;
@@ -87,6 +92,8 @@ import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.orchestrator.Orchestrator;
+import org.apache.hc.client5.http.ssl.DefaultHostnameVerifier;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -112,6 +119,7 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
+import static ai.vespa.http.HttpURL.Path.parse;
 import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER;
 import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER;
 import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse;
@@ -141,6 +149,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private final Optional<InfraDeployer> infraDeployer;
     private final ConfigConvergenceChecker convergeChecker;
     private final HttpProxy httpProxy;
+    private final EndpointsChecker endpointsChecker;
     private final Clock clock;
     private final ConfigserverConfig configserverConfig;
     private final FileDistributionStatus fileDistributionStatus = new FileDistributionStatus();
@@ -169,6 +178,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
              infraDeployerProvider.getInfraDeployer(),
              configConvergenceChecker,
              httpProxy,
+             createEndpointsChecker(),
              configserverConfig,
              orchestrator,
              new LogRetriever(),
@@ -185,6 +195,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                   Optional<InfraDeployer> infraDeployer,
                                   ConfigConvergenceChecker configConvergenceChecker,
                                   HttpProxy httpProxy,
+                                  EndpointsChecker endpointsChecker,
                                   ConfigserverConfig configserverConfig,
                                   Orchestrator orchestrator,
                                   LogRetriever logRetriever,
@@ -199,6 +210,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         this.infraDeployer = Objects.requireNonNull(infraDeployer);
         this.convergeChecker = Objects.requireNonNull(configConvergenceChecker);
         this.httpProxy = Objects.requireNonNull(httpProxy);
+        this.endpointsChecker = Objects.requireNonNull(endpointsChecker);
         this.configserverConfig = Objects.requireNonNull(configserverConfig);
         this.orchestrator = Objects.requireNonNull(orchestrator);
         this.logRetriever = Objects.requireNonNull(logRetriever);
@@ -215,6 +227,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         private TenantRepository tenantRepository;
         private Optional<Provisioner> hostProvisioner;
         private HttpProxy httpProxy = new HttpProxy(new SimpleHttpFetcher(Duration.ofSeconds(30)));
+        private EndpointsChecker endpointsChecker = __ -> { throw new UnsupportedOperationException(); };
         private Clock clock = Clock.systemUTC();
         private ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder().build();
         private Orchestrator orchestrator;
@@ -292,12 +305,18 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
             return this;
         }
 
+        public Builder withEndpointsChecker(EndpointsChecker endpointsChecker) {
+            this.endpointsChecker = endpointsChecker;
+            return this;
+        }
+
         public ApplicationRepository build() {
             return new ApplicationRepository(tenantRepository,
                                              hostProvisioner,
                                              InfraDeployerProvider.empty().getInfraDeployer(),
                                              configConvergenceChecker,
                                              httpProxy,
+                                             endpointsChecker,
                                              configserverConfig,
                                              orchestrator,
                                              logRetriever,
@@ -730,6 +749,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
 
     public ConfigConvergenceChecker configConvergenceChecker() { return convergeChecker; }
 
+    public Availability verifyEndpoints(List<Endpoint> endpoints) {
+        return endpointsChecker.endpointsAvailable(endpoints);
+    }
+
     // ---------------- Logs ----------------------------------------------------------------
 
     public HttpResponse getLogs(ApplicationId applicationId, Optional<DomainName> hostname, String apiParams) {
@@ -1211,4 +1234,24 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
 
     }
 
+    private static EndpointsChecker createEndpointsChecker() {
+        HttpFetcher fetcher = new SimpleHttpFetcher(Duration.ofSeconds(10), new DefaultHostnameVerifier()::verify);
+        return EndpointsChecker.of(endpoint -> {
+            int remainingFailures = 3;
+            int remainingSuccesses = 100;
+            while (remainingSuccesses > 0 && remainingFailures > 0) {
+                try {
+                    HttpResponse response = fetcher.get(new Params(3000),
+                                                        endpoint.url().withPath(parse("/status.html")).asURI());
+                    if (response.getStatus() == 200) remainingSuccesses--;
+                    else remainingFailures--;
+                }
+                catch (Exception e) {
+                    remainingFailures--;
+                }
+            }
+            return remainingSuccesses == 0;
+        });
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 68d0b81dc2b..9619ad69b3c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -10,6 +10,9 @@ import com.yahoo.config.application.api.ApplicationFile;
 import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.EndpointsChecker.Availability;
+import com.yahoo.config.provision.EndpointsChecker.Endpoint;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.Zone;
@@ -20,7 +23,10 @@ import com.yahoo.jdisc.Response;
 import com.yahoo.restapi.ErrorResponse;
 import com.yahoo.restapi.MessageResponse;
 import com.yahoo.restapi.Path;
+import com.yahoo.restapi.SlimeJsonResponse;
+import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
 import com.yahoo.slime.SlimeUtils;
 import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.config.server.ApplicationRepository;
@@ -43,9 +49,12 @@ import com.yahoo.vespa.config.server.tenant.Tenant;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.net.InetAddress;
 import java.net.URI;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -104,6 +113,7 @@ public class ApplicationHandler extends HttpHandler {
     public HttpResponse handlePOST(HttpRequest request) {
         Path path = new Path(request.getUri());
 
+        if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/verify-endpoints")) return verifyEndpoints(request);
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindex")) return triggerReindexing(applicationId(path), request);
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return enableReindexing(applicationId(path));
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/restart")) return restart(applicationId(path), request);
@@ -322,6 +332,32 @@ public class ApplicationHandler extends HttpHandler {
         return new MessageResponse("Success");
     }
 
+    private HttpResponse verifyEndpoints(HttpRequest request) {
+        byte[] data = uncheck(() -> request.getData().readAllBytes());
+        List<Endpoint> endpoints = new ArrayList<>();
+        SlimeUtils.jsonToSlime(data).get()
+                  .field("endpoints")
+                  .traverse((ArrayTraverser) (__, endpointObject) -> {
+                      endpoints.add(new Endpoint(ClusterSpec.Id.from(endpointObject.field("clusterName").asString()),
+                                                 HttpURL.from(URI.create(endpointObject.field("url").asString())),
+                                                 SlimeUtils.optionalString(endpointObject.field("ipAddress")).map(uncheck(InetAddress::getByName)),
+                                                 SlimeUtils.optionalString(endpointObject.field("canonicalName")).map(DomainName::of),
+                                                 endpointObject.field("public").asBool()));
+                  });
+        if (endpoints.isEmpty()) throw new IllegalArgumentException("No endpoints in request " + request);
+
+        Availability availability = applicationRepository.verifyEndpoints(endpoints);
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        root.setString("status", switch (availability.status()) {
+            case available -> "available";
+            case endpointsUnavailable -> "endpointsUnavailable";
+            case containersUnhealthy -> "containersUnhealthy";
+        });
+        root.setString("message", availability.message());
+        return new SlimeJsonResponse(slime);
+    }
+
     private HttpResponse testerStartTests(ApplicationId applicationId, String suite, HttpRequest request) {
         byte[] data;
         try {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index 3925899e1cd..c270b4559f9 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.http.v2;
 
+import ai.vespa.http.DomainName;
 import ai.vespa.http.HttpURL;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.Version;
@@ -9,6 +10,10 @@ import com.yahoo.config.model.api.PortInfo;
 import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.EndpointsChecker;
+import com.yahoo.config.provision.EndpointsChecker.Availability;
+import com.yahoo.config.provision.EndpointsChecker.Endpoint;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
@@ -53,6 +58,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
+import java.net.InetAddress;
 import java.net.URI;
 import java.net.URLEncoder;
 import java.time.Duration;
@@ -60,6 +66,7 @@ import java.time.Instant;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
@@ -77,10 +84,12 @@ import static com.yahoo.vespa.config.server.http.HandlerTest.assertHttpStatusCod
 import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString;
 import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServiceListResponse;
 import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServiceResponse.createResponse;
+import static com.yahoo.yolean.Exceptions.uncheck;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -107,6 +116,8 @@ public class ApplicationHandlerTest {
     private MockProvisioner provisioner;
     private OrchestratorMock orchestrator;
     private ManualClock clock;
+    private List<Endpoint> expectedEndpoints;
+    private Availability availability;
 
     @Rule
     public TemporaryFolder temporaryFolder = new TemporaryFolder();
@@ -139,6 +150,7 @@ public class ApplicationHandlerTest {
                 .withLogRetriever(logRetriever)
                 .withConfigserverConfig(configserverConfig)
                 .withSecretStoreValidator(secretStoreValidator)
+                .withEndpointsChecker(endpoints -> { assertEquals(expectedEndpoints, endpoints); return availability; })
                 .build();
     }
 
@@ -504,6 +516,35 @@ public class ApplicationHandlerTest {
     }
 
     @Test
+    public void testVerifyEndpoints() {
+        expectedEndpoints = List.of(new Endpoint(ClusterSpec.Id.from("bluster"),
+                                                 HttpURL.from(URI.create("https://bluster.tld:1234")),
+                                                 Optional.of(uncheck(() -> InetAddress.getByName("4.3.2.1"))),
+                                                 Optional.of(DomainName.of("fluster.tld")),
+                                                 false));
+        availability = new Availability(EndpointsChecker.Status.available, "Endpoints are ready");
+        ApplicationHandler handler = createApplicationHandler();
+        HttpRequest request = createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/verify-endpoints",
+                                                POST,
+                                                new ByteArrayInputStream("""
+                                                                         {
+                                                                           "endpoints": [
+                                                                             {
+                                                                               "clusterName": "bluster",
+                                                                               "url": "https://bluster.tld:1234",
+                                                                               "ipAddress": "4.3.2.1",
+                                                                               "canonicalName": "fluster.tld",
+                                                                               "public": false
+                                                                             }
+                                                                           ]
+                                                                         }""".getBytes(UTF_8)));
+        HttpResponse response = handler.handle(request);
+        assertEquals(200, response.getStatus());
+        assertEquals("{\"status\":\"available\",\"message\":\"Endpoints are ready\"}",
+                     new ByteArrayOutputStream() {{ uncheck(() -> response.render(this)); }}.toString(UTF_8));
+    }
+
+    @Test
     public void testClusterReindexingStateSerialization() {
         Stream.of(ClusterReindexing.State.values()).forEach(ClusterReindexing.State::toString);
     }
@@ -592,12 +633,12 @@ public class ApplicationHandlerTest {
                                                    hostAndPort,
                                                    uri);
             assertResponse("{\n" +
-                                   "  \"url\": \"" + uri.toString() + "\",\n" +
-                                   "  \"host\": \"" + hostAndPort + "\",\n" +
-                                   "  \"wantedGeneration\": 3,\n" +
-                                   "  \"converged\": true,\n" +
-                                   "  \"currentGeneration\": 3\n" +
-                                   "}",
+                           "  \"url\": \"" + uri.toString() + "\",\n" +
+                           "  \"host\": \"" + hostAndPort + "\",\n" +
+                           "  \"wantedGeneration\": 3,\n" +
+                           "  \"converged\": true,\n" +
+                           "  \"currentGeneration\": 3\n" +
+                           "}",
                            200,
                            response);
         }
@@ -609,11 +650,11 @@ public class ApplicationHandlerTest {
                                                    uri);
 
             assertResponse("{\n" +
-                                   "  \"url\": \"" + uri.toString() + "\",\n" +
-                                   "  \"host\": \"" + hostAndPort + "\",\n" +
-                                   "  \"wantedGeneration\": 3,\n" +
-                                   "  \"problem\": \"Host:port (service) no longer part of application, refetch list of services.\"\n" +
-                                   "}",
+                           "  \"url\": \"" + uri.toString() + "\",\n" +
+                           "  \"host\": \"" + hostAndPort + "\",\n" +
+                           "  \"wantedGeneration\": 3,\n" +
+                           "  \"problem\": \"Host:port (service) no longer part of application, refetch list of services.\"\n" +
+                           "}",
                            410,
                            response);
         }
@@ -635,20 +676,20 @@ public class ApplicationHandlerTest {
                                                                         3L),
                                                 requestUrl);
             assertResponse("{\n" +
-                                   "  \"services\": [\n" +
-                                   "    {\n" +
-                                   "      \"host\": \"" + hostname + "\",\n" +
-                                   "      \"port\": " + port + ",\n" +
-                                   "      \"type\": \"container\",\n" +
-                                   "      \"url\": \"" + serviceUrl.toString() + "\",\n" +
-                                   "      \"currentGeneration\":" + 3 + "\n" +
-                                   "    }\n" +
-                                   "  ],\n" +
-                                   "  \"url\": \"" + requestUrl.toString() + "\",\n" +
-                                   "  \"currentGeneration\": 3,\n" +
-                                   "  \"wantedGeneration\": 3,\n" +
-                                   "  \"converged\": true\n" +
-                                   "}",
+                           "  \"services\": [\n" +
+                           "    {\n" +
+                           "      \"host\": \"" + hostname + "\",\n" +
+                           "      \"port\": " + port + ",\n" +
+                           "      \"type\": \"container\",\n" +
+                           "      \"url\": \"" + serviceUrl.toString() + "\",\n" +
+                           "      \"currentGeneration\":" + 3 + "\n" +
+                           "    }\n" +
+                           "  ],\n" +
+                           "  \"url\": \"" + requestUrl.toString() + "\",\n" +
+                           "  \"currentGeneration\": 3,\n" +
+                           "  \"wantedGeneration\": 3,\n" +
+                           "  \"converged\": true\n" +
+                           "}",
                            200,
                            response);
         }
@@ -669,27 +710,27 @@ public class ApplicationHandlerTest {
                                                                         3L),
                                                 requestUrl);
             assertResponse("{\n" +
-                                   "  \"services\": [\n" +
-                                   "    {\n" +
-                                   "      \"host\": \"" + hostname + "\",\n" +
-                                   "      \"port\": " + port + ",\n" +
-                                   "      \"type\": \"container\",\n" +
-                                   "      \"url\": \"" + serviceUrl.toString() + "\",\n" +
-                                   "      \"currentGeneration\":" + 4 + "\n" +
-                                   "    },\n" +
-                                   "    {\n" +
-                                   "      \"host\": \"" + hostname2 + "\",\n" +
-                                   "      \"port\": " + port2 + ",\n" +
-                                   "      \"type\": \"container\",\n" +
-                                   "      \"url\": \"" + serviceUrl2.toString() + "\",\n" +
-                                   "      \"currentGeneration\":" + 3 + "\n" +
-                                   "    }\n" +
-                                   "  ],\n" +
-                                   "  \"url\": \"" + requestUrl.toString() + "\",\n" +
-                                   "  \"currentGeneration\": 3,\n" +
-                                   "  \"wantedGeneration\": 4,\n" +
-                                   "  \"converged\": false\n" +
-                                   "}",
+                           "  \"services\": [\n" +
+                           "    {\n" +
+                           "      \"host\": \"" + hostname + "\",\n" +
+                           "      \"port\": " + port + ",\n" +
+                           "      \"type\": \"container\",\n" +
+                           "      \"url\": \"" + serviceUrl.toString() + "\",\n" +
+                           "      \"currentGeneration\":" + 4 + "\n" +
+                           "    },\n" +
+                           "    {\n" +
+                           "      \"host\": \"" + hostname2 + "\",\n" +
+                           "      \"port\": " + port2 + ",\n" +
+                           "      \"type\": \"container\",\n" +
+                           "      \"url\": \"" + serviceUrl2.toString() + "\",\n" +
+                           "      \"currentGeneration\":" + 3 + "\n" +
+                           "    }\n" +
+                           "  ],\n" +
+                           "  \"url\": \"" + requestUrl.toString() + "\",\n" +
+                           "  \"currentGeneration\": 3,\n" +
+                           "  \"wantedGeneration\": 4,\n" +
+                           "  \"converged\": false\n" +
+                           "}",
                            200,
                            response);
         }
@@ -707,11 +748,11 @@ public class ApplicationHandlerTest {
                                                uri);
 
         assertResponse("{\n" +
-                               "  \"url\": \"" + uri.toString() + "\",\n" +
-                               "  \"host\": \"" + hostAndPort + "\",\n" +
-                               "  \"wantedGeneration\": 3,\n" +
-                               "  \"error\": \"some error message\"" +
-                               "}",
+                       "  \"url\": \"" + uri + "\",\n" +
+                       "  \"host\": \"" + hostAndPort + "\",\n" +
+                       "  \"wantedGeneration\": 3,\n" +
+                       "  \"error\": \"some error message\"" +
+                       "}",
                        404,
                        response);
     }