diff options
40 files changed, 323 insertions, 558 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java b/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java deleted file mode 100644 index d9ced0177e5..00000000000 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java +++ /dev/null @@ -1,133 +0,0 @@ -package com.yahoo.config.provision; - -import ai.vespa.http.DomainName; -import ai.vespa.http.HttpURL; - -import javax.naming.NamingException; -import javax.naming.directory.Attribute; -import javax.naming.directory.Attributes; -import javax.naming.directory.InitialDirContext; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Collections; -import java.util.Enumeration; -import java.util.List; -import java.util.Optional; - -/** - * @author jonmv - */ -public interface EndpointsChecker { - - record Endpoint(ClusterSpec.Id clusterName, - HttpURL url, - Optional<InetAddress> ipAddress, - Optional<DomainName> canonicalName, - boolean isPublic) { } - - /** Status sorted by increasing readiness. */ - enum Status { endpointsUnavailable, containersUnhealthy, available } - - record Availability(Status status, String message) { } - - interface HostNameResolver { Optional<InetAddress> resolve(DomainName hostName); } - - interface CNameResolver { Optional<DomainName> resolve(DomainName hostName); } - - interface ContainerHealthChecker { boolean healthy(Endpoint endpoint); } - - static EndpointsChecker of(ContainerHealthChecker containerHealthChecker) { - return zoneEndpoints -> endpointsAvailable(zoneEndpoints, EndpointsChecker::resolveHostName, EndpointsChecker::resolveCname, containerHealthChecker); - } - - static EndpointsChecker mock(HostNameResolver hostNameResolver, CNameResolver cNameResolver, ContainerHealthChecker containerHealthChecker) { - return zoneEndpoints -> endpointsAvailable(zoneEndpoints, hostNameResolver, cNameResolver, containerHealthChecker); - } - - Availability endpointsAvailable(List<Endpoint> zoneEndpoints); - - private static Availability endpointsAvailable(List<Endpoint> zoneEndpoints, - HostNameResolver hostNameResolver, - CNameResolver cNameResolver, - ContainerHealthChecker containerHealthChecker) { - if (zoneEndpoints.isEmpty()) - return new Availability(Status.endpointsUnavailable, "Endpoints not yet ready."); - - for (Endpoint endpoint : zoneEndpoints) { - Optional<InetAddress> resolvedIpAddress = hostNameResolver.resolve(endpoint.url().domain()); - if (resolvedIpAddress.isEmpty()) - return new Availability(Status.endpointsUnavailable, "DNS lookup yielded no IP address for '" + endpoint.url().domain() + "'."); - - if (resolvedIpAddress.equals(endpoint.ipAddress())) // We expect a certain IP address, and that's what we got, so we're good. - continue; - - if (endpoint.ipAddress().isPresent()) // We expect a certain IP address, but that's not what we got. - return new Availability(Status.endpointsUnavailable, - "IP address of '" + endpoint.url().domain() + "' (" + - resolvedIpAddress.get().getHostAddress() + ") and load balancer " + - "' (" + endpoint.ipAddress().get().getHostAddress() + ") are not equal"); - - if (endpoint.canonicalName().isEmpty()) // We have no expected IP address, and no canonical name, so there's nothing more to check. - continue; - - Optional<DomainName> cNameValue = cNameResolver.resolve(endpoint.url().domain()); - if (cNameValue.filter(endpoint.canonicalName().get()::equals).isEmpty()) { - return new Availability(Status.endpointsUnavailable, - "CNAME '" + endpoint.url().domain() + "' points at " + - cNameValue.map(name -> "'" + name + "'").orElse("nothing") + - " but should point at load balancer " + - endpoint.canonicalName().map(name -> "'" + name + "'").orElse("nothing")); - } - - Optional<InetAddress> loadBalancerAddress = hostNameResolver.resolve(endpoint.canonicalName().get()); - if ( ! loadBalancerAddress.equals(resolvedIpAddress)) { - return new Availability(Status.endpointsUnavailable, - "IP address of CNAME '" + endpoint.url().domain() + "' (" + - resolvedIpAddress.get().getHostAddress() + ") and load balancer '" + - endpoint.canonicalName().get() + "' (" + - loadBalancerAddress.map(InetAddress::getHostAddress).orElse("empty") + ") are not equal"); - } - } - - for (Endpoint endpoint : zoneEndpoints) - if ( ! containerHealthChecker.healthy(endpoint)) - return new Availability(Status.containersUnhealthy, "Failed to get enough healthy responses from " + endpoint.url()); - - return new Availability(Status.available, "Endpoints are ready"); - } - - /** Returns the IP address of the given host name, if any. */ - private static Optional<InetAddress> resolveHostName(DomainName hostname) { - try { - return Optional.of(InetAddress.getByName(hostname.value())); - } - catch (UnknownHostException ignored) { - return Optional.empty(); - } - } - - /** Returns the host name of the given CNAME, if any. */ - private static Optional<DomainName> resolveCname(DomainName endpoint) { - try { - InitialDirContext ctx = new InitialDirContext(); - try { - Attributes attrs = ctx.getAttributes("dns:/" + endpoint.value(), new String[]{ "CNAME" }); - for (Attribute attribute : Collections.list(attrs.getAll())) { - Enumeration<?> vals = attribute.getAll(); - if (vals.hasMoreElements()) { - String hostname = vals.nextElement().toString(); - return Optional.of(hostname.substring(0, hostname.length() - 1)).map(DomainName::of); - } - } - } - finally { - ctx.close(); - } - } - catch (NamingException e) { - throw new RuntimeException(e); - } - return Optional.empty(); - } - -} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java index 09b71c6a982..10e22f8df06 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java @@ -8,6 +8,9 @@ import java.util.Objects; /** * Settings for a zone endpoint of a deployment. * + * TODO: Fix isEmpty + * Inline empty and constructor + * * @author jonmv */ public class ZoneEndpoint { @@ -18,6 +21,10 @@ public class ZoneEndpoint { private final boolean isPrivateEndpoint; private final List<AllowedUrn> allowedUrns; + public ZoneEndpoint(List<String> allowedUrns) { + this(true, true, allowedUrns.stream().map(arn -> new AllowedUrn(AccessType.awsPrivateLink, arn)).toList()); + } + public ZoneEndpoint(boolean isPublicEndpoint, boolean isPrivateEndpoint, List<AllowedUrn> allowedUrns) { if ( ! allowedUrns.isEmpty() && ! isPrivateEndpoint) throw new IllegalArgumentException("cannot list allowed urns, without also enabling private visibility"); diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java index 5e30e8fa99c..3404d7ed55e 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java @@ -9,8 +9,6 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NetworkPorts; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ZoneEndpoint; -import com.yahoo.config.provision.ZoneEndpoint.AccessType; -import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -70,7 +68,7 @@ public class AllocatedHostsSerializerTest { bigSlowDiskSpeedNode, anyDiskSpeedNode, ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), - Optional.empty(), new ZoneEndpoint(true, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "burn")))), + Optional.empty(), new ZoneEndpoint(List.of("burn"))), Optional.empty(), Optional.empty(), Optional.empty())); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index edcffcca878..ca06fe202d9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.config.server; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Query; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; -import com.yahoo.component.annotation.Inject; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; @@ -17,9 +17,6 @@ import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.EndpointsChecker; -import com.yahoo.config.provision.EndpointsChecker.Availability; -import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.InfraDeployer; @@ -60,8 +57,6 @@ import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.deploy.Deployment; import com.yahoo.vespa.config.server.deploy.InfraDeployerProvider; import com.yahoo.vespa.config.server.filedistribution.FileDirectory; -import com.yahoo.vespa.config.server.http.HttpFetcher; -import com.yahoo.vespa.config.server.http.HttpFetcher.Params; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.SecretStoreValidator; @@ -92,8 +87,6 @@ import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.Orchestrator; -import org.apache.hc.client5.http.ssl.DefaultHostnameVerifier; - import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -119,7 +112,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import static ai.vespa.http.HttpURL.Path.parse; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; @@ -149,7 +141,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final Optional<InfraDeployer> infraDeployer; private final ConfigConvergenceChecker convergeChecker; private final HttpProxy httpProxy; - private final EndpointsChecker endpointsChecker; private final Clock clock; private final ConfigserverConfig configserverConfig; private final FileDistributionStatus fileDistributionStatus = new FileDistributionStatus(); @@ -178,7 +169,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye infraDeployerProvider.getInfraDeployer(), configConvergenceChecker, httpProxy, - createEndpointsChecker(), configserverConfig, orchestrator, new LogRetriever(), @@ -195,7 +185,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Optional<InfraDeployer> infraDeployer, ConfigConvergenceChecker configConvergenceChecker, HttpProxy httpProxy, - EndpointsChecker endpointsChecker, ConfigserverConfig configserverConfig, Orchestrator orchestrator, LogRetriever logRetriever, @@ -210,7 +199,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.infraDeployer = Objects.requireNonNull(infraDeployer); this.convergeChecker = Objects.requireNonNull(configConvergenceChecker); this.httpProxy = Objects.requireNonNull(httpProxy); - this.endpointsChecker = Objects.requireNonNull(endpointsChecker); this.configserverConfig = Objects.requireNonNull(configserverConfig); this.orchestrator = Objects.requireNonNull(orchestrator); this.logRetriever = Objects.requireNonNull(logRetriever); @@ -227,7 +215,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private TenantRepository tenantRepository; private Optional<Provisioner> hostProvisioner; private HttpProxy httpProxy = new HttpProxy(new SimpleHttpFetcher(Duration.ofSeconds(30))); - private EndpointsChecker endpointsChecker = __ -> { throw new UnsupportedOperationException(); }; private Clock clock = Clock.systemUTC(); private ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder().build(); private Orchestrator orchestrator; @@ -305,18 +292,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return this; } - public Builder withEndpointsChecker(EndpointsChecker endpointsChecker) { - this.endpointsChecker = endpointsChecker; - return this; - } - public ApplicationRepository build() { return new ApplicationRepository(tenantRepository, hostProvisioner, InfraDeployerProvider.empty().getInfraDeployer(), configConvergenceChecker, httpProxy, - endpointsChecker, configserverConfig, orchestrator, logRetriever, @@ -749,10 +730,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public ConfigConvergenceChecker configConvergenceChecker() { return convergeChecker; } - public Availability verifyEndpoints(List<Endpoint> endpoints) { - return endpointsChecker.endpointsAvailable(endpoints); - } - // ---------------- Logs ---------------------------------------------------------------- public HttpResponse getLogs(ApplicationId applicationId, Optional<DomainName> hostname, String apiParams) { @@ -1234,24 +1211,4 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } - private static EndpointsChecker createEndpointsChecker() { - HttpFetcher fetcher = new SimpleHttpFetcher(Duration.ofSeconds(10), new DefaultHostnameVerifier()::verify); - return EndpointsChecker.of(endpoint -> { - int remainingFailures = 3; - int remainingSuccesses = 100; - while (remainingSuccesses > 0 && remainingFailures > 0) { - try { - HttpResponse response = fetcher.get(new Params(3000), - endpoint.url().withPath(parse("/status.html")).asURI()); - if (response.getStatus() == 200) remainingSuccesses--; - else remainingFailures--; - } - catch (Exception e) { - remainingFailures--; - } - } - return remainingSuccesses == 0; - }); - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java index 22236281a93..0aa86ab211a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java @@ -27,7 +27,6 @@ import java.util.List; import static java.nio.charset.StandardCharsets.UTF_8; public class HttpProxy { - private final HttpFetcher fetcher; @Inject public HttpProxy(NodeHostnameVerifier verifier) { this(new SimpleHttpFetcher(Duration.ofSeconds(30), verifier)); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java index a7f4ef5d513..5b332d3f434 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java @@ -19,7 +19,6 @@ import java.util.logging.Level; import java.util.logging.Logger; public class SimpleHttpFetcher implements HttpFetcher { - private static final Logger logger = Logger.getLogger(SimpleHttpFetcher.class.getName()); private final CloseableHttpClient client; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 9619ad69b3c..68d0b81dc2b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -10,9 +10,6 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.EndpointsChecker.Availability; -import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Zone; @@ -23,10 +20,7 @@ import com.yahoo.jdisc.Response; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.server.ApplicationRepository; @@ -49,12 +43,9 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import java.io.IOException; import java.io.UncheckedIOException; -import java.net.InetAddress; import java.net.URI; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -113,7 +104,6 @@ public class ApplicationHandler extends HttpHandler { public HttpResponse handlePOST(HttpRequest request) { Path path = new Path(request.getUri()); - if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/verify-endpoints")) return verifyEndpoints(request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindex")) return triggerReindexing(applicationId(path), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return enableReindexing(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/restart")) return restart(applicationId(path), request); @@ -332,32 +322,6 @@ public class ApplicationHandler extends HttpHandler { return new MessageResponse("Success"); } - private HttpResponse verifyEndpoints(HttpRequest request) { - byte[] data = uncheck(() -> request.getData().readAllBytes()); - List<Endpoint> endpoints = new ArrayList<>(); - SlimeUtils.jsonToSlime(data).get() - .field("endpoints") - .traverse((ArrayTraverser) (__, endpointObject) -> { - endpoints.add(new Endpoint(ClusterSpec.Id.from(endpointObject.field("clusterName").asString()), - HttpURL.from(URI.create(endpointObject.field("url").asString())), - SlimeUtils.optionalString(endpointObject.field("ipAddress")).map(uncheck(InetAddress::getByName)), - SlimeUtils.optionalString(endpointObject.field("canonicalName")).map(DomainName::of), - endpointObject.field("public").asBool())); - }); - if (endpoints.isEmpty()) throw new IllegalArgumentException("No endpoints in request " + request); - - Availability availability = applicationRepository.verifyEndpoints(endpoints); - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("status", switch (availability.status()) { - case available -> "available"; - case endpointsUnavailable -> "endpointsUnavailable"; - case containersUnhealthy -> "containersUnhealthy"; - }); - root.setString("message", availability.message()); - return new SlimeJsonResponse(slime); - } - private HttpResponse testerStartTests(ApplicationId applicationId, String suite, HttpRequest request) { byte[] data; try { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index c270b4559f9..3925899e1cd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; @@ -10,10 +9,6 @@ import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.EndpointsChecker; -import com.yahoo.config.provision.EndpointsChecker.Availability; -import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -58,7 +53,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; -import java.net.InetAddress; import java.net.URI; import java.net.URLEncoder; import java.time.Duration; @@ -66,7 +60,6 @@ import java.time.Instant; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -84,12 +77,10 @@ import static com.yahoo.vespa.config.server.http.HandlerTest.assertHttpStatusCod import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString; import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServiceListResponse; import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServiceResponse.createResponse; -import static com.yahoo.yolean.Exceptions.uncheck; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -116,8 +107,6 @@ public class ApplicationHandlerTest { private MockProvisioner provisioner; private OrchestratorMock orchestrator; private ManualClock clock; - private List<Endpoint> expectedEndpoints; - private Availability availability; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -150,7 +139,6 @@ public class ApplicationHandlerTest { .withLogRetriever(logRetriever) .withConfigserverConfig(configserverConfig) .withSecretStoreValidator(secretStoreValidator) - .withEndpointsChecker(endpoints -> { assertEquals(expectedEndpoints, endpoints); return availability; }) .build(); } @@ -516,35 +504,6 @@ public class ApplicationHandlerTest { } @Test - public void testVerifyEndpoints() { - expectedEndpoints = List.of(new Endpoint(ClusterSpec.Id.from("bluster"), - HttpURL.from(URI.create("https://bluster.tld:1234")), - Optional.of(uncheck(() -> InetAddress.getByName("4.3.2.1"))), - Optional.of(DomainName.of("fluster.tld")), - false)); - availability = new Availability(EndpointsChecker.Status.available, "Endpoints are ready"); - ApplicationHandler handler = createApplicationHandler(); - HttpRequest request = createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/verify-endpoints", - POST, - new ByteArrayInputStream(""" - { - "endpoints": [ - { - "clusterName": "bluster", - "url": "https://bluster.tld:1234", - "ipAddress": "4.3.2.1", - "canonicalName": "fluster.tld", - "public": false - } - ] - }""".getBytes(UTF_8))); - HttpResponse response = handler.handle(request); - assertEquals(200, response.getStatus()); - assertEquals("{\"status\":\"available\",\"message\":\"Endpoints are ready\"}", - new ByteArrayOutputStream() {{ uncheck(() -> response.render(this)); }}.toString(UTF_8)); - } - - @Test public void testClusterReindexingStateSerialization() { Stream.of(ClusterReindexing.State.values()).forEach(ClusterReindexing.State::toString); } @@ -633,12 +592,12 @@ public class ApplicationHandlerTest { hostAndPort, uri); assertResponse("{\n" + - " \"url\": \"" + uri.toString() + "\",\n" + - " \"host\": \"" + hostAndPort + "\",\n" + - " \"wantedGeneration\": 3,\n" + - " \"converged\": true,\n" + - " \"currentGeneration\": 3\n" + - "}", + " \"url\": \"" + uri.toString() + "\",\n" + + " \"host\": \"" + hostAndPort + "\",\n" + + " \"wantedGeneration\": 3,\n" + + " \"converged\": true,\n" + + " \"currentGeneration\": 3\n" + + "}", 200, response); } @@ -650,11 +609,11 @@ public class ApplicationHandlerTest { uri); assertResponse("{\n" + - " \"url\": \"" + uri.toString() + "\",\n" + - " \"host\": \"" + hostAndPort + "\",\n" + - " \"wantedGeneration\": 3,\n" + - " \"problem\": \"Host:port (service) no longer part of application, refetch list of services.\"\n" + - "}", + " \"url\": \"" + uri.toString() + "\",\n" + + " \"host\": \"" + hostAndPort + "\",\n" + + " \"wantedGeneration\": 3,\n" + + " \"problem\": \"Host:port (service) no longer part of application, refetch list of services.\"\n" + + "}", 410, response); } @@ -676,20 +635,20 @@ public class ApplicationHandlerTest { 3L), requestUrl); assertResponse("{\n" + - " \"services\": [\n" + - " {\n" + - " \"host\": \"" + hostname + "\",\n" + - " \"port\": " + port + ",\n" + - " \"type\": \"container\",\n" + - " \"url\": \"" + serviceUrl.toString() + "\",\n" + - " \"currentGeneration\":" + 3 + "\n" + - " }\n" + - " ],\n" + - " \"url\": \"" + requestUrl.toString() + "\",\n" + - " \"currentGeneration\": 3,\n" + - " \"wantedGeneration\": 3,\n" + - " \"converged\": true\n" + - "}", + " \"services\": [\n" + + " {\n" + + " \"host\": \"" + hostname + "\",\n" + + " \"port\": " + port + ",\n" + + " \"type\": \"container\",\n" + + " \"url\": \"" + serviceUrl.toString() + "\",\n" + + " \"currentGeneration\":" + 3 + "\n" + + " }\n" + + " ],\n" + + " \"url\": \"" + requestUrl.toString() + "\",\n" + + " \"currentGeneration\": 3,\n" + + " \"wantedGeneration\": 3,\n" + + " \"converged\": true\n" + + "}", 200, response); } @@ -710,27 +669,27 @@ public class ApplicationHandlerTest { 3L), requestUrl); assertResponse("{\n" + - " \"services\": [\n" + - " {\n" + - " \"host\": \"" + hostname + "\",\n" + - " \"port\": " + port + ",\n" + - " \"type\": \"container\",\n" + - " \"url\": \"" + serviceUrl.toString() + "\",\n" + - " \"currentGeneration\":" + 4 + "\n" + - " },\n" + - " {\n" + - " \"host\": \"" + hostname2 + "\",\n" + - " \"port\": " + port2 + ",\n" + - " \"type\": \"container\",\n" + - " \"url\": \"" + serviceUrl2.toString() + "\",\n" + - " \"currentGeneration\":" + 3 + "\n" + - " }\n" + - " ],\n" + - " \"url\": \"" + requestUrl.toString() + "\",\n" + - " \"currentGeneration\": 3,\n" + - " \"wantedGeneration\": 4,\n" + - " \"converged\": false\n" + - "}", + " \"services\": [\n" + + " {\n" + + " \"host\": \"" + hostname + "\",\n" + + " \"port\": " + port + ",\n" + + " \"type\": \"container\",\n" + + " \"url\": \"" + serviceUrl.toString() + "\",\n" + + " \"currentGeneration\":" + 4 + "\n" + + " },\n" + + " {\n" + + " \"host\": \"" + hostname2 + "\",\n" + + " \"port\": " + port2 + ",\n" + + " \"type\": \"container\",\n" + + " \"url\": \"" + serviceUrl2.toString() + "\",\n" + + " \"currentGeneration\":" + 3 + "\n" + + " }\n" + + " ],\n" + + " \"url\": \"" + requestUrl.toString() + "\",\n" + + " \"currentGeneration\": 3,\n" + + " \"wantedGeneration\": 4,\n" + + " \"converged\": false\n" + + "}", 200, response); } @@ -748,11 +707,11 @@ public class ApplicationHandlerTest { uri); assertResponse("{\n" + - " \"url\": \"" + uri + "\",\n" + - " \"host\": \"" + hostAndPort + "\",\n" + - " \"wantedGeneration\": 3,\n" + - " \"error\": \"some error message\"" + - "}", + " \"url\": \"" + uri.toString() + "\",\n" + + " \"host\": \"" + hostAndPort + "\",\n" + + " \"wantedGeneration\": 3,\n" + + " \"error\": \"some error message\"" + + "}", 404, response); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 93ac16c606d..cc05bc01d99 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import ai.vespa.http.HttpURL.Query; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.EndpointsChecker.Availability; -import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.zone.ZoneId; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL.Path; @@ -147,8 +145,6 @@ public interface ConfigServer { Optional<TestReport> getTestReport(DeploymentId deployment); - Availability verifyEndpoints(DeploymentId deploymentId, List<Endpoint> zoneEndpoints); - /** Get maximum resources consumed */ QuotaUsage getQuotaUsage(DeploymentId deploymentId); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java index 0c81fbd3670..26330f11d65 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java @@ -8,30 +8,74 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import java.util.List; +import java.util.Objects; import java.util.Optional; -import static java.util.Objects.requireNonNull; - /** * Represents an exclusive load balancer, assigned to an application's cluster. * * @author mortent */ -public record LoadBalancer(String id, ApplicationId application, ClusterSpec.Id cluster, - Optional<DomainName> hostname, Optional<String> ipAddress, - State state, Optional<String> dnsZone, Optional<CloudAccount> cloudAccount, - Optional<PrivateServiceInfo> service, boolean isPublic) { - - public LoadBalancer { - requireNonNull(id, "id must be non-null"); - requireNonNull(application, "application must be non-null"); - requireNonNull(cluster, "cluster must be non-null"); - requireNonNull(hostname, "hostname must be non-null"); - requireNonNull(ipAddress, "ipAddress must be non-null"); - requireNonNull(state, "state must be non-null"); - requireNonNull(dnsZone, "dnsZone must be non-null"); - requireNonNull(cloudAccount, "cloudAccount must be non-null"); - requireNonNull(service, "service must be non-null"); +public class LoadBalancer { + + private final String id; + private final ApplicationId application; + private final ClusterSpec.Id cluster; + private final Optional<DomainName> hostname; + private final Optional<String> ipAddress; + private final State state; + private final Optional<String> dnsZone; + private final Optional<CloudAccount> cloudAccount; + private final Optional<PrivateServiceInfo> service; + + public LoadBalancer(String id, ApplicationId application, ClusterSpec.Id cluster, Optional<DomainName> hostname, + Optional<String> ipAddress, State state, Optional<String> dnsZone, + Optional<CloudAccount> cloudAccount, Optional<PrivateServiceInfo> service) { + this.id = Objects.requireNonNull(id, "id must be non-null"); + this.application = Objects.requireNonNull(application, "application must be non-null"); + this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); + this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); + this.ipAddress = Objects.requireNonNull(ipAddress, "ipAddress must be non-null"); + this.state = Objects.requireNonNull(state, "state must be non-null"); + this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); + this.cloudAccount = Objects.requireNonNull(cloudAccount, "cloudAccount must be non-null"); + this.service = Objects.requireNonNull(service, "service must be non-null"); + } + + public String id() { + return id; + } + + public ApplicationId application() { + return application; + } + + public ClusterSpec.Id cluster() { + return cluster; + } + + public Optional<DomainName> hostname() { + return hostname; + } + + public Optional<String> ipAddress() { + return ipAddress; + } + + public Optional<String> dnsZone() { + return dnsZone; + } + + public State state() { + return state; + } + + public Optional<CloudAccount> cloudAccount() { + return cloudAccount; + } + + public Optional<PrivateServiceInfo> service() { + return service; } public enum State { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java index 4095e4b03fd..b4d9dd49880 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; -import com.yahoo.config.provision.EndpointsChecker.Endpoint; -import com.yahoo.config.provision.EndpointsChecker.Availability; +import ai.vespa.http.DomainName; +import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; +import java.net.InetAddress; import java.net.URI; import java.util.List; import java.util.Optional; @@ -26,10 +27,17 @@ public interface TesterCloud { /** Returns the current status of the tester. */ Status getStatus(DeploymentId deploymentId); + /** Returns whether the container is ready to serve. */ + boolean ready(URI endpointUrl); + /** Returns whether the test container is ready to serve */ boolean testerReady(DeploymentId deploymentId); - Availability verifyEndpoints(DeploymentId deploymentId, List<Endpoint> endpoints); + /** Returns the IP address of the given host name, if any. */ + Optional<InetAddress> resolveHostName(DomainName hostname); + + /** Returns the host name of the given CNAME, if any. */ + Optional<DomainName> resolveCname(DomainName hostName); /** Returns the test report as JSON if available */ Optional<TestReport> getTestReport(DeploymentId deploymentId); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java index e29e8086c80..939c74fe61d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java @@ -3,9 +3,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.stubs; import ai.vespa.http.DomainName; import com.google.common.net.InetAddresses; -import com.yahoo.config.provision.EndpointsChecker; -import com.yahoo.config.provision.EndpointsChecker.Endpoint; -import com.yahoo.config.provision.EndpointsChecker.Availability; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; @@ -19,6 +16,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.NOT_STARTED; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.RUNNING; @@ -26,7 +24,6 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Teste public class MockTesterCloud implements TesterCloud { private final NameService nameService; - private final EndpointsChecker endpointsChecker = EndpointsChecker.mock(this::resolveHostName, this::resolveCname, __ -> true); private List<LogEntry> log = new ArrayList<>(); private Status status = NOT_STARTED; @@ -52,23 +49,25 @@ public class MockTesterCloud implements TesterCloud { public Status getStatus(DeploymentId deploymentId) { return status; } @Override - public boolean testerReady(DeploymentId deploymentId) { + public boolean ready(URI testerUrl) { return true; } @Override - public Availability verifyEndpoints(DeploymentId deploymentId, List<Endpoint> endpoints) { - return endpointsChecker.endpointsAvailable(endpoints); + public boolean testerReady(DeploymentId deploymentId) { + return true; } - private Optional<InetAddress> resolveHostName(DomainName hostname) { + @Override + public Optional<InetAddress> resolveHostName(DomainName hostname) { return nameService.findRecords(Record.Type.A, RecordName.from(hostname.value())).stream() .findFirst() .map(record -> InetAddresses.forString(record.data().asString())) .or(() -> Optional.of(InetAddresses.forString("1.2.3.4"))); } - private Optional<DomainName> resolveCname(DomainName hostName) { + @Override + public Optional<DomainName> resolveCname(DomainName hostName) { return nameService.findRecords(Record.Type.CNAME, RecordName.from(hostName.value())).stream() .findFirst() .map(record -> DomainName.of(record.data().asString().substring(0, record.data().asString().length() - 1))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 14f2b38f24a..1c5f98cb7f9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -2,16 +2,13 @@ package com.yahoo.vespa.hosted.controller.deployment; import ai.vespa.http.DomainName; -import ai.vespa.http.HttpURL; +import com.google.common.net.InetAddresses; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.Notifications; import com.yahoo.config.application.api.Notifications.When; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.EndpointsChecker; -import com.yahoo.config.provision.EndpointsChecker.Availability; -import com.yahoo.config.provision.EndpointsChecker.Status; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -48,7 +45,6 @@ import com.yahoo.yolean.Exceptions; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.io.UncheckedIOException; -import java.net.InetAddress; import java.security.cert.CertificateExpiredException; import java.security.cert.CertificateNotYetValidException; import java.security.cert.X509Certificate; @@ -91,7 +87,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; -import static com.yahoo.yolean.Exceptions.uncheck; import static java.util.Objects.requireNonNull; import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; @@ -356,13 +351,13 @@ public class InternalStepRunner implements StepRunner { } if (summary.converged()) { controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); - Availability availability = endpointsAvailable(id.application(), id.type().zone(), logger); - if (availability.status() == Status.available) { + if (endpointsAvailable(id.application(), id.type().zone(), logger)) { + if (containersAreUp(id.application(), id.type().zone(), logger)) { logger.log("Installation succeeded!"); return Optional.of(running); + } } - logger.log(availability.message()); - if (availability.status() == Status.endpointsUnavailable && timedOut(id, deployment.get(), timeouts.endpoint())) { + else if (timedOut(id, deployment.get(), timeouts.endpoint())) { logger.log(WARNING, "Endpoints failed to show up within " + timeouts.endpoint().toMinutes() + " minutes!"); return Optional.of(error); } @@ -481,6 +476,21 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); } + /** Returns true iff all calls to endpoint in the deployment give 100 consecutive 200 OK responses on /status.html. */ + private boolean containersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { + var endpoints = controller.routing().readTestRunnerEndpointsOf(Set.of(new DeploymentId(id, zoneId))); + if ( ! endpoints.containsKey(zoneId)) + return false; + + return endpoints.get(zoneId).parallelStream().allMatch(endpoint -> { + boolean ready = controller.jobController().cloud().ready(endpoint.url()); + if (!ready) { + logger.log("Failed to get 100 consecutive OKs from " + endpoint); + } + return ready; + }); + } + /** Returns true iff all containers in the tester deployment give 100 consecutive 200 OK responses on /status.html. */ private boolean testerContainersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { DeploymentId deploymentId = new DeploymentId(id, zoneId); @@ -492,25 +502,50 @@ public class InternalStepRunner implements StepRunner { } } - private Availability endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { + private boolean endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { DeploymentId deployment = new DeploymentId(id, zone); Map<ZoneId, List<Endpoint>> endpoints = controller.routing().readTestRunnerEndpointsOf(Set.of(deployment)); + if ( ! endpoints.containsKey(zone)) { + logger.log("Endpoints not yet ready."); + return false; + } + for (var endpoint : endpoints.get(zone)) { + DomainName endpointName = DomainName.of(endpoint.dnsName()); + var ipAddress = controller.jobController().cloud().resolveHostName(endpointName); + if (ipAddress.isEmpty()) { + logger.log(INFO, "DNS lookup yielded no IP address for '" + endpointName + "'."); + return false; + } + DeploymentRoutingContext context = controller.routing().of(deployment); + if (context.routingMethod() == RoutingMethod.exclusive) { + RoutingPolicy policy = context.routingPolicy(ClusterSpec.Id.from(endpoint.name())) + .orElseThrow(() -> new IllegalStateException(endpoint + " has no matching policy")); + if (policy.ipAddress().isPresent()) { + if (ipAddress.equals(policy.ipAddress().map(InetAddresses::forString))) continue; + logger.log(INFO, "IP address of '" + endpointName + "' (" + + ipAddress.map(InetAddresses::toAddrString).get() + ") and load balancer " + + "' (" + policy.ipAddress().orElseThrow() + ") are not equal"); + return false; + } + + var cNameValue = controller.jobController().cloud().resolveCname(endpointName); + if ( ! cNameValue.map(policy.canonicalName().get()::equals).orElse(false)) { + logger.log(INFO, "CNAME '" + endpointName + "' points at " + + cNameValue.map(name -> "'" + name + "'").orElse("nothing") + + " but should point at load balancer '" + policy.canonicalName() + "'"); + return false; + } + var loadBalancerAddress = controller.jobController().cloud().resolveHostName(policy.canonicalName().get()); + if ( ! loadBalancerAddress.equals(ipAddress)) { + logger.log(INFO, "IP address of CNAME '" + endpointName + "' (" + ipAddress.get() + ") and load balancer '" + + policy.canonicalName().get() + "' (" + loadBalancerAddress.orElse(null) + ") are not equal"); + return false; + } + } + } + logEndpoints(endpoints, logger); - DeploymentRoutingContext context = controller.routing().of(deployment); - boolean resolveEndpoints = context.routingMethod() == RoutingMethod.exclusive; - return controller.serviceRegistry().testerCloud().verifyEndpoints( - deployment, - endpoints.getOrDefault(zone, List.of()) - .stream() - .map(endpoint -> { - ClusterSpec.Id cluster = ClusterSpec.Id.from(endpoint.name()); - RoutingPolicy policy = context.routingPolicy(cluster).get(); - return new EndpointsChecker.Endpoint(cluster, - HttpURL.from(endpoint.url()), - policy.ipAddress().filter(__ -> resolveEndpoints).map(uncheck(InetAddress::getByName)), - policy.canonicalName().filter(__ -> resolveEndpoints), - policy.isPublic()); - }).toList()); + return true; } private void logEndpoints(Map<ZoneId, List<Endpoint>> zoneEndpoints, DualLogger logger) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/AbstractNameServiceRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/AbstractNameServiceRequest.java deleted file mode 100644 index 9d21f5b26bd..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/AbstractNameServiceRequest.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.yahoo.vespa.hosted.controller.dns; - -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; - -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -/** - * @author jonmv - */ -public abstract class AbstractNameServiceRequest implements NameServiceRequest { - - private final Optional<TenantAndApplicationId> owner; - private final RecordName name; - - AbstractNameServiceRequest(Optional<TenantAndApplicationId> owner, RecordName name) { - this.owner = requireNonNull(owner); - this.name = requireNonNull(name); - } - - @Override - public RecordName name() { - return name; - } - - @Override - public Optional<TenantAndApplicationId> owner() { - return owner; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java index 6f4ee3dfc06..f1e4ca3b82b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java @@ -15,13 +15,14 @@ import java.util.Optional; * * @author mpolden */ -public class CreateRecord extends AbstractNameServiceRequest { +public class CreateRecord implements NameServiceRequest { + private final Optional<TenantAndApplicationId> owner; private final Record record; /** DO NOT USE. Public for serialization purposes */ public CreateRecord(Optional<TenantAndApplicationId> owner, Record record) { - super(owner, record.name()); + this.owner = Objects.requireNonNull(owner, "owner must be non-null"); this.record = Objects.requireNonNull(record, "record must be non-null"); if (record.type() != Record.Type.CNAME && record.type() != Record.Type.A) { throw new IllegalArgumentException("Record of type " + record.type() + " is not supported: " + record); @@ -33,6 +34,16 @@ public class CreateRecord extends AbstractNameServiceRequest { } @Override + public Optional<RecordName> name() { + return Optional.of(record.name()); + } + + @Override + public Optional<TenantAndApplicationId> owner() { + return owner; + } + + @Override public void dispatchTo(NameService nameService) { List<Record> records = nameService.findRecords(record.type(), record.name()); records.forEach(r -> { @@ -56,12 +67,12 @@ public class CreateRecord extends AbstractNameServiceRequest { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; CreateRecord that = (CreateRecord) o; - return owner().equals(that.owner()) && record.equals(that.record); + return owner.equals(that.owner) && record.equals(that.record); } @Override public int hashCode() { - return Objects.hash(owner(), record); + return Objects.hash(owner, record); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java index ef7b74a4d4b..a668c408794 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java @@ -20,14 +20,17 @@ import java.util.stream.Collectors; * * @author mpolden */ -public class CreateRecords extends AbstractNameServiceRequest { +public class CreateRecords implements NameServiceRequest { + private final Optional<TenantAndApplicationId> owner; + private final RecordName name; private final Record.Type type; private final List<Record> records; /** DO NOT USE. Public for serialization purposes */ public CreateRecords(Optional<TenantAndApplicationId> owner, List<Record> records) { - super(owner, requireOneOf(Record::name, records)); + this.owner = Objects.requireNonNull(owner, "owner must be non-null"); + this.name = requireOneOf(Record::name, records); this.type = requireOneOf(Record::type, records); this.records = List.copyOf(Objects.requireNonNull(records, "records must be non-null")); if (type != Record.Type.ALIAS && type != Record.Type.TXT && type != Record.Type.DIRECT) { @@ -40,19 +43,29 @@ public class CreateRecords extends AbstractNameServiceRequest { } @Override + public Optional<RecordName> name() { + return Optional.of(name); + } + + @Override + public Optional<TenantAndApplicationId> owner() { + return owner; + } + + @Override public void dispatchTo(NameService nameService) { switch (type) { case ALIAS -> { var targets = records.stream().map(Record::data).map(AliasTarget::unpack).collect(Collectors.toSet()); - nameService.createAlias(name(), targets); + nameService.createAlias(name, targets); } case DIRECT -> { var targets = records.stream().map(Record::data).map(DirectTarget::unpack).collect(Collectors.toSet()); - nameService.createDirect(name(), targets); + nameService.createDirect(name, targets); } case TXT -> { var dataFields = records.stream().map(Record::data).toList(); - nameService.createTxtRecords(name(), dataFields); + nameService.createTxtRecords(name, dataFields); } } } @@ -67,12 +80,12 @@ public class CreateRecords extends AbstractNameServiceRequest { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; CreateRecords that = (CreateRecords) o; - return owner().equals(that.owner()) && records.equals(that.records); + return owner.equals(that.owner) && records.equals(that.records); } @Override public int hashCode() { - return Objects.hash(owner(), records); + return Objects.hash(owner, records); } /** Find exactly one distinct value of field in given list */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java index dd3cca9a4fa..c43130ce4e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java @@ -14,8 +14,7 @@ import java.util.Optional; */ public interface NameServiceRequest { - /** The record name this request pertains to. */ - RecordName name(); + Optional<RecordName> name(); /** The application owning this request */ Optional<TenantAndApplicationId> owner(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java index 273136ba0a6..2ff2edf11f4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java @@ -21,9 +21,11 @@ import java.util.Optional; * * @author mpolden */ -public class RemoveRecords extends AbstractNameServiceRequest { +public class RemoveRecords implements NameServiceRequest { + private final Optional<TenantAndApplicationId> owner; private final Record.Type type; + private final RecordName name; private final Optional<RecordData> data; public RemoveRecords(Optional<TenantAndApplicationId> owner, Record.Type type, RecordName name) { @@ -36,8 +38,9 @@ public class RemoveRecords extends AbstractNameServiceRequest { /** DO NOT USE. Public for serialization purposes */ public RemoveRecords(Optional<TenantAndApplicationId> owner, Record.Type type, RecordName name, Optional<RecordData> data) { - super(owner, name); + this.owner = Objects.requireNonNull(owner, "owner must be non-null"); this.type = Objects.requireNonNull(type, "type must be non-null"); + this.name = Objects.requireNonNull(name, "name must be non-null"); this.data = Objects.requireNonNull(data, "data must be non-null"); } @@ -45,6 +48,16 @@ public class RemoveRecords extends AbstractNameServiceRequest { return type; } + @Override + public Optional<RecordName> name() { + return Optional.of(name); + } + + @Override + public Optional<TenantAndApplicationId> owner() { + return owner; + } + public Optional<RecordData> data() { return data; } @@ -53,7 +66,7 @@ public class RemoveRecords extends AbstractNameServiceRequest { public void dispatchTo(NameService nameService) { // Deletions require all records fields to match exactly, data may be incomplete even if present. To ensure // completeness we search for the record(s) first - List<Record> completeRecords = nameService.findRecords(type, name()).stream() + List<Record> completeRecords = nameService.findRecords(type, name).stream() .filter(record -> data.isEmpty() || matchingFqdnIn(data.get(), record)) .toList(); nameService.removeRecords(completeRecords); @@ -61,7 +74,7 @@ public class RemoveRecords extends AbstractNameServiceRequest { @Override public String toString() { - return "remove records of type " + type + ", by name " + name() + + return "remove records of type " + type + ", by name " + name + data.map(d -> " data " + d).orElse(""); } @@ -70,12 +83,12 @@ public class RemoveRecords extends AbstractNameServiceRequest { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RemoveRecords that = (RemoveRecords) o; - return owner().equals(that.owner()) && type == that.type && name().equals(that.name()) && data.equals(that.data); + return owner.equals(that.owner) && type == that.type && name.equals(that.name) && data.equals(that.data); } @Override public int hashCode() { - return Objects.hash(owner(), type, name(), data); + return Objects.hash(owner, type, name, data); } private static boolean matchingFqdnIn(RecordData data, Record record) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java index d02d27b5293..89230969164 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java @@ -92,7 +92,7 @@ public class NameServiceQueueSerializer { private void toSlime(Cursor object, RemoveRecords removeRecords) { object.setString(requestType, Request.removeRecords.name()); object.setString(typeField, removeRecords.type().name()); - object.setString(nameField, removeRecords.name().asString()); + removeRecords.name().ifPresent(name -> object.setString(nameField, name.asString())); removeRecords.data().ifPresent(data -> object.setString(dataField, data.asString())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 47b27aac79a..4d759056dfc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -50,7 +50,6 @@ public class RoutingPolicySerializer { private static final String agentField = "agent"; private static final String changedAtField = "changedAt"; private static final String statusField = "status"; - private static final String privateOnlyField = "private"; public Slime toSlime(List<RoutingPolicy> routingPolicies) { var slime = new Slime(); @@ -69,7 +68,6 @@ public class RoutingPolicySerializer { policy.applicationEndpoints().forEach(endpointId -> applicationEndpointsArray.addString(endpointId.id())); policyObject.setBool(loadBalancerActiveField, policy.status().isActive()); globalRoutingToSlime(policy.status().routingStatus(), policyObject.setObject(globalRoutingField)); - if ( ! policy.isPublic()) policyObject.setBool(privateOnlyField, true); }); return slime; } @@ -86,7 +84,6 @@ public class RoutingPolicySerializer { RoutingPolicyId id = new RoutingPolicyId(owner, ClusterSpec.Id.from(inspect.field(clusterField).asString()), ZoneId.from(inspect.field(zoneField).asString())); - boolean isPublic = ! inspect.field(privateOnlyField).asBool(); policies.add(new RoutingPolicy(id, SlimeUtils.optionalString(inspect.field(canonicalNameField)).map(DomainName::of), SlimeUtils.optionalString(inspect.field(ipAddressField)), @@ -94,8 +91,7 @@ public class RoutingPolicySerializer { instanceEndpoints, applicationEndpoints, new RoutingPolicy.Status(inspect.field(loadBalancerActiveField).asBool(), - globalRoutingFromSlime(inspect.field(globalRoutingField))), - isPublic)); + globalRoutingFromSlime(inspect.field(globalRoutingField))))); }); return Collections.unmodifiableList(policies); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 1c4916b9bed..c737f9b58ef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -361,8 +361,7 @@ public class RoutingPolicies { var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.ipAddress(), dnsZone, allocation.instanceEndpointsOf(loadBalancer), allocation.applicationEndpointsOf(loadBalancer), - new RoutingPolicy.Status(isActive(loadBalancer), RoutingStatus.DEFAULT), - loadBalancer.isPublic()); + new RoutingPolicy.Status(isActive(loadBalancer), RoutingStatus.DEFAULT)); // Preserve global routing status for existing policy if (existingPolicy != null) { newPolicy = newPolicy.with(newPolicy.status().with(existingPolicy.status().routingStatus())); @@ -400,7 +399,7 @@ public class RoutingPolicies { while (controller.clock().instant().isBefore(doom)) { try (Mutex lock = controller.curator().lockNameServiceQueue()) { if (controller.curator().readNameServiceQueue().requests().stream() - .noneMatch(request -> request.name().equals(challenge.name()))) { + .noneMatch(request -> request.name().equals(Optional.of(challenge.name())))) { try { challenge.trigger().run(); } finally { nameServiceForwarderIn(deploymentId.zoneId()).removeRecords(Type.TXT, challenge.name(), Priority.normal, ownerOf(deploymentId)); } return; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 38ecff452c8..3d43e42af27 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.EndpointId; @@ -28,12 +29,11 @@ public record RoutingPolicy(RoutingPolicyId id, Optional<String> dnsZone, Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, - Status status, - boolean isPublic) { + Status status) { /** DO NOT USE. Public for serialization purposes */ public RoutingPolicy(RoutingPolicyId id, Optional<DomainName> canonicalName, Optional<String> ipAddress, Optional<String> dnsZone, - Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, Status status, boolean isPublic) { + Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, Status status) { this.id = Objects.requireNonNull(id, "id must be non-null"); this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); this.ipAddress = Objects.requireNonNull(ipAddress, "ipAddress must be non-null"); @@ -41,15 +41,10 @@ public record RoutingPolicy(RoutingPolicyId id, this.instanceEndpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(instanceEndpoints, "instanceEndpoints must be non-null")); this.applicationEndpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(applicationEndpoints, "applicationEndpoints must be non-null")); this.status = Objects.requireNonNull(status, "status must be non-null"); - this.isPublic = isPublic; if (canonicalName.isEmpty() == ipAddress.isEmpty()) throw new IllegalArgumentException("Exactly 1 of canonicalName=%s and ipAddress=%s must be set".formatted( canonicalName.map(DomainName::value).orElse("<empty>"), ipAddress.orElse("<empty>"))); - if ( ! instanceEndpoints.isEmpty() && ! isPublic) - throw new IllegalArgumentException("Non-public zone endpoint cannot be part of any global endpoint, but was in: " + instanceEndpoints); - if ( ! applicationEndpoints.isEmpty() && ! isPublic) - throw new IllegalArgumentException("Non-public zone endpoint cannot be part of any application endpoint, but was in: " + applicationEndpoints); } /** The ID of this */ @@ -87,11 +82,6 @@ public record RoutingPolicy(RoutingPolicyId id, return status; } - /** Returns whether this has a load balancer which is available from public internet. */ - public boolean isPublic() { - return isPublic; - } - /** Returns whether this policy applies to given deployment */ public boolean appliesTo(DeploymentId deployment) { return id.owner().equals(deployment.applicationId()) && @@ -100,7 +90,7 @@ public record RoutingPolicy(RoutingPolicyId id, /** Returns a copy of this with status set to given status */ public RoutingPolicy with(Status status) { - return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, status, isPublic); + return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, status); } /** Returns the zone endpoints of this */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 95c22480c0f..7edb458c154 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -277,8 +277,7 @@ public class DeploymentContext { Optional.empty(), Set.of(EndpointId.of("default")), Set.of(), - new RoutingPolicy.Status(false, RoutingStatus.DEFAULT), - true)); + new RoutingPolicy.Status(false, RoutingStatus.DEFAULT))); tester.controller().curator().writeRoutingPolicies(instanceId, List.copyOf(policies.values())); return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 5704af75cb9..6f859ff3d15 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -12,8 +12,6 @@ import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.EndpointsChecker.Availability; -import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; @@ -44,10 +42,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.QuotaUsage import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; -import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; @@ -86,7 +82,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; */ public class ConfigServerMock extends AbstractComponent implements ConfigServer { - private final MockTesterCloud mockTesterCloud; private final Map<DeploymentId, Application> applications = new LinkedHashMap<>(); private final Set<ZoneId> inactiveZones = new HashSet<>(); private final Map<DeploymentId, EndpointStatus> endpoints = new HashMap<>(); @@ -110,9 +105,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private Consumer<ApplicationId> prepareException = null; private Supplier<String> log = () -> "INFO - All good"; - public ConfigServerMock(ZoneRegistryMock zoneRegistry, NameService nameService) { + @Inject + public ConfigServerMock(ZoneRegistryMock zoneRegistry) { bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.notController()); - this.mockTesterCloud = new MockTesterCloud(nameService); } /** Assigns a reserved tenant node to the given deployment, with initial versions. */ @@ -375,10 +370,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer public Optional<TestReport> getTestReport(DeploymentId deployment) { return Optional.ofNullable(testReport.get(deployment)); } - - @Override - public Availability verifyEndpoints(DeploymentId deploymentId, List<Endpoint> zoneEndpoints) { - return mockTesterCloud.verifyEndpoints(deploymentId, zoneEndpoints); // Wraps the same name service mock, which is updated by test harness. + public void setTestReport(DeploymentId deploymentId, TestReport report) { + testReport.put(deploymentId, report); } /** Add any of given loadBalancers that do not already exist to the load balancers in zone */ @@ -426,8 +419,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer LoadBalancer.State.active, Optional.of("dns-zone-1"), Optional.empty(), - Optional.of(new PrivateServiceInfo("service", List.of(new AllowedUrn(AccessType.awsPrivateLink, "arne")))), - true))); + Optional.of(new PrivateServiceInfo("service", List.of(new AllowedUrn(AccessType.awsPrivateLink, "arne"))))))); } Application application = applications.get(id); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 0ba8866c990..382a697c4cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -100,8 +100,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); - this.configServerMock = new ConfigServerMock(zoneRegistryMock, memoryNameService); - this.mockTesterCloud = new MockTesterCloud(memoryNameService); + this.configServerMock = new ConfigServerMock(zoneRegistryMock); + this.mockTesterCloud = new MockTesterCloud(nameService()); this.clock.setInstant(Instant.ofEpochSecond(1600000000)); this.controllerVersion = new ControllerVersion(Version.fromString("6.1.0"), "badb01", clock.instant()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java index ad9ebd2a968..582514163e1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.CreateRecord; import com.yahoo.vespa.hosted.controller.dns.CreateRecords; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; -import com.yahoo.vespa.hosted.controller.dns.NameServiceRequest; import com.yahoo.vespa.hosted.controller.dns.RemoveRecords; import org.junit.jupiter.api.Test; @@ -33,7 +32,7 @@ public class NameServiceQueueSerializerTest { Optional<TenantAndApplicationId> owner = Optional.of(TenantAndApplicationId.from("t", "a")); var record1 = new Record(Record.Type.CNAME, RecordName.from("cname.example.com"), RecordData.from("example.com")); var record2 = new Record(Record.Type.TXT, RecordName.from("txt.example.com"), RecordData.from("text")); - var requests = List.<NameServiceRequest>of( + var requests = List.of( new CreateRecord(owner, record1), new CreateRecords(owner, List.of(record2)), new CreateRecords(owner, List.of(new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index 8e0b1dd1d4e..6285c5c4aac 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -32,38 +32,35 @@ public class RoutingPolicySerializerTest { var instanceEndpoints = Set.of(EndpointId.of("r1"), EndpointId.of("r2")); var applicationEndpoints = Set.of(EndpointId.of("a1")); var id1 = new RoutingPolicyId(owner, - ClusterSpec.Id.from("my-cluster1"), - ZoneId.from("prod", "us-north-1")); + ClusterSpec.Id.from("my-cluster1"), + ZoneId.from("prod", "us-north-1")); var id2 = new RoutingPolicyId(owner, - ClusterSpec.Id.from("my-cluster2"), - ZoneId.from("prod", "us-north-2")); + ClusterSpec.Id.from("my-cluster2"), + ZoneId.from("prod", "us-north-2")); var policies = List.of(new RoutingPolicy(id1, - Optional.of(HostName.of("long-and-ugly-name")), - Optional.empty(), - Optional.of("zone1"), - Set.of(), - Set.of(), - new RoutingPolicy.Status(true, RoutingStatus.DEFAULT), - false), - new RoutingPolicy(id2, - Optional.of(HostName.of("long-and-ugly-name-2")), - Optional.empty(), - Optional.empty(), - instanceEndpoints, - Set.of(), - new RoutingPolicy.Status(false, - new RoutingStatus(RoutingStatus.Value.out, - RoutingStatus.Agent.tenant, - Instant.ofEpochSecond(123))), - true), - new RoutingPolicy(id1, - Optional.empty(), - Optional.of("127.0.0.1"), - Optional.of("zone2"), - instanceEndpoints, - applicationEndpoints, - new RoutingPolicy.Status(true, RoutingStatus.DEFAULT), - true)); + Optional.of(HostName.of("long-and-ugly-name")), + Optional.empty(), + Optional.of("zone1"), + instanceEndpoints, + applicationEndpoints, + new RoutingPolicy.Status(true, RoutingStatus.DEFAULT)), + new RoutingPolicy(id2, + Optional.of(HostName.of("long-and-ugly-name-2")), + Optional.empty(), + Optional.empty(), + instanceEndpoints, + Set.of(), + new RoutingPolicy.Status(false, + new RoutingStatus(RoutingStatus.Value.out, + RoutingStatus.Agent.tenant, + Instant.ofEpochSecond(123)))), + new RoutingPolicy(id1, + Optional.empty(), + Optional.of("127.0.0.1"), + Optional.of("zone2"), + instanceEndpoints, + applicationEndpoints, + new RoutingPolicy.Status(true, RoutingStatus.DEFAULT))); var serialized = serializer.fromSlime(owner, serializer.toSlime(policies)); assertEquals(policies.size(), serialized.size()); for (Iterator<RoutingPolicy> it1 = policies.iterator(), it2 = serialized.iterator(); it1.hasNext(); ) { @@ -76,7 +73,6 @@ public class RoutingPolicySerializerTest { assertEquals(expected.instanceEndpoints(), actual.instanceEndpoints()); assertEquals(expected.applicationEndpoints(), actual.applicationEndpoints()); assertEquals(expected.status(), actual.status()); - assertEquals(expected.isPublic(), actual.isPublic()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 1efafb81685..03322d7c962 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -364,8 +364,8 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { ControllerTester wrapped = new ControllerTester(tester); wrapped.upgradeSystem(Version.fromString("7.1")); new DeploymentTester(wrapped).newDeploymentContext(ApplicationId.from(tenantName, applicationName, InstanceName.defaultName())) - .submit() - .deploy(); + .submit() + .deploy(); tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), (response) -> assertFalse(response.getBodyAsString().contains("archiveAccessRole")), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 2932860efaa..9c1253c7520 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -497,8 +497,7 @@ public class RoutingPoliciesTest { LoadBalancer.State.active, Optional.of("dns-zone-1"), Optional.empty(), - Optional.empty(), - true); + Optional.empty()); tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer)); // Application redeployment preserves DNS record @@ -953,8 +952,7 @@ public class RoutingPoliciesTest { LoadBalancer.State.active, Optional.of("dns-zone-1").filter(__ -> lbHostname.isPresent()), Optional.empty(), - Optional.empty(), - true)); + Optional.empty())); } return loadBalancers; } diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java b/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java index c5ebafb2425..4f2bdfb213e 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java +++ b/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java @@ -31,7 +31,6 @@ import static com.yahoo.security.tls.TransportSecurityUtils.isTransportSecurityE * @author jonmv */ public class VespaHttpClientBuilder { - private HttpClientConnectionManagerFactory connectionManagerFactory = PoolingHttpClientConnectionManager::new; private HostnameVerifier hostnameVerifier = new NoopHostnameVerifier(); private boolean rewriteHttpToHttps = true; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index bf5b735c4a0..15a799c06d8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -89,7 +89,6 @@ public class LoadBalancersResponse extends SlimeJsonResponse { } } instance.serviceId().ifPresent(serviceId -> lbObject.setString("serviceId", serviceId.value())); - lbObject.setBool("public", instance.settings().isPublicEndpoint()); }); lb.instance() .map(LoadBalancerInstance::cloudAccount) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 6cbb2ba1fb4..29914993bae 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -21,8 +21,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.ZoneEndpoint; -import com.yahoo.config.provision.ZoneEndpoint.AccessType; -import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.mock.MockCurator; @@ -192,10 +190,7 @@ public class MockNodeRepository extends NodeRepository { activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), null), zoneApp, provisioner); ApplicationId app1Id = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1")); - ClusterSpec cluster1Id = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1")) - .vespaVersion("6.42") - .loadBalancerSettings(new ZoneEndpoint(false, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "arne")))) - .build(); + ClusterSpec cluster1Id = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1")).vespaVersion("6.42").loadBalancerSettings(new ZoneEndpoint(List.of("arne"))).build(); activate(provisioner.prepare(app1Id, cluster1Id, Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1)), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index 16e82e116e1..dee895b02d2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -7,8 +7,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.ZoneEndpoint; -import com.yahoo.config.provision.ZoneEndpoint.AccessType; -import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import com.yahoo.vespa.hosted.provision.lb.DnsZone; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; @@ -48,7 +46,7 @@ public class LoadBalancerSerializerTest { new Real(DomainName.of("real-2"), "127.0.0.2", 4080)), - new ZoneEndpoint(false, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "123"))), + new ZoneEndpoint(List.of("123")), Optional.of(PrivateServiceId.of("foo")), CloudAccount.from("012345678912"))), LoadBalancer.State.active, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 9ba8c4d2d75..d9c27ae29ca 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -13,8 +13,6 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ZoneEndpoint; -import com.yahoo.config.provision.ZoneEndpoint.AccessType; -import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -326,7 +324,7 @@ public class LoadBalancerProvisionerTest { assertEquals(ZoneEndpoint.defaultEndpoint, loadBalancers.first().get().instance().get().settings()); // Next deployment contains new settings - ZoneEndpoint settings = new ZoneEndpoint(true, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "alice"), new AllowedUrn(AccessType.gcpServiceConnect, "bob"))); + ZoneEndpoint settings = new ZoneEndpoint(List.of("alice", "bob")); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1"), Optional.empty(), settings))); loadBalancers = tester.nodeRepository().loadBalancers().list(); assertEquals(1, loadBalancers.size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json index f6d9c1f079c..a9a728bab15 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json @@ -28,8 +28,7 @@ "ipAddress": "127.0.14.1", "port": 4443 } - ], - "public": true + ] } ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json index eef381e8df7..becca98a913 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json @@ -37,8 +37,7 @@ } ] }, - "serviceId": "service", - "public": false + "serviceId": "service" }, { "id": "cfg:cfg:cfg:configservers", @@ -57,8 +56,7 @@ "ports": [ 4443 ], - "reals": [], - "public": true + "reals": [] }, { "id": "tenant4:application4:instance4:id4", @@ -88,8 +86,7 @@ "ipAddress": "127.0.14.1", "port": 4443 } - ], - "public": true + ] } ] } diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index c3b87278345..418f3ed5911 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -3646,19 +3646,6 @@ ], "fields" : [ ] }, - "com.yahoo.yolean.Exceptions$FunctionThrowingIOException" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "interface", - "abstract" - ], - "methods" : [ - "public abstract java.lang.Object map(java.lang.Object)" - ], - "fields" : [ ] - }, "com.yahoo.yolean.Exceptions$RunnableThrowingIOException" : { "superClass" : "java.lang.Object", "interfaces" : [ ], @@ -3713,7 +3700,6 @@ "public static void uncheckInterruptedAndRestoreFlag(com.yahoo.yolean.Exceptions$RunnableThrowingInterruptedException)", "public static varargs void uncheck(com.yahoo.yolean.Exceptions$RunnableThrowingIOException, java.lang.String, java.lang.String[])", "public static void uncheckAndIgnore(com.yahoo.yolean.Exceptions$RunnableThrowingIOException, java.lang.Class)", - "public static java.util.function.Function uncheck(com.yahoo.yolean.Exceptions$FunctionThrowingIOException)", "public static java.lang.Object uncheck(com.yahoo.yolean.Exceptions$SupplierThrowingIOException)", "public static varargs java.lang.Object uncheck(com.yahoo.yolean.Exceptions$SupplierThrowingIOException, java.lang.String, java.lang.String[])", "public static java.lang.Object uncheckAndIgnore(com.yahoo.yolean.Exceptions$SupplierThrowingIOException, java.lang.Class)", diff --git a/vespajlib/src/main/java/ai/vespa/http/HttpURL.java b/vespajlib/src/main/java/ai/vespa/http/HttpURL.java index ba1a8e08740..9641ea2a8fd 100644 --- a/vespajlib/src/main/java/ai/vespa/http/HttpURL.java +++ b/vespajlib/src/main/java/ai/vespa/http/HttpURL.java @@ -239,7 +239,7 @@ public class HttpURL { return parse(raw, HttpURL::requirePathSegment); } - /** Parses the given raw, normalized path string; this ignores whether the path is absolute or relative. */ + /** Parses the given raw, normalized path string; this ignores whether the path is absolute or relative.) */ public static Path parse(String raw, Consumer<String> validator) { Path path = new Path(null, 0, raw.endsWith("/"), segmentValidator(validator)); if (raw.startsWith("/")) raw = raw.substring(1); diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8.java b/vespajlib/src/main/java/com/yahoo/text/Utf8.java index 3a7ecaa727a..2a42cb5cdee 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Utf8.java +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8.java @@ -10,8 +10,7 @@ import java.nio.ReadOnlyBufferException; import java.nio.charset.Charset; import java.nio.charset.CharsetEncoder; import java.nio.charset.CodingErrorAction; - -import static java.nio.charset.StandardCharsets.UTF_8; +import java.nio.charset.StandardCharsets; /** * Utility class with functions for handling UTF-8 @@ -24,16 +23,16 @@ public final class Utf8 { private static final byte [] TRUE = {(byte) 't', (byte) 'r', (byte) 'u', (byte) 'e'}; private static final byte [] FALSE = {(byte) 'f', (byte) 'a', (byte) 'l', (byte) 's', (byte) 'e'}; - private static final byte[] LONG_MIN_VALUE_BYTES = String.valueOf(Long.MIN_VALUE).getBytes(UTF_8); + private static final byte[] LONG_MIN_VALUE_BYTES = String.valueOf(Long.MIN_VALUE).getBytes(StandardCharsets.UTF_8); /** Returns the Charset instance for UTF-8 */ public static Charset getCharset() { - return UTF_8; + return StandardCharsets.UTF_8; } /** To be used instead of String.String(byte[] bytes) */ public static String toStringStd(byte[] data) { - return new String(data, UTF_8); + return new String(data, StandardCharsets.UTF_8); } /** @@ -61,7 +60,7 @@ public final class Utf8 { * @return a decoded String */ public static String toString(ByteBuffer data) { - CharBuffer c = UTF_8.decode(data); + CharBuffer c = StandardCharsets.UTF_8.decode(data); return c.toString(); } @@ -69,7 +68,7 @@ public final class Utf8 { * Uses String.getBytes directly. */ public static byte[] toBytesStd(String str) { - return str.getBytes(UTF_8); + return str.getBytes(StandardCharsets.UTF_8); } /** @@ -113,7 +112,7 @@ public final class Utf8 { */ public static byte[] toBytes(String string) { // This is just wrapper for String::getBytes. Pre-Java 9 this had a more efficient approach for ASCII-only strings. - return string.getBytes(UTF_8); + return string.getBytes(StandardCharsets.UTF_8); } /** * Decode a UTF-8 string. @@ -123,7 +122,7 @@ public final class Utf8 { */ public static String toString(byte[] utf8) { // This is just wrapper for String::new. Pre-Java 9 this had a more efficient approach for ASCII-onlu strings. - return new String(utf8, UTF_8); + return new String(utf8, StandardCharsets.UTF_8); } /** @@ -139,7 +138,7 @@ public final class Utf8 { */ public static byte[] toBytes(String str, int offset, int length) { CharBuffer c = CharBuffer.wrap(str, offset, offset + length); - ByteBuffer b = UTF_8.encode(c); + ByteBuffer b = StandardCharsets.UTF_8.encode(c); byte[] result = new byte[b.remaining()]; b.get(result); return result; @@ -162,7 +161,7 @@ public final class Utf8 { */ public static int toBytes(String str, int srcOffset, int srcLen, byte[] dst, int dstOffset) { CharBuffer c = CharBuffer.wrap(str, srcOffset, srcOffset + srcLen); - ByteBuffer b = UTF_8.encode(c); + ByteBuffer b = StandardCharsets.UTF_8.encode(c); int encoded = b.remaining(); b.get(dst, dstOffset, encoded); return encoded; @@ -207,7 +206,7 @@ public final class Utf8 { * @see Utf8#toBytes(String, int, int, ByteBuffer, CharsetEncoder) */ public static CharsetEncoder getNewEncoder() { - return UTF_8.newEncoder().onMalformedInput(CodingErrorAction.REPLACE) + return StandardCharsets.UTF_8.newEncoder().onMalformedInput(CodingErrorAction.REPLACE) .onUnmappableCharacter(CodingErrorAction.REPLACE); } diff --git a/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java b/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java index 4f3f048eb0c..89b4e76368b 100644 --- a/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java +++ b/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java @@ -4,7 +4,6 @@ package com.yahoo.yolean; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Optional; -import java.util.function.Function; /** * Helper methods for handling exceptions @@ -130,14 +129,6 @@ public class Exceptions { @FunctionalInterface public interface RunnableThrowingInterruptedException { void run() throws InterruptedException; } /** - * Wraps any IOException thrown from a function in an UncheckedIOException. - */ - public static <T, R> Function<T, R> uncheck(FunctionThrowingIOException<T, R> function) { - return t -> uncheck(() -> function.map(t)); - } - @FunctionalInterface public interface FunctionThrowingIOException<T, R> { R map(T t) throws IOException; } - - /** * Wraps any IOException thrown from a supplier in an UncheckedIOException. */ public static <T> T uncheck(SupplierThrowingIOException<T> supplier) { |