diff options
56 files changed, 722 insertions, 388 deletions
diff --git a/config-provisioning/pom.xml b/config-provisioning/pom.xml index b447cb792a6..ab9418ec488 100644 --- a/config-provisioning/pom.xml +++ b/config-provisioning/pom.xml @@ -16,6 +16,13 @@ <dependencies> <dependency> + <!-- required for bundle-plugin to generate import-package statements for Java's standard library --> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>annotations</artifactId> <version>${project.version}</version> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java b/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java new file mode 100644 index 00000000000..d9ced0177e5 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java @@ -0,0 +1,133 @@ +package com.yahoo.config.provision; + +import ai.vespa.http.DomainName; +import ai.vespa.http.HttpURL; + +import javax.naming.NamingException; +import javax.naming.directory.Attribute; +import javax.naming.directory.Attributes; +import javax.naming.directory.InitialDirContext; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; +import java.util.Optional; + +/** + * @author jonmv + */ +public interface EndpointsChecker { + + record Endpoint(ClusterSpec.Id clusterName, + HttpURL url, + Optional<InetAddress> ipAddress, + Optional<DomainName> canonicalName, + boolean isPublic) { } + + /** Status sorted by increasing readiness. */ + enum Status { endpointsUnavailable, containersUnhealthy, available } + + record Availability(Status status, String message) { } + + interface HostNameResolver { Optional<InetAddress> resolve(DomainName hostName); } + + interface CNameResolver { Optional<DomainName> resolve(DomainName hostName); } + + interface ContainerHealthChecker { boolean healthy(Endpoint endpoint); } + + static EndpointsChecker of(ContainerHealthChecker containerHealthChecker) { + return zoneEndpoints -> endpointsAvailable(zoneEndpoints, EndpointsChecker::resolveHostName, EndpointsChecker::resolveCname, containerHealthChecker); + } + + static EndpointsChecker mock(HostNameResolver hostNameResolver, CNameResolver cNameResolver, ContainerHealthChecker containerHealthChecker) { + return zoneEndpoints -> endpointsAvailable(zoneEndpoints, hostNameResolver, cNameResolver, containerHealthChecker); + } + + Availability endpointsAvailable(List<Endpoint> zoneEndpoints); + + private static Availability endpointsAvailable(List<Endpoint> zoneEndpoints, + HostNameResolver hostNameResolver, + CNameResolver cNameResolver, + ContainerHealthChecker containerHealthChecker) { + if (zoneEndpoints.isEmpty()) + return new Availability(Status.endpointsUnavailable, "Endpoints not yet ready."); + + for (Endpoint endpoint : zoneEndpoints) { + Optional<InetAddress> resolvedIpAddress = hostNameResolver.resolve(endpoint.url().domain()); + if (resolvedIpAddress.isEmpty()) + return new Availability(Status.endpointsUnavailable, "DNS lookup yielded no IP address for '" + endpoint.url().domain() + "'."); + + if (resolvedIpAddress.equals(endpoint.ipAddress())) // We expect a certain IP address, and that's what we got, so we're good. + continue; + + if (endpoint.ipAddress().isPresent()) // We expect a certain IP address, but that's not what we got. + return new Availability(Status.endpointsUnavailable, + "IP address of '" + endpoint.url().domain() + "' (" + + resolvedIpAddress.get().getHostAddress() + ") and load balancer " + + "' (" + endpoint.ipAddress().get().getHostAddress() + ") are not equal"); + + if (endpoint.canonicalName().isEmpty()) // We have no expected IP address, and no canonical name, so there's nothing more to check. + continue; + + Optional<DomainName> cNameValue = cNameResolver.resolve(endpoint.url().domain()); + if (cNameValue.filter(endpoint.canonicalName().get()::equals).isEmpty()) { + return new Availability(Status.endpointsUnavailable, + "CNAME '" + endpoint.url().domain() + "' points at " + + cNameValue.map(name -> "'" + name + "'").orElse("nothing") + + " but should point at load balancer " + + endpoint.canonicalName().map(name -> "'" + name + "'").orElse("nothing")); + } + + Optional<InetAddress> loadBalancerAddress = hostNameResolver.resolve(endpoint.canonicalName().get()); + if ( ! loadBalancerAddress.equals(resolvedIpAddress)) { + return new Availability(Status.endpointsUnavailable, + "IP address of CNAME '" + endpoint.url().domain() + "' (" + + resolvedIpAddress.get().getHostAddress() + ") and load balancer '" + + endpoint.canonicalName().get() + "' (" + + loadBalancerAddress.map(InetAddress::getHostAddress).orElse("empty") + ") are not equal"); + } + } + + for (Endpoint endpoint : zoneEndpoints) + if ( ! containerHealthChecker.healthy(endpoint)) + return new Availability(Status.containersUnhealthy, "Failed to get enough healthy responses from " + endpoint.url()); + + return new Availability(Status.available, "Endpoints are ready"); + } + + /** Returns the IP address of the given host name, if any. */ + private static Optional<InetAddress> resolveHostName(DomainName hostname) { + try { + return Optional.of(InetAddress.getByName(hostname.value())); + } + catch (UnknownHostException ignored) { + return Optional.empty(); + } + } + + /** Returns the host name of the given CNAME, if any. */ + private static Optional<DomainName> resolveCname(DomainName endpoint) { + try { + InitialDirContext ctx = new InitialDirContext(); + try { + Attributes attrs = ctx.getAttributes("dns:/" + endpoint.value(), new String[]{ "CNAME" }); + for (Attribute attribute : Collections.list(attrs.getAll())) { + Enumeration<?> vals = attribute.getAll(); + if (vals.hasMoreElements()) { + String hostname = vals.nextElement().toString(); + return Optional.of(hostname.substring(0, hostname.length() - 1)).map(DomainName::of); + } + } + } + finally { + ctx.close(); + } + } + catch (NamingException e) { + throw new RuntimeException(e); + } + return Optional.empty(); + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java index 10e22f8df06..09b71c6a982 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java @@ -8,9 +8,6 @@ import java.util.Objects; /** * Settings for a zone endpoint of a deployment. * - * TODO: Fix isEmpty - * Inline empty and constructor - * * @author jonmv */ public class ZoneEndpoint { @@ -21,10 +18,6 @@ public class ZoneEndpoint { private final boolean isPrivateEndpoint; private final List<AllowedUrn> allowedUrns; - public ZoneEndpoint(List<String> allowedUrns) { - this(true, true, allowedUrns.stream().map(arn -> new AllowedUrn(AccessType.awsPrivateLink, arn)).toList()); - } - public ZoneEndpoint(boolean isPublicEndpoint, boolean isPrivateEndpoint, List<AllowedUrn> allowedUrns) { if ( ! allowedUrns.isEmpty() && ! isPrivateEndpoint) throw new IllegalArgumentException("cannot list allowed urns, without also enabling private visibility"); diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java index 3404d7ed55e..5e30e8fa99c 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java @@ -9,6 +9,8 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NetworkPorts; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ZoneEndpoint; +import com.yahoo.config.provision.ZoneEndpoint.AccessType; +import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -68,7 +70,7 @@ public class AllocatedHostsSerializerTest { bigSlowDiskSpeedNode, anyDiskSpeedNode, ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), - Optional.empty(), new ZoneEndpoint(List.of("burn"))), + Optional.empty(), new ZoneEndpoint(true, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "burn")))), Optional.empty(), Optional.empty(), Optional.empty())); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index ca06fe202d9..edcffcca878 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.config.server; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Query; -import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; @@ -17,6 +17,9 @@ import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.EndpointsChecker; +import com.yahoo.config.provision.EndpointsChecker.Availability; +import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.InfraDeployer; @@ -57,6 +60,8 @@ import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.deploy.Deployment; import com.yahoo.vespa.config.server.deploy.InfraDeployerProvider; import com.yahoo.vespa.config.server.filedistribution.FileDirectory; +import com.yahoo.vespa.config.server.http.HttpFetcher; +import com.yahoo.vespa.config.server.http.HttpFetcher.Params; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.SecretStoreValidator; @@ -87,6 +92,8 @@ import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.Orchestrator; +import org.apache.hc.client5.http.ssl.DefaultHostnameVerifier; + import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -112,6 +119,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static ai.vespa.http.HttpURL.Path.parse; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; @@ -141,6 +149,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final Optional<InfraDeployer> infraDeployer; private final ConfigConvergenceChecker convergeChecker; private final HttpProxy httpProxy; + private final EndpointsChecker endpointsChecker; private final Clock clock; private final ConfigserverConfig configserverConfig; private final FileDistributionStatus fileDistributionStatus = new FileDistributionStatus(); @@ -169,6 +178,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye infraDeployerProvider.getInfraDeployer(), configConvergenceChecker, httpProxy, + createEndpointsChecker(), configserverConfig, orchestrator, new LogRetriever(), @@ -185,6 +195,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Optional<InfraDeployer> infraDeployer, ConfigConvergenceChecker configConvergenceChecker, HttpProxy httpProxy, + EndpointsChecker endpointsChecker, ConfigserverConfig configserverConfig, Orchestrator orchestrator, LogRetriever logRetriever, @@ -199,6 +210,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.infraDeployer = Objects.requireNonNull(infraDeployer); this.convergeChecker = Objects.requireNonNull(configConvergenceChecker); this.httpProxy = Objects.requireNonNull(httpProxy); + this.endpointsChecker = Objects.requireNonNull(endpointsChecker); this.configserverConfig = Objects.requireNonNull(configserverConfig); this.orchestrator = Objects.requireNonNull(orchestrator); this.logRetriever = Objects.requireNonNull(logRetriever); @@ -215,6 +227,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private TenantRepository tenantRepository; private Optional<Provisioner> hostProvisioner; private HttpProxy httpProxy = new HttpProxy(new SimpleHttpFetcher(Duration.ofSeconds(30))); + private EndpointsChecker endpointsChecker = __ -> { throw new UnsupportedOperationException(); }; private Clock clock = Clock.systemUTC(); private ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder().build(); private Orchestrator orchestrator; @@ -292,12 +305,18 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return this; } + public Builder withEndpointsChecker(EndpointsChecker endpointsChecker) { + this.endpointsChecker = endpointsChecker; + return this; + } + public ApplicationRepository build() { return new ApplicationRepository(tenantRepository, hostProvisioner, InfraDeployerProvider.empty().getInfraDeployer(), configConvergenceChecker, httpProxy, + endpointsChecker, configserverConfig, orchestrator, logRetriever, @@ -730,6 +749,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public ConfigConvergenceChecker configConvergenceChecker() { return convergeChecker; } + public Availability verifyEndpoints(List<Endpoint> endpoints) { + return endpointsChecker.endpointsAvailable(endpoints); + } + // ---------------- Logs ---------------------------------------------------------------- public HttpResponse getLogs(ApplicationId applicationId, Optional<DomainName> hostname, String apiParams) { @@ -1211,4 +1234,24 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } + private static EndpointsChecker createEndpointsChecker() { + HttpFetcher fetcher = new SimpleHttpFetcher(Duration.ofSeconds(10), new DefaultHostnameVerifier()::verify); + return EndpointsChecker.of(endpoint -> { + int remainingFailures = 3; + int remainingSuccesses = 100; + while (remainingSuccesses > 0 && remainingFailures > 0) { + try { + HttpResponse response = fetcher.get(new Params(3000), + endpoint.url().withPath(parse("/status.html")).asURI()); + if (response.getStatus() == 200) remainingSuccesses--; + else remainingFailures--; + } + catch (Exception e) { + remainingFailures--; + } + } + return remainingSuccesses == 0; + }); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java index 0aa86ab211a..22236281a93 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java @@ -27,6 +27,7 @@ import java.util.List; import static java.nio.charset.StandardCharsets.UTF_8; public class HttpProxy { + private final HttpFetcher fetcher; @Inject public HttpProxy(NodeHostnameVerifier verifier) { this(new SimpleHttpFetcher(Duration.ofSeconds(30), verifier)); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java index 5b332d3f434..a7f4ef5d513 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java @@ -19,6 +19,7 @@ import java.util.logging.Level; import java.util.logging.Logger; public class SimpleHttpFetcher implements HttpFetcher { + private static final Logger logger = Logger.getLogger(SimpleHttpFetcher.class.getName()); private final CloseableHttpClient client; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 68d0b81dc2b..9619ad69b3c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -10,6 +10,9 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.EndpointsChecker.Availability; +import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Zone; @@ -20,7 +23,10 @@ import com.yahoo.jdisc.Response; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.server.ApplicationRepository; @@ -43,9 +49,12 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import java.io.IOException; import java.io.UncheckedIOException; +import java.net.InetAddress; import java.net.URI; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -104,6 +113,7 @@ public class ApplicationHandler extends HttpHandler { public HttpResponse handlePOST(HttpRequest request) { Path path = new Path(request.getUri()); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/verify-endpoints")) return verifyEndpoints(request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindex")) return triggerReindexing(applicationId(path), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return enableReindexing(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/restart")) return restart(applicationId(path), request); @@ -322,6 +332,32 @@ public class ApplicationHandler extends HttpHandler { return new MessageResponse("Success"); } + private HttpResponse verifyEndpoints(HttpRequest request) { + byte[] data = uncheck(() -> request.getData().readAllBytes()); + List<Endpoint> endpoints = new ArrayList<>(); + SlimeUtils.jsonToSlime(data).get() + .field("endpoints") + .traverse((ArrayTraverser) (__, endpointObject) -> { + endpoints.add(new Endpoint(ClusterSpec.Id.from(endpointObject.field("clusterName").asString()), + HttpURL.from(URI.create(endpointObject.field("url").asString())), + SlimeUtils.optionalString(endpointObject.field("ipAddress")).map(uncheck(InetAddress::getByName)), + SlimeUtils.optionalString(endpointObject.field("canonicalName")).map(DomainName::of), + endpointObject.field("public").asBool())); + }); + if (endpoints.isEmpty()) throw new IllegalArgumentException("No endpoints in request " + request); + + Availability availability = applicationRepository.verifyEndpoints(endpoints); + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("status", switch (availability.status()) { + case available -> "available"; + case endpointsUnavailable -> "endpointsUnavailable"; + case containersUnhealthy -> "containersUnhealthy"; + }); + root.setString("message", availability.message()); + return new SlimeJsonResponse(slime); + } + private HttpResponse testerStartTests(ApplicationId applicationId, String suite, HttpRequest request) { byte[] data; try { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 3925899e1cd..c270b4559f9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; @@ -9,6 +10,10 @@ import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.EndpointsChecker; +import com.yahoo.config.provision.EndpointsChecker.Availability; +import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -53,6 +58,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; +import java.net.InetAddress; import java.net.URI; import java.net.URLEncoder; import java.time.Duration; @@ -60,6 +66,7 @@ import java.time.Instant; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -77,10 +84,12 @@ import static com.yahoo.vespa.config.server.http.HandlerTest.assertHttpStatusCod import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString; import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServiceListResponse; import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServiceResponse.createResponse; +import static com.yahoo.yolean.Exceptions.uncheck; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -107,6 +116,8 @@ public class ApplicationHandlerTest { private MockProvisioner provisioner; private OrchestratorMock orchestrator; private ManualClock clock; + private List<Endpoint> expectedEndpoints; + private Availability availability; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -139,6 +150,7 @@ public class ApplicationHandlerTest { .withLogRetriever(logRetriever) .withConfigserverConfig(configserverConfig) .withSecretStoreValidator(secretStoreValidator) + .withEndpointsChecker(endpoints -> { assertEquals(expectedEndpoints, endpoints); return availability; }) .build(); } @@ -504,6 +516,35 @@ public class ApplicationHandlerTest { } @Test + public void testVerifyEndpoints() { + expectedEndpoints = List.of(new Endpoint(ClusterSpec.Id.from("bluster"), + HttpURL.from(URI.create("https://bluster.tld:1234")), + Optional.of(uncheck(() -> InetAddress.getByName("4.3.2.1"))), + Optional.of(DomainName.of("fluster.tld")), + false)); + availability = new Availability(EndpointsChecker.Status.available, "Endpoints are ready"); + ApplicationHandler handler = createApplicationHandler(); + HttpRequest request = createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/verify-endpoints", + POST, + new ByteArrayInputStream(""" + { + "endpoints": [ + { + "clusterName": "bluster", + "url": "https://bluster.tld:1234", + "ipAddress": "4.3.2.1", + "canonicalName": "fluster.tld", + "public": false + } + ] + }""".getBytes(UTF_8))); + HttpResponse response = handler.handle(request); + assertEquals(200, response.getStatus()); + assertEquals("{\"status\":\"available\",\"message\":\"Endpoints are ready\"}", + new ByteArrayOutputStream() {{ uncheck(() -> response.render(this)); }}.toString(UTF_8)); + } + + @Test public void testClusterReindexingStateSerialization() { Stream.of(ClusterReindexing.State.values()).forEach(ClusterReindexing.State::toString); } @@ -592,12 +633,12 @@ public class ApplicationHandlerTest { hostAndPort, uri); assertResponse("{\n" + - " \"url\": \"" + uri.toString() + "\",\n" + - " \"host\": \"" + hostAndPort + "\",\n" + - " \"wantedGeneration\": 3,\n" + - " \"converged\": true,\n" + - " \"currentGeneration\": 3\n" + - "}", + " \"url\": \"" + uri.toString() + "\",\n" + + " \"host\": \"" + hostAndPort + "\",\n" + + " \"wantedGeneration\": 3,\n" + + " \"converged\": true,\n" + + " \"currentGeneration\": 3\n" + + "}", 200, response); } @@ -609,11 +650,11 @@ public class ApplicationHandlerTest { uri); assertResponse("{\n" + - " \"url\": \"" + uri.toString() + "\",\n" + - " \"host\": \"" + hostAndPort + "\",\n" + - " \"wantedGeneration\": 3,\n" + - " \"problem\": \"Host:port (service) no longer part of application, refetch list of services.\"\n" + - "}", + " \"url\": \"" + uri.toString() + "\",\n" + + " \"host\": \"" + hostAndPort + "\",\n" + + " \"wantedGeneration\": 3,\n" + + " \"problem\": \"Host:port (service) no longer part of application, refetch list of services.\"\n" + + "}", 410, response); } @@ -635,20 +676,20 @@ public class ApplicationHandlerTest { 3L), requestUrl); assertResponse("{\n" + - " \"services\": [\n" + - " {\n" + - " \"host\": \"" + hostname + "\",\n" + - " \"port\": " + port + ",\n" + - " \"type\": \"container\",\n" + - " \"url\": \"" + serviceUrl.toString() + "\",\n" + - " \"currentGeneration\":" + 3 + "\n" + - " }\n" + - " ],\n" + - " \"url\": \"" + requestUrl.toString() + "\",\n" + - " \"currentGeneration\": 3,\n" + - " \"wantedGeneration\": 3,\n" + - " \"converged\": true\n" + - "}", + " \"services\": [\n" + + " {\n" + + " \"host\": \"" + hostname + "\",\n" + + " \"port\": " + port + ",\n" + + " \"type\": \"container\",\n" + + " \"url\": \"" + serviceUrl.toString() + "\",\n" + + " \"currentGeneration\":" + 3 + "\n" + + " }\n" + + " ],\n" + + " \"url\": \"" + requestUrl.toString() + "\",\n" + + " \"currentGeneration\": 3,\n" + + " \"wantedGeneration\": 3,\n" + + " \"converged\": true\n" + + "}", 200, response); } @@ -669,27 +710,27 @@ public class ApplicationHandlerTest { 3L), requestUrl); assertResponse("{\n" + - " \"services\": [\n" + - " {\n" + - " \"host\": \"" + hostname + "\",\n" + - " \"port\": " + port + ",\n" + - " \"type\": \"container\",\n" + - " \"url\": \"" + serviceUrl.toString() + "\",\n" + - " \"currentGeneration\":" + 4 + "\n" + - " },\n" + - " {\n" + - " \"host\": \"" + hostname2 + "\",\n" + - " \"port\": " + port2 + ",\n" + - " \"type\": \"container\",\n" + - " \"url\": \"" + serviceUrl2.toString() + "\",\n" + - " \"currentGeneration\":" + 3 + "\n" + - " }\n" + - " ],\n" + - " \"url\": \"" + requestUrl.toString() + "\",\n" + - " \"currentGeneration\": 3,\n" + - " \"wantedGeneration\": 4,\n" + - " \"converged\": false\n" + - "}", + " \"services\": [\n" + + " {\n" + + " \"host\": \"" + hostname + "\",\n" + + " \"port\": " + port + ",\n" + + " \"type\": \"container\",\n" + + " \"url\": \"" + serviceUrl.toString() + "\",\n" + + " \"currentGeneration\":" + 4 + "\n" + + " },\n" + + " {\n" + + " \"host\": \"" + hostname2 + "\",\n" + + " \"port\": " + port2 + ",\n" + + " \"type\": \"container\",\n" + + " \"url\": \"" + serviceUrl2.toString() + "\",\n" + + " \"currentGeneration\":" + 3 + "\n" + + " }\n" + + " ],\n" + + " \"url\": \"" + requestUrl.toString() + "\",\n" + + " \"currentGeneration\": 3,\n" + + " \"wantedGeneration\": 4,\n" + + " \"converged\": false\n" + + "}", 200, response); } @@ -707,11 +748,11 @@ public class ApplicationHandlerTest { uri); assertResponse("{\n" + - " \"url\": \"" + uri.toString() + "\",\n" + - " \"host\": \"" + hostAndPort + "\",\n" + - " \"wantedGeneration\": 3,\n" + - " \"error\": \"some error message\"" + - "}", + " \"url\": \"" + uri + "\",\n" + + " \"host\": \"" + hostAndPort + "\",\n" + + " \"wantedGeneration\": 3,\n" + + " \"error\": \"some error message\"" + + "}", 404, response); } diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index 90194f3ba6a..92ce6abb319 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -15,6 +15,9 @@ import java.util.TreeSet; import static com.yahoo.text.Lowercase.toLowerCase; /** + * NOTE: We are in the process of moving the functionality of this over to {@link com.yahoo.search.schema.SchemaInfo} - + * see if you can use that before adding usage of this. + * * A central repository for information about indices. Standard usage is * * <pre><code> @@ -24,9 +27,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; * * @author Steinar Knutsen */ -// TODO: We should replace this with a better representation of search definitions -// which is immutable, models clusters and search definitions inside clusters properly, -// and uses better names. -bratseth +// TODO: Complete migration to SchemaInfo public class IndexFacts { private Map<String, List<String>> clusterByDocument; diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index e8ea6f88fce..0c65f4bf8dc 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -475,7 +475,7 @@ public class SelectParser implements Parser { Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); String field = children.get(0).asString(); String property = children.get(1).asString(); - NearestNeighborItem item = new NearestNeighborItem(field, property); + NearestNeighborItem item = new NearestNeighborItem(indexFactsSession.getCanonicName(field), property); Inspector annotations = getAnnotations(value); if (annotations != null){ annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 76ceb60b3d5..639e5b592c3 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -448,7 +448,7 @@ public class YqlParser implements Parser { Preconditions.checkArgument(args.size() == 2, "Expected 2 arguments, got %s.", args.size()); String field = fetchFieldName(args.get(0)); String property = fetchLiteral(args.get(1)); - NearestNeighborItem item = new NearestNeighborItem(field, property); + NearestNeighborItem item = new NearestNeighborItem(indexFactsSession.getCanonicName(field), property); Integer targetNumHits = getAnnotation(ast, TARGET_HITS, Integer.class, null, "desired minimum hits to produce"); if (targetNumHits == null) { diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java index 9a7e3915d19..e85280914d8 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java @@ -63,12 +63,12 @@ public class ValidateNearestNeighborTestCase { )); } - private static TensorType tt_dense_dvector_42 = TensorType.fromSpec("tensor(x[42])"); - private static TensorType tt_dense_dvector_3 = TensorType.fromSpec("tensor(x[3])"); - private static TensorType tt_dense_dvector_2 = TensorType.fromSpec("tensor(x[2])"); - private static TensorType tt_dense_fvector_3 = TensorType.fromSpec("tensor<float>(x[3])"); - private static TensorType tt_dense_matrix_xy = TensorType.fromSpec("tensor(x[3],y[1])"); - private static TensorType tt_sparse_vector_x = TensorType.fromSpec("tensor(x{})"); + private static final TensorType tt_dense_dvector_42 = TensorType.fromSpec("tensor(x[42])"); + private static final TensorType tt_dense_dvector_3 = TensorType.fromSpec("tensor(x[3])"); + private static final TensorType tt_dense_dvector_2 = TensorType.fromSpec("tensor(x[2])"); + private static final TensorType tt_dense_fvector_3 = TensorType.fromSpec("tensor<float>(x[3])"); + private static final TensorType tt_dense_matrix_xy = TensorType.fromSpec("tensor(x[3],y[1])"); + private static final TensorType tt_sparse_vector_x = TensorType.fromSpec("tensor(x{})"); private Tensor makeTensor(TensorType tensorType) { return makeTensor(tensorType, 3); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index c15c3b2c5ea..bc9448f39d8 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -15,6 +15,7 @@ import com.yahoo.prelude.query.FuzzyItem; import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.MarkerWordItem; +import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.PhraseSegmentItem; import com.yahoo.prelude.query.PrefixItem; @@ -990,21 +991,27 @@ public class YqlParserTestCase { .name("music") .command(new Command.Builder().indexname("title").command("index")) .command(new Command.Builder().indexname("year").command("attribute")) + .command(new Command.Builder().indexname("embedding").command("attribute")) .alias(new Alias.Builder().alias("song").indexname("title")) - .alias(new Alias.Builder().alias("from").indexname("year")))); + .alias(new Alias.Builder().alias("from").indexname("year")) + .alias(new Alias.Builder().alias("vector").indexname("embedding")))); IndexModel model = new IndexModel(modelConfig, (QrSearchersConfig) null); IndexFacts indexFacts = new IndexFacts(model); ParserEnvironment parserEnvironment = new ParserEnvironment().setIndexFacts(indexFacts); YqlParser configuredParser = new YqlParser(parserEnvironment); - QueryTree x = configuredParser.parse(new Parsable() - .setQuery("select * from sources * where title contains \"a\" and song contains \"b\" order by \"from\"")); - List<IndexedItem> terms = QueryTree.getPositiveTerms(x); + QueryTree query = configuredParser.parse(new Parsable() + .setQuery("select * from sources * where title contains \"a\" and song contains \"b\"" + + "and nearestNeighbor(vector, queryVector)" + + "order by \"from\"")); + List<IndexedItem> terms = QueryTree.getPositiveTerms(query); assertEquals(2, terms.size()); for (IndexedItem term : terms) assertEquals("title", term.getIndexName()); assertEquals(1, configuredParser.getSorting().fieldOrders().size()); assertEquals("year", configuredParser.getSorting().fieldOrders().get(0).getFieldName()); + var nnItem = (NearestNeighborItem)((AndItem)query.getRoot()).getItem(2); + assertEquals("embedding", nnItem.getIndexName()); } @Test diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 887a1a3a033..10d3a5aeabe 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -546,7 +546,6 @@ public class SelectTestCase { } @Test - @SuppressWarnings("deprecation") void testWeakAnd() { assertParse("{ \"weakAnd\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }", "WEAKAND(100) a:A b:B"); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index cc05bc01d99..93ac16c606d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import ai.vespa.http.HttpURL.Query; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.EndpointsChecker.Availability; +import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.zone.ZoneId; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL.Path; @@ -145,6 +147,8 @@ public interface ConfigServer { Optional<TestReport> getTestReport(DeploymentId deployment); + Availability verifyEndpoints(DeploymentId deploymentId, List<Endpoint> zoneEndpoints); + /** Get maximum resources consumed */ QuotaUsage getQuotaUsage(DeploymentId deploymentId); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java index 26330f11d65..0c81fbd3670 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java @@ -8,74 +8,30 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import java.util.List; -import java.util.Objects; import java.util.Optional; +import static java.util.Objects.requireNonNull; + /** * Represents an exclusive load balancer, assigned to an application's cluster. * * @author mortent */ -public class LoadBalancer { - - private final String id; - private final ApplicationId application; - private final ClusterSpec.Id cluster; - private final Optional<DomainName> hostname; - private final Optional<String> ipAddress; - private final State state; - private final Optional<String> dnsZone; - private final Optional<CloudAccount> cloudAccount; - private final Optional<PrivateServiceInfo> service; - - public LoadBalancer(String id, ApplicationId application, ClusterSpec.Id cluster, Optional<DomainName> hostname, - Optional<String> ipAddress, State state, Optional<String> dnsZone, - Optional<CloudAccount> cloudAccount, Optional<PrivateServiceInfo> service) { - this.id = Objects.requireNonNull(id, "id must be non-null"); - this.application = Objects.requireNonNull(application, "application must be non-null"); - this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); - this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); - this.ipAddress = Objects.requireNonNull(ipAddress, "ipAddress must be non-null"); - this.state = Objects.requireNonNull(state, "state must be non-null"); - this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); - this.cloudAccount = Objects.requireNonNull(cloudAccount, "cloudAccount must be non-null"); - this.service = Objects.requireNonNull(service, "service must be non-null"); - } - - public String id() { - return id; - } - - public ApplicationId application() { - return application; - } - - public ClusterSpec.Id cluster() { - return cluster; - } - - public Optional<DomainName> hostname() { - return hostname; - } - - public Optional<String> ipAddress() { - return ipAddress; - } - - public Optional<String> dnsZone() { - return dnsZone; - } - - public State state() { - return state; - } - - public Optional<CloudAccount> cloudAccount() { - return cloudAccount; - } - - public Optional<PrivateServiceInfo> service() { - return service; +public record LoadBalancer(String id, ApplicationId application, ClusterSpec.Id cluster, + Optional<DomainName> hostname, Optional<String> ipAddress, + State state, Optional<String> dnsZone, Optional<CloudAccount> cloudAccount, + Optional<PrivateServiceInfo> service, boolean isPublic) { + + public LoadBalancer { + requireNonNull(id, "id must be non-null"); + requireNonNull(application, "application must be non-null"); + requireNonNull(cluster, "cluster must be non-null"); + requireNonNull(hostname, "hostname must be non-null"); + requireNonNull(ipAddress, "ipAddress must be non-null"); + requireNonNull(state, "state must be non-null"); + requireNonNull(dnsZone, "dnsZone must be non-null"); + requireNonNull(cloudAccount, "cloudAccount must be non-null"); + requireNonNull(service, "service must be non-null"); } public enum State { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java index b4d9dd49880..4095e4b03fd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; -import ai.vespa.http.DomainName; -import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.EndpointsChecker.Endpoint; +import com.yahoo.config.provision.EndpointsChecker.Availability; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; -import java.net.InetAddress; import java.net.URI; import java.util.List; import java.util.Optional; @@ -27,17 +26,10 @@ public interface TesterCloud { /** Returns the current status of the tester. */ Status getStatus(DeploymentId deploymentId); - /** Returns whether the container is ready to serve. */ - boolean ready(URI endpointUrl); - /** Returns whether the test container is ready to serve */ boolean testerReady(DeploymentId deploymentId); - /** Returns the IP address of the given host name, if any. */ - Optional<InetAddress> resolveHostName(DomainName hostname); - - /** Returns the host name of the given CNAME, if any. */ - Optional<DomainName> resolveCname(DomainName hostName); + Availability verifyEndpoints(DeploymentId deploymentId, List<Endpoint> endpoints); /** Returns the test report as JSON if available */ Optional<TestReport> getTestReport(DeploymentId deploymentId); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java index 939c74fe61d..e29e8086c80 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java @@ -3,6 +3,9 @@ package com.yahoo.vespa.hosted.controller.api.integration.stubs; import ai.vespa.http.DomainName; import com.google.common.net.InetAddresses; +import com.yahoo.config.provision.EndpointsChecker; +import com.yahoo.config.provision.EndpointsChecker.Endpoint; +import com.yahoo.config.provision.EndpointsChecker.Availability; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; @@ -16,7 +19,6 @@ import java.net.URI; import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.NOT_STARTED; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.RUNNING; @@ -24,6 +26,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Teste public class MockTesterCloud implements TesterCloud { private final NameService nameService; + private final EndpointsChecker endpointsChecker = EndpointsChecker.mock(this::resolveHostName, this::resolveCname, __ -> true); private List<LogEntry> log = new ArrayList<>(); private Status status = NOT_STARTED; @@ -49,25 +52,23 @@ public class MockTesterCloud implements TesterCloud { public Status getStatus(DeploymentId deploymentId) { return status; } @Override - public boolean ready(URI testerUrl) { + public boolean testerReady(DeploymentId deploymentId) { return true; } @Override - public boolean testerReady(DeploymentId deploymentId) { - return true; + public Availability verifyEndpoints(DeploymentId deploymentId, List<Endpoint> endpoints) { + return endpointsChecker.endpointsAvailable(endpoints); } - @Override - public Optional<InetAddress> resolveHostName(DomainName hostname) { + private Optional<InetAddress> resolveHostName(DomainName hostname) { return nameService.findRecords(Record.Type.A, RecordName.from(hostname.value())).stream() .findFirst() .map(record -> InetAddresses.forString(record.data().asString())) .or(() -> Optional.of(InetAddresses.forString("1.2.3.4"))); } - @Override - public Optional<DomainName> resolveCname(DomainName hostName) { + private Optional<DomainName> resolveCname(DomainName hostName) { return nameService.findRecords(Record.Type.CNAME, RecordName.from(hostName.value())).stream() .findFirst() .map(record -> DomainName.of(record.data().asString().substring(0, record.data().asString().length() - 1))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 7da61f9bc63..14f2b38f24a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -2,13 +2,16 @@ package com.yahoo.vespa.hosted.controller.deployment; import ai.vespa.http.DomainName; -import com.google.common.net.InetAddresses; +import ai.vespa.http.HttpURL; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.Notifications; import com.yahoo.config.application.api.Notifications.When; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.EndpointsChecker; +import com.yahoo.config.provision.EndpointsChecker.Availability; +import com.yahoo.config.provision.EndpointsChecker.Status; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -45,6 +48,7 @@ import com.yahoo.yolean.Exceptions; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.io.UncheckedIOException; +import java.net.InetAddress; import java.security.cert.CertificateExpiredException; import java.security.cert.CertificateNotYetValidException; import java.security.cert.X509Certificate; @@ -87,6 +91,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static com.yahoo.yolean.Exceptions.uncheck; import static java.util.Objects.requireNonNull; import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; @@ -275,7 +280,8 @@ public class InternalStepRunner implements StepRunner { switch (e.type()) { case CERT_NOT_AVAILABLE: // Same as CERTIFICATE_NOT_READY above, only from the controller - logger.log("Validating CA signed certificate requested for app: not yet available"); + logger.log("Creating a CA signed certificate for the application. " + + "This may take up to " + timeouts.endpointCertificate() + " on first deployment."); if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) { logger.log(WARNING, "CA signed certificate for app not available within " + timeouts.endpointCertificate() + ": " + Exceptions.toMessageString(e)); @@ -350,13 +356,13 @@ public class InternalStepRunner implements StepRunner { } if (summary.converged()) { controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); - if (endpointsAvailable(id.application(), id.type().zone(), logger)) { - if (containersAreUp(id.application(), id.type().zone(), logger)) { + Availability availability = endpointsAvailable(id.application(), id.type().zone(), logger); + if (availability.status() == Status.available) { logger.log("Installation succeeded!"); return Optional.of(running); - } } - else if (timedOut(id, deployment.get(), timeouts.endpoint())) { + logger.log(availability.message()); + if (availability.status() == Status.endpointsUnavailable && timedOut(id, deployment.get(), timeouts.endpoint())) { logger.log(WARNING, "Endpoints failed to show up within " + timeouts.endpoint().toMinutes() + " minutes!"); return Optional.of(error); } @@ -475,21 +481,6 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); } - /** Returns true iff all calls to endpoint in the deployment give 100 consecutive 200 OK responses on /status.html. */ - private boolean containersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { - var endpoints = controller.routing().readTestRunnerEndpointsOf(Set.of(new DeploymentId(id, zoneId))); - if ( ! endpoints.containsKey(zoneId)) - return false; - - return endpoints.get(zoneId).parallelStream().allMatch(endpoint -> { - boolean ready = controller.jobController().cloud().ready(endpoint.url()); - if (!ready) { - logger.log("Failed to get 100 consecutive OKs from " + endpoint); - } - return ready; - }); - } - /** Returns true iff all containers in the tester deployment give 100 consecutive 200 OK responses on /status.html. */ private boolean testerContainersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { DeploymentId deploymentId = new DeploymentId(id, zoneId); @@ -501,50 +492,25 @@ public class InternalStepRunner implements StepRunner { } } - private boolean endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { + private Availability endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { DeploymentId deployment = new DeploymentId(id, zone); Map<ZoneId, List<Endpoint>> endpoints = controller.routing().readTestRunnerEndpointsOf(Set.of(deployment)); - if ( ! endpoints.containsKey(zone)) { - logger.log("Endpoints not yet ready."); - return false; - } - for (var endpoint : endpoints.get(zone)) { - DomainName endpointName = DomainName.of(endpoint.dnsName()); - var ipAddress = controller.jobController().cloud().resolveHostName(endpointName); - if (ipAddress.isEmpty()) { - logger.log(INFO, "DNS lookup yielded no IP address for '" + endpointName + "'."); - return false; - } - DeploymentRoutingContext context = controller.routing().of(deployment); - if (context.routingMethod() == RoutingMethod.exclusive) { - RoutingPolicy policy = context.routingPolicy(ClusterSpec.Id.from(endpoint.name())) - .orElseThrow(() -> new IllegalStateException(endpoint + " has no matching policy")); - if (policy.ipAddress().isPresent()) { - if (ipAddress.equals(policy.ipAddress().map(InetAddresses::forString))) continue; - logger.log(INFO, "IP address of '" + endpointName + "' (" + - ipAddress.map(InetAddresses::toAddrString).get() + ") and load balancer " - + "' (" + policy.ipAddress().orElseThrow() + ") are not equal"); - return false; - } - - var cNameValue = controller.jobController().cloud().resolveCname(endpointName); - if ( ! cNameValue.map(policy.canonicalName().get()::equals).orElse(false)) { - logger.log(INFO, "CNAME '" + endpointName + "' points at " + - cNameValue.map(name -> "'" + name + "'").orElse("nothing") + - " but should point at load balancer '" + policy.canonicalName() + "'"); - return false; - } - var loadBalancerAddress = controller.jobController().cloud().resolveHostName(policy.canonicalName().get()); - if ( ! loadBalancerAddress.equals(ipAddress)) { - logger.log(INFO, "IP address of CNAME '" + endpointName + "' (" + ipAddress.get() + ") and load balancer '" + - policy.canonicalName().get() + "' (" + loadBalancerAddress.orElse(null) + ") are not equal"); - return false; - } - } - } - logEndpoints(endpoints, logger); - return true; + DeploymentRoutingContext context = controller.routing().of(deployment); + boolean resolveEndpoints = context.routingMethod() == RoutingMethod.exclusive; + return controller.serviceRegistry().testerCloud().verifyEndpoints( + deployment, + endpoints.getOrDefault(zone, List.of()) + .stream() + .map(endpoint -> { + ClusterSpec.Id cluster = ClusterSpec.Id.from(endpoint.name()); + RoutingPolicy policy = context.routingPolicy(cluster).get(); + return new EndpointsChecker.Endpoint(cluster, + HttpURL.from(endpoint.url()), + policy.ipAddress().filter(__ -> resolveEndpoints).map(uncheck(InetAddress::getByName)), + policy.canonicalName().filter(__ -> resolveEndpoints), + policy.isPublic()); + }).toList()); } private void logEndpoints(Map<ZoneId, List<Endpoint>> zoneEndpoints, DualLogger logger) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/AbstractNameServiceRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/AbstractNameServiceRequest.java new file mode 100644 index 00000000000..9d21f5b26bd --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/AbstractNameServiceRequest.java @@ -0,0 +1,33 @@ +package com.yahoo.vespa.hosted.controller.dns; + +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; + +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +/** + * @author jonmv + */ +public abstract class AbstractNameServiceRequest implements NameServiceRequest { + + private final Optional<TenantAndApplicationId> owner; + private final RecordName name; + + AbstractNameServiceRequest(Optional<TenantAndApplicationId> owner, RecordName name) { + this.owner = requireNonNull(owner); + this.name = requireNonNull(name); + } + + @Override + public RecordName name() { + return name; + } + + @Override + public Optional<TenantAndApplicationId> owner() { + return owner; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java index f1e4ca3b82b..6f4ee3dfc06 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java @@ -15,14 +15,13 @@ import java.util.Optional; * * @author mpolden */ -public class CreateRecord implements NameServiceRequest { +public class CreateRecord extends AbstractNameServiceRequest { - private final Optional<TenantAndApplicationId> owner; private final Record record; /** DO NOT USE. Public for serialization purposes */ public CreateRecord(Optional<TenantAndApplicationId> owner, Record record) { - this.owner = Objects.requireNonNull(owner, "owner must be non-null"); + super(owner, record.name()); this.record = Objects.requireNonNull(record, "record must be non-null"); if (record.type() != Record.Type.CNAME && record.type() != Record.Type.A) { throw new IllegalArgumentException("Record of type " + record.type() + " is not supported: " + record); @@ -34,16 +33,6 @@ public class CreateRecord implements NameServiceRequest { } @Override - public Optional<RecordName> name() { - return Optional.of(record.name()); - } - - @Override - public Optional<TenantAndApplicationId> owner() { - return owner; - } - - @Override public void dispatchTo(NameService nameService) { List<Record> records = nameService.findRecords(record.type(), record.name()); records.forEach(r -> { @@ -67,12 +56,12 @@ public class CreateRecord implements NameServiceRequest { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; CreateRecord that = (CreateRecord) o; - return owner.equals(that.owner) && record.equals(that.record); + return owner().equals(that.owner()) && record.equals(that.record); } @Override public int hashCode() { - return Objects.hash(owner, record); + return Objects.hash(owner(), record); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java index a668c408794..ef7b74a4d4b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java @@ -20,17 +20,14 @@ import java.util.stream.Collectors; * * @author mpolden */ -public class CreateRecords implements NameServiceRequest { +public class CreateRecords extends AbstractNameServiceRequest { - private final Optional<TenantAndApplicationId> owner; - private final RecordName name; private final Record.Type type; private final List<Record> records; /** DO NOT USE. Public for serialization purposes */ public CreateRecords(Optional<TenantAndApplicationId> owner, List<Record> records) { - this.owner = Objects.requireNonNull(owner, "owner must be non-null"); - this.name = requireOneOf(Record::name, records); + super(owner, requireOneOf(Record::name, records)); this.type = requireOneOf(Record::type, records); this.records = List.copyOf(Objects.requireNonNull(records, "records must be non-null")); if (type != Record.Type.ALIAS && type != Record.Type.TXT && type != Record.Type.DIRECT) { @@ -43,29 +40,19 @@ public class CreateRecords implements NameServiceRequest { } @Override - public Optional<RecordName> name() { - return Optional.of(name); - } - - @Override - public Optional<TenantAndApplicationId> owner() { - return owner; - } - - @Override public void dispatchTo(NameService nameService) { switch (type) { case ALIAS -> { var targets = records.stream().map(Record::data).map(AliasTarget::unpack).collect(Collectors.toSet()); - nameService.createAlias(name, targets); + nameService.createAlias(name(), targets); } case DIRECT -> { var targets = records.stream().map(Record::data).map(DirectTarget::unpack).collect(Collectors.toSet()); - nameService.createDirect(name, targets); + nameService.createDirect(name(), targets); } case TXT -> { var dataFields = records.stream().map(Record::data).toList(); - nameService.createTxtRecords(name, dataFields); + nameService.createTxtRecords(name(), dataFields); } } } @@ -80,12 +67,12 @@ public class CreateRecords implements NameServiceRequest { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; CreateRecords that = (CreateRecords) o; - return owner.equals(that.owner) && records.equals(that.records); + return owner().equals(that.owner()) && records.equals(that.records); } @Override public int hashCode() { - return Objects.hash(owner, records); + return Objects.hash(owner(), records); } /** Find exactly one distinct value of field in given list */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java index c43130ce4e9..dd3cca9a4fa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java @@ -14,7 +14,8 @@ import java.util.Optional; */ public interface NameServiceRequest { - Optional<RecordName> name(); + /** The record name this request pertains to. */ + RecordName name(); /** The application owning this request */ Optional<TenantAndApplicationId> owner(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java index 2ff2edf11f4..273136ba0a6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java @@ -21,11 +21,9 @@ import java.util.Optional; * * @author mpolden */ -public class RemoveRecords implements NameServiceRequest { +public class RemoveRecords extends AbstractNameServiceRequest { - private final Optional<TenantAndApplicationId> owner; private final Record.Type type; - private final RecordName name; private final Optional<RecordData> data; public RemoveRecords(Optional<TenantAndApplicationId> owner, Record.Type type, RecordName name) { @@ -38,9 +36,8 @@ public class RemoveRecords implements NameServiceRequest { /** DO NOT USE. Public for serialization purposes */ public RemoveRecords(Optional<TenantAndApplicationId> owner, Record.Type type, RecordName name, Optional<RecordData> data) { - this.owner = Objects.requireNonNull(owner, "owner must be non-null"); + super(owner, name); this.type = Objects.requireNonNull(type, "type must be non-null"); - this.name = Objects.requireNonNull(name, "name must be non-null"); this.data = Objects.requireNonNull(data, "data must be non-null"); } @@ -48,16 +45,6 @@ public class RemoveRecords implements NameServiceRequest { return type; } - @Override - public Optional<RecordName> name() { - return Optional.of(name); - } - - @Override - public Optional<TenantAndApplicationId> owner() { - return owner; - } - public Optional<RecordData> data() { return data; } @@ -66,7 +53,7 @@ public class RemoveRecords implements NameServiceRequest { public void dispatchTo(NameService nameService) { // Deletions require all records fields to match exactly, data may be incomplete even if present. To ensure // completeness we search for the record(s) first - List<Record> completeRecords = nameService.findRecords(type, name).stream() + List<Record> completeRecords = nameService.findRecords(type, name()).stream() .filter(record -> data.isEmpty() || matchingFqdnIn(data.get(), record)) .toList(); nameService.removeRecords(completeRecords); @@ -74,7 +61,7 @@ public class RemoveRecords implements NameServiceRequest { @Override public String toString() { - return "remove records of type " + type + ", by name " + name + + return "remove records of type " + type + ", by name " + name() + data.map(d -> " data " + d).orElse(""); } @@ -83,12 +70,12 @@ public class RemoveRecords implements NameServiceRequest { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RemoveRecords that = (RemoveRecords) o; - return owner.equals(that.owner) && type == that.type && name.equals(that.name) && data.equals(that.data); + return owner().equals(that.owner()) && type == that.type && name().equals(that.name()) && data.equals(that.data); } @Override public int hashCode() { - return Objects.hash(owner, type, name, data); + return Objects.hash(owner(), type, name(), data); } private static boolean matchingFqdnIn(RecordData data, Record record) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java index 89230969164..d02d27b5293 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java @@ -92,7 +92,7 @@ public class NameServiceQueueSerializer { private void toSlime(Cursor object, RemoveRecords removeRecords) { object.setString(requestType, Request.removeRecords.name()); object.setString(typeField, removeRecords.type().name()); - removeRecords.name().ifPresent(name -> object.setString(nameField, name.asString())); + object.setString(nameField, removeRecords.name().asString()); removeRecords.data().ifPresent(data -> object.setString(dataField, data.asString())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 4d759056dfc..47b27aac79a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -50,6 +50,7 @@ public class RoutingPolicySerializer { private static final String agentField = "agent"; private static final String changedAtField = "changedAt"; private static final String statusField = "status"; + private static final String privateOnlyField = "private"; public Slime toSlime(List<RoutingPolicy> routingPolicies) { var slime = new Slime(); @@ -68,6 +69,7 @@ public class RoutingPolicySerializer { policy.applicationEndpoints().forEach(endpointId -> applicationEndpointsArray.addString(endpointId.id())); policyObject.setBool(loadBalancerActiveField, policy.status().isActive()); globalRoutingToSlime(policy.status().routingStatus(), policyObject.setObject(globalRoutingField)); + if ( ! policy.isPublic()) policyObject.setBool(privateOnlyField, true); }); return slime; } @@ -84,6 +86,7 @@ public class RoutingPolicySerializer { RoutingPolicyId id = new RoutingPolicyId(owner, ClusterSpec.Id.from(inspect.field(clusterField).asString()), ZoneId.from(inspect.field(zoneField).asString())); + boolean isPublic = ! inspect.field(privateOnlyField).asBool(); policies.add(new RoutingPolicy(id, SlimeUtils.optionalString(inspect.field(canonicalNameField)).map(DomainName::of), SlimeUtils.optionalString(inspect.field(ipAddressField)), @@ -91,7 +94,8 @@ public class RoutingPolicySerializer { instanceEndpoints, applicationEndpoints, new RoutingPolicy.Status(inspect.field(loadBalancerActiveField).asBool(), - globalRoutingFromSlime(inspect.field(globalRoutingField))))); + globalRoutingFromSlime(inspect.field(globalRoutingField))), + isPublic)); }); return Collections.unmodifiableList(policies); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index c737f9b58ef..1c4916b9bed 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -361,7 +361,8 @@ public class RoutingPolicies { var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.ipAddress(), dnsZone, allocation.instanceEndpointsOf(loadBalancer), allocation.applicationEndpointsOf(loadBalancer), - new RoutingPolicy.Status(isActive(loadBalancer), RoutingStatus.DEFAULT)); + new RoutingPolicy.Status(isActive(loadBalancer), RoutingStatus.DEFAULT), + loadBalancer.isPublic()); // Preserve global routing status for existing policy if (existingPolicy != null) { newPolicy = newPolicy.with(newPolicy.status().with(existingPolicy.status().routingStatus())); @@ -399,7 +400,7 @@ public class RoutingPolicies { while (controller.clock().instant().isBefore(doom)) { try (Mutex lock = controller.curator().lockNameServiceQueue()) { if (controller.curator().readNameServiceQueue().requests().stream() - .noneMatch(request -> request.name().equals(Optional.of(challenge.name())))) { + .noneMatch(request -> request.name().equals(challenge.name()))) { try { challenge.trigger().run(); } finally { nameServiceForwarderIn(deploymentId.zoneId()).removeRecords(Type.TXT, challenge.name(), Priority.normal, ownerOf(deploymentId)); } return; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 3d43e42af27..38ecff452c8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.EndpointId; @@ -29,11 +28,12 @@ public record RoutingPolicy(RoutingPolicyId id, Optional<String> dnsZone, Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, - Status status) { + Status status, + boolean isPublic) { /** DO NOT USE. Public for serialization purposes */ public RoutingPolicy(RoutingPolicyId id, Optional<DomainName> canonicalName, Optional<String> ipAddress, Optional<String> dnsZone, - Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, Status status) { + Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, Status status, boolean isPublic) { this.id = Objects.requireNonNull(id, "id must be non-null"); this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); this.ipAddress = Objects.requireNonNull(ipAddress, "ipAddress must be non-null"); @@ -41,10 +41,15 @@ public record RoutingPolicy(RoutingPolicyId id, this.instanceEndpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(instanceEndpoints, "instanceEndpoints must be non-null")); this.applicationEndpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(applicationEndpoints, "applicationEndpoints must be non-null")); this.status = Objects.requireNonNull(status, "status must be non-null"); + this.isPublic = isPublic; if (canonicalName.isEmpty() == ipAddress.isEmpty()) throw new IllegalArgumentException("Exactly 1 of canonicalName=%s and ipAddress=%s must be set".formatted( canonicalName.map(DomainName::value).orElse("<empty>"), ipAddress.orElse("<empty>"))); + if ( ! instanceEndpoints.isEmpty() && ! isPublic) + throw new IllegalArgumentException("Non-public zone endpoint cannot be part of any global endpoint, but was in: " + instanceEndpoints); + if ( ! applicationEndpoints.isEmpty() && ! isPublic) + throw new IllegalArgumentException("Non-public zone endpoint cannot be part of any application endpoint, but was in: " + applicationEndpoints); } /** The ID of this */ @@ -82,6 +87,11 @@ public record RoutingPolicy(RoutingPolicyId id, return status; } + /** Returns whether this has a load balancer which is available from public internet. */ + public boolean isPublic() { + return isPublic; + } + /** Returns whether this policy applies to given deployment */ public boolean appliesTo(DeploymentId deployment) { return id.owner().equals(deployment.applicationId()) && @@ -90,7 +100,7 @@ public record RoutingPolicy(RoutingPolicyId id, /** Returns a copy of this with status set to given status */ public RoutingPolicy with(Status status) { - return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, status); + return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, status, isPublic); } /** Returns the zone endpoints of this */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 7edb458c154..95c22480c0f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -277,7 +277,8 @@ public class DeploymentContext { Optional.empty(), Set.of(EndpointId.of("default")), Set.of(), - new RoutingPolicy.Status(false, RoutingStatus.DEFAULT))); + new RoutingPolicy.Status(false, RoutingStatus.DEFAULT), + true)); tester.controller().curator().writeRoutingPolicies(instanceId, List.copyOf(policies.values())); return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 6f859ff3d15..5704af75cb9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -12,6 +12,8 @@ import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.EndpointsChecker.Availability; +import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; @@ -42,8 +44,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.QuotaUsage import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; +import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; @@ -82,6 +86,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; */ public class ConfigServerMock extends AbstractComponent implements ConfigServer { + private final MockTesterCloud mockTesterCloud; private final Map<DeploymentId, Application> applications = new LinkedHashMap<>(); private final Set<ZoneId> inactiveZones = new HashSet<>(); private final Map<DeploymentId, EndpointStatus> endpoints = new HashMap<>(); @@ -105,9 +110,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private Consumer<ApplicationId> prepareException = null; private Supplier<String> log = () -> "INFO - All good"; - @Inject - public ConfigServerMock(ZoneRegistryMock zoneRegistry) { + public ConfigServerMock(ZoneRegistryMock zoneRegistry, NameService nameService) { bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.notController()); + this.mockTesterCloud = new MockTesterCloud(nameService); } /** Assigns a reserved tenant node to the given deployment, with initial versions. */ @@ -370,8 +375,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer public Optional<TestReport> getTestReport(DeploymentId deployment) { return Optional.ofNullable(testReport.get(deployment)); } - public void setTestReport(DeploymentId deploymentId, TestReport report) { - testReport.put(deploymentId, report); + + @Override + public Availability verifyEndpoints(DeploymentId deploymentId, List<Endpoint> zoneEndpoints) { + return mockTesterCloud.verifyEndpoints(deploymentId, zoneEndpoints); // Wraps the same name service mock, which is updated by test harness. } /** Add any of given loadBalancers that do not already exist to the load balancers in zone */ @@ -419,7 +426,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer LoadBalancer.State.active, Optional.of("dns-zone-1"), Optional.empty(), - Optional.of(new PrivateServiceInfo("service", List.of(new AllowedUrn(AccessType.awsPrivateLink, "arne"))))))); + Optional.of(new PrivateServiceInfo("service", List.of(new AllowedUrn(AccessType.awsPrivateLink, "arne")))), + true))); } Application application = applications.get(id); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 382a697c4cd..0ba8866c990 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -100,8 +100,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); - this.configServerMock = new ConfigServerMock(zoneRegistryMock); - this.mockTesterCloud = new MockTesterCloud(nameService()); + this.configServerMock = new ConfigServerMock(zoneRegistryMock, memoryNameService); + this.mockTesterCloud = new MockTesterCloud(memoryNameService); this.clock.setInstant(Instant.ofEpochSecond(1600000000)); this.controllerVersion = new ControllerVersion(Version.fromString("6.1.0"), "badb01", clock.instant()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java index 582514163e1..ad9ebd2a968 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.CreateRecord; import com.yahoo.vespa.hosted.controller.dns.CreateRecords; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; +import com.yahoo.vespa.hosted.controller.dns.NameServiceRequest; import com.yahoo.vespa.hosted.controller.dns.RemoveRecords; import org.junit.jupiter.api.Test; @@ -32,7 +33,7 @@ public class NameServiceQueueSerializerTest { Optional<TenantAndApplicationId> owner = Optional.of(TenantAndApplicationId.from("t", "a")); var record1 = new Record(Record.Type.CNAME, RecordName.from("cname.example.com"), RecordData.from("example.com")); var record2 = new Record(Record.Type.TXT, RecordName.from("txt.example.com"), RecordData.from("text")); - var requests = List.of( + var requests = List.<NameServiceRequest>of( new CreateRecord(owner, record1), new CreateRecords(owner, List.of(record2)), new CreateRecords(owner, List.of(new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index 6285c5c4aac..8e0b1dd1d4e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -32,35 +32,38 @@ public class RoutingPolicySerializerTest { var instanceEndpoints = Set.of(EndpointId.of("r1"), EndpointId.of("r2")); var applicationEndpoints = Set.of(EndpointId.of("a1")); var id1 = new RoutingPolicyId(owner, - ClusterSpec.Id.from("my-cluster1"), - ZoneId.from("prod", "us-north-1")); + ClusterSpec.Id.from("my-cluster1"), + ZoneId.from("prod", "us-north-1")); var id2 = new RoutingPolicyId(owner, - ClusterSpec.Id.from("my-cluster2"), - ZoneId.from("prod", "us-north-2")); + ClusterSpec.Id.from("my-cluster2"), + ZoneId.from("prod", "us-north-2")); var policies = List.of(new RoutingPolicy(id1, - Optional.of(HostName.of("long-and-ugly-name")), - Optional.empty(), - Optional.of("zone1"), - instanceEndpoints, - applicationEndpoints, - new RoutingPolicy.Status(true, RoutingStatus.DEFAULT)), - new RoutingPolicy(id2, - Optional.of(HostName.of("long-and-ugly-name-2")), - Optional.empty(), - Optional.empty(), - instanceEndpoints, - Set.of(), - new RoutingPolicy.Status(false, - new RoutingStatus(RoutingStatus.Value.out, - RoutingStatus.Agent.tenant, - Instant.ofEpochSecond(123)))), - new RoutingPolicy(id1, - Optional.empty(), - Optional.of("127.0.0.1"), - Optional.of("zone2"), - instanceEndpoints, - applicationEndpoints, - new RoutingPolicy.Status(true, RoutingStatus.DEFAULT))); + Optional.of(HostName.of("long-and-ugly-name")), + Optional.empty(), + Optional.of("zone1"), + Set.of(), + Set.of(), + new RoutingPolicy.Status(true, RoutingStatus.DEFAULT), + false), + new RoutingPolicy(id2, + Optional.of(HostName.of("long-and-ugly-name-2")), + Optional.empty(), + Optional.empty(), + instanceEndpoints, + Set.of(), + new RoutingPolicy.Status(false, + new RoutingStatus(RoutingStatus.Value.out, + RoutingStatus.Agent.tenant, + Instant.ofEpochSecond(123))), + true), + new RoutingPolicy(id1, + Optional.empty(), + Optional.of("127.0.0.1"), + Optional.of("zone2"), + instanceEndpoints, + applicationEndpoints, + new RoutingPolicy.Status(true, RoutingStatus.DEFAULT), + true)); var serialized = serializer.fromSlime(owner, serializer.toSlime(policies)); assertEquals(policies.size(), serialized.size()); for (Iterator<RoutingPolicy> it1 = policies.iterator(), it2 = serialized.iterator(); it1.hasNext(); ) { @@ -73,6 +76,7 @@ public class RoutingPolicySerializerTest { assertEquals(expected.instanceEndpoints(), actual.instanceEndpoints()); assertEquals(expected.applicationEndpoints(), actual.applicationEndpoints()); assertEquals(expected.status(), actual.status()); + assertEquals(expected.isPublic(), actual.isPublic()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 03322d7c962..1efafb81685 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -364,8 +364,8 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { ControllerTester wrapped = new ControllerTester(tester); wrapped.upgradeSystem(Version.fromString("7.1")); new DeploymentTester(wrapped).newDeploymentContext(ApplicationId.from(tenantName, applicationName, InstanceName.defaultName())) - .submit() - .deploy(); + .submit() + .deploy(); tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), (response) -> assertFalse(response.getBodyAsString().contains("archiveAccessRole")), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 9c1253c7520..2932860efaa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -497,7 +497,8 @@ public class RoutingPoliciesTest { LoadBalancer.State.active, Optional.of("dns-zone-1"), Optional.empty(), - Optional.empty()); + Optional.empty(), + true); tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer)); // Application redeployment preserves DNS record @@ -952,7 +953,8 @@ public class RoutingPoliciesTest { LoadBalancer.State.active, Optional.of("dns-zone-1").filter(__ -> lbHostname.isPresent()), Optional.empty(), - Optional.empty())); + Optional.empty(), + true)); } return loadBalancers; } diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 6589e5bf1a3..bce3a7dc703 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -36,6 +36,7 @@ <target_jdk_version>17</target_jdk_version> <maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version> <maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version> + <maven-dependency-plugin.version>3.5.0</maven-dependency-plugin.version> <!-- NOTE: this must not be overriden by users, and must be in sync with junit version specified in 'tenant-cd-api' --> <vespa.junit.version>5.8.1</vespa.junit.version> <test.categories>!integration</test.categories> @@ -443,6 +444,15 @@ </profiles> <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>${maven-dependency-plugin.version}</version> + </plugin> + </plugins> + </pluginManagement> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java b/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java index 4f2bdfb213e..c5ebafb2425 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java +++ b/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java @@ -31,6 +31,7 @@ import static com.yahoo.security.tls.TransportSecurityUtils.isTransportSecurityE * @author jonmv */ public class VespaHttpClientBuilder { + private HttpClientConnectionManagerFactory connectionManagerFactory = PoolingHttpClientConnectionManager::new; private HostnameVerifier hostnameVerifier = new NoopHostnameVerifier(); private boolean rewriteHttpToHttps = true; diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index bc93ff58719..c6f41406f4f 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -18,16 +18,16 @@ org.apache.commons:commons-collections4:4.2 org.apache.commons:commons-compress:1.22 org.apache.commons:commons-lang3:3.12.0 org.apache.maven:maven-archiver:3.6.0 -org.apache.maven:maven-artifact:3.8.6 -org.apache.maven:maven-builder-support:3.8.6 -org.apache.maven:maven-core:3.8.6 -org.apache.maven:maven-model:3.8.6 -org.apache.maven:maven-model-builder:3.8.6 -org.apache.maven:maven-plugin-api:3.8.6 -org.apache.maven:maven-repository-metadata:3.8.6 -org.apache.maven:maven-resolver-provider:3.8.6 -org.apache.maven:maven-settings:3.8.6 -org.apache.maven:maven-settings-builder:3.8.6 +org.apache.maven:maven-artifact:3.8.7 +org.apache.maven:maven-builder-support:3.8.7 +org.apache.maven:maven-core:3.8.7 +org.apache.maven:maven-model:3.8.7 +org.apache.maven:maven-model-builder:3.8.7 +org.apache.maven:maven-plugin-api:3.8.7 +org.apache.maven:maven-repository-metadata:3.8.7 +org.apache.maven:maven-resolver-provider:3.8.7 +org.apache.maven:maven-settings:3.8.7 +org.apache.maven:maven-settings-builder:3.8.7 org.apache.maven.enforcer:enforcer-api:3.0.0 org.apache.maven.plugin-tools:maven-plugin-annotations:3.6.4 org.apache.maven.plugins:maven-shade-plugin:3.4.1 diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index c19aebcda6e..e74e3068674 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -62,8 +62,8 @@ public class LoadBalancerServiceMock implements LoadBalancerService { Collections.singleton(4443), ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"), spec.reals(), - spec.settings().orElse(ZoneEndpoint.defaultEndpoint), - spec.settings().map(__ -> PrivateServiceId.of("service")), + spec.settings(), + spec.settings().isPrivateEndpoint() ? Optional.of(PrivateServiceId.of("service")) : Optional.empty(), spec.cloudAccount()); instances.put(id, instance); return instance; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java index e0ef6739542..6be8bd71553 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java @@ -21,7 +21,7 @@ public class LoadBalancerSpec { private final ApplicationId application; private final ClusterSpec.Id cluster; private final Set<Real> reals; - private final Optional<ZoneEndpoint> settings; + private final ZoneEndpoint settings; private final CloudAccount cloudAccount; public LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, @@ -29,7 +29,7 @@ public class LoadBalancerSpec { this.application = Objects.requireNonNull(application); this.cluster = Objects.requireNonNull(cluster); this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals)); - this.settings = Optional.ofNullable(settings); + this.settings = Objects.requireNonNull(settings); this.cloudAccount = Objects.requireNonNull(cloudAccount); } @@ -49,7 +49,7 @@ public class LoadBalancerSpec { } /** Static user-configured settings for this load balancer. */ - public Optional<ZoneEndpoint> settings() { + public ZoneEndpoint settings() { return settings; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index 0722bb8e1dc..fa672c0fd1e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -29,13 +29,15 @@ public class SharedLoadBalancerService implements LoadBalancerService { @Override public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { + if ( ! spec.settings().isPublicEndpoint()) + throw new IllegalArgumentException("non-public endpoints is not supported with " + getClass()); return new LoadBalancerInstance(Optional.of(DomainName.of(vipHostname)), Optional.empty(), Optional.empty(), Set.of(4443), Set.of(), spec.reals(), - spec.settings().orElse(ZoneEndpoint.defaultEndpoint), + spec.settings(), Optional.empty(), spec.cloudAccount()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 92fdb1d2e52..fc03ddcc3b0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -81,7 +81,7 @@ public class LoadBalancerProvisioner { * Prepare a load balancer for given application and cluster. * <p> * If a load balancer for the cluster already exists, it will be reconfigured based on the currently allocated - * nodes. It's state will remain unchanged. + * nodes. Its state will remain unchanged. * <p> * If no load balancer exists, a new one will be provisioned in {@link LoadBalancer.State#reserved}. * <p> @@ -93,7 +93,7 @@ public class LoadBalancerProvisioner { ClusterSpec.Id clusterId = effectiveId(cluster); LoadBalancerId loadBalancerId = requireNonClashing(new LoadBalancerId(application, clusterId)); NodeList nodes = nodesOf(clusterId, application); - prepare(loadBalancerId, nodes, requestedNodes.cloudAccount()); + prepare(loadBalancerId, nodes, cluster.zoneEndpoint(), requestedNodes.cloudAccount()); } } @@ -189,10 +189,10 @@ public class LoadBalancerProvisioner { return loadBalancerId; } - private void prepare(LoadBalancerId id, NodeList nodes, CloudAccount cloudAccount) { + private void prepare(LoadBalancerId id, NodeList nodes, ZoneEndpoint zoneEndpoint, CloudAccount cloudAccount) { Instant now = nodeRepository.clock().instant(); Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); - Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, null, cloudAccount); + Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, zoneEndpoint, cloudAccount, true); LoadBalancer newLoadBalancer; LoadBalancer.State fromState = null; if (loadBalancer.isEmpty()) { @@ -206,9 +206,19 @@ public class LoadBalancerProvisioner { // provision a new one in the wanted account newLoadBalancer = newLoadBalancer.with(LoadBalancer.State.removable, now); } + if (!hasCorrectVisibility(newLoadBalancer, zoneEndpoint)) { + // We have a load balancer, but with the wrong visibility. Load balancer must be removed before we can + // provision a new one with the wanted visibility + newLoadBalancer = newLoadBalancer.with(LoadBalancer.State.removable, now); + } // Always store the load balancer. LoadBalancerExpirer will remove unwanted ones db.writeLoadBalancer(newLoadBalancer, fromState); - requireInstance(id, instance, cloudAccount); + requireInstance(id, newLoadBalancer, cloudAccount, zoneEndpoint); + } + + private static boolean hasCorrectVisibility(LoadBalancer newLoadBalancer, ZoneEndpoint zoneEndpoint) { + return newLoadBalancer.instance().isEmpty() + || newLoadBalancer.instance().get().settings().isPublicEndpoint() == zoneEndpoint.isPublicEndpoint(); } private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, ZoneEndpoint settings, NodeList nodes) { @@ -218,18 +228,19 @@ public class LoadBalancerProvisioner { if (loadBalancer.isEmpty()) throw new IllegalArgumentException("Could not activate load balancer that was never prepared: " + id); if (loadBalancer.get().instance().isEmpty()) throw new IllegalArgumentException("Activating " + id + ", but prepare never provisioned a load balancer instance"); - Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, settings, loadBalancer.get().instance().get().cloudAccount()); + Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, settings, loadBalancer.get().instance().get().cloudAccount(), false); LoadBalancer.State state = instance.isPresent() ? LoadBalancer.State.active : loadBalancer.get().state(); LoadBalancer newLoadBalancer = loadBalancer.get().with(instance).with(state, now); db.writeLoadBalancers(List.of(newLoadBalancer), loadBalancer.get().state(), transaction.nested()); - requireInstance(id, instance, loadBalancer.get().instance().get().cloudAccount()); + requireInstance(id, newLoadBalancer, loadBalancer.get().instance().get().cloudAccount(), settings); } /** Provision or reconfigure a load balancer instance, if necessary */ private Optional<LoadBalancerInstance> provisionInstance(LoadBalancerId id, NodeList nodes, Optional<LoadBalancer> currentLoadBalancer, ZoneEndpoint zoneEndpoint, - CloudAccount cloudAccount) { + CloudAccount cloudAccount, + boolean preparing) { boolean shouldDeactivateRouting = deactivateRouting.with(FetchVector.Dimension.APPLICATION_ID, id.application().serializedForm()) .value(); @@ -239,15 +250,15 @@ public class LoadBalancerProvisioner { } else { reals = realsOf(nodes); } - if (isUpToDate(currentLoadBalancer, reals, zoneEndpoint)) + if (isUpToDate(currentLoadBalancer, reals, zoneEndpoint, preparing)) return currentLoadBalancer.get().instance(); log.log(Level.INFO, () -> "Provisioning instance for " + id + ", targeting: " + reals); try { // Override settings at activation, otherwise keep existing ones. - ZoneEndpoint settings = zoneEndpoint != null ? zoneEndpoint - : currentLoadBalancer.flatMap(LoadBalancer::instance) - .map(LoadBalancerInstance::settings) - .orElse(null); + ZoneEndpoint settings = ! preparing ? zoneEndpoint + : currentLoadBalancer.flatMap(LoadBalancer::instance) + .map(LoadBalancerInstance::settings) + .orElse(zoneEndpoint); LoadBalancerInstance created = service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals, settings, cloudAccount), shouldDeactivateRouting || allowEmptyReals(currentLoadBalancer)); if (created.serviceId().isEmpty() && currentLoadBalancer.flatMap(LoadBalancer::instance).flatMap(LoadBalancerInstance::serviceId).isPresent()) @@ -307,12 +318,12 @@ public class LoadBalancerProvisioner { return loadBalancer.instance().isEmpty() || loadBalancer.instance().get().cloudAccount().equals(cloudAccount); } - /** Returns whether load balancer has given reals, and settings if specified*/ - private static boolean isUpToDate(Optional<LoadBalancer> loadBalancer, Set<Real> reals, ZoneEndpoint zoneEndpoint) { + /** Returns whether load balancer has given reals, and settings if specified */ + private static boolean isUpToDate(Optional<LoadBalancer> loadBalancer, Set<Real> reals, ZoneEndpoint zoneEndpoint, boolean preparing) { if (loadBalancer.isEmpty()) return false; if (loadBalancer.get().instance().isEmpty()) return false; return loadBalancer.get().instance().get().reals().equals(reals) - && (zoneEndpoint == null || loadBalancer.get().instance().get().settings().equals(zoneEndpoint)); + && (preparing || loadBalancer.get().instance().get().settings().equals(zoneEndpoint)); } /** Returns whether to allow given load balancer to have no reals */ @@ -331,14 +342,17 @@ public class LoadBalancerProvisioner { return reachable; } - private static void requireInstance(LoadBalancerId id, Optional<LoadBalancerInstance> instance, CloudAccount cloudAccount) { - if (instance.isEmpty()) { + private static void requireInstance(LoadBalancerId id, LoadBalancer loadBalancer, CloudAccount cloudAccount, ZoneEndpoint zoneEndpoint) { + if (loadBalancer.instance().isEmpty()) { // Signal that load balancer is not ready yet throw new LoadBalancerServiceException("Could not (re)configure " + id + ". The operation will be retried on next deployment"); } - if (!instance.get().cloudAccount().equals(cloudAccount)) { + if (!inAccount(cloudAccount, loadBalancer)) { throw new LoadBalancerServiceException("Could not (re)configure " + id + " due to change in cloud account. The operation will be retried on next deployment"); } + if (!hasCorrectVisibility(loadBalancer, zoneEndpoint)) { + throw new LoadBalancerServiceException("Could not (re)configure " + id + " due to change in load balancer visibility. The operation will be retried on next deployment"); + } } private static ClusterSpec.Id effectiveId(ClusterSpec cluster) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index 15a799c06d8..bf5b735c4a0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -89,6 +89,7 @@ public class LoadBalancersResponse extends SlimeJsonResponse { } } instance.serviceId().ifPresent(serviceId -> lbObject.setString("serviceId", serviceId.value())); + lbObject.setBool("public", instance.settings().isPublicEndpoint()); }); lb.instance() .map(LoadBalancerInstance::cloudAccount) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 29914993bae..6cbb2ba1fb4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -21,6 +21,8 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.ZoneEndpoint; +import com.yahoo.config.provision.ZoneEndpoint.AccessType; +import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.mock.MockCurator; @@ -190,7 +192,10 @@ public class MockNodeRepository extends NodeRepository { activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), null), zoneApp, provisioner); ApplicationId app1Id = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1")); - ClusterSpec cluster1Id = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1")).vespaVersion("6.42").loadBalancerSettings(new ZoneEndpoint(List.of("arne"))).build(); + ClusterSpec cluster1Id = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1")) + .vespaVersion("6.42") + .loadBalancerSettings(new ZoneEndpoint(false, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "arne")))) + .build(); activate(provisioner.prepare(app1Id, cluster1Id, Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1)), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index dee895b02d2..16e82e116e1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -7,6 +7,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.ZoneEndpoint; +import com.yahoo.config.provision.ZoneEndpoint.AccessType; +import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import com.yahoo.vespa.hosted.provision.lb.DnsZone; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; @@ -46,7 +48,7 @@ public class LoadBalancerSerializerTest { new Real(DomainName.of("real-2"), "127.0.0.2", 4080)), - new ZoneEndpoint(List.of("123")), + new ZoneEndpoint(false, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "123"))), Optional.of(PrivateServiceId.of("foo")), CloudAccount.from("012345678912"))), LoadBalancer.State.active, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index d9c27ae29ca..0aa2a001bfa 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -13,6 +13,8 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ZoneEndpoint; +import com.yahoo.config.provision.ZoneEndpoint.AccessType; +import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -41,6 +43,7 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -324,13 +327,46 @@ public class LoadBalancerProvisionerTest { assertEquals(ZoneEndpoint.defaultEndpoint, loadBalancers.first().get().instance().get().settings()); // Next deployment contains new settings - ZoneEndpoint settings = new ZoneEndpoint(List.of("alice", "bob")); + ZoneEndpoint settings = new ZoneEndpoint(true, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "alice"), new AllowedUrn(AccessType.gcpServiceConnect, "bob"))); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1"), Optional.empty(), settings))); loadBalancers = tester.nodeRepository().loadBalancers().list(); assertEquals(1, loadBalancers.size()); assertEquals(settings, loadBalancers.first().get().instance().get().settings()); } + @Test + public void load_balancer_with_changing_visibility() { + ClusterResources resources = new ClusterResources(3, 1, nodeResources); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty)); + tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); + LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); + assertEquals(1, loadBalancers.size()); + assertEquals(ZoneEndpoint.defaultEndpoint, loadBalancers.first().get().instance().get().settings()); + + // Next deployment has only a private endpoint + ZoneEndpoint settings = new ZoneEndpoint(false, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "alice"), new AllowedUrn(AccessType.gcpServiceConnect, "bob"))); + assertEquals("Could not (re)configure load balancer tenant1:application1:default:c1 due to change in load balancer visibility. The operation will be retried on next deployment", + assertThrows(LoadBalancerServiceException.class, + () -> prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1"), Optional.empty(), settings))) + .getMessage()); + + // Existing LB is removed + loadBalancers = tester.nodeRepository().loadBalancers().list(); + assertEquals(1, loadBalancers.size()); + assertSame(LoadBalancer.State.removable, loadBalancers.first().get().state()); + new LoadBalancerExpirer(tester.nodeRepository(), + Duration.ofDays(1), + tester.loadBalancerService(), + new TestMetric()) + .run(); + assertEquals(0, tester.nodeRepository().loadBalancers().list().in(LoadBalancer.State.removable).size()); + + // Next deployment provisions a new LB + tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1"), Optional.empty(), settings))); + loadBalancers = tester.nodeRepository().loadBalancers().list(); + assertEquals(1, loadBalancers.size()); + assertEquals(settings, loadBalancers.first().get().instance().get().settings()); + } @Test public void load_balancer_with_custom_cloud_account() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json index a9a728bab15..f6d9c1f079c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json @@ -28,7 +28,8 @@ "ipAddress": "127.0.14.1", "port": 4443 } - ] + ], + "public": true } ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json index becca98a913..eef381e8df7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json @@ -37,7 +37,8 @@ } ] }, - "serviceId": "service" + "serviceId": "service", + "public": false }, { "id": "cfg:cfg:cfg:configservers", @@ -56,7 +57,8 @@ "ports": [ 4443 ], - "reals": [] + "reals": [], + "public": true }, { "id": "tenant4:application4:instance4:id4", @@ -86,7 +88,8 @@ "ipAddress": "127.0.14.1", "port": 4443 } - ] + ], + "public": true } ] } diff --git a/parent/pom.xml b/parent/pom.xml index 612ec7aef19..93945575588 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1136,15 +1136,15 @@ <maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version> <maven-bundle-plugin.version>5.1.2</maven-bundle-plugin.version> <maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version> - <maven-core.version>3.8.6</maven-core.version> - <maven-dependency-plugin.version>3.3.0</maven-dependency-plugin.version> <!-- NOTE: When upgrading, also update explicit versions in tenant base poms! --> + <maven-core.version>3.8.7</maven-core.version> + <maven-dependency-plugin.version>3.5.0</maven-dependency-plugin.version> <!-- NOTE: When upgrading, also update explicit versions in tenant base poms! --> <maven-deploy-plugin.version>2.8.2</maven-deploy-plugin.version> <maven-enforcer-plugin.version>3.0.0</maven-enforcer-plugin.version> <maven-failsafe-plugin.version>3.0.0-M6</maven-failsafe-plugin.version> <maven-install-plugin.version>3.0.0-M1</maven-install-plugin.version> <maven-jar-plugin.version>3.2.0</maven-jar-plugin.version> <maven-javadoc-plugin.version>3.3.1</maven-javadoc-plugin.version> - <maven-plugin-api.version>3.8.6</maven-plugin-api.version> + <maven-plugin-api.version>${maven-core.version}</maven-plugin-api.version> <maven-plugin-tools.version>3.6.4</maven-plugin-tools.version> <maven-resources-plugin.version>3.2.0</maven-resources-plugin.version> <maven-shade-plugin.version>3.4.1</maven-shade-plugin.version> diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index f4db22eeeaa..31d547807d9 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -99,21 +99,21 @@ org.apache.httpcomponents.client5:httpclient5:5.2.1 org.apache.httpcomponents.core5:httpcore5:5.2.1 org.apache.httpcomponents.core5:httpcore5-h2:5.2.1 org.apache.maven:maven-archiver:3.6.0 -org.apache.maven:maven-artifact:3.8.6 +org.apache.maven:maven-artifact:3.8.7 org.apache.maven:maven-artifact-manager:2.2.1 -org.apache.maven:maven-builder-support:3.8.6 +org.apache.maven:maven-builder-support:3.8.7 org.apache.maven:maven-compat:3.0 -org.apache.maven:maven-core:3.8.6 -org.apache.maven:maven-model:3.8.6 -org.apache.maven:maven-model-builder:3.8.6 -org.apache.maven:maven-plugin-api:3.8.6 +org.apache.maven:maven-core:3.8.7 +org.apache.maven:maven-model:3.8.7 +org.apache.maven:maven-model-builder:3.8.7 +org.apache.maven:maven-plugin-api:3.8.7 org.apache.maven:maven-plugin-registry:2.2.1 org.apache.maven:maven-profile:2.2.1 org.apache.maven:maven-project:2.2.1 -org.apache.maven:maven-repository-metadata:3.8.6 -org.apache.maven:maven-resolver-provider:3.8.6 -org.apache.maven:maven-settings:3.8.6 -org.apache.maven:maven-settings-builder:3.8.6 +org.apache.maven:maven-repository-metadata:3.8.7 +org.apache.maven:maven-resolver-provider:3.8.7 +org.apache.maven:maven-settings:3.8.7 +org.apache.maven:maven-settings-builder:3.8.7 org.apache.maven.plugin-tools:maven-plugin-annotations:3.6.4 org.apache.maven.plugins:maven-jar-plugin:3.2.0 org.apache.maven.resolver:maven-resolver-api:1.6.3 diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 418f3ed5911..c3b87278345 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -3646,6 +3646,19 @@ ], "fields" : [ ] }, + "com.yahoo.yolean.Exceptions$FunctionThrowingIOException" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract java.lang.Object map(java.lang.Object)" + ], + "fields" : [ ] + }, "com.yahoo.yolean.Exceptions$RunnableThrowingIOException" : { "superClass" : "java.lang.Object", "interfaces" : [ ], @@ -3700,6 +3713,7 @@ "public static void uncheckInterruptedAndRestoreFlag(com.yahoo.yolean.Exceptions$RunnableThrowingInterruptedException)", "public static varargs void uncheck(com.yahoo.yolean.Exceptions$RunnableThrowingIOException, java.lang.String, java.lang.String[])", "public static void uncheckAndIgnore(com.yahoo.yolean.Exceptions$RunnableThrowingIOException, java.lang.Class)", + "public static java.util.function.Function uncheck(com.yahoo.yolean.Exceptions$FunctionThrowingIOException)", "public static java.lang.Object uncheck(com.yahoo.yolean.Exceptions$SupplierThrowingIOException)", "public static varargs java.lang.Object uncheck(com.yahoo.yolean.Exceptions$SupplierThrowingIOException, java.lang.String, java.lang.String[])", "public static java.lang.Object uncheckAndIgnore(com.yahoo.yolean.Exceptions$SupplierThrowingIOException, java.lang.Class)", diff --git a/vespajlib/src/main/java/ai/vespa/http/HttpURL.java b/vespajlib/src/main/java/ai/vespa/http/HttpURL.java index 9641ea2a8fd..ba1a8e08740 100644 --- a/vespajlib/src/main/java/ai/vespa/http/HttpURL.java +++ b/vespajlib/src/main/java/ai/vespa/http/HttpURL.java @@ -239,7 +239,7 @@ public class HttpURL { return parse(raw, HttpURL::requirePathSegment); } - /** Parses the given raw, normalized path string; this ignores whether the path is absolute or relative.) */ + /** Parses the given raw, normalized path string; this ignores whether the path is absolute or relative. */ public static Path parse(String raw, Consumer<String> validator) { Path path = new Path(null, 0, raw.endsWith("/"), segmentValidator(validator)); if (raw.startsWith("/")) raw = raw.substring(1); diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8.java b/vespajlib/src/main/java/com/yahoo/text/Utf8.java index 2a42cb5cdee..3a7ecaa727a 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Utf8.java +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8.java @@ -10,7 +10,8 @@ import java.nio.ReadOnlyBufferException; import java.nio.charset.Charset; import java.nio.charset.CharsetEncoder; import java.nio.charset.CodingErrorAction; -import java.nio.charset.StandardCharsets; + +import static java.nio.charset.StandardCharsets.UTF_8; /** * Utility class with functions for handling UTF-8 @@ -23,16 +24,16 @@ public final class Utf8 { private static final byte [] TRUE = {(byte) 't', (byte) 'r', (byte) 'u', (byte) 'e'}; private static final byte [] FALSE = {(byte) 'f', (byte) 'a', (byte) 'l', (byte) 's', (byte) 'e'}; - private static final byte[] LONG_MIN_VALUE_BYTES = String.valueOf(Long.MIN_VALUE).getBytes(StandardCharsets.UTF_8); + private static final byte[] LONG_MIN_VALUE_BYTES = String.valueOf(Long.MIN_VALUE).getBytes(UTF_8); /** Returns the Charset instance for UTF-8 */ public static Charset getCharset() { - return StandardCharsets.UTF_8; + return UTF_8; } /** To be used instead of String.String(byte[] bytes) */ public static String toStringStd(byte[] data) { - return new String(data, StandardCharsets.UTF_8); + return new String(data, UTF_8); } /** @@ -60,7 +61,7 @@ public final class Utf8 { * @return a decoded String */ public static String toString(ByteBuffer data) { - CharBuffer c = StandardCharsets.UTF_8.decode(data); + CharBuffer c = UTF_8.decode(data); return c.toString(); } @@ -68,7 +69,7 @@ public final class Utf8 { * Uses String.getBytes directly. */ public static byte[] toBytesStd(String str) { - return str.getBytes(StandardCharsets.UTF_8); + return str.getBytes(UTF_8); } /** @@ -112,7 +113,7 @@ public final class Utf8 { */ public static byte[] toBytes(String string) { // This is just wrapper for String::getBytes. Pre-Java 9 this had a more efficient approach for ASCII-only strings. - return string.getBytes(StandardCharsets.UTF_8); + return string.getBytes(UTF_8); } /** * Decode a UTF-8 string. @@ -122,7 +123,7 @@ public final class Utf8 { */ public static String toString(byte[] utf8) { // This is just wrapper for String::new. Pre-Java 9 this had a more efficient approach for ASCII-onlu strings. - return new String(utf8, StandardCharsets.UTF_8); + return new String(utf8, UTF_8); } /** @@ -138,7 +139,7 @@ public final class Utf8 { */ public static byte[] toBytes(String str, int offset, int length) { CharBuffer c = CharBuffer.wrap(str, offset, offset + length); - ByteBuffer b = StandardCharsets.UTF_8.encode(c); + ByteBuffer b = UTF_8.encode(c); byte[] result = new byte[b.remaining()]; b.get(result); return result; @@ -161,7 +162,7 @@ public final class Utf8 { */ public static int toBytes(String str, int srcOffset, int srcLen, byte[] dst, int dstOffset) { CharBuffer c = CharBuffer.wrap(str, srcOffset, srcOffset + srcLen); - ByteBuffer b = StandardCharsets.UTF_8.encode(c); + ByteBuffer b = UTF_8.encode(c); int encoded = b.remaining(); b.get(dst, dstOffset, encoded); return encoded; @@ -206,7 +207,7 @@ public final class Utf8 { * @see Utf8#toBytes(String, int, int, ByteBuffer, CharsetEncoder) */ public static CharsetEncoder getNewEncoder() { - return StandardCharsets.UTF_8.newEncoder().onMalformedInput(CodingErrorAction.REPLACE) + return UTF_8.newEncoder().onMalformedInput(CodingErrorAction.REPLACE) .onUnmappableCharacter(CodingErrorAction.REPLACE); } diff --git a/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java b/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java index 89b4e76368b..4f3f048eb0c 100644 --- a/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java +++ b/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java @@ -4,6 +4,7 @@ package com.yahoo.yolean; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Optional; +import java.util.function.Function; /** * Helper methods for handling exceptions @@ -129,6 +130,14 @@ public class Exceptions { @FunctionalInterface public interface RunnableThrowingInterruptedException { void run() throws InterruptedException; } /** + * Wraps any IOException thrown from a function in an UncheckedIOException. + */ + public static <T, R> Function<T, R> uncheck(FunctionThrowingIOException<T, R> function) { + return t -> uncheck(() -> function.map(t)); + } + @FunctionalInterface public interface FunctionThrowingIOException<T, R> { R map(T t) throws IOException; } + + /** * Wraps any IOException thrown from a supplier in an UncheckedIOException. */ public static <T> T uncheck(SupplierThrowingIOException<T> supplier) { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java index 738697d4521..ba541ab2cd6 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java @@ -102,6 +102,30 @@ public class TensorTypeTestCase { assertEquals("tensor<int8>(x[])", TensorType.fromSpec("tensor<int8>(x[])").toString()); } + @Test + public void testIndexedSubtype() { + assertEquals(TensorType.fromSpec("tensor(x[10])"), + TensorType.fromSpec("tensor(x[10])").indexedSubtype()); + assertEquals(TensorType.fromSpec("tensor(x[10])"), + TensorType.fromSpec("tensor(x[10],a{})").indexedSubtype()); + assertEquals(TensorType.fromSpec("tensor(x[10],y[5])"), + TensorType.fromSpec("tensor(x[10],y[5],a{},b{})").indexedSubtype()); + assertEquals(TensorType.fromSpec("tensor()"), + TensorType.fromSpec("tensor(a{})").indexedSubtype()); + } + + @Test + public void testMappedSubtype() { + assertEquals(TensorType.fromSpec("tensor(a{})"), + TensorType.fromSpec("tensor(a{})").mappedSubtype()); + assertEquals(TensorType.fromSpec("tensor(a{})"), + TensorType.fromSpec("tensor(x[10],a{})").mappedSubtype()); + assertEquals(TensorType.fromSpec("tensor(a{},b{})"), + TensorType.fromSpec("tensor(x[10],y[5],a{},b{})").mappedSubtype()); + assertEquals(TensorType.fromSpec("tensor()"), + TensorType.fromSpec("tensor(x[10])").mappedSubtype()); + } + private static void assertTensorType(String typeSpec) { assertTensorType(typeSpec, typeSpec); } |