diff options
30 files changed, 431 insertions, 66 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java index 83106e75627..fd355d427a3 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java @@ -11,9 +11,10 @@ import java.util.stream.Collectors; /** * Represents an application- or instance-level endpoint in deployments.xml. - * - * - An instance-level endpoint is global and can span multiple regions within a single instance. - * - An application-level endpoint points can span multiple instances within a single region. + * <p> + * - An instance-level endpoint is global and may span multiple regions within a single instance. + * - An application-level endpoint may span multiple instances within a single region, or + * even multiple instances across multiple regions, depending on the name service used for the cloud. * * @author ogronnesby * @author mpolden @@ -44,7 +45,8 @@ public class Endpoint { this.level = Objects.requireNonNull(level, "level must be non-null"); this.targets = List.copyOf(Objects.requireNonNull(targets, "targets must be non-null")); if (endpointId().length() > endpointMaxLength || !endpointPattern.matcher(endpointId()).matches()) { - throw new IllegalArgumentException("Invalid endpoint ID: '" + endpointId() + "'"); + throw new IllegalArgumentException("Endpoint ID must be all lowercase, alphanumeric, with no consecutive dashes, " + + "of length 1 to 12, and begin with a character; but got '" + endpointId() + "'"); } if (targets.isEmpty()) throw new IllegalArgumentException("targets must be non-empty"); for (int i = 0; i < targets.size(); i++) { @@ -66,7 +68,7 @@ public class Endpoint { } } - /** The unique identifer of this */ + /** The unique identifier of this */ public String endpointId() { return endpointId; } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 1c93ce4bd6d..d2fd6d8cfc4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -25,6 +25,7 @@ import java.util.ListIterator; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -67,6 +68,7 @@ public class InMemoryProvisioner implements HostProvisioner { private final boolean alwaysReturnOneNode; private Provisioned provisioned = new Provisioned(); + private final Set<ClusterSpec> clusters = new TreeSet<>(Comparator.comparing(cluster -> cluster.id().value())); private Environment environment = Environment.prod; @@ -146,6 +148,7 @@ public class InMemoryProvisioner implements HostProvisioner { @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { provisioned.add(cluster.id(), requested); + clusters.add(cluster); if (environment == Environment.dev) { requested = requested.withLimits(requested.minResources().withNodes(1), requested.maxResources().withNodes(1)); @@ -197,6 +200,7 @@ public class InMemoryProvisioner implements HostProvisioner { /** Create a new provisioned instance to record provision requests to this and returns it */ public Provisioned startProvisionedRecording() { provisioned = new Provisioned(); + clusters.clear(); return provisioned; } @@ -287,4 +291,7 @@ public class InMemoryProvisioner implements HostProvisioner { return 0; } } + + public Set<ClusterSpec> provisionedClusters() { return clusters; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index b46f474aa7c..a31e3fcce71 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.LoadBalancerSettings; import com.yahoo.config.provision.NodeResources; import com.yahoo.text.XML; import com.yahoo.vespa.model.HostResource; @@ -255,6 +256,15 @@ public class NodesSpecification { ClusterSpec.Id clusterId, DeployLogger logger, boolean stateful) { + return provision(hostSystem, clusterType, clusterId, LoadBalancerSettings.empty, logger, stateful); + } + + public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, + ClusterSpec.Type clusterType, + ClusterSpec.Id clusterId, + LoadBalancerSettings loadBalancerSettings, + DeployLogger logger, + boolean stateful) { if (combinedId.isPresent()) clusterType = ClusterSpec.Type.combined; ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId) @@ -262,6 +272,7 @@ public class NodesSpecification { .exclusive(exclusive) .combinedId(combinedId.map(ClusterSpec.Id::from)) .dockerImageRepository(dockerImageRepo) + .loadBalancerSettings(loadBalancerSettings) .stateful(stateful) .build(); return hostSystem.allocateHosts(cluster, Capacity.from(min, max, required, canFail, cloudAccount), logger); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 5dc937fe585..03c9335bbc4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -27,6 +27,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.LoadBalancerSettings; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; @@ -734,6 +735,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } + private LoadBalancerSettings loadBalancerSettings(Element loadBalancerElement) { + List<String> allowedUrnElements = XML.getChildren(XML.getChild(loadBalancerElement, "private-access"), + "allowed-urn") + .stream().map(XML::getValue).toList(); + return new LoadBalancerSettings(allowedUrnElements); + } + private static Map<String, String> getEnvironmentVariables(Element environmentVariables) { var map = new LinkedHashMap<String, String>(); if (environmentVariables != null) { @@ -745,7 +753,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return map; } - private List<ApplicationContainer> createNodes(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { + private List<ApplicationContainer> createNodes(ApplicationContainerCluster cluster, Element containerElement, + Element nodesElement, ConfigModelContext context) { if (nodesElement.hasAttribute("type")) // internal use for hosted system infrastructure nodes return createNodesFromNodeType(cluster, nodesElement, context); else if (nodesElement.hasAttribute("of")) {// hosted node spec referencing a content cluster @@ -818,9 +827,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { NodesSpecification nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); + LoadBalancerSettings loadBalancerSettings = loadBalancerSettings(XML.getChild(containerElement, "load-balancer")); Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().hostSystem(), ClusterSpec.Type.container, - ClusterSpec.Id.from(cluster.getName()), + ClusterSpec.Id.from(cluster.getName()), + loadBalancerSettings, log, getZooKeeper(containerElement) != null); return createNodesFromHosts(hosts, cluster, context.getDeployState()); diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 1d31435dad1..8cd7071462e 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -6,7 +6,8 @@ ContainerCluster = element container { ContainerServices & DocumentBinding* & NodesOfContainerCluster? & - ClientAuthorize? + ClientAuthorize? & + LoadBalancer? } ContainerServices = @@ -255,6 +256,14 @@ NodesOfContainerCluster = element nodes { ) } +LoadBalancer = element load-balancer { + element private-access { + element allow-urn { + xsd:string + }* + }? +} + #DOCUMENT BINDINGS: diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 251c18dbdd6..fafd4bb1ff5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -197,6 +197,57 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + void load_balancers_can_be_set() throws IOException, SAXException { + // No load-balancer or nodes elements + verifyAllowedUrns(""); + + // No load-balancer element + verifyAllowedUrns("<nodes count='2' />"); + + // No nodes element + verifyAllowedUrns(""" + <load-balancer> + <private-access> + <allowed-urn>foo</allowed-urn> + <allowed-urn>bar</allowed-urn> + </private-access> + </load-balancer> + """); + + // Both load-balancer and nodes + verifyAllowedUrns(""" + <load-balancer> + <private-access> + <allowed-urn>foo</allowed-urn> + <allowed-urn>bar</allowed-urn> + </private-access> + </load-balancer> + <nodes count='2' /> + """, + "foo", "bar"); + } + + private void verifyAllowedUrns(String containerXml, String... expectedAllowedUrns) throws IOException, SAXException { + String servicesXml = """ + <container id='default' version='1.0'> + %s + </container> + """.formatted(containerXml); + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); + InMemoryProvisioner provisioner = new InMemoryProvisioner(true, false, "host1.yahoo.com", "host2.yahoo.com"); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() + .modelHostProvisioner(provisioner) + .provisioned(provisioner.startProvisionedRecording()) + .applicationPackage(applicationPackage) + .properties(new TestProperties().setMultitenant(true).setHostedVespa(true)) + .build()); + assertEquals(2, model.hostSystem().getHosts().size()); + assertEquals(1, provisioner.provisionedClusters().size()); + assertEquals(List.of(expectedAllowedUrns), + provisioner.provisionedClusters().iterator().next().loadBalancerSettings().allowedUrns()); + } + + @Test void builtin_handlers_get_default_threadpool() { createBasicContainerModel(); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index dd07b29c2de..5fae9497f69 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -24,10 +24,12 @@ public final class ClusterSpec { private final boolean exclusive; private final Optional<Id> combinedId; private final Optional<DockerImage> dockerImageRepo; + private final LoadBalancerSettings loadBalancerSettings; private final boolean stateful; private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion, boolean exclusive, - Optional<Id> combinedId, Optional<DockerImage> dockerImageRepo, boolean stateful) { + Optional<Id> combinedId, Optional<DockerImage> dockerImageRepo, + LoadBalancerSettings loadBalancerSettings, boolean stateful) { this.type = type; this.id = id; this.groupId = groupId; @@ -45,6 +47,7 @@ public final class ClusterSpec { if (type.isContent() && !stateful) { throw new IllegalArgumentException("Cluster of type " + type + " must be stateful"); } + this.loadBalancerSettings = Objects.requireNonNull(loadBalancerSettings); this.stateful = stateful; } @@ -60,6 +63,9 @@ public final class ClusterSpec { /** Returns the docker image (repository + vespa version) we want this cluster to run */ public Optional<String> dockerImage() { return dockerImageRepo.map(repo -> repo.withTag(vespaVersion).asString()); } + /** Returns any additional load balancer settings for application container clusters. */ + public LoadBalancerSettings loadBalancerSettings() { return loadBalancerSettings; } + /** Returns the version of Vespa that we want this cluster to run */ public Version vespaVersion() { return vespaVersion; } @@ -81,11 +87,11 @@ public final class ClusterSpec { public boolean isStateful() { return stateful; } public ClusterSpec with(Optional<Group> newGroup) { - return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive, combinedId, dockerImageRepo, stateful); + return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive, combinedId, dockerImageRepo, loadBalancerSettings, stateful); } public ClusterSpec exclusive(boolean exclusive) { - return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, stateful); + return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, loadBalancerSettings, stateful); } /** Creates a ClusterSpec when requesting a cluster */ @@ -103,13 +109,14 @@ public final class ClusterSpec { private final Type type; private final Id id; private final boolean specification; - private boolean stateful; private Optional<Group> groupId = Optional.empty(); private Optional<DockerImage> dockerImageRepo = Optional.empty(); private Version vespaVersion; private boolean exclusive = false; private Optional<Id> combinedId = Optional.empty(); + private LoadBalancerSettings loadBalancerSettings = LoadBalancerSettings.empty; + private boolean stateful; private Builder(Type type, Id id, boolean specification) { this.type = type; @@ -124,7 +131,7 @@ public final class ClusterSpec { if (vespaVersion == null) throw new IllegalArgumentException("vespaVersion is required to be set when creating a ClusterSpec with specification()"); } else if (groupId.isPresent()) throw new IllegalArgumentException("groupId is not allowed to be set when creating a ClusterSpec with request()"); - return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, stateful); + return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, loadBalancerSettings, stateful); } public Builder group(Group groupId) { @@ -157,6 +164,11 @@ public final class ClusterSpec { return this; } + public Builder loadBalancerSettings(LoadBalancerSettings loadBalancerSettings) { + this.loadBalancerSettings = loadBalancerSettings; + return this; + } + public Builder stateful(boolean stateful) { this.stateful = stateful; return this; @@ -181,12 +193,13 @@ public final class ClusterSpec { groupId.equals(that.groupId) && vespaVersion.equals(that.vespaVersion) && combinedId.equals(that.combinedId) && - dockerImageRepo.equals(that.dockerImageRepo); + dockerImageRepo.equals(that.dockerImageRepo) && + loadBalancerSettings.equals(that.loadBalancerSettings); } @Override public int hashCode() { - return Objects.hash(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, stateful); + return Objects.hash(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, loadBalancerSettings, stateful); } /** @@ -203,7 +216,7 @@ public final class ClusterSpec { /** A cluster type */ public enum Type { - // These enum values are stored in ZooKeeper - do not change + // These enum names are written to ZooKeeper - do not change admin, container, content, @@ -220,13 +233,13 @@ public final class ClusterSpec { } public static Type from(String typeName) { - switch (typeName) { - case "admin" : return admin; - case "container" : return container; - case "content" : return content; - case "combined" : return combined; - default: throw new IllegalArgumentException("Illegal cluster type '" + typeName + "'"); - } + return switch (typeName) { + case "admin" -> admin; + case "container" -> container; + case "content" -> content; + case "combined" -> combined; + default -> throw new IllegalArgumentException("Illegal cluster type '" + typeName + "'"); + }; } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/LoadBalancerSettings.java b/config-provisioning/src/main/java/com/yahoo/config/provision/LoadBalancerSettings.java new file mode 100644 index 00000000000..723de25fa87 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/LoadBalancerSettings.java @@ -0,0 +1,20 @@ +package com.yahoo.config.provision; + +import java.util.List; + +/** + * Settings for a load balancer provisioned for an application container cluster. + * + * @author jonmv + */ +public record LoadBalancerSettings(List<String> allowedUrns) { + + public static final LoadBalancerSettings empty = new LoadBalancerSettings(List.of()); + + public LoadBalancerSettings(List<String> allowedUrns) { + this.allowedUrns = List.copyOf(allowedUrns); + } + + public boolean isEmpty() { return allowedUrns.isEmpty(); } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index e49e9c7998e..923304d4c55 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationS import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.api.integration.horizon.HorizonClient; import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; @@ -57,6 +58,8 @@ public interface ServiceRegistry { NameService nameService(); + VpcEndpointService vpcEndpointService(); + Mailer mailer(); EndpointCertificateProvider endpointCertificateProvider(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index 9a9270bdf7f..aff58989231 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.ConcurrentSkipListSet; import java.util.stream.Collectors; /** @@ -17,7 +18,7 @@ import java.util.stream.Collectors; */ public class MemoryNameService implements NameService { - private final Set<Record> records = new TreeSet<>(); + private final Set<Record> records = new ConcurrentSkipListSet<>(); public Set<Record> records() { return Collections.unmodifiableSet(records); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java new file mode 100644 index 00000000000..e4f14c7a7b6 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java @@ -0,0 +1,23 @@ +package com.yahoo.vespa.hosted.controller.api.integration.dns; + +import ai.vespa.http.DomainName; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; + +import java.util.Optional; + +/** + * @author jonmv + */ +public class MockVpcEndpointService implements VpcEndpointService { + + public static final VpcEndpointService empty = (name, cluster, account) -> Optional.empty(); + + public VpcEndpointService delegate = empty; + + @Override + public Optional<DnsChallenge> setPrivateDns(DomainName privateDnsName, ClusterId clusterId, Optional<CloudAccount> account) { + return delegate.setPrivateDns(privateDnsName, clusterId, account); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java new file mode 100644 index 00000000000..109b084f672 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java @@ -0,0 +1,20 @@ +package com.yahoo.vespa.hosted.controller.api.integration.dns; + +import ai.vespa.http.DomainName; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; + +import java.util.Optional; + +/** + * @author jonmv + */ +public interface VpcEndpointService { + + /** Sets the private DNS name for any VPC endpoint for the given cluster, potentially guarded by a challenge. */ + Optional<DnsChallenge> setPrivateDns(DomainName privateDnsName, ClusterId clusterId, Optional<CloudAccount> account); + + /** Create a TXT record with this name and token, then run the trigger, to pass this challenge. */ + record DnsChallenge(RecordName name, RecordData data, Runnable trigger) { } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java index 344ffad80e9..e20c30d2745 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java @@ -3,9 +3,11 @@ package com.yahoo.vespa.hosted.controller.dns; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import java.util.List; import java.util.Objects; +import java.util.Optional; /** * Create or update a record. @@ -29,6 +31,11 @@ public class CreateRecord implements NameServiceRequest { } @Override + public Optional<RecordName> name() { + return Optional.of(record.name()); + } + + @Override public void dispatchTo(NameService nameService) { List<Record> records = nameService.findRecords(record.type(), record.name()); records.forEach(r -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java index b97fdde560e..88e4f351f1f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -39,6 +40,11 @@ public class CreateRecords implements NameServiceRequest { } @Override + public Optional<RecordName> name() { + return Optional.of(name); + } + + @Override public void dispatchTo(NameService nameService) { switch (type) { case ALIAS -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java index d42b9efbdb3..42ee8a7d2d5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java @@ -2,6 +2,9 @@ package com.yahoo.vespa.hosted.controller.dns; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; + +import java.util.Optional; /** * Interface for requests to a {@link NameService}. @@ -10,6 +13,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; */ public interface NameServiceRequest { + Optional<RecordName> name(); + /** Send this to given name service */ void dispatchTo(NameService nameService); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index b0d16126600..4e2eb03f01d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.routing; import ai.vespa.http.DomainName; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.RoutingMethod; @@ -9,6 +10,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; @@ -27,7 +29,9 @@ import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; import com.yahoo.vespa.hosted.controller.dns.NameServiceRequest; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.yolean.UncheckedInterruptedException; +import java.time.Instant; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -349,7 +353,7 @@ public class RoutingPolicies { if (existingPolicy != null) { newPolicy = newPolicy.with(newPolicy.status().with(existingPolicy.status().routingStatus())); } - updateZoneDnsOf(newPolicy); + updateZoneDnsOf(newPolicy, allocation); policies.put(newPolicy.id(), newPolicy); } RoutingPolicyList updated = RoutingPolicyList.copyOf(policies.values()); @@ -358,16 +362,42 @@ public class RoutingPolicies { } /** Update zone DNS record for given policy */ - private void updateZoneDnsOf(RoutingPolicy policy) { + private void updateZoneDnsOf(RoutingPolicy policy, LoadBalancerAllocation allocation) { for (var endpoint : policy.zoneEndpointsIn(controller.system(), RoutingMethod.exclusive, controller.zoneRegistry())) { var name = RecordName.from(endpoint.dnsName()); var record = policy.canonicalName().isPresent() ? new Record(Record.Type.CNAME, name, RecordData.fqdn(policy.canonicalName().get().value())) : new Record(Record.Type.A, name, RecordData.from(policy.ipAddress().orElseThrow())); nameServiceForwarderIn(policy.id().zone()).createRecord(record, Priority.normal); + setPrivateDns(endpoint, allocation); } } + private void setPrivateDns(Endpoint endpoint, LoadBalancerAllocation allocation) { + controller.serviceRegistry().vpcEndpointService() + .setPrivateDns(DomainName.of(endpoint.dnsName()), + new ClusterId(allocation.deployment, endpoint.cluster()), + controller.applications().decideCloudAccountOf(allocation.deployment, allocation.deploymentSpec)) + .ifPresent(challenge -> { + try { + nameServiceForwarderIn(allocation.deployment.zoneId()).createTxt(challenge.name(), List.of(challenge.data()), Priority.high); + Instant doom = controller.clock().instant().plusSeconds(30); + while (controller.clock().instant().isBefore(doom)) { + if (controller.curator().readNameServiceQueue().requests().stream() + .noneMatch(request -> request.name().equals(Optional.of(challenge.name())))) { + challenge.trigger().run(); + return; + } + Thread.sleep(100); + } + throw new UncheckedTimeoutException("timed out waiting for DNS challenge to be processed"); + } + catch (InterruptedException e) { + throw new UncheckedInterruptedException("interrupted waiting for DNS challenge to be processed", e, true); + } + }); + } + /** * Remove policies and zone DNS records unreferenced by given load balancers * diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 46c731e6e49..eb16ecaab81 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -1,10 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; +import ai.vespa.http.DomainName; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.component.annotation.Inject; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; @@ -28,6 +30,11 @@ import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCe import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateValidator; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateValidatorMock; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; +import com.yahoo.vespa.hosted.controller.api.integration.dns.MockVpcEndpointService; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; import com.yahoo.vespa.hosted.controller.api.integration.horizon.HorizonClient; import com.yahoo.vespa.hosted.controller.api.integration.horizon.MockHorizonClient; @@ -64,6 +71,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ZoneRegistryMock zoneRegistryMock; private final ConfigServerMock configServerMock; private final MemoryNameService memoryNameService = new MemoryNameService(); + private final MockVpcEndpointService vpcEndpointService = new MockVpcEndpointService(); private final MockMailer mockMailer = new MockMailer(); private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(clock); private final EndpointCertificateValidatorMock endpointCertificateValidatorMock = new EndpointCertificateValidatorMock(); @@ -201,6 +209,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override + public MockVpcEndpointService vpcEndpointService() { + return vpcEndpointService; + } + + @Override public ZoneRegistryMock zoneRegistry() { return zoneRegistryMock; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 7b00e7040b5..867e03258f9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; @@ -21,8 +22,11 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record.Type; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; @@ -47,11 +51,16 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * @author mortent @@ -502,6 +511,48 @@ public class RoutingPoliciesTest { } @Test + void private_dns_for_vpc_endpoint() { + // Challenge answered for endpoint + RoutingPoliciesTester tester = new RoutingPoliciesTester(); + Map<RecordName, RecordData> challenges = new ConcurrentHashMap<>(); + tester.tester.controllerTester().serviceRegistry().vpcEndpointService().delegate = + (name, cluster, account) -> { + RecordName recordName = RecordName.from("challenge--" + name.value()); + if (challenges.containsKey(recordName)) return Optional.empty(); + RecordData recordData = RecordData.from(account.map(CloudAccount::value).orElse("system")); + return Optional.of(new DnsChallenge(recordName, recordData, () -> challenges.put(recordName, recordData))); + }; + + DeploymentContext app = tester.newDeploymentContext("t", "a", "default"); + ApplicationPackage appPackage = applicationPackageBuilder().region(zone3.region()).build(); + app.submit(appPackage); + + AtomicBoolean done = new AtomicBoolean(); + new Thread(() -> { + while ( ! done.get()) { + app.flushDnsUpdates(); + try { Thread.sleep(10); } catch (InterruptedException e) { break; } + } + }).start(); + app.deploy(); + done.set(true); + + assertEquals(Set.of(new Record(Type.CNAME, + RecordName.from("a.t.aws-us-east-1a.vespa.oath.cloud"), + RecordData.from("lb-0--t.a.default--prod.aws-us-east-1a.")), + new Record(Type.TXT, + RecordName.from("challenge--a.t.aws-us-east-1a.vespa.oath.cloud"), + RecordData.from("system")), + new Record(Type.TXT, + RecordName.from("challenge--a.t.us-east-1.test.vespa.oath.cloud"), + RecordData.from("system")), + new Record(Type.TXT, + RecordName.from("challenge--a.t.us-east-3.staging.vespa.oath.cloud"), + RecordData.from("system"))), + tester.controllerTester().nameService().records()); + } + + @Test void set_global_endpoint_status() { var tester = new RoutingPoliciesTester(); var context = tester.newDeploymentContext("tenant1", "app1", "default"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java index aa7d388c78f..fde3c85bdab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.lb; import ai.vespa.http.DomainName; import com.google.common.collect.ImmutableSortedSet; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.LoadBalancerSettings; import java.util.Objects; import java.util.Optional; @@ -23,16 +24,18 @@ public class LoadBalancerInstance { private final Set<Integer> ports; private final Set<String> networks; private final Set<Real> reals; + private final LoadBalancerSettings settings; private final CloudAccount cloudAccount; public LoadBalancerInstance(Optional<DomainName> hostname, Optional<String> ipAddress, Optional<DnsZone> dnsZone, Set<Integer> ports, - Set<String> networks, Set<Real> reals, CloudAccount cloudAccount) { + Set<String> networks, Set<Real> reals, LoadBalancerSettings settings, CloudAccount cloudAccount) { this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); this.ipAddress = Objects.requireNonNull(ipAddress, "ip must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); this.ports = ImmutableSortedSet.copyOf(requirePorts(ports)); this.networks = ImmutableSortedSet.copyOf(Objects.requireNonNull(networks, "networks must be non-null")); this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals, "targets must be non-null")); + this.settings = Objects.requireNonNull(settings, "settings must be non-null"); this.cloudAccount = Objects.requireNonNull(cloudAccount, "cloudAccount must be non-null"); if (hostname.isEmpty() == ipAddress.isEmpty()) { @@ -71,6 +74,11 @@ public class LoadBalancerInstance { return reals; } + /** Static user-configured settings of this load balancer */ + public LoadBalancerSettings settings() { + return settings; + } + /** Cloud account of this load balancer */ public CloudAccount cloudAccount() { return cloudAccount; @@ -78,7 +86,7 @@ public class LoadBalancerInstance { /** Returns a copy of this with reals set to given reals */ public LoadBalancerInstance withReals(Set<Real> reals) { - return new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, cloudAccount); + return new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, settings, cloudAccount); } private static Set<Integer> requirePorts(Set<Integer> ports) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index ac5330dce12..2b6f64012b1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -61,6 +61,7 @@ public class LoadBalancerServiceMock implements LoadBalancerService { Collections.singleton(4443), ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"), spec.reals(), + spec.settings(), spec.cloudAccount()); instances.put(id, instance); return instance; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java index 18c6ed52046..c4a6f6b2a37 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSortedSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.LoadBalancerSettings; import java.util.Objects; import java.util.Set; @@ -19,13 +20,15 @@ public class LoadBalancerSpec { private final ApplicationId application; private final ClusterSpec.Id cluster; private final Set<Real> reals; + private final LoadBalancerSettings settings; private final CloudAccount cloudAccount; public LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, - CloudAccount cloudAccount) { + LoadBalancerSettings settings, CloudAccount cloudAccount) { this.application = Objects.requireNonNull(application); this.cluster = Objects.requireNonNull(cluster); this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals)); + this.settings = Objects.requireNonNull(settings); this.cloudAccount = Objects.requireNonNull(cloudAccount); } @@ -44,6 +47,11 @@ public class LoadBalancerSpec { return reals; } + /** Static user-configured settings for this load balancer. */ + public LoadBalancerSettings settings() { + return settings; + } + /** Cloud account to use when satisfying this */ public CloudAccount cloudAccount() { return cloudAccount; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index c126b3969fa..d9032bd0a5b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.lb; import ai.vespa.http.DomainName; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.LoadBalancerSettings; import com.yahoo.config.provision.NodeType; import java.util.Objects; @@ -12,7 +13,7 @@ import java.util.Set; /** * This implementation of {@link LoadBalancerService} returns the load balancer(s) that exist by default in the shared * routing layer. - * + * <p> * Since such load balancers always exist, we can return the hostname of the routing layer VIP directly. Nothing has to * be provisioned. * @@ -28,12 +29,14 @@ public class SharedLoadBalancerService implements LoadBalancerService { @Override public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { + if ( ! spec.settings().isEmpty()) throw new IllegalArgumentException("custom load balancer settings are not supported with " + getClass()); return new LoadBalancerInstance(Optional.of(DomainName.of(vipHostname)), Optional.empty(), Optional.empty(), Set.of(4443), Set.of(), spec.reals(), + spec.settings(), spec.cloudAccount()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index f0abe184614..0264d0df837 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -111,7 +111,9 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { try { attempts.add(1); LOG.log(Level.INFO, () -> "Removing reals from inactive load balancer " + lb.id() + ": " + Sets.difference(lb.instance().get().reals(), reals)); - service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals, lb.instance().get().cloudAccount()), true); + service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals, + lb.instance().get().settings(), lb.instance().get().cloudAccount()), + true); db.writeLoadBalancer(lb.with(lb.instance().map(instance -> instance.withReals(reals))), lb.state()); } catch (Exception e) { failed.add(lb.id()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index c0bf9926ae1..9eccb21b756 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.persistence; import ai.vespa.http.DomainName; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.LoadBalancerSettings; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -48,6 +49,8 @@ public class LoadBalancerSerializer { private static final String ipAddressField = "ipAddress"; private static final String portField = "port"; private static final String cloudAccountField = "cloudAccount"; + private static final String settingsField = "settings"; + private static final String allowedUrnsField = "allowedUrns"; public static byte[] toJson(LoadBalancer loadBalancer) { Slime slime = new Slime(); @@ -71,6 +74,11 @@ public class LoadBalancerSerializer { realObject.setLong(portField, real.port()); })); loadBalancer.instance() + .map(LoadBalancerInstance::settings) + .filter(settings -> ! settings.isEmpty()) + .ifPresent(settings -> settings.allowedUrns().forEach(root.setObject(settingsField) + .setArray(allowedUrnsField)::addString)); + loadBalancer.instance() .map(LoadBalancerInstance::cloudAccount) .filter(cloudAccount -> !cloudAccount.isUnspecified()) .ifPresent(cloudAccount -> root.setString(cloudAccountField, cloudAccount.value())); @@ -101,9 +109,10 @@ public class LoadBalancerSerializer { Optional<DomainName> hostname = optionalString(object.field(hostnameField), Function.identity()).filter(s -> !s.isEmpty()).map(DomainName::of); Optional<String> ipAddress = optionalString(object.field(lbIpAddressField), Function.identity()).filter(s -> !s.isEmpty()); Optional<DnsZone> dnsZone = optionalString(object.field(dnsZoneField), DnsZone::new); + LoadBalancerSettings settings = loadBalancerSettings(object.field(settingsField)); CloudAccount cloudAccount = optionalString(object.field(cloudAccountField), CloudAccount::from).orElse(CloudAccount.empty); Optional<LoadBalancerInstance> instance = hostname.isEmpty() && ipAddress.isEmpty() ? Optional.empty() : - Optional.of(new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, cloudAccount)); + Optional.of(new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, settings, cloudAccount)); return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()), instance, @@ -111,6 +120,13 @@ public class LoadBalancerSerializer { Instant.ofEpochMilli(object.field(changedAtField).asLong())); } + private static LoadBalancerSettings loadBalancerSettings(Inspector settingsObject) { + if ( ! settingsObject.valid()) return LoadBalancerSettings.empty; + return new LoadBalancerSettings(SlimeUtils.entriesStream(settingsObject.field(allowedUrnsField)) + .map(Inspector::asString) + .toList()); + } + private static <T> Optional<T> optionalValue(Inspector field, Function<Inspector, T> fieldMapper) { return Optional.of(field).filter(Inspector::valid).map(fieldMapper); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 0763d19bae3..c9c2199a234 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -6,7 +6,9 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.ClusterSpec.Type; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.LoadBalancerSettings; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.exception.LoadBalancerServiceException; @@ -39,6 +41,9 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.reducing; + /** * Provisions and configures application load balancers. * @@ -75,12 +80,12 @@ public class LoadBalancerProvisioner { /** * Prepare a load balancer for given application and cluster. - * + * <p> * If a load balancer for the cluster already exists, it will be reconfigured based on the currently allocated * nodes. It's state will remain unchanged. - * + * <p> * If no load balancer exists, a new one will be provisioned in {@link LoadBalancer.State#reserved}. - * + * <p> * Calling this for irrelevant node or cluster types is a no-op. */ public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { @@ -89,33 +94,33 @@ public class LoadBalancerProvisioner { ClusterSpec.Id clusterId = effectiveId(cluster); LoadBalancerId loadBalancerId = requireNonClashing(new LoadBalancerId(application, clusterId)); NodeList nodes = nodesOf(clusterId, application); - prepare(loadBalancerId, nodes, requestedNodes.cloudAccount()); + prepare(loadBalancerId, nodes, cluster.loadBalancerSettings(), requestedNodes.cloudAccount()); } } /** * Activate load balancer for given application and cluster. - * + * <p> * If a load balancer for the cluster already exists, it will be reconfigured based on the currently allocated * nodes and the load balancer itself will be moved to {@link LoadBalancer.State#active}. - * + * <p> * Load balancers for clusters that are no longer in given clusters are deactivated. - * + * <p> * Calling this when no load balancer has been prepared for given cluster is a no-op. */ public void activate(Set<ClusterSpec> clusters, NodeList newActive, ApplicationTransaction transaction) { - Set<ClusterSpec.Id> activatingClusters = clusters.stream() - .map(LoadBalancerProvisioner::effectiveId) - .collect(Collectors.toSet()); + Map<ClusterSpec.Id, ClusterSpec> activatingClusters = clusters.stream() + .collect(groupingBy(LoadBalancerProvisioner::effectiveId, + reducing(null, (o, n) -> o == null || o.type() != Type.container ? n : o))); for (var cluster : loadBalancedClustersOf(newActive).entrySet()) { - if (!activatingClusters.contains(cluster.getKey())) continue; + if (!activatingClusters.containsKey(cluster.getKey())) continue; Node clusterNode = cluster.getValue().first().get(); if (!shouldProvision(transaction.application(), clusterNode.type(), clusterNode.allocation().get().membership().cluster().type())) continue; - activate(transaction, cluster.getKey(), cluster.getValue()); + activate(transaction, cluster.getKey(), activatingClusters.get(cluster.getKey()).loadBalancerSettings(), cluster.getValue()); } // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed - var surplusLoadBalancers = surplusLoadBalancersOf(transaction.application(), activatingClusters); + var surplusLoadBalancers = surplusLoadBalancersOf(transaction.application(), activatingClusters.keySet()); deactivate(surplusLoadBalancers, transaction.nested()); } @@ -180,10 +185,10 @@ public class LoadBalancerProvisioner { return loadBalancerId; } - private void prepare(LoadBalancerId id, NodeList nodes, CloudAccount cloudAccount) { + private void prepare(LoadBalancerId id, NodeList nodes, LoadBalancerSettings loadBalancerSettings, CloudAccount cloudAccount) { Instant now = nodeRepository.clock().instant(); Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); - Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, cloudAccount); + Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, loadBalancerSettings, cloudAccount); LoadBalancer newLoadBalancer; LoadBalancer.State fromState = null; if (loadBalancer.isEmpty()) { @@ -202,14 +207,14 @@ public class LoadBalancerProvisioner { requireInstance(id, instance, cloudAccount); } - private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, NodeList nodes) { + private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, LoadBalancerSettings loadBalancerSettings, NodeList nodes) { Instant now = nodeRepository.clock().instant(); LoadBalancerId id = new LoadBalancerId(transaction.application(), cluster); Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); - if (loadBalancer.isEmpty()) throw new IllegalArgumentException("Could not active load balancer that was never prepared: " + id); + if (loadBalancer.isEmpty()) throw new IllegalArgumentException("Could not activate load balancer that was never prepared: " + id); if (loadBalancer.get().instance().isEmpty()) throw new IllegalArgumentException("Activating " + id + ", but prepare never provisioned a load balancer instance"); - Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, loadBalancer.get().instance().get().cloudAccount()); + Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, loadBalancerSettings, loadBalancer.get().instance().get().cloudAccount()); LoadBalancer.State state = instance.isPresent() ? LoadBalancer.State.active : loadBalancer.get().state(); LoadBalancer newLoadBalancer = loadBalancer.get().with(instance).with(state, now); db.writeLoadBalancers(List.of(newLoadBalancer), loadBalancer.get().state(), transaction.nested()); @@ -219,6 +224,7 @@ public class LoadBalancerProvisioner { /** Provision or reconfigure a load balancer instance, if necessary */ private Optional<LoadBalancerInstance> provisionInstance(LoadBalancerId id, NodeList nodes, Optional<LoadBalancer> currentLoadBalancer, + LoadBalancerSettings loadBalancerSettings, CloudAccount cloudAccount) { boolean shouldDeactivateRouting = deactivateRouting.with(FetchVector.Dimension.APPLICATION_ID, id.application().serializedForm()) @@ -229,10 +235,10 @@ public class LoadBalancerProvisioner { } else { reals = realsOf(nodes); } - if (hasReals(currentLoadBalancer, reals)) return currentLoadBalancer.get().instance(); + if (isUpToDate(currentLoadBalancer, reals, loadBalancerSettings)) return currentLoadBalancer.get().instance(); log.log(Level.INFO, () -> "Provisioning instance for " + id + ", targeting: " + reals); try { - return Optional.of(service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals, cloudAccount), + return Optional.of(service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals, loadBalancerSettings, cloudAccount), shouldDeactivateRouting || allowEmptyReals(currentLoadBalancer))); } catch (Exception e) { log.log(Level.WARNING, e, () -> "Could not (re)configure " + id + ", targeting: " + @@ -288,11 +294,12 @@ public class LoadBalancerProvisioner { return loadBalancer.instance().isEmpty() || loadBalancer.instance().get().cloudAccount().equals(cloudAccount); } - /** Returns whether load balancer has given reals */ - private static boolean hasReals(Optional<LoadBalancer> loadBalancer, Set<Real> reals) { + /** Returns whether load balancer has given reals and settings */ + private static boolean isUpToDate(Optional<LoadBalancer> loadBalancer, Set<Real> reals, LoadBalancerSettings loadBalancerSettings) { if (loadBalancer.isEmpty()) return false; if (loadBalancer.get().instance().isEmpty()) return false; - return loadBalancer.get().instance().get().reals().equals(reals); + return loadBalancer.get().instance().get().reals().equals(reals) + && loadBalancer.get().instance().get().settings().equals(loadBalancerSettings); } /** Returns whether to allow given load balancer to have no reals */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java index be3216c79a4..92c7ba7fe27 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.LoadBalancerSettings; import org.junit.Test; import java.util.Optional; @@ -27,7 +28,9 @@ public class SharedLoadBalancerServiceTest { @Test public void test_create_lb() { - var lb = loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals, CloudAccount.empty), false); + var lb = loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals, + LoadBalancerSettings.empty, CloudAccount.empty), + false); assertEquals(Optional.of(HostName.of("vip.example.com")), lb.hostname()); assertEquals(Optional.empty(), lb.dnsZone()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index d2d805a1bd5..1e0385d152a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -6,6 +6,7 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.LoadBalancerSettings; import com.yahoo.vespa.hosted.provision.lb.DnsZone; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; @@ -14,6 +15,7 @@ import com.yahoo.vespa.hosted.provision.lb.Real; import org.junit.Test; import java.time.Instant; +import java.util.List; import java.util.Optional; import static java.time.temporal.ChronoUnit.MILLIS; @@ -43,6 +45,7 @@ public class LoadBalancerSerializerTest { new Real(DomainName.of("real-2"), "127.0.0.2", 4080)), + new LoadBalancerSettings(List.of("123")), CloudAccount.from("012345678912"))), LoadBalancer.State.active, now); @@ -56,6 +59,7 @@ public class LoadBalancerSerializerTest { assertEquals(loadBalancer.state(), serialized.state()); assertEquals(loadBalancer.changedAt().truncatedTo(MILLIS), serialized.changedAt()); assertEquals(loadBalancer.instance().get().reals(), serialized.instance().get().reals()); + assertEquals(loadBalancer.instance().get().settings(), serialized.instance().get().settings()); assertEquals(loadBalancer.instance().get().cloudAccount(), serialized.instance().get().cloudAccount()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 325e4b58174..4635eb6bc7c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.LoadBalancerSettings; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.exception.LoadBalancerServiceException; @@ -215,7 +216,7 @@ public class LoadBalancerProvisionerTest { public void provision_load_balancer_combined_cluster() { Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().list(app1).asList(); var combinedId = ClusterSpec.Id.from("container1"); - var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, ClusterSpec.Id.from("content1"), Optional.of(combinedId))); + var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, ClusterSpec.Id.from("content1"), Optional.of(combinedId), LoadBalancerSettings.empty)); assertEquals(1, lbs.get().size()); assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().get().reals().size()); tester.activate(app1, nodes); @@ -314,6 +315,24 @@ public class LoadBalancerProvisionerTest { } @Test + public void load_balancer_with_custom_settings() { + ClusterResources resources = new ClusterResources(3, 1, nodeResources); + Capacity capacity = Capacity.from(resources, resources, false, true, Optional.of(CloudAccount.empty)); + tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); + LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); + assertEquals(1, loadBalancers.size()); + assertEquals(LoadBalancerSettings.empty, loadBalancers.first().get().instance().get().settings()); + + // Next deployment contains new settings + LoadBalancerSettings settings = new LoadBalancerSettings(List.of("alice", "bob")); + tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1"), Optional.empty(), settings))); + loadBalancers = tester.nodeRepository().loadBalancers().list(); + assertEquals(1, loadBalancers.size()); + assertEquals(settings, loadBalancers.first().get().instance().get().settings()); + } + + + @Test public void load_balancer_with_custom_cloud_account() { ClusterResources resources = new ClusterResources(3, 1, nodeResources); CloudAccount cloudAccount0 = CloudAccount.empty; @@ -412,11 +431,11 @@ public class LoadBalancerProvisionerTest { } private static ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id) { - return clusterRequest(type, id, Optional.empty()); + return clusterRequest(type, id, Optional.empty(), LoadBalancerSettings.empty); } - private static ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id, Optional<ClusterSpec.Id> combinedId) { - return ClusterSpec.request(type, id).vespaVersion("6.42").combinedId(combinedId).build(); + private static ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id, Optional<ClusterSpec.Id> combinedId, LoadBalancerSettings settings) { + return ClusterSpec.request(type, id).vespaVersion("6.42").combinedId(combinedId).loadBalancerSettings(settings).build(); } private static <T> T get(Set<T> set, int position) { diff --git a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java index b7c51a0c5c9..e8cb5344aff 100644 --- a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java +++ b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java @@ -22,10 +22,17 @@ public interface FeedClientBuilder { String PREFERRED_IMPLEMENTATION_PROPERTY = "vespa.feed.client.builder.implementation"; - /** Creates a builder for a single container endpoint **/ + /** + * Creates a builder for a single feed container endpoint. + * This is for feeding against a container cluster with a load balancer in front of it. + **/ static FeedClientBuilder create(URI endpoint) { return create(Collections.singletonList(endpoint)); } - /** Creates a builder for multiple container endpoints **/ + /** + * Creates a builder which <em>distributes</em> the feed across the given feed container endpoints. + * This is for feeding directly against container nodes, i.e., when no load balancer sits in front of these. + * Each feed operation is sent to <em>one</em> of the endpoints, <strong>not all of them</strong>! + */ static FeedClientBuilder create(List<URI> endpoints) { return Helper.getFeedClientBuilderSupplier().get().setEndpointUris(endpoints); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/SingletonManager.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/SingletonManager.java index 42400b82ab6..8eda57b0476 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/SingletonManager.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/SingletonManager.java @@ -327,7 +327,6 @@ class SingletonManager { Instant ourDoom = doom.get(); boolean shouldBeActive = ourDoom != null && ourDoom != INVALID && ! clock.instant().isAfter(ourDoom); if ( ! active && shouldBeActive) { - logger.log(INFO, "Activating singleton for ID: " + id); try { active = true; if ( ! singletons.isEmpty()) metrics.activation(singletons.peek()::activate); @@ -338,7 +337,6 @@ class SingletonManager { } } if (active && ! shouldBeActive) { - logger.log(INFO, "Deactivating singleton for ID: " + id); logger.log(FINE, () -> "Doom value is " + doom); try { if ( ! singletons.isEmpty()) metrics.deactivation(singletons.peek()::deactivate); @@ -415,6 +413,7 @@ class SingletonManager { Instant start = clock.instant(); boolean failed = false; metric.add(ACTIVATION, 1, context); + logger.log(INFO, "Activating singleton for ID: " + id); try { activation.run(); } @@ -423,7 +422,9 @@ class SingletonManager { throw e; } finally { - metric.set(ACTIVATION_MILLIS, Duration.between(start, clock.instant()).toMillis(), context); + long durationMillis = Duration.between(start, clock.instant()).toMillis(); + metric.set(ACTIVATION_MILLIS, durationMillis, context); + logger.log(INFO, "Activation completed in %.3f seconds".formatted(durationMillis * 1e-3)); if (failed) metric.add(ACTIVATION_FAILURES, 1, context); else isActive = true; ping(); @@ -434,6 +435,7 @@ class SingletonManager { Instant start = clock.instant(); boolean failed = false; metric.add(DEACTIVATION, 1, context); + logger.log(INFO, "Deactivating singleton for ID: " + id); try { deactivation.run(); } @@ -442,7 +444,9 @@ class SingletonManager { throw e; } finally { - metric.set(DEACTIVATION_MILLIS, Duration.between(start, clock.instant()).toMillis(), context); + long durationMillis = Duration.between(start, clock.instant()).toMillis(); + metric.set(DEACTIVATION_MILLIS, durationMillis, context); + logger.log(INFO, "Deactivation completed in %.3f seconds".formatted(durationMillis * 1e-3)); if (failed) metric.add(DEACTIVATION_FAILURES, 1, context); isActive = false; ping(); |