diff options
author | Martin Polden <mpolden@mpolden.no> | 2022-10-06 14:22:06 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-10-06 14:22:06 +0200 |
commit | eea644eaae153312e05d415dc747584ccb54d898 (patch) | |
tree | b14275f720c81c12bf0f7e4cef41d968890f0631 | |
parent | 6bba1403ceb472e01a184528fb0129e0bf5ee36b (diff) | |
parent | f37c417e4a545f4155148ee6f083e489235e6d6c (diff) |
Merge pull request #24324 from vespa-engine/freva/lb-ip
Propagate load balancer IP to routing policies
22 files changed, 201 insertions, 87 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java index 6199c8c28b9..8b0b674d682 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java @@ -19,15 +19,17 @@ public class LoadBalancer { private final ApplicationId application; private final ClusterSpec.Id cluster; private final Optional<DomainName> hostname; + private final Optional<String> ipAddress; private final State state; private final Optional<String> dnsZone; - public LoadBalancer(String id, ApplicationId application, ClusterSpec.Id cluster, Optional<DomainName> hostname, State state, - Optional<String> dnsZone) { + public LoadBalancer(String id, ApplicationId application, ClusterSpec.Id cluster, Optional<DomainName> hostname, + Optional<String> ipAddress, State state, Optional<String> dnsZone) { this.id = Objects.requireNonNull(id, "id must be non-null"); this.application = Objects.requireNonNull(application, "application must be non-null"); this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); + this.ipAddress = Objects.requireNonNull(ipAddress, "ipAddress must be non-null"); this.state = Objects.requireNonNull(state, "state must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); } @@ -48,6 +50,10 @@ public class LoadBalancer { return hostname; } + public Optional<String> ipAddress() { + return ipAddress; + } + public Optional<String> dnsZone() { return dnsZone; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index 18ff3f18137..18d7bc53035 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -33,8 +33,8 @@ public class MemoryNameService implements NameService { } @Override - public Record createCname(RecordName name, RecordData canonicalName) { - var record = new Record(Record.Type.CNAME, name, canonicalName); + public Record createRecord(Record.Type type, RecordName name, RecordData canonicalName) { + var record = new Record(type, name, canonicalName); add(record); return record; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java index eac657d8b75..505ff3850ab 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java @@ -12,13 +12,14 @@ import java.util.Set; public interface NameService { /** - * Create a new CNAME record + * Create a new record * - * @param name The alias to create (lhs of the record) - * @param canonicalName The canonical name which the alias should point to (rhs of the record). This must be a FQDN. + * @param type The DNS type of record to make, only a small set of types are supported, check with the implementation + * @param name Name of the record, e.g. a FQDN for records of type A + * @param data Data of the record, e.g. IP address for records of type A * @return The created record */ - Record createCname(RecordName name, RecordData canonicalName); + Record createRecord(Record.Type type, RecordName name, RecordData data); /** * Create a non-standard ALIAS record pointing to given targets. Implementations of this are expected to be diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java index 63c2388b461..e53be91f94b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.stubs; import ai.vespa.http.DomainName; +import com.google.common.net.InetAddresses; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; @@ -12,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import java.net.InetAddress; import java.net.URI; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -60,12 +60,10 @@ public class MockTesterCloud implements TesterCloud { @Override public Optional<InetAddress> resolveHostName(DomainName hostname) { - try { - return Optional.of(InetAddress.getByAddress(new byte[]{ 1, 2, 3, 4 })); - } - catch (UnknownHostException e) { - throw new IllegalStateException("should not happen"); - } + return nameService.findRecords(Record.Type.A, RecordName.from(hostname.value())).stream() + .findFirst() + .map(record -> InetAddresses.forString(record.data().asString())) + .or(() -> Optional.of(InetAddresses.forString("1.2.3.4"))); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 071d8a4d11f..91bca1e481c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -268,9 +268,9 @@ public class RoutingController { // Register names in DNS Rotation rotation = rotationRepository.requireRotation(assignedRotation.rotationId()); for (var endpoint : rotationEndpoints) { - controller.nameServiceForwarder().createCname(RecordName.from(endpoint.dnsName()), - RecordData.fqdn(rotation.name()), - Priority.normal); + controller.nameServiceForwarder().createRecord( + new Record(Record.Type.CNAME, RecordName.from(endpoint.dnsName()), RecordData.fqdn(rotation.name())), + Priority.normal); List<String> names = List.of(endpoint.dnsName(), // Include rotation ID as a valid name of this container endpoint // (required by global routing health checks) @@ -305,9 +305,9 @@ public class RoutingController { ZoneId targetZone = targetZones.iterator().next(); String vipHostname = controller.zoneRegistry().getVipHostname(targetZone) .orElseThrow(() -> new IllegalArgumentException("No VIP configured for zone " + targetZone)); - controller.nameServiceForwarder().createCname(RecordName.from(endpoint.dnsName()), - RecordData.fqdn(vipHostname), - Priority.normal); + controller.nameServiceForwarder().createRecord( + new Record(Record.Type.CNAME, RecordName.from(endpoint.dnsName()), RecordData.fqdn(vipHostname)), + Priority.normal); } Map<ClusterSpec.Id, EndpointList> applicationEndpointsByCluster = applicationEndpoints.groupingBy(Endpoint::cluster); for (var kv : applicationEndpointsByCluster.entrySet()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 965f1b09819..4d7b84be65e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import ai.vespa.http.DomainName; +import com.google.common.net.InetAddresses; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.Notifications; @@ -510,18 +511,25 @@ public class InternalStepRunner implements StepRunner { if (context.routingMethod() == RoutingMethod.exclusive) { RoutingPolicy policy = context.routingPolicy(ClusterSpec.Id.from(endpoint.name())) .orElseThrow(() -> new IllegalStateException(endpoint + " has no matching policy")); + if (policy.ipAddress().isPresent()) { + if (ipAddress.equals(policy.ipAddress().map(InetAddresses::forString))) continue; + logger.log(INFO, "IP address of '" + endpointName + "' (" + + ipAddress.map(InetAddresses::toAddrString).get() + ") and load balancer " + + "' (" + policy.ipAddress().orElseThrow() + ") are not equal"); + return false; + } var cNameValue = controller.jobController().cloud().resolveCname(endpointName); - if ( ! cNameValue.map(policy.canonicalName()::equals).orElse(false)) { + if ( ! cNameValue.map(policy.canonicalName().get()::equals).orElse(false)) { logger.log(INFO, "CNAME '" + endpointName + "' points at " + cNameValue.map(name -> "'" + name + "'").orElse("nothing") + " but should point at load balancer '" + policy.canonicalName() + "'"); return false; } - var loadBalancerAddress = controller.jobController().cloud().resolveHostName(policy.canonicalName()); + var loadBalancerAddress = controller.jobController().cloud().resolveHostName(policy.canonicalName().get()); if ( ! loadBalancerAddress.equals(ipAddress)) { logger.log(INFO, "IP address of CNAME '" + endpointName + "' (" + ipAddress.get() + ") and load balancer '" + - policy.canonicalName() + "' (" + loadBalancerAddress.orElse(null) + ") are not equal"); + policy.canonicalName().get() + "' (" + loadBalancerAddress.orElse(null) + ") are not equal"); return false; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java index 464e3eff203..344ffad80e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java @@ -19,7 +19,7 @@ public class CreateRecord implements NameServiceRequest { /** DO NOT USE. Public for serialization purposes */ public CreateRecord(Record record) { this.record = Objects.requireNonNull(record, "record must be non-null"); - if (record.type() != Record.Type.CNAME) { + if (record.type() != Record.Type.CNAME && record.type() != Record.Type.A) { throw new IllegalArgumentException("Record of type " + record.type() + " is not supported: " + record); } } @@ -38,7 +38,7 @@ public class CreateRecord implements NameServiceRequest { } }); if (records.isEmpty()) { - nameService.createCname(record.name(), record.data()); + nameService.createRecord(record.type(), record.name(), record.data()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index 540e8489e6d..9d2c7918252 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.dns; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; @@ -43,9 +42,9 @@ public class NameServiceForwarder { this.db = Objects.requireNonNull(db, "db must be non-null"); } - /** Create or update a CNAME record with given name and data */ - public void createCname(RecordName name, RecordData canonicalName, NameServiceQueue.Priority priority) { - forward(new CreateRecord(new Record(Record.Type.CNAME, name, canonicalName)), priority); + /** Create or update a given record */ + public void createRecord(Record record, NameServiceQueue.Priority priority) { + forward(new CreateRecord(record), priority); } /** Create or update an ALIAS record with given name and targets */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 72c16ae0110..4d759056dfc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -40,6 +40,7 @@ public class RoutingPolicySerializer { private static final String routingPoliciesField = "routingPolicies"; private static final String clusterField = "cluster"; private static final String canonicalNameField = "canonicalName"; + private static final String ipAddressField = "ipAddress"; private static final String zoneField = "zone"; private static final String dnsZoneField = "dnsZone"; private static final String instanceEndpointsField = "rotations"; @@ -58,7 +59,8 @@ public class RoutingPolicySerializer { var policyObject = policyArray.addObject(); policyObject.setString(clusterField, policy.id().cluster().value()); policyObject.setString(zoneField, policy.id().zone().value()); - policyObject.setString(canonicalNameField, policy.canonicalName().value()); + policy.canonicalName().map(DomainName::value).ifPresent(name -> policyObject.setString(canonicalNameField, name)); + policy.ipAddress().ifPresent(ipAddress -> policyObject.setString(ipAddressField, ipAddress)); policy.dnsZone().ifPresent(dnsZone -> policyObject.setString(dnsZoneField, dnsZone)); var instanceEndpointsArray = policyObject.setArray(instanceEndpointsField); policy.instanceEndpoints().forEach(endpointId -> instanceEndpointsArray.addString(endpointId.id())); @@ -83,7 +85,8 @@ public class RoutingPolicySerializer { ClusterSpec.Id.from(inspect.field(clusterField).asString()), ZoneId.from(inspect.field(zoneField).asString())); policies.add(new RoutingPolicy(id, - DomainName.of(inspect.field(canonicalNameField).asString()), + SlimeUtils.optionalString(inspect.field(canonicalNameField)).map(DomainName::of), + SlimeUtils.optionalString(inspect.field(ipAddressField)), SlimeUtils.optionalString(inspect.field(dnsZoneField)), instanceEndpoints, applicationEndpoints, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index ac29f8952a0..f76d04c9e1d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -205,20 +205,24 @@ public class RoutingPolicies { for (var policy : policies) { if (policy.dnsZone().isEmpty()) continue; if (controller.zoneRegistry().routingMethod(policy.id().zone()) != RoutingMethod.exclusive) continue; - Endpoint regionEndpoint = policy.regionEndpointIn(controller.system(), RoutingMethod.exclusive); + Endpoint endpoint = policy.regionEndpointIn(controller.system(), RoutingMethod.exclusive); var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); long weight = 1; if (isConfiguredOut(zonePolicy, policy, inactiveZones)) { weight = 0; // A record with 0 weight will not receive traffic. If all records within a group have 0 // weight, traffic is routed to all records with equal probability. } - var weightedTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(), - policy.id().zone(), weight); - endpoints.computeIfAbsent(regionEndpoint, (k) -> new RegionEndpoint(new LatencyAliasTarget(DomainName.of(regionEndpoint.dnsName()), - policy.dnsZone().get(), - policy.id().zone()))) - .zoneTargets() - .add(weightedTarget); + + RegionEndpoint regionEndpoint = endpoints.computeIfAbsent(endpoint, (k) -> new RegionEndpoint( + new LatencyAliasTarget(DomainName.of(endpoint.dnsName()), policy.dnsZone().get(), policy.id().zone()))); + + if (policy.canonicalName().isPresent()) { + var weightedTarget = new WeightedAliasTarget( + policy.canonicalName().get(), policy.dnsZone().get(), policy.id().zone(), weight); + regionEndpoint.zoneTargets().add(weightedTarget); + } else { + // TODO (freva): Add direct weighted record + } } return endpoints.values(); } @@ -250,8 +254,9 @@ public class RoutingPolicies { for (var target : endpoint.targets()) { if (!policy.appliesTo(target.deployment())) continue; if (policy.dnsZone().isEmpty()) continue; // Does not support ALIAS records + if (policy.canonicalName().isEmpty()) continue; // TODO (freva): Handle DIRECT records ZoneRoutingPolicy zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); - WeightedAliasTarget weightedAliasTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(), + WeightedAliasTarget weightedAliasTarget = new WeightedAliasTarget(policy.canonicalName().get(), policy.dnsZone().get(), target.deployment().zoneId(), target.weight()); Set<AliasTarget> activeTargets = targetsByEndpoint.computeIfAbsent(endpoint, (k) -> new LinkedHashSet<>()); Set<AliasTarget> inactiveTargets = inactiveTargetsByEndpoint.computeIfAbsent(endpoint, (k) -> new LinkedHashSet<>()); @@ -309,10 +314,10 @@ public class RoutingPolicies { private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) { Map<RoutingPolicyId, RoutingPolicy> policies = new LinkedHashMap<>(instancePolicies.asMap()); for (LoadBalancer loadBalancer : allocation.loadBalancers) { - if (loadBalancer.hostname().isEmpty()) continue; + if (loadBalancer.hostname().isEmpty() && loadBalancer.ipAddress().isEmpty()) continue; var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), allocation.deployment.zoneId()); var existingPolicy = policies.get(policyId); - var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname().get(), loadBalancer.dnsZone(), + var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.ipAddress(), loadBalancer.dnsZone(), allocation.instanceEndpointsOf(loadBalancer), allocation.applicationEndpointsOf(loadBalancer), new RoutingPolicy.Status(isActive(loadBalancer), RoutingStatus.DEFAULT)); @@ -332,8 +337,10 @@ public class RoutingPolicies { private void updateZoneDnsOf(RoutingPolicy policy) { for (var endpoint : policy.zoneEndpointsIn(controller.system(), RoutingMethod.exclusive, controller.zoneRegistry())) { var name = RecordName.from(endpoint.dnsName()); - var data = RecordData.fqdn(policy.canonicalName().value()); - nameServiceForwarderIn(policy.id().zone()).createCname(name, data, Priority.normal); + var record = policy.canonicalName().isPresent() ? + new Record(Record.Type.CNAME, name, RecordData.fqdn(policy.canonicalName().get().value())) : + new Record(Record.Type.A, name, RecordData.from(policy.ipAddress().orElseThrow())); + nameServiceForwarderIn(policy.id().zone()).createRecord(record, Priority.normal); } } @@ -393,10 +400,16 @@ public class RoutingPolicies { for (var policy : policies) { if (!policy.appliesTo(allocation.deployment)) continue; NameServiceForwarder forwarder = nameServiceForwarderIn(policy.id().zone()); - endpoints.forEach(endpoint -> forwarder.removeRecords(Record.Type.ALIAS, - RecordName.from(endpoint.dnsName()), - RecordData.fqdn(policy.canonicalName().value()), - Priority.normal)); + for (Endpoint endpoint : endpoints) { + if (policy.canonicalName().isPresent()) { + forwarder.removeRecords(Record.Type.ALIAS, + RecordName.from(endpoint.dnsName()), + RecordData.fqdn(policy.canonicalName().get().value()), + Priority.normal); + } else { + // TODO (freva): Remove DIRECT records + } + } } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 585cda65e66..04c32590a4c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -24,21 +24,27 @@ import java.util.Set; * @author mpolden */ public record RoutingPolicy(RoutingPolicyId id, - DomainName canonicalName, + Optional<DomainName> canonicalName, + Optional<String> ipAddress, Optional<String> dnsZone, Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, Status status) { /** DO NOT USE. Public for serialization purposes */ - public RoutingPolicy(RoutingPolicyId id, DomainName canonicalName, Optional<String> dnsZone, + public RoutingPolicy(RoutingPolicyId id, Optional<DomainName> canonicalName, Optional<String> ipAddress, Optional<String> dnsZone, Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, Status status) { this.id = Objects.requireNonNull(id, "id must be non-null"); this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); + this.ipAddress = Objects.requireNonNull(ipAddress, "ipAddress must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); this.instanceEndpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(instanceEndpoints, "instanceEndpoints must be non-null")); this.applicationEndpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(applicationEndpoints, "applicationEndpoints must be non-null")); this.status = Objects.requireNonNull(status, "status must be non-null"); + + if (canonicalName.isEmpty() == ipAddress.isEmpty()) + throw new IllegalArgumentException("Exactly 1 of canonicalName=%s and ipAddress=%s must be set".formatted( + canonicalName.map(DomainName::value).orElse("<empty>"), ipAddress.orElse("<empty>"))); } /** The ID of this */ @@ -47,10 +53,15 @@ public record RoutingPolicy(RoutingPolicyId id, } /** The canonical name for the load balancer this applies to (rhs of a CNAME or ALIAS record) */ - public DomainName canonicalName() { + public Optional<DomainName> canonicalName() { return canonicalName; } + /** The IP address for the load balancer this applies to (rhs of an A or DIRECT record) */ + public Optional<String> ipAddress() { + return ipAddress; + } + /** DNS zone for the load balancer this applies to, if any. Used when creating ALIAS records. */ public Optional<String> dnsZone() { return dnsZone; @@ -79,7 +90,7 @@ public record RoutingPolicy(RoutingPolicyId id, /** Returns a copy of this with status set to given status */ public RoutingPolicy with(Status status) { - return new RoutingPolicy(id, canonicalName, dnsZone, instanceEndpoints, applicationEndpoints, status); + return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, status); } /** Returns the zone endpoints of this */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index bd43a9dafbc..b38bdbb1eaf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -271,7 +271,8 @@ public class DeploymentContext { var clusterId = "default-inactive"; var id = new RoutingPolicyId(instanceId, Id.from(clusterId), zone); var policies = new LinkedHashMap<>(tester.controller().routing().policies().read(instanceId).asMap()); - policies.put(id, new RoutingPolicy(id, HostName.of("lb-host"), + policies.put(id, new RoutingPolicy(id, Optional.of(HostName.of("lb-host")), + Optional.empty(), Optional.empty(), Set.of(EndpointId.of("default")), Set.of(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index d7f83979054..5a8d6a45e43 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -402,6 +402,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer id.applicationId(), cluster, Optional.of(HostName.of("lb-0--" + id.applicationId().toFullString() + "--" + id.zoneId().toString())), + Optional.empty(), LoadBalancer.State.active, Optional.of("dns-zone-1")))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index 3df459513d7..6285c5c4aac 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -38,20 +38,29 @@ public class RoutingPolicySerializerTest { ClusterSpec.Id.from("my-cluster2"), ZoneId.from("prod", "us-north-2")); var policies = List.of(new RoutingPolicy(id1, - HostName.of("long-and-ugly-name"), + Optional.of(HostName.of("long-and-ugly-name")), + Optional.empty(), Optional.of("zone1"), instanceEndpoints, applicationEndpoints, new RoutingPolicy.Status(true, RoutingStatus.DEFAULT)), new RoutingPolicy(id2, - HostName.of("long-and-ugly-name-2"), + Optional.of(HostName.of("long-and-ugly-name-2")), + Optional.empty(), Optional.empty(), instanceEndpoints, Set.of(), new RoutingPolicy.Status(false, new RoutingStatus(RoutingStatus.Value.out, RoutingStatus.Agent.tenant, - Instant.ofEpochSecond(123))))); + Instant.ofEpochSecond(123)))), + new RoutingPolicy(id1, + Optional.empty(), + Optional.of("127.0.0.1"), + Optional.of("zone2"), + instanceEndpoints, + applicationEndpoints, + new RoutingPolicy.Status(true, RoutingStatus.DEFAULT))); var serialized = serializer.fromSlime(owner, serializer.toSlime(policies)); assertEquals(policies.size(), serialized.size()); for (Iterator<RoutingPolicy> it1 = policies.iterator(), it2 = serialized.iterator(); it1.hasNext(); ) { @@ -59,6 +68,7 @@ public class RoutingPolicySerializerTest { var actual = it2.next(); assertEquals(expected.id(), actual.id()); assertEquals(expected.canonicalName(), actual.canonicalName()); + assertEquals(expected.ipAddress(), actual.ipAddress()); assertEquals(expected.dnsZone(), actual.dnsZone()); assertEquals(expected.instanceEndpoints(), actual.instanceEndpoints()); assertEquals(expected.applicationEndpoints(), actual.applicationEndpoints()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index c5d74e0b01d..2761c736e11 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; +import ai.vespa.http.DomainName; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import com.yahoo.config.application.api.DeploymentSpec; @@ -351,6 +352,32 @@ public class RoutingPoliciesTest { } @Test + void cross_cloud_policies() { + var tester = new RoutingPoliciesTester(SystemName.Public); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + var zone1 = ZoneId.from("prod", "aws-us-east-1c"); + var zone2 = ZoneId.from("prod", "gcp-us-south1-b"); + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + + var applicationPackage = applicationPackageBuilder() + .region(zone1.region().value()) + .region(zone2.region().value()) + .endpoint("r0", "c0") + .trustDefaultCertificate() + .build(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + List<String> expectedRecords = List.of("c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", + "c0.app1.tenant1.gcp-us-south1-b.z.vespa-app.cloud", + "c0.app1.tenant1.aws-us-east-1.w.vespa-app.cloud", + "r0.app1.tenant1.g.vespa-app.cloud"); + assertEquals(Set.copyOf(expectedRecords), tester.recordNames()); + + assertEquals(List.of("lb-0--tenant1.app1.default--prod.aws-us-east-1c."), tester.recordDataOf(Record.Type.CNAME, expectedRecords.get(0))); + assertEquals(List.of("10.0.0.0"), tester.recordDataOf(Record.Type.A, expectedRecords.get(1))); + } + + @Test void global_routing_policies_in_public() { var tester = new RoutingPoliciesTester(SystemName.Public); var context = tester.newDeploymentContext("tenant1", "app1", "default"); @@ -359,8 +386,8 @@ public class RoutingPoliciesTest { ZoneId zone2 = prodZones.get(1); var applicationPackage = applicationPackageBuilder() - .region(zone1.region().value()) - .region(zone2.region().value()) + .region(zone1.region()) + .region(zone2.region()) .endpoint("default", "default") .trustDefaultCertificate() .build(); @@ -452,6 +479,7 @@ public class RoutingPoliciesTest { context.instanceId(), ClusterSpec.Id.from("c0"), Optional.of(newHostname), + Optional.empty(), LoadBalancer.State.active, Optional.of("dns-zone-1")); tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer)); @@ -461,8 +489,8 @@ public class RoutingPoliciesTest { assertEquals(expectedRecords, tester.recordNames()); assertEquals(1, tester.policiesOf(context.instanceId()).size()); assertEquals(newHostname.value() + ".", - tester.cnameDataOf(expectedRecords.iterator().next()).get(0), - "CNAME points to current load blancer"); + tester.recordDataOf(Record.Type.CNAME, expectedRecords.iterator().next()).get(0), + "CNAME points to current load balancer"); } @Test @@ -836,20 +864,24 @@ public class RoutingPoliciesTest { private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, boolean shared, int count) { List<LoadBalancer> loadBalancers = new ArrayList<>(); for (int i = 0; i < count; i++) { - HostName lbHostname; - if (shared) { - lbHostname = HostName.of("shared-lb--" + zone.value()); + Optional<DomainName> lbHostname; + Optional<String> ipAddress; + if (zone.region().value().startsWith("gcp-")) { + lbHostname = Optional.empty(); + ipAddress = Optional.of("10.0.0." + i); } else { - lbHostname = HostName.of("lb-" + i + "--" + application.toFullString() + - "--" + zone.value()); + String hostname = shared ? "shared-lb--" + zone.value() : "lb-" + i + "--" + application.toFullString() + "--" + zone.value(); + lbHostname = Optional.of(DomainName.of(hostname)); + ipAddress = Optional.empty(); } loadBalancers.add( new LoadBalancer("LB-" + i + "-Z-" + zone.value(), application, ClusterSpec.Id.from("c" + i), - Optional.of(lbHostname), + lbHostname, + ipAddress, LoadBalancer.State.active, - Optional.of("dns-zone-1"))); + Optional.of("dns-zone-1").filter(__ -> lbHostname.isPresent()))); } return loadBalancers; } @@ -858,6 +890,7 @@ public class RoutingPoliciesTest { var sharedRegion = RegionName.from("aws-us-east-1c"); return List.of(ZoneId.from(Environment.prod, sharedRegion), ZoneId.from(Environment.prod, RegionName.from("aws-eu-west-1a")), + ZoneId.from(Environment.prod, RegionName.from("gcp-us-south1-b")), ZoneId.from(Environment.staging, RegionName.from("us-east-3")), ZoneId.from(Environment.test, RegionName.from("us-east-1"))); } @@ -936,8 +969,8 @@ public class RoutingPoliciesTest { .collect(Collectors.toSet()); } - private List<String> cnameDataOf(String name) { - return tester.controllerTester().nameService().findRecords(Record.Type.CNAME, RecordName.from(name)).stream() + private List<String> recordDataOf(Record.Type type, String name) { + return tester.controllerTester().nameService().findRecords(type, RecordName.from(name)).stream() .map(Record::data) .map(RecordData::asString) .collect(Collectors.toList()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java index edffa817f64..b8e7cfb5441 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java @@ -17,28 +17,39 @@ import java.util.Set; */ public class LoadBalancerInstance { - private final DomainName hostname; + private final Optional<DomainName> hostname; + private final Optional<String> ipAddress; private final Optional<DnsZone> dnsZone; private final Set<Integer> ports; private final Set<String> networks; private final Set<Real> reals; private final Optional<CloudAccount> cloudAccount; - public LoadBalancerInstance(DomainName hostname, Optional<DnsZone> dnsZone, Set<Integer> ports, Set<String> networks, - Set<Real> reals, Optional<CloudAccount> cloudAccount) { + public LoadBalancerInstance(Optional<DomainName> hostname, Optional<String> ipAddress, Optional<DnsZone> dnsZone, Set<Integer> ports, + Set<String> networks, Set<Real> reals, Optional<CloudAccount> cloudAccount) { this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); + this.ipAddress = Objects.requireNonNull(ipAddress, "ip must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); this.ports = ImmutableSortedSet.copyOf(requirePorts(ports)); this.networks = ImmutableSortedSet.copyOf(Objects.requireNonNull(networks, "networks must be non-null")); this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals, "targets must be non-null")); this.cloudAccount = Objects.requireNonNull(cloudAccount, "cloudAccount must be non-null"); + + if (hostname.isEmpty() == ipAddress.isEmpty()) + throw new IllegalArgumentException("Exactly 1 of hostname=%s and ipAddress=%s must be set".formatted( + hostname.map(DomainName::value).orElse("<empty>"), ipAddress.orElse("<empty>"))); } /** Fully-qualified domain name of this load balancer. This hostname can be used for query and feed */ - public DomainName hostname() { + public Optional<DomainName> hostname() { return hostname; } + /** IP address of this load balancer */ + public Optional<String> ipAddress() { + return ipAddress; + } + /** ID of the DNS zone associated with this */ public Optional<DnsZone> dnsZone() { return dnsZone; @@ -66,7 +77,7 @@ public class LoadBalancerInstance { /** Returns a copy of this with reals set to given reals */ public LoadBalancerInstance withReals(Set<Real> reals) { - return new LoadBalancerInstance(hostname, dnsZone, ports, networks, reals, cloudAccount); + return new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, cloudAccount); } private static Set<Integer> requirePorts(Set<Integer> ports) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index df92a6ee44d..ac5330dce12 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -55,7 +55,8 @@ public class LoadBalancerServiceMock implements LoadBalancerService { throw new IllegalArgumentException("Refusing to remove all reals from load balancer " + id); } var instance = new LoadBalancerInstance( - DomainName.of("lb-" + spec.application().toShortString() + "-" + spec.cluster().value()), + Optional.of(DomainName.of("lb-" + spec.application().toShortString() + "-" + spec.cluster().value())), + Optional.empty(), Optional.of(new DnsZone("zone-id-1")), Collections.singleton(4443), ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index 3d0cd3ef1e1..c126b3969fa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -28,7 +28,8 @@ public class SharedLoadBalancerService implements LoadBalancerService { @Override public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { - return new LoadBalancerInstance(DomainName.of(vipHostname), + return new LoadBalancerInstance(Optional.of(DomainName.of(vipHostname)), + Optional.empty(), Optional.empty(), Set.of(4443), Set.of(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index 83180b2b136..30c7e79ab02 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -38,6 +38,7 @@ public class LoadBalancerSerializer { private static final String idField = "id"; private static final String hostnameField = "hostname"; + private static final String lbIpAddressField = "ipAddress"; private static final String stateField = "state"; private static final String changedAtField = "changedAt"; private static final String dnsZoneField = "dnsZone"; @@ -53,7 +54,8 @@ public class LoadBalancerSerializer { Cursor root = slime.setObject(); root.setString(idField, loadBalancer.id().serializedForm()); - loadBalancer.instance().ifPresent(instance -> root.setString(hostnameField, instance.hostname().value())); + loadBalancer.instance().flatMap(LoadBalancerInstance::hostname).ifPresent(hostname -> root.setString(hostnameField, hostname.value())); + loadBalancer.instance().flatMap(LoadBalancerInstance::ipAddress).ifPresent(ip -> root.setString(lbIpAddressField, ip)); root.setString(stateField, asString(loadBalancer.state())); root.setLong(changedAtField, loadBalancer.changedAt().toEpochMilli()); loadBalancer.instance().flatMap(LoadBalancerInstance::dnsZone).ifPresent(dnsZone -> root.setString(dnsZoneField, dnsZone.id())); @@ -94,10 +96,11 @@ public class LoadBalancerSerializer { object.field(networksField).traverse((ArrayTraverser) (i, network) -> networks.add(network.asString())); Optional<DomainName> hostname = optionalString(object.field(hostnameField), Function.identity()).filter(s -> !s.isEmpty()).map(DomainName::of); + Optional<String> ipAddress = optionalString(object.field(lbIpAddressField), Function.identity()).filter(s -> !s.isEmpty()); Optional<DnsZone> dnsZone = optionalString(object.field(dnsZoneField), DnsZone::new); Optional<CloudAccount> cloudAccount = optionalString(object.field(cloudAccountField), CloudAccount::new); - Optional<LoadBalancerInstance> instance = hostname.map(h -> new LoadBalancerInstance(h, dnsZone, ports, - networks, reals, cloudAccount)); + Optional<LoadBalancerInstance> instance = hostname.isEmpty() && ipAddress.isEmpty() ? Optional.empty() : + Optional.of(new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, cloudAccount)); return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()), instance, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index 7686a9a4885..13489db9f62 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -56,7 +56,8 @@ public class LoadBalancersResponse extends SlimeJsonResponse { lbObject.setString("tenant", lb.id().application().tenant().value()); lbObject.setString("instance", lb.id().application().instance().value()); lbObject.setString("cluster", lb.id().cluster().value()); - lb.instance().ifPresent(instance -> lbObject.setString("hostname", instance.hostname().value())); + lb.instance().flatMap(LoadBalancerInstance::hostname).ifPresent(hostname -> lbObject.setString("hostname", hostname.value())); + lb.instance().flatMap(LoadBalancerInstance::ipAddress).ifPresent(ipAddress -> lbObject.setString("ipAddress", ipAddress)); lb.instance().flatMap(LoadBalancerInstance::dnsZone).ifPresent(dnsZone -> lbObject.setString("dnsZone", dnsZone.id())); Cursor networkArray = lbObject.setArray("networks"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java index 825e46865fe..1b5fceecbf9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java @@ -28,7 +28,7 @@ public class SharedLoadBalancerServiceTest { public void test_create_lb() { var lb = loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals, Optional.empty()), false); - assertEquals(HostName.of("vip.example.com"), lb.hostname()); + assertEquals(Optional.of(HostName.of("vip.example.com")), lb.hostname()); assertEquals(Optional.empty(), lb.dnsZone()); assertEquals(Set.of(), lb.networks()); assertEquals(Set.of(4443), lb.ports()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index ce8fc2e9d03..5bb9b71a223 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -24,15 +24,16 @@ import static org.junit.Assert.assertEquals; */ public class LoadBalancerSerializerTest { + private static final LoadBalancerId loadBalancerId = new LoadBalancerId( + ApplicationId.from("tenant1", "application1", "default"), ClusterSpec.Id.from("qrs"));; + @Test public void test_serialization() { var now = Instant.now(); - var loadBalancer = new LoadBalancer(new LoadBalancerId(ApplicationId.from("tenant1", - "application1", - "default"), - ClusterSpec.Id.from("qrs")), + var loadBalancer = new LoadBalancer(loadBalancerId, Optional.of(new LoadBalancerInstance( - DomainName.of("lb-host"), + Optional.of(DomainName.of("lb-host")), + Optional.empty(), Optional.of(new DnsZone("zone-id-1")), ImmutableSet.of(4080, 4443), ImmutableSet.of("10.2.3.4/24"), @@ -58,4 +59,16 @@ public class LoadBalancerSerializerTest { assertEquals(loadBalancer.instance().get().cloudAccount(), serialized.instance().get().cloudAccount()); } + @Test + public void no_instance_serialization() { + var now = Instant.now(); + var loadBalancer = new LoadBalancer(loadBalancerId, Optional.empty(), LoadBalancer.State.reserved, now); + + var serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer)); + assertEquals(loadBalancer.id(), serialized.id()); + assertEquals(loadBalancer.instance(), serialized.instance()); + assertEquals(loadBalancer.state(), serialized.state()); + assertEquals(loadBalancer.changedAt().truncatedTo(MILLIS), serialized.changedAt()); + } + } |