From 533707520186207e7959c4ce86e6694086104619 Mon Sep 17 00:00:00 2001 From: jonmv Date: Fri, 10 Nov 2023 11:01:58 +0100 Subject: Pre-provision load balancers, and use for apps --- .../hosted/provision/lb/LoadBalancerInstance.java | 7 +- .../hosted/provision/lb/LoadBalancerService.java | 14 +++- .../provision/lb/LoadBalancerServiceMock.java | 10 +-- .../hosted/provision/lb/LoadBalancerSpec.java | 9 +-- .../provision/lb/SharedLoadBalancerService.java | 5 +- .../maintenance/LoadBalancerPreProvisioner.java | 28 +++++++ .../maintenance/NodeRepositoryMaintenance.java | 8 +- .../hosted/provision/persistence/CuratorDb.java | 22 ++++++ .../persistence/LoadBalancerSerializer.java | 5 +- .../provisioning/LoadBalancerProvisioner.java | 89 +++++++++++++++++++--- .../provisioning/ProvisionServiceProvider.java | 4 +- .../provision/restapi/LoadBalancersResponse.java | 2 +- .../provision/testutils/MockNodeRepository.java | 9 ++- .../persistence/LoadBalancerSerializerTest.java | 2 +- .../restapi/responses/load-balancers-single.json | 17 +++-- .../restapi/responses/load-balancers.json | 4 +- 16 files changed, 179 insertions(+), 56 deletions(-) create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerPreProvisioner.java (limited to 'node-repository/src') diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java index 04ea9d20edf..4dbf891b1b7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java @@ -11,7 +11,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.UUID; /** * Represents a load balancer instance. This contains the fields that are owned by a {@link LoadBalancerService} and is @@ -21,7 +20,7 @@ import java.util.UUID; */ public class LoadBalancerInstance { - private final Optional idSeed; + private final Optional idSeed; private final Optional hostname; private final Optional ip4Address; private final Optional ip6Address; @@ -33,7 +32,7 @@ public class LoadBalancerInstance { private final List serviceIds; private final CloudAccount cloudAccount; - public LoadBalancerInstance(Optional idSeed, Optional hostname, Optional ip4Address, Optional ip6Address, + public LoadBalancerInstance(Optional idSeed, Optional hostname, Optional ip4Address, Optional ip6Address, Optional dnsZone, Set ports, Set networks, Set reals, ZoneEndpoint settings, List serviceIds, CloudAccount cloudAccount) { this.idSeed = Objects.requireNonNull(idSeed, "idSeed must be non-null"); @@ -55,7 +54,7 @@ public class LoadBalancerInstance { } /** A unique seed to use when generating cloud-specific resource IDs for this load balancer instance. */ - public Optional idSeed() { + public Optional idSeed() { return idSeed; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index 6649b18ed02..6ddde1151dd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.EndpointsChecker.HealthChecker; import com.yahoo.config.provision.NodeType; import java.util.Optional; -import java.util.UUID; /** * A managed load balance service. @@ -23,7 +22,16 @@ public interface LoadBalancerService { * @param spec Load balancer specification * @return The provisioned load balancer instance */ - LoadBalancerInstance provision(LoadBalancerSpec spec); + default LoadBalancerInstance provision(LoadBalancerSpec spec) { return provision(spec, Optional.empty()); } + + /** + * Provisions load balancers from the given specification. Implementations are expected to be idempotent + * + * @param spec Load balancer specification + * @param idSeed Seed for generating a unique ID for the load balancer instance + * @return The provisioned load balancer instance + */ + LoadBalancerInstance provision(LoadBalancerSpec spec, Optional idSeed); /** * Configures load balancers for the given specification. Implementations are expected to be idempotent @@ -48,7 +56,7 @@ public interface LoadBalancerService { boolean supports(NodeType nodeType, ClusterSpec.Type clusterType); /** See {@link HealthChecker#healthy(Endpoint)}. */ - Availability healthy(Endpoint endpoint, Optional idSeed); + Availability healthy(Endpoint endpoint, Optional idSeed); /** Load balancer protocols */ enum Protocol { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index 554ec26d5f4..03ff17c6ebc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -15,7 +15,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import static java.util.Objects.requireNonNull; @@ -26,7 +25,7 @@ import static java.util.stream.Collectors.toMap; */ public class LoadBalancerServiceMock implements LoadBalancerService { - private record Key(ApplicationId application, ClusterSpec.Id cluster, UUID idSeed) { + private record Key(ApplicationId application, ClusterSpec.Id cluster, String idSeed) { @Override public int hashCode() { return idSeed == null ? Objects.hash(application, cluster) : Objects.hash(idSeed); } @Override public boolean equals(Object o) { if (o == this) return true; @@ -69,9 +68,8 @@ public class LoadBalancerServiceMock implements LoadBalancerService { } @Override - public LoadBalancerInstance provision(LoadBalancerSpec spec) { + public LoadBalancerInstance provision(LoadBalancerSpec spec, Optional idSeed) { if (throwOnCreate) throw new IllegalStateException("Did not expect a new load balancer to be created"); - Optional idSeed = uuid.getAndSet(false) ? Optional.of(UUID.fromString("c11272ab-d20e-4c86-b808-ffedaa00c480")) : Optional.empty(); var instance = new LoadBalancerInstance( idSeed, Optional.of(DomainName.of("lb-" + spec.application().toShortString() + "-" + spec.cluster().value())), @@ -104,7 +102,7 @@ public class LoadBalancerServiceMock implements LoadBalancerService { @Override public void reallocate(LoadBalancerInstance provisioned, LoadBalancerSpec spec) { - instances.put(new Key(spec.application(), spec.cluster(), null), + instances.put(new Key(spec.application(), spec.cluster(), provisioned.idSeed().get()), requireNonNull(instances.remove(new Key(null, null, provisioned.idSeed().get())))); // ᕙ༼◕_◕༽ᕤ } @@ -117,7 +115,7 @@ public class LoadBalancerServiceMock implements LoadBalancerService { } @Override - public Availability healthy(Endpoint endpoint, Optional idSeed) { + public Availability healthy(Endpoint endpoint, Optional idSeed) { return Availability.ready; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java index 199578d66f2..49a5c9c3c5e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.ZoneEndpoint; import java.util.Objects; -import java.util.Optional; import java.util.Set; /** @@ -20,12 +19,8 @@ public record LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster ZoneEndpoint settings, CloudAccount cloudAccount) { public static final ApplicationId preProvisionOwner = ApplicationId.from("hosted-vespa", "pre-provision", "default"); - public static LoadBalancerSpec preProvisionSpec(int slot) { - return new LoadBalancerSpec(preProvisionOwner, - ClusterSpec.Id.from("slot-" + slot), - Set.of(), - ZoneEndpoint.defaultEndpoint, - CloudAccount.empty); + public static LoadBalancerSpec preProvisionSpec(ClusterSpec.Id slot) { + return new LoadBalancerSpec(preProvisionOwner, slot, Set.of(), ZoneEndpoint.defaultEndpoint, CloudAccount.empty); } public LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set reals, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index 9124b012dfe..883b8dec944 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -11,7 +11,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.UUID; /** * This implementation of {@link LoadBalancerService} returns the load balancer(s) that exist by default in the shared @@ -31,7 +30,7 @@ public class SharedLoadBalancerService implements LoadBalancerService { } @Override - public LoadBalancerInstance provision(LoadBalancerSpec spec) { + public LoadBalancerInstance provision(LoadBalancerSpec spec, Optional idSeed) { return create(spec); } @@ -79,7 +78,7 @@ public class SharedLoadBalancerService implements LoadBalancerService { } @Override - public Availability healthy(Endpoint endpoint, Optional idSeed) { + public Availability healthy(Endpoint endpoint, Optional idSeed) { return Availability.ready; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerPreProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerPreProvisioner.java new file mode 100644 index 00000000000..50c58f6c834 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerPreProvisioner.java @@ -0,0 +1,28 @@ +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; +import com.yahoo.vespa.hosted.provision.provisioning.LoadBalancerProvisioner; + +import java.time.Duration; + +/** + * @author jonmv + */ +public class LoadBalancerPreProvisioner extends NodeRepositoryMaintainer { + + private final LoadBalancerProvisioner provisioner; + + public LoadBalancerPreProvisioner(NodeRepository nodeRepository, Duration interval, LoadBalancerService service, Metric metric) { + super(nodeRepository, interval, metric); + this.provisioner = new LoadBalancerProvisioner(nodeRepository, service); + } + + @Override + protected double maintain() { + provisioner.refreshPool(); + return 0; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 433a37e9686..de67cc07a1b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -63,8 +63,10 @@ public class NodeRepositoryMaintenance extends AbstractComponent { maintainers.add(new DeprovisionedExpirer(nodeRepository, defaults.deprovisionedExpiry, metric)); provisionServiceProvider.getLoadBalancerService() - .map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric)) - .ifPresent(maintainers::add); + .ifPresent(lbService -> { + maintainers.add(new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric)); + maintainers.add(new LoadBalancerPreProvisioner(nodeRepository, defaults.loadBalancerPreProvisionerInterval, lbService, metric)); + }); provisionServiceProvider.getHostProvisioner() .map(hostProvisioner -> List.of( new HostCapacityMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource, metric), @@ -110,6 +112,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration retiredInterval; private final Duration infrastructureProvisionInterval; private final Duration loadBalancerExpirerInterval; + private final Duration loadBalancerPreProvisionerInterval; private final Duration dynamicProvisionerInterval; private final Duration hostDeprovisionerInterval; private final Duration hostResumeProvisionerInterval; @@ -140,6 +143,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { failGrace = Duration.ofMinutes(10); infrastructureProvisionInterval = Duration.ofMinutes(3); loadBalancerExpirerInterval = Duration.ofMinutes(5); + loadBalancerPreProvisionerInterval = Duration.ofMinutes(1); metricsInterval = Duration.ofMinutes(1); nodeFailerInterval = Duration.ofMinutes(4); nodeFailureStatusUpdateInterval = Duration.ofMinutes(2); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index 4b81e580b64..d511570881b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -83,6 +83,8 @@ public class CuratorDb { private final CachingCurator db; private final Clock clock; private final CuratorCounter provisionIndexCounter; + private final CuratorCounter loadBalancerPoolHead; + private final CuratorCounter loadBalancerPoolTail; /** Simple cache for deserialized node objects, based on their ZK node version. */ private final Cache> cachedNodes = CacheBuilder.newBuilder().recordStats().build(); @@ -92,6 +94,8 @@ public class CuratorDb { this.db = new CachingCurator(curator, root, useCache); this.clock = clock; this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter")); + this.loadBalancerPoolHead = new CuratorCounter(curator, root.append("loadBalancerPoolHead")); + this.loadBalancerPoolTail = new CuratorCounter(curator, root.append("loadBalancerPoolTail")); initZK(); } @@ -110,6 +114,8 @@ public class CuratorDb { db.create(archiveUrisPath); db.create(loadBalancersPath); provisionIndexCounter.initialize(100); + loadBalancerPoolHead.initialize(1); + loadBalancerPoolTail.initialize(1); } /** Adds a set of nodes. Rollbacks/fails transaction if any node is not in the expected state. */ @@ -504,6 +510,22 @@ public class CuratorDb { .toList(); } + public long readLoadBalancerPoolHead() { + return loadBalancerPoolHead.get(); + } + + public long incrementLoadBalancerPoolHead() { + return loadBalancerPoolHead.add(1); + } + + public long readLoadBalancerPoolTail() { + return loadBalancerPoolTail.get(); + } + + public long incrementLoadBalancerPoolTail() { + return loadBalancerPoolTail.add(1); + } + public CacheStats cacheStats() { return db.cacheStats(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index 9e81cd3d3b7..756692917e3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -26,7 +26,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.function.Function; import java.util.function.Predicate; @@ -72,7 +71,7 @@ public class LoadBalancerSerializer { Cursor root = slime.setObject(); root.setString(idField, loadBalancer.id().serializedForm()); - loadBalancer.instance().flatMap(LoadBalancerInstance::idSeed).ifPresent(idSeed -> root.setString(idSeedField, idSeed.toString())); + loadBalancer.instance().flatMap(LoadBalancerInstance::idSeed).ifPresent(idSeed -> root.setString(idSeedField, idSeed)); loadBalancer.instance().flatMap(LoadBalancerInstance::hostname).ifPresent(hostname -> root.setString(hostnameField, hostname.value())); loadBalancer.instance().flatMap(LoadBalancerInstance::ip4Address).ifPresent(ip -> root.setString(lbIpAddressField, ip)); loadBalancer.instance().flatMap(LoadBalancerInstance::ip6Address).ifPresent(ip -> root.setString(lbIp6AddressField, ip)); @@ -128,7 +127,7 @@ public class LoadBalancerSerializer { Set networks = new LinkedHashSet<>(); object.field(networksField).traverse((ArrayTraverser) (i, network) -> networks.add(network.asString())); - Optional idSeed = SlimeUtils.optionalString(object.field(idSeedField)).map(UUID::fromString); + Optional idSeed = SlimeUtils.optionalString(object.field(idSeedField)); Optional hostname = SlimeUtils.optionalString(object.field(hostnameField)).map(DomainName::of); Optional ip4Address = SlimeUtils.optionalString(object.field(lbIpAddressField)); Optional ip6Address = SlimeUtils.optionalString(object.field(lbIp6AddressField)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 4ee1573269a..968e963f1a8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -12,9 +12,11 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.ZoneEndpoint; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -42,6 +44,9 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.vespa.applicationmodel.TenantId.HOSTED_VESPA; +import static com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec.preProvisionOwner; +import static com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec.preProvisionSpec; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.reducing; @@ -65,6 +70,7 @@ public class LoadBalancerProvisioner { private final LoadBalancerService service; private final BooleanFlag deactivateRouting; private final BooleanFlag ipv6AwsTargetGroups; + private final IntFlag preProvisionPoolSize; public LoadBalancerProvisioner(NodeRepository nodeRepository, LoadBalancerService service) { this.nodeRepository = nodeRepository; @@ -72,8 +78,9 @@ public class LoadBalancerProvisioner { this.service = service; this.deactivateRouting = PermanentFlags.DEACTIVATE_ROUTING.bindTo(nodeRepository.flagSource()); this.ipv6AwsTargetGroups = Flags.IPV6_AWS_TARGET_GROUPS.bindTo(nodeRepository.flagSource()); - // Read and write all load balancers to make sure they are stored in the latest version of the serialization format + this.preProvisionPoolSize = PermanentFlags.PRE_PROVISIONED_LB_COUNT.bindTo(nodeRepository.flagSource()); + // Read and write all load balancers to make sure they are stored in the latest version of the serialization format for (var id : db.readLoadBalancerIds()) { try (var lock = db.lock(id.application())) { var loadBalancer = db.readLoadBalancer(id); @@ -97,7 +104,7 @@ public class LoadBalancerProvisioner { try (var lock = db.lock(application)) { ClusterSpec.Id clusterId = effectiveId(cluster); LoadBalancerId loadBalancerId = requireNonClashing(new LoadBalancerId(application, clusterId)); - prepare(loadBalancerId, cluster.zoneEndpoint(), requested.cloudAccount()); + prepare(loadBalancerId, cluster.zoneEndpoint(), requested); } } @@ -192,8 +199,9 @@ public class LoadBalancerProvisioner { return loadBalancerId; } - private void prepare(LoadBalancerId id, ZoneEndpoint zoneEndpoint, CloudAccount cloudAccount) { + private void prepare(LoadBalancerId id, ZoneEndpoint zoneEndpoint, NodeSpec requested) { Instant now = nodeRepository.clock().instant(); + CloudAccount cloudAccount = requested.cloudAccount(); Optional loadBalancer = db.readLoadBalancer(id); LoadBalancer newLoadBalancer = loadBalancer.orElse(new LoadBalancer(id, Optional.empty(), LoadBalancer.State.reserved, now)); LoadBalancer.State fromState = loadBalancer.map(LoadBalancer::state).orElse(null); @@ -206,7 +214,7 @@ public class LoadBalancerProvisioner { newLoadBalancer = newLoadBalancer.with(State.removable, now); throw new LoadBalancerServiceException("Could not (re)configure " + id + " due to change in load balancer visibility. The operation will be retried on next deployment"); } - LoadBalancerInstance instance = provisionInstance(id, loadBalancer, zoneEndpoint, cloudAccount); + LoadBalancerInstance instance = provisionInstance(id, loadBalancer, zoneEndpoint, requested); newLoadBalancer = newLoadBalancer.with(instance); } catch (LoadBalancerServiceException e) { log.log(Level.WARNING, "Failed to provision load balancer", e); @@ -242,7 +250,7 @@ public class LoadBalancerProvisioner { private LoadBalancerInstance provisionInstance(LoadBalancerId id, Optional currentLoadBalancer, ZoneEndpoint zoneEndpoint, - CloudAccount cloudAccount) { + NodeSpec requested) { Set reals = currentLoadBalancer.flatMap(LoadBalancer::instance) .map(LoadBalancerInstance::reals) .orElse(Set.of()); // Targeted reals are changed on activation. @@ -259,20 +267,79 @@ public class LoadBalancerProvisioner { log.log(Level.INFO, () -> "Provisioning instance for " + id); try { - return service.provision(new LoadBalancerSpec(id.application(), id.cluster(), reals, settings, cloudAccount)) - // Provisioning a private endpoint service requires hard resources to be ready, so we delay it until activation. - .withServiceIds(currentLoadBalancer.flatMap(LoadBalancer::instance).map(LoadBalancerInstance::serviceIds).orElse(List.of())); + LoadBalancerSpec spec = new LoadBalancerSpec(id.application(), id.cluster(), reals, settings, requested.cloudAccount()); + return provisionFromPool(spec, requested.type()).orElseGet(() -> service.provision(spec)) + // Provisioning a private endpoint service requires hard resources to be ready, so we delay it until activation. + .withServiceIds(currentLoadBalancer.flatMap(LoadBalancer::instance).map(LoadBalancerInstance::serviceIds).orElse(List.of())); } catch (Exception e) { throw new LoadBalancerServiceException("Could not provision " + id + ". The operation will be retried on next deployment.", e); } } + private Optional provisionFromPool(LoadBalancerSpec spec, NodeType type) { + if (type != NodeType.tenant) return Optional.empty(); + if ( ! spec.settings().isDefault()) return Optional.empty(); + if ( ! spec.cloudAccount().isUnspecified()) return Optional.empty(); + if (preProvisionPoolSize.value() == 0) return Optional.empty(); + + try (Lock lock = db.lock(preProvisionOwner)) { + long tail = db.readLoadBalancerPoolTail(); + if (tail >= db.readLoadBalancerPoolHead()) return Optional.empty(); + ClusterSpec.Id slot = slotId(tail); + db.incrementLoadBalancerPoolTail(); // Acquire now; if we fail below, no one else will use the possibly inconsistent instance. + LoadBalancer chosen = db.readLoadBalancer(new LoadBalancerId(preProvisionOwner, slotId(tail))) + .orElseThrow(() -> new IllegalStateException("could not find load balancer " + slot + " in pre-provisioned pool")); + if (chosen.state() != State.active || chosen.instance().isEmpty()) + throw new IllegalStateException("expected active load balancer in pre-provisioned pool, but got " + chosen); + service.reallocate(chosen.instance().get(), spec); + db.removeLoadBalancer(chosen.id()); // Using a transaction to remove this, and write the instance, would be better, but much hassle. + return chosen.instance(); // Should be immediately written again outside of this! + } + catch (Exception e) { + log.log(Level.WARNING, "Failed to provision load balancer from pool", e); + } + return Optional.empty(); + } + + static ClusterSpec.Id slotId(long counter) { + return ClusterSpec.Id.from(String.valueOf(counter)); + } + + static long slotOf(ClusterSpec.Id id) { + return Long.parseLong(id.value()); + } + + /** Evict surplus and failed load balancers, and pre-provision deficit ones. Should only be run by a maintenance job. */ + public void refreshPool() { + int size = preProvisionPoolSize.value(); + long head = db.readLoadBalancerPoolHead(); + long tail = db.readLoadBalancerPoolTail(); + try (Lock lock = db.lock(preProvisionOwner)) { + while (head - tail > size) tail = db.incrementLoadBalancerPoolTail(); + // Mark surplus load balancers, and ones we failed to move to an application, for removal. + for (LoadBalancer lb : db.readLoadBalancers(l -> l.application().equals(preProvisionOwner)).values()) { + long slot = slotOf(lb.id().cluster()); + if (slot < tail) db.writeLoadBalancer(lb.with(State.removable, nodeRepository.clock().instant()), lb.state()); + } + } + // No need for lock while we provision, since we'll write atomically only after we're done, and the job lock ensures single writer. + while (head - tail < size) { + ClusterSpec.Id slot = slotId(head); + db.writeLoadBalancer(new LoadBalancer(new LoadBalancerId(preProvisionOwner, slot), + Optional.of(service.provision(preProvisionSpec(slot), Optional.of(slot.value()))), + State.active, // Keep the expirer away. + nodeRepository.clock().instant()), + null); + head = db.incrementLoadBalancerPoolHead(); + } + } + /** Reconfigure a load balancer instance, if necessary */ private LoadBalancerInstance configureInstance(LoadBalancerId id, NodeList nodes, - LoadBalancer currentLoadBalancer, - ZoneEndpoint zoneEndpoint, - CloudAccount cloudAccount) { + LoadBalancer currentLoadBalancer, + ZoneEndpoint zoneEndpoint, + CloudAccount cloudAccount) { boolean shouldDeactivateRouting = deactivateRouting.with(FetchVector.Dimension.INSTANCE_ID, id.application().serializedForm()) .value(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java index 7b7971318cd..20202ca7a74 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java @@ -3,11 +3,9 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.EndpointsChecker.Availability; import com.yahoo.config.provision.EndpointsChecker.Endpoint; -import com.yahoo.config.provision.EndpointsChecker.HealthCheckerProvider; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import java.util.Optional; -import java.util.UUID; /** * Injectable component that provides provision service for load-balancers and hosts @@ -27,7 +25,7 @@ public interface ProvisionServiceProvider { } interface ProtoHealthChecker { - Availability healthy(Endpoint endpoint, Optional idSeed); + Availability healthy(Endpoint endpoint, Optional idSeed); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index e40e1ba5951..f1a10134c7a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -90,7 +90,7 @@ public class LoadBalancersResponse extends SlimeJsonResponse { } instance.serviceId().ifPresent(serviceId -> lbObject.setString("serviceId", serviceId.value())); lbObject.setBool("public", instance.settings().isPublicEndpoint()); - instance.idSeed().ifPresent(idSeed -> lbObject.setString("idSeed", idSeed.toString())); + instance.idSeed().ifPresent(idSeed -> lbObject.setString("idSeed", idSeed)); }); lb.instance() .map(LoadBalancerInstance::cloudAccount) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 9a6ab16b4fc..fe6b204ed31 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -31,6 +31,7 @@ import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -39,10 +40,12 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling; import com.yahoo.vespa.hosted.provision.autoscale.Load; import com.yahoo.vespa.hosted.provision.autoscale.MemoryMetricsDb; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; +import com.yahoo.vespa.hosted.provision.provisioning.LoadBalancerProvisioner; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.service.duper.ConfigServerApplication; import com.yahoo.vespa.service.duper.InfraApplication; @@ -92,7 +95,7 @@ public class MockNodeRepository extends NodeRepository { DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), Optional.empty(), Optional.empty(), - new InMemoryFlagSource(), + new InMemoryFlagSource().withIntFlag(PermanentFlags.PRE_PROVISIONED_LB_COUNT.id(), 1), new MemoryMetricsDb(Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z"))), new OrchestratorMock(), true, @@ -105,7 +108,9 @@ public class MockNodeRepository extends NodeRepository { } private void populate() { - NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, Zone.defaultZone(), new MockProvisionServiceProvider(), new MockMetric()); + MockProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(); + provisionServiceProvider.getLoadBalancerService().ifPresent(service -> new LoadBalancerProvisioner(this, service).refreshPool()); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, Zone.defaultZone(), provisionServiceProvider, new MockMetric()); List nodes = new ArrayList<>(); // Regular nodes diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index 146b9df1722..0d3c1994ad8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -39,7 +39,7 @@ public class LoadBalancerSerializerTest { { var loadBalancer = new LoadBalancer(loadBalancerId, Optional.of(new LoadBalancerInstance( - Optional.of(UUID.randomUUID()), + Optional.of("1"), Optional.of(DomainName.of("lb-host")), Optional.empty(), Optional.empty(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json index f6d9c1f079c..620275bb033 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json @@ -1,15 +1,14 @@ { "loadBalancers": [ { - "id": "tenant4:application4:instance4:id4", - "state": "active", - "changedAt": 123, "application": "application4", - "tenant": "tenant4", - "instance": "instance4", + "changedAt": 123, "cluster": "id4", - "hostname": "lb-tenant4.application4.instance4-id4", "dnsZone": "zone-id-1", + "hostname": "lb-hosted-vespa.pre-provision-1", + "id": "tenant4:application4:instance4:id4", + "idSeed": "1", + "instance": "instance4", "networks": [ "10.2.3.0/24", "10.4.5.0/24" @@ -17,6 +16,7 @@ "ports": [ 4443 ], + "public": true, "reals": [ { "hostname": "host13.yahoo.com", @@ -29,7 +29,8 @@ "port": 4443 } ], - "public": true + "state": "active", + "tenant": "tenant4" } ] -} +} \ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json index e0c370fc376..e8392d92522 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json @@ -47,7 +47,6 @@ "dnsZone": "zone-id-1", "hostname": "lb-hosted-vespa.zone-config-servers-zone-config-servers", "id": "hosted-vespa:zone-config-servers:default:zone-config-servers", - "idSeed": "c11272ab-d20e-4c86-b808-ffedaa00c480", "instance": "default", "networks": [ "10.2.3.0/24", @@ -77,8 +76,9 @@ "changedAt": 123, "cluster": "id4", "dnsZone": "zone-id-1", - "hostname": "lb-tenant4.application4.instance4-id4", + "hostname": "lb-hosted-vespa.pre-provision-1", "id": "tenant4:application4:instance4:id4", + "idSeed": "1", "instance": "instance4", "networks": [ "10.2.3.0/24", -- cgit v1.2.3