Pre-provision load balancers, and use for apps

16 files changed, 179 insertions, 56 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java
index 04ea9d20edf..4dbf891b1b7 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java
@@ -11,7 +11,6 @@ import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.UUID;
 
 /**
  * Represents a load balancer instance. This contains the fields that are owned by a {@link LoadBalancerService} and is
@@ -21,7 +20,7 @@ import java.util.UUID;
  */
 public class LoadBalancerInstance {
 
-    private final Optional<UUID> idSeed;
+    private final Optional<String> idSeed;
     private final Optional<DomainName> hostname;
     private final Optional<String> ip4Address;
     private final Optional<String> ip6Address;
@@ -33,7 +32,7 @@ public class LoadBalancerInstance {
     private final List<PrivateServiceId> serviceIds;
     private final CloudAccount cloudAccount;
 
-    public LoadBalancerInstance(Optional<UUID> idSeed, Optional<DomainName> hostname, Optional<String> ip4Address, Optional<String> ip6Address,
+    public LoadBalancerInstance(Optional<String> idSeed, Optional<DomainName> hostname, Optional<String> ip4Address, Optional<String> ip6Address,
                                 Optional<DnsZone> dnsZone, Set<Integer> ports, Set<String> networks, Set<Real> reals,
                                 ZoneEndpoint settings, List<PrivateServiceId> serviceIds, CloudAccount cloudAccount) {
         this.idSeed = Objects.requireNonNull(idSeed, "idSeed must be non-null");
@@ -55,7 +54,7 @@ public class LoadBalancerInstance {
     }
 
     /** A unique seed to use when generating cloud-specific resource IDs for this load balancer instance. */
-    public Optional<UUID> idSeed() {
+    public Optional<String> idSeed() {
         return idSeed;
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java
index 6649b18ed02..6ddde1151dd 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java
@@ -8,7 +8,6 @@ import com.yahoo.config.provision.EndpointsChecker.HealthChecker;
 import com.yahoo.config.provision.NodeType;
 
 import java.util.Optional;
-import java.util.UUID;
 
 /**
  * A managed load balance service.
@@ -23,7 +22,16 @@ public interface LoadBalancerService {
      * @param spec        Load balancer specification
      * @return The provisioned load balancer instance
      */
-    LoadBalancerInstance provision(LoadBalancerSpec spec);
+    default LoadBalancerInstance provision(LoadBalancerSpec spec) { return provision(spec, Optional.empty()); }
+
+    /**
+     * Provisions load balancers from the given specification. Implementations are expected to be idempotent
+     *
+     * @param spec        Load balancer specification
+     * @param idSeed      Seed for generating a unique ID for the load balancer instance
+     * @return The provisioned load balancer instance
+     */
+    LoadBalancerInstance provision(LoadBalancerSpec spec, Optional<String> idSeed);
 
     /**
      * Configures load balancers for the given specification. Implementations are expected to be idempotent
@@ -48,7 +56,7 @@ public interface LoadBalancerService {
     boolean supports(NodeType nodeType, ClusterSpec.Type clusterType);
 
     /** See {@link HealthChecker#healthy(Endpoint)}. */
-    Availability healthy(Endpoint endpoint, Optional<UUID> idSeed);
+    Availability healthy(Endpoint endpoint, Optional<String> idSeed);
 
     /** Load balancer protocols */
     enum Protocol {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java
index 554ec26d5f4..03ff17c6ebc 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java
@@ -15,7 +15,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static java.util.Objects.requireNonNull;
@@ -26,7 +25,7 @@ import static java.util.stream.Collectors.toMap;
  */
 public class LoadBalancerServiceMock implements LoadBalancerService {
 
-    private record Key(ApplicationId application, ClusterSpec.Id cluster, UUID idSeed) {
+    private record Key(ApplicationId application, ClusterSpec.Id cluster, String idSeed) {
         @Override public int hashCode() { return idSeed == null ? Objects.hash(application, cluster) : Objects.hash(idSeed); }
         @Override public boolean equals(Object o) {
             if (o == this) return true;
@@ -69,9 +68,8 @@ public class LoadBalancerServiceMock implements LoadBalancerService {
     }
 
     @Override
-    public LoadBalancerInstance provision(LoadBalancerSpec spec) {
+    public LoadBalancerInstance provision(LoadBalancerSpec spec, Optional<String> idSeed) {
         if (throwOnCreate) throw new IllegalStateException("Did not expect a new load balancer to be created");
-        Optional<UUID> idSeed = uuid.getAndSet(false) ? Optional.of(UUID.fromString("c11272ab-d20e-4c86-b808-ffedaa00c480")) : Optional.empty();
         var instance = new LoadBalancerInstance(
                 idSeed,
                 Optional.of(DomainName.of("lb-" + spec.application().toShortString() + "-" + spec.cluster().value())),
@@ -104,7 +102,7 @@ public class LoadBalancerServiceMock implements LoadBalancerService {
 
     @Override
     public void reallocate(LoadBalancerInstance provisioned, LoadBalancerSpec spec) {
-        instances.put(new Key(spec.application(), spec.cluster(), null),
+        instances.put(new Key(spec.application(), spec.cluster(), provisioned.idSeed().get()),
                       requireNonNull(instances.remove(new Key(null, null, provisioned.idSeed().get())))); // ᕙ༼◕_◕༽ᕤ
     }
 
@@ -117,7 +115,7 @@ public class LoadBalancerServiceMock implements LoadBalancerService {
     }
 
     @Override
-    public Availability healthy(Endpoint endpoint, Optional<UUID> idSeed) {
+    public Availability healthy(Endpoint endpoint, Optional<String> idSeed) {
         return Availability.ready;
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java
index 199578d66f2..49a5c9c3c5e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java
@@ -8,7 +8,6 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.ZoneEndpoint;
 
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 
 /**
@@ -20,12 +19,8 @@ public record LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster
                                ZoneEndpoint settings, CloudAccount cloudAccount) {
 
     public static final ApplicationId preProvisionOwner = ApplicationId.from("hosted-vespa", "pre-provision", "default");
-    public static LoadBalancerSpec preProvisionSpec(int slot) {
-        return new LoadBalancerSpec(preProvisionOwner,
-                                    ClusterSpec.Id.from("slot-" + slot),
-                                    Set.of(),
-                                    ZoneEndpoint.defaultEndpoint,
-                                    CloudAccount.empty);
+    public static LoadBalancerSpec preProvisionSpec(ClusterSpec.Id slot) {
+        return new LoadBalancerSpec(preProvisionOwner, slot, Set.of(), ZoneEndpoint.defaultEndpoint, CloudAccount.empty);
     }
 
     public LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals,
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java
index 9124b012dfe..883b8dec944 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java
@@ -11,7 +11,6 @@ import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.UUID;
 
 /**
  * This implementation of {@link LoadBalancerService} returns the load balancer(s) that exist by default in the shared
@@ -31,7 +30,7 @@ public class SharedLoadBalancerService implements LoadBalancerService {
     }
 
     @Override
-    public LoadBalancerInstance provision(LoadBalancerSpec spec) {
+    public LoadBalancerInstance provision(LoadBalancerSpec spec, Optional<String> idSeed) {
         return create(spec);
     }
 
@@ -79,7 +78,7 @@ public class SharedLoadBalancerService implements LoadBalancerService {
     }
 
     @Override
-    public Availability healthy(Endpoint endpoint, Optional<UUID> idSeed) {
+    public Availability healthy(Endpoint endpoint, Optional<String> idSeed) {
         return Availability.ready;
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerPreProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerPreProvisioner.java
new file mode 100644
index 00000000000..50c58f6c834
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerPreProvisioner.java
@@ -0,0 +1,28 @@
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService;
+import com.yahoo.vespa.hosted.provision.provisioning.LoadBalancerProvisioner;
+
+import java.time.Duration;
+
+/**
+ * @author jonmv
+ */
+public class LoadBalancerPreProvisioner extends NodeRepositoryMaintainer {
+
+    private final LoadBalancerProvisioner provisioner;
+
+    public LoadBalancerPreProvisioner(NodeRepository nodeRepository, Duration interval, LoadBalancerService service, Metric metric) {
+        super(nodeRepository, interval, metric);
+        this.provisioner = new LoadBalancerProvisioner(nodeRepository, service);
+    }
+
+    @Override
+    protected double maintain() {
+        provisioner.refreshPool();
+        return 0;
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 433a37e9686..de67cc07a1b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -63,8 +63,10 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         maintainers.add(new DeprovisionedExpirer(nodeRepository, defaults.deprovisionedExpiry, metric));
 
         provisionServiceProvider.getLoadBalancerService()
-                                .map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric))
-                                .ifPresent(maintainers::add);
+                .ifPresent(lbService -> {
+                    maintainers.add(new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric));
+                    maintainers.add(new LoadBalancerPreProvisioner(nodeRepository, defaults.loadBalancerPreProvisionerInterval, lbService, metric));
+                });
         provisionServiceProvider.getHostProvisioner()
                                 .map(hostProvisioner -> List.of(
                                         new HostCapacityMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource, metric),
@@ -110,6 +112,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         private final Duration retiredInterval;
         private final Duration infrastructureProvisionInterval;
         private final Duration loadBalancerExpirerInterval;
+        private final Duration loadBalancerPreProvisionerInterval;
         private final Duration dynamicProvisionerInterval;
         private final Duration hostDeprovisionerInterval;
         private final Duration hostResumeProvisionerInterval;
@@ -140,6 +143,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
             failGrace = Duration.ofMinutes(10);
             infrastructureProvisionInterval = Duration.ofMinutes(3);
             loadBalancerExpirerInterval = Duration.ofMinutes(5);
+            loadBalancerPreProvisionerInterval = Duration.ofMinutes(1);
             metricsInterval = Duration.ofMinutes(1);
             nodeFailerInterval = Duration.ofMinutes(4);
             nodeFailureStatusUpdateInterval = Duration.ofMinutes(2);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java
index 4b81e580b64..d511570881b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java
@@ -83,6 +83,8 @@ public class CuratorDb {
     private final CachingCurator db;
     private final Clock clock;
     private final CuratorCounter provisionIndexCounter;
+    private final CuratorCounter loadBalancerPoolHead;
+    private final CuratorCounter loadBalancerPoolTail;
 
     /** Simple cache for deserialized node objects, based on their ZK node version. */
     private final Cache<Path, Pair<Integer, Node>> cachedNodes = CacheBuilder.newBuilder().recordStats().build();
@@ -92,6 +94,8 @@ public class CuratorDb {
         this.db = new CachingCurator(curator, root, useCache);
         this.clock = clock;
         this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter"));
+        this.loadBalancerPoolHead = new CuratorCounter(curator, root.append("loadBalancerPoolHead"));
+        this.loadBalancerPoolTail = new CuratorCounter(curator, root.append("loadBalancerPoolTail"));
         initZK();
     }
 
@@ -110,6 +114,8 @@ public class CuratorDb {
         db.create(archiveUrisPath);
         db.create(loadBalancersPath);
         provisionIndexCounter.initialize(100);
+        loadBalancerPoolHead.initialize(1);
+        loadBalancerPoolTail.initialize(1);
     }
 
     /** Adds a set of nodes. Rollbacks/fails transaction if any node is not in the expected state. */
@@ -504,6 +510,22 @@ public class CuratorDb {
                         .toList();
     }
 
+    public long readLoadBalancerPoolHead() {
+        return loadBalancerPoolHead.get();
+    }
+
+    public long incrementLoadBalancerPoolHead() {
+        return loadBalancerPoolHead.add(1);
+    }
+
+    public long readLoadBalancerPoolTail() {
+        return loadBalancerPoolTail.get();
+    }
+
+    public long incrementLoadBalancerPoolTail() {
+        return loadBalancerPoolTail.add(1);
+    }
+
     public CacheStats cacheStats() {
         return db.cacheStats();
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
index 9e81cd3d3b7..756692917e3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
@@ -26,7 +26,6 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
-import java.util.UUID;
 import java.util.function.Function;
 import java.util.function.Predicate;
 
@@ -72,7 +71,7 @@ public class LoadBalancerSerializer {
         Cursor root = slime.setObject();
 
         root.setString(idField, loadBalancer.id().serializedForm());
-        loadBalancer.instance().flatMap(LoadBalancerInstance::idSeed).ifPresent(idSeed -> root.setString(idSeedField, idSeed.toString()));
+        loadBalancer.instance().flatMap(LoadBalancerInstance::idSeed).ifPresent(idSeed -> root.setString(idSeedField, idSeed));
         loadBalancer.instance().flatMap(LoadBalancerInstance::hostname).ifPresent(hostname -> root.setString(hostnameField, hostname.value()));
         loadBalancer.instance().flatMap(LoadBalancerInstance::ip4Address).ifPresent(ip -> root.setString(lbIpAddressField, ip));
         loadBalancer.instance().flatMap(LoadBalancerInstance::ip6Address).ifPresent(ip -> root.setString(lbIp6AddressField, ip));
@@ -128,7 +127,7 @@ public class LoadBalancerSerializer {
         Set<String> networks = new LinkedHashSet<>();
         object.field(networksField).traverse((ArrayTraverser) (i, network) -> networks.add(network.asString()));
 
-        Optional<UUID> idSeed = SlimeUtils.optionalString(object.field(idSeedField)).map(UUID::fromString);
+        Optional<String> idSeed = SlimeUtils.optionalString(object.field(idSeedField));
         Optional<DomainName> hostname = SlimeUtils.optionalString(object.field(hostnameField)).map(DomainName::of);
         Optional<String> ip4Address = SlimeUtils.optionalString(object.field(lbIpAddressField));
         Optional<String> ip6Address = SlimeUtils.optionalString(object.field(lbIp6AddressField));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
index 4ee1573269a..968e963f1a8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
@@ -12,9 +12,11 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.ZoneEndpoint;
 import com.yahoo.config.provision.exception.LoadBalancerServiceException;
 import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.flags.BooleanFlag;
 import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.IntFlag;
 import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
@@ -42,6 +44,9 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
+import static com.yahoo.vespa.applicationmodel.TenantId.HOSTED_VESPA;
+import static com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec.preProvisionOwner;
+import static com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec.preProvisionSpec;
 import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.reducing;
 
@@ -65,6 +70,7 @@ public class LoadBalancerProvisioner {
     private final LoadBalancerService service;
     private final BooleanFlag deactivateRouting;
     private final BooleanFlag ipv6AwsTargetGroups;
+    private final IntFlag preProvisionPoolSize;
 
     public LoadBalancerProvisioner(NodeRepository nodeRepository, LoadBalancerService service) {
         this.nodeRepository = nodeRepository;
@@ -72,8 +78,9 @@ public class LoadBalancerProvisioner {
         this.service = service;
         this.deactivateRouting = PermanentFlags.DEACTIVATE_ROUTING.bindTo(nodeRepository.flagSource());
         this.ipv6AwsTargetGroups = Flags.IPV6_AWS_TARGET_GROUPS.bindTo(nodeRepository.flagSource());
-        // Read and write all load balancers to make sure they are stored in the latest version of the serialization format
+        this.preProvisionPoolSize = PermanentFlags.PRE_PROVISIONED_LB_COUNT.bindTo(nodeRepository.flagSource());
 
+        // Read and write all load balancers to make sure they are stored in the latest version of the serialization format
         for (var id : db.readLoadBalancerIds()) {
             try (var lock = db.lock(id.application())) {
                 var loadBalancer = db.readLoadBalancer(id);
@@ -97,7 +104,7 @@ public class LoadBalancerProvisioner {
         try (var lock = db.lock(application)) {
             ClusterSpec.Id clusterId = effectiveId(cluster);
             LoadBalancerId loadBalancerId = requireNonClashing(new LoadBalancerId(application, clusterId));
-            prepare(loadBalancerId, cluster.zoneEndpoint(), requested.cloudAccount());
+            prepare(loadBalancerId, cluster.zoneEndpoint(), requested);
         }
     }
 
@@ -192,8 +199,9 @@ public class LoadBalancerProvisioner {
         return loadBalancerId;
     }
 
-    private void prepare(LoadBalancerId id, ZoneEndpoint zoneEndpoint, CloudAccount cloudAccount) {
+    private void prepare(LoadBalancerId id, ZoneEndpoint zoneEndpoint, NodeSpec requested) {
         Instant now = nodeRepository.clock().instant();
+        CloudAccount cloudAccount = requested.cloudAccount();
         Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id);
         LoadBalancer newLoadBalancer = loadBalancer.orElse(new LoadBalancer(id, Optional.empty(), LoadBalancer.State.reserved, now));
         LoadBalancer.State fromState = loadBalancer.map(LoadBalancer::state).orElse(null);
@@ -206,7 +214,7 @@ public class LoadBalancerProvisioner {
                 newLoadBalancer = newLoadBalancer.with(State.removable, now);
                 throw new LoadBalancerServiceException("Could not (re)configure " + id + " due to change in load balancer visibility. The operation will be retried on next deployment");
             }
-            LoadBalancerInstance instance = provisionInstance(id, loadBalancer, zoneEndpoint, cloudAccount);
+            LoadBalancerInstance instance = provisionInstance(id, loadBalancer, zoneEndpoint, requested);
             newLoadBalancer = newLoadBalancer.with(instance);
         } catch (LoadBalancerServiceException e) {
             log.log(Level.WARNING, "Failed to provision load balancer", e);
@@ -242,7 +250,7 @@ public class LoadBalancerProvisioner {
     private LoadBalancerInstance provisionInstance(LoadBalancerId id,
                                                    Optional<LoadBalancer> currentLoadBalancer,
                                                    ZoneEndpoint zoneEndpoint,
-                                                   CloudAccount cloudAccount) {
+                                                   NodeSpec requested) {
         Set<Real> reals = currentLoadBalancer.flatMap(LoadBalancer::instance)
                                              .map(LoadBalancerInstance::reals)
                                              .orElse(Set.of()); // Targeted reals are changed on activation.
@@ -259,20 +267,79 @@ public class LoadBalancerProvisioner {
 
         log.log(Level.INFO, () -> "Provisioning instance for " + id);
         try {
-            return service.provision(new LoadBalancerSpec(id.application(), id.cluster(), reals, settings, cloudAccount))
-                          // Provisioning a private endpoint service requires hard resources to be ready, so we delay it until activation.
-                          .withServiceIds(currentLoadBalancer.flatMap(LoadBalancer::instance).map(LoadBalancerInstance::serviceIds).orElse(List.of()));
+            LoadBalancerSpec spec = new LoadBalancerSpec(id.application(), id.cluster(), reals, settings, requested.cloudAccount());
+            return provisionFromPool(spec, requested.type()).orElseGet(() -> service.provision(spec))
+                                          // Provisioning a private endpoint service requires hard resources to be ready, so we delay it until activation.
+                                          .withServiceIds(currentLoadBalancer.flatMap(LoadBalancer::instance).map(LoadBalancerInstance::serviceIds).orElse(List.of()));
         }
         catch (Exception e) {
             throw new LoadBalancerServiceException("Could not provision " + id + ". The operation will be retried on next deployment.", e);
         }
     }
 
+    private Optional<LoadBalancerInstance> provisionFromPool(LoadBalancerSpec spec, NodeType type) {
+        if (type != NodeType.tenant) return Optional.empty();
+        if ( ! spec.settings().isDefault()) return Optional.empty();
+        if ( ! spec.cloudAccount().isUnspecified()) return Optional.empty();
+        if (preProvisionPoolSize.value() == 0) return Optional.empty();
+
+        try (Lock lock = db.lock(preProvisionOwner)) {
+            long tail = db.readLoadBalancerPoolTail();
+            if (tail >= db.readLoadBalancerPoolHead()) return Optional.empty();
+            ClusterSpec.Id slot = slotId(tail);
+            db.incrementLoadBalancerPoolTail(); // Acquire now; if we fail below, no one else will use the possibly inconsistent instance.
+            LoadBalancer chosen = db.readLoadBalancer(new LoadBalancerId(preProvisionOwner, slotId(tail)))
+                    .orElseThrow(() -> new IllegalStateException("could not find load balancer " + slot + " in pre-provisioned pool"));
+            if (chosen.state() != State.active || chosen.instance().isEmpty())
+                throw new IllegalStateException("expected active load balancer in pre-provisioned pool, but got " + chosen);
+            service.reallocate(chosen.instance().get(), spec);
+            db.removeLoadBalancer(chosen.id()); // Using a transaction to remove this, and write the instance, would be better, but much hassle.
+            return chosen.instance(); // Should be immediately written again outside of this!
+        }
+        catch (Exception e) {
+            log.log(Level.WARNING, "Failed to provision load balancer from pool", e);
+        }
+        return Optional.empty();
+    }
+
+    static ClusterSpec.Id slotId(long counter) {
+        return ClusterSpec.Id.from(String.valueOf(counter));
+    }
+
+    static long slotOf(ClusterSpec.Id id) {
+        return Long.parseLong(id.value());
+    }
+
+    /** Evict surplus and failed load balancers, and pre-provision deficit ones. Should only be run by a maintenance job. */
+    public void refreshPool() {
+        int size = preProvisionPoolSize.value();
+        long head = db.readLoadBalancerPoolHead();
+        long tail = db.readLoadBalancerPoolTail();
+        try (Lock lock = db.lock(preProvisionOwner)) {
+            while (head - tail > size) tail = db.incrementLoadBalancerPoolTail();
+            // Mark surplus load balancers, and ones we failed to move to an application, for removal.
+            for (LoadBalancer lb : db.readLoadBalancers(l -> l.application().equals(preProvisionOwner)).values()) {
+                long slot = slotOf(lb.id().cluster());
+                if (slot < tail) db.writeLoadBalancer(lb.with(State.removable, nodeRepository.clock().instant()), lb.state());
+            }
+        }
+        // No need for lock while we provision, since we'll write atomically only after we're done, and the job lock ensures single writer.
+        while (head - tail < size) {
+            ClusterSpec.Id slot = slotId(head);
+            db.writeLoadBalancer(new LoadBalancer(new LoadBalancerId(preProvisionOwner, slot),
+                                                  Optional.of(service.provision(preProvisionSpec(slot), Optional.of(slot.value()))),
+                                                  State.active, // Keep the expirer away.
+                                                  nodeRepository.clock().instant()),
+                                 null);
+            head = db.incrementLoadBalancerPoolHead();
+        }
+    }
+
     /** Reconfigure a load balancer instance, if necessary */
     private LoadBalancerInstance configureInstance(LoadBalancerId id, NodeList nodes,
-                                                             LoadBalancer currentLoadBalancer,
-                                                             ZoneEndpoint zoneEndpoint,
-                                                             CloudAccount cloudAccount) {
+                                                   LoadBalancer currentLoadBalancer,
+                                                   ZoneEndpoint zoneEndpoint,
+                                                   CloudAccount cloudAccount) {
         boolean shouldDeactivateRouting = deactivateRouting.with(FetchVector.Dimension.INSTANCE_ID,
                                                                  id.application().serializedForm())
                                                            .value();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java
index 7b7971318cd..20202ca7a74 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java
@@ -3,11 +3,9 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 
 import com.yahoo.config.provision.EndpointsChecker.Availability;
 import com.yahoo.config.provision.EndpointsChecker.Endpoint;
-import com.yahoo.config.provision.EndpointsChecker.HealthCheckerProvider;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService;
 
 import java.util.Optional;
-import java.util.UUID;
 
 /**
  * Injectable component that provides provision service for load-balancers and hosts
@@ -27,7 +25,7 @@ public interface ProvisionServiceProvider {
     }
 
     interface ProtoHealthChecker {
-        Availability healthy(Endpoint endpoint, Optional<UUID> idSeed);
+        Availability healthy(Endpoint endpoint, Optional<String> idSeed);
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java
index e40e1ba5951..f1a10134c7a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java
@@ -90,7 +90,7 @@ public class LoadBalancersResponse extends SlimeJsonResponse {
                 }
                 instance.serviceId().ifPresent(serviceId -> lbObject.setString("serviceId", serviceId.value()));
                 lbObject.setBool("public", instance.settings().isPublicEndpoint());
-                instance.idSeed().ifPresent(idSeed -> lbObject.setString("idSeed", idSeed.toString()));
+                instance.idSeed().ifPresent(idSeed -> lbObject.setString("idSeed", idSeed));
             });
             lb.instance()
               .map(LoadBalancerInstance::cloudAccount)
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 9a6ab16b4fc..fe6b204ed31 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -31,6 +31,7 @@ import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeMutex;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -39,10 +40,12 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster;
 import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling;
 import com.yahoo.vespa.hosted.provision.autoscale.Load;
 import com.yahoo.vespa.hosted.provision.autoscale.MemoryMetricsDb;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.node.Status;
 import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider;
+import com.yahoo.vespa.hosted.provision.provisioning.LoadBalancerProvisioner;
 import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
 import com.yahoo.vespa.service.duper.ConfigServerApplication;
 import com.yahoo.vespa.service.duper.InfraApplication;
@@ -92,7 +95,7 @@ public class MockNodeRepository extends NodeRepository {
               DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
               Optional.empty(),
               Optional.empty(),
-              new InMemoryFlagSource(),
+              new InMemoryFlagSource().withIntFlag(PermanentFlags.PRE_PROVISIONED_LB_COUNT.id(), 1),
               new MemoryMetricsDb(Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z"))),
               new OrchestratorMock(),
               true,
@@ -105,7 +108,9 @@ public class MockNodeRepository extends NodeRepository {
     }
 
     private void populate() {
-        NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, Zone.defaultZone(), new MockProvisionServiceProvider(), new MockMetric());
+        MockProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider();
+        provisionServiceProvider.getLoadBalancerService().ifPresent(service -> new LoadBalancerProvisioner(this, service).refreshPool());
+        NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, Zone.defaultZone(), provisionServiceProvider, new MockMetric());
         List<Node> nodes = new ArrayList<>();
 
         // Regular nodes
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java
index 146b9df1722..0d3c1994ad8 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java
@@ -39,7 +39,7 @@ public class LoadBalancerSerializerTest {
         {
             var loadBalancer = new LoadBalancer(loadBalancerId,
                                                 Optional.of(new LoadBalancerInstance(
-                                                        Optional.of(UUID.randomUUID()),
+                                                        Optional.of("1"),
                                                         Optional.of(DomainName.of("lb-host")),
                                                         Optional.empty(),
                                                         Optional.empty(),
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json
index f6d9c1f079c..620275bb033 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json
@@ -1,15 +1,14 @@
 {
   "loadBalancers": [
     {
-      "id": "tenant4:application4:instance4:id4",
-      "state": "active",
-      "changedAt": 123,
       "application": "application4",
-      "tenant": "tenant4",
-      "instance": "instance4",
+      "changedAt": 123,
       "cluster": "id4",
-      "hostname": "lb-tenant4.application4.instance4-id4",
       "dnsZone": "zone-id-1",
+      "hostname": "lb-hosted-vespa.pre-provision-1",
+      "id": "tenant4:application4:instance4:id4",
+      "idSeed": "1",
+      "instance": "instance4",
       "networks": [
         "10.2.3.0/24",
         "10.4.5.0/24"
@@ -17,6 +16,7 @@
       "ports": [
         4443
       ],
+      "public": true,
       "reals": [
         {
           "hostname": "host13.yahoo.com",
@@ -29,7 +29,8 @@
           "port": 4443
         }
       ],
-      "public": true
+      "state": "active",
+      "tenant": "tenant4"
     }
   ]
-}
+}
\ No newline at end of file
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json
index e0c370fc376..e8392d92522 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json
@@ -47,7 +47,6 @@
       "dnsZone": "zone-id-1",
       "hostname": "lb-hosted-vespa.zone-config-servers-zone-config-servers",
       "id": "hosted-vespa:zone-config-servers:default:zone-config-servers",
-      "idSeed": "c11272ab-d20e-4c86-b808-ffedaa00c480",
       "instance": "default",
       "networks": [
         "10.2.3.0/24",
@@ -77,8 +76,9 @@
       "changedAt": 123,
       "cluster": "id4",
       "dnsZone": "zone-id-1",
-      "hostname": "lb-tenant4.application4.instance4-id4",
+      "hostname": "lb-hosted-vespa.pre-provision-1",
       "id": "tenant4:application4:instance4:id4",
+      "idSeed": "1",
       "instance": "instance4",
       "networks": [
         "10.2.3.0/24",