diff options
16 files changed, 103 insertions, 93 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 5406d820498..52b68708eee 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -263,9 +263,8 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr loadBalancers.list(endpoint.applicationId()) .cluster(endpoint.clusterName()) .first() - .flatMap(LoadBalancer::instance) - .map(LoadBalancerInstance::idSeed) - .orElseThrow(() -> new IllegalArgumentException("no load balancer instance for '" + endpoint + "'"))); + .map(LoadBalancer::idSeed) + .orElseThrow(() -> new IllegalArgumentException("no load balancer for '" + endpoint + "'"))); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java index 36f2eb4bb2c..f8de6e1db16 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java @@ -17,12 +17,14 @@ import java.util.Set; public class LoadBalancer { private final LoadBalancerId id; + private final String idSeed; private final Optional<LoadBalancerInstance> instance; private final State state; private final Instant changedAt; - public LoadBalancer(LoadBalancerId id, Optional<LoadBalancerInstance> instance, State state, Instant changedAt) { + public LoadBalancer(LoadBalancerId id, String idSeed, Optional<LoadBalancerInstance> instance, State state, Instant changedAt) { this.id = Objects.requireNonNull(id, "id must be non-null"); + this.idSeed = Objects.requireNonNull(idSeed, "idSeed must be non-null"); this.instance = Objects.requireNonNull(instance, "instance must be non-null"); this.state = Objects.requireNonNull(state, "state must be non-null"); this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); @@ -40,6 +42,11 @@ public class LoadBalancer { return id; } + /** Seed to use for generating resource IDs for provisioned resources in this. */ + public String idSeed() { + return idSeed; + } + /** The instance associated with this */ public Optional<LoadBalancerInstance> instance() { return instance; @@ -64,12 +71,12 @@ public class LoadBalancer { if (this.state != State.reserved && state == State.reserved) { throw new IllegalArgumentException("Invalid state transition: " + this.state + " -> " + state); } - return new LoadBalancer(id, instance, state, changedAt); + return new LoadBalancer(id, idSeed, instance, state, changedAt); } /** Returns a copy of this with instance set to given instance */ public LoadBalancer with(LoadBalancerInstance instance) { - return new LoadBalancer(id, Optional.of(instance), state, changedAt); + return new LoadBalancer(id, idSeed, Optional.of(instance), state, changedAt); } public enum State { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java index 818858ef385..c0931ecbc70 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java @@ -20,7 +20,6 @@ import java.util.Set; */ public class LoadBalancerInstance { - private final String idSeed; private final Optional<DomainName> hostname; private final Optional<String> ip4Address; private final Optional<String> ip6Address; @@ -32,10 +31,9 @@ public class LoadBalancerInstance { private final List<PrivateServiceId> serviceIds; private final CloudAccount cloudAccount; - public LoadBalancerInstance(String idSeed, Optional<DomainName> hostname, Optional<String> ip4Address, Optional<String> ip6Address, + public LoadBalancerInstance(Optional<DomainName> hostname, Optional<String> ip4Address, Optional<String> ip6Address, Optional<DnsZone> dnsZone, Set<Integer> ports, Set<String> networks, Set<Real> reals, ZoneEndpoint settings, List<PrivateServiceId> serviceIds, CloudAccount cloudAccount) { - this.idSeed = Objects.requireNonNull(idSeed, "idSeed must be non-null"); this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); this.ip4Address = Objects.requireNonNull(ip4Address, "ip4Address must be non-null"); this.ip6Address = Objects.requireNonNull(ip6Address, "ip6Address must be non-null"); @@ -53,11 +51,6 @@ public class LoadBalancerInstance { } } - /** A unique seed to use when generating cloud-specific resource IDs for this load balancer instance. */ - public String idSeed() { - return idSeed; - } - /** Fully-qualified domain name of this load balancer. This hostname can be used for query and feed */ public Optional<DomainName> hostname() { return hostname; @@ -128,7 +121,7 @@ public class LoadBalancerInstance { public LoadBalancerInstance with(Set<Real> reals, ZoneEndpoint settings, Optional<PrivateServiceId> serviceId) { List<PrivateServiceId> ids = new ArrayList<>(serviceIds); serviceId.filter(id -> ! ids.contains(id)).ifPresent(ids::add); - return new LoadBalancerInstance(idSeed, hostname, ip4Address, ip6Address, dnsZone, ports, networks, + return new LoadBalancerInstance(hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, ids, cloudAccount); } @@ -137,7 +130,7 @@ public class LoadBalancerInstance { public LoadBalancerInstance withServiceIds(List<PrivateServiceId> serviceIds) { List<PrivateServiceId> ids = new ArrayList<>(serviceIds); for (PrivateServiceId id : this.serviceIds) if ( ! ids.contains(id)) ids.add(id); - return new LoadBalancerInstance(idSeed, hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, ids, cloudAccount); + return new LoadBalancerInstance(hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, ids, cloudAccount); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index 9e62edf8d4b..efd1536d108 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -18,10 +18,9 @@ public interface LoadBalancerService { * Provisions load balancers from the given specification. Implementations are expected to be idempotent * * @param spec Load balancer specification - * @param idSeed Seed for generating a unique ID for the load balancer instance * @return The provisioned load balancer instance */ - LoadBalancerInstance provision(LoadBalancerSpec spec, String idSeed); + LoadBalancerInstance provision(LoadBalancerSpec spec); /** * Configures load balancers for the given specification. Implementations are expected to be idempotent @@ -34,7 +33,7 @@ public interface LoadBalancerService { */ LoadBalancerInstance configure(LoadBalancerInstance instance, LoadBalancerSpec spec, boolean force); - void reallocate(LoadBalancerInstance provisioned, LoadBalancerSpec spec); + void reallocate(LoadBalancerSpec spec); /** Permanently remove given load balancer */ void remove(LoadBalancer loadBalancer); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index 2b1007f75d4..2c672d0eada 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -36,7 +36,6 @@ public class LoadBalancerServiceMock implements LoadBalancerService { private final Map<Key, LoadBalancerInstance> instances = new HashMap<>(); private boolean throwOnCreate = false; private boolean supportsProvisioning = true; - private final AtomicBoolean uuid = new AtomicBoolean(true); public Map<LoadBalancerId, LoadBalancerInstance> instances() { return instances.entrySet().stream().collect(toMap(e -> new LoadBalancerId(e.getKey().application, e.getKey().cluster), @@ -66,10 +65,9 @@ public class LoadBalancerServiceMock implements LoadBalancerService { } @Override - public LoadBalancerInstance provision(LoadBalancerSpec spec, String idSeed) { + public LoadBalancerInstance provision(LoadBalancerSpec spec) { if (throwOnCreate) throw new IllegalStateException("Did not expect a new load balancer to be created"); var instance = new LoadBalancerInstance( - idSeed, Optional.of(DomainName.of("lb-" + spec.application().toShortString() + "-" + spec.cluster().value())), Optional.empty(), Optional.empty(), @@ -80,13 +78,13 @@ public class LoadBalancerServiceMock implements LoadBalancerService { spec.settings(), spec.settings().isPrivateEndpoint() ? List.of(PrivateServiceId.of("service")) : List.of(), spec.cloudAccount()); - instances.put(new Key(spec.application(), spec.cluster(), idSeed), instance); + instances.put(new Key(spec.application(), spec.cluster(), spec.idSeed()), instance); return instance; } @Override public LoadBalancerInstance configure(LoadBalancerInstance instance, LoadBalancerSpec spec, boolean force) { - var id = new Key(spec.application(), spec.cluster(), instance.idSeed()); + var id = new Key(spec.application(), spec.cluster(), spec.idSeed()); var oldInstance = requireNonNull(instances.get(id), "expected existing load balancer " + id); if (!force && !oldInstance.reals().isEmpty() && spec.reals().isEmpty()) { throw new IllegalArgumentException("Refusing to remove all reals from load balancer " + id); @@ -99,16 +97,16 @@ public class LoadBalancerServiceMock implements LoadBalancerService { } @Override - public void reallocate(LoadBalancerInstance provisioned, LoadBalancerSpec spec) { - instances.put(new Key(spec.application(), spec.cluster(), provisioned.idSeed()), - requireNonNull(instances.remove(new Key(null, null, provisioned.idSeed())))); // ᕙ༼◕_◕༽ᕤ + public void reallocate(LoadBalancerSpec spec) { + instances.put(new Key(spec.application(), spec.cluster(), spec.idSeed()), + requireNonNull(instances.remove(new Key(null, null, spec.idSeed())))); // ᕙ༼◕_◕༽ᕤ } @Override public void remove(LoadBalancer loadBalancer) { requireNonNull(instances.remove(new Key(loadBalancer.id().application(), loadBalancer.id().cluster(), - loadBalancer.instance().get().idSeed())), + loadBalancer.idSeed())), "expected load balancer to exist: " + loadBalancer.id()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java index bde736e7a28..8c0aee2b4c6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java @@ -16,20 +16,21 @@ import java.util.Set; * @author mpolden */ public record LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, - ZoneEndpoint settings, CloudAccount cloudAccount) { + ZoneEndpoint settings, CloudAccount cloudAccount, String idSeed) { public static final ApplicationId preProvisionOwner = ApplicationId.from("hosted-vespa", "pre-provision", "default"); - public static LoadBalancerSpec preProvisionSpec(ClusterSpec.Id slot, CloudAccount account) { - return new LoadBalancerSpec(preProvisionOwner, slot, Set.of(), ZoneEndpoint.defaultEndpoint, account); + public static LoadBalancerSpec preProvisionSpec(ClusterSpec.Id slot, CloudAccount account, String idSeed) { + return new LoadBalancerSpec(preProvisionOwner, slot, Set.of(), ZoneEndpoint.defaultEndpoint, account, idSeed); } public LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, - ZoneEndpoint settings, CloudAccount cloudAccount) { + ZoneEndpoint settings, CloudAccount cloudAccount, String idSeed) { this.application = Objects.requireNonNull(application); this.cluster = Objects.requireNonNull(cluster); this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals)); this.settings = Objects.requireNonNull(settings); this.cloudAccount = Objects.requireNonNull(cloudAccount); + this.idSeed = Objects.requireNonNull(idSeed); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index f6de166ed01..1ca7442adf5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -30,12 +30,11 @@ public class SharedLoadBalancerService implements LoadBalancerService { } @Override - public LoadBalancerInstance provision(LoadBalancerSpec spec, String idSeed) { + public LoadBalancerInstance provision(LoadBalancerSpec spec) { if ( ! spec.settings().isPublicEndpoint()) throw new IllegalArgumentException("non-public endpoints is not supported with " + getClass()); - return new LoadBalancerInstance(idSeed, - Optional.of(DomainName.of(vipHostname)), + return new LoadBalancerInstance(Optional.of(DomainName.of(vipHostname)), Optional.empty(), Optional.empty(), Optional.empty(), @@ -53,7 +52,7 @@ public class SharedLoadBalancerService implements LoadBalancerService { } @Override - public void reallocate(LoadBalancerInstance provisioned, LoadBalancerSpec spec) { + public void reallocate(LoadBalancerSpec spec) { throw new UnsupportedOperationException("reallocate is not supported with " + getClass()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 04ea831c45c..105928c503e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -114,7 +114,8 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { LOG.log(Level.INFO, () -> "Removing reals from inactive load balancer " + lb.id() + ": " + Sets.difference(lb.instance().get().reals(), reals)); LoadBalancerInstance instance = service.configure(lb.instance().get(), new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals, - lb.instance().get().settings(), lb.instance().get().cloudAccount()), + lb.instance().get().settings(), + lb.instance().get().cloudAccount(), lb.idSeed()), true); db.writeLoadBalancer(lb.with(instance), lb.state()); } catch (Exception e) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index e351a4cc59f..99cc0d1e601 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -72,7 +72,7 @@ public class LoadBalancerSerializer { Cursor root = slime.setObject(); root.setString(idField, loadBalancer.id().serializedForm()); - loadBalancer.instance().map(LoadBalancerInstance::idSeed).ifPresent(idSeed -> root.setString(idSeedField, idSeed)); + root.setString(idSeedField, loadBalancer.idSeed()); loadBalancer.instance().flatMap(LoadBalancerInstance::hostname).ifPresent(hostname -> root.setString(hostnameField, hostname.value())); loadBalancer.instance().flatMap(LoadBalancerInstance::ip4Address).ifPresent(ip -> root.setString(lbIpAddressField, ip)); loadBalancer.instance().flatMap(LoadBalancerInstance::ip6Address).ifPresent(ip -> root.setString(lbIp6AddressField, ip)); @@ -142,9 +142,10 @@ public class LoadBalancerSerializer { CloudAccount cloudAccount = optionalString(object.field(cloudAccountField), CloudAccount::from).orElse(CloudAccount.empty); Optional<LoadBalancerInstance> instance = hostname.isEmpty() && ip4Address.isEmpty() && ip6Address.isEmpty() ? Optional.empty() - : Optional.of(new LoadBalancerInstance(idSeed, hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, serviceIds, cloudAccount)); + : Optional.of(new LoadBalancerInstance(hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, serviceIds, cloudAccount)); return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()), + idSeed, instance, stateFromString(object.field(stateField).asString()), Instant.ofEpochMilli(object.field(changedAtField).asLong())); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 0964a9d9779..e6f2dc0fbfe 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.ZoneEndpoint; @@ -200,22 +201,21 @@ public class LoadBalancerProvisioner { } private void prepare(LoadBalancerId id, ZoneEndpoint zoneEndpoint, NodeSpec requested) { - Instant now = nodeRepository.clock().instant(); CloudAccount cloudAccount = requested.cloudAccount(); Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); - LoadBalancer newLoadBalancer = loadBalancer.orElse(new LoadBalancer(id, Optional.empty(), LoadBalancer.State.reserved, now)); + LoadBalancer newLoadBalancer = null; LoadBalancer.State fromState = loadBalancer.map(LoadBalancer::state).orElse(null); try { if (loadBalancer.isPresent() && ! inAccount(cloudAccount, loadBalancer.get())) { - newLoadBalancer = newLoadBalancer.with(State.removable, now); + newLoadBalancer = loadBalancer.get().with(State.removable, nodeRepository.clock().instant()); throw new LoadBalancerServiceException("Could not (re)configure " + id + " due to change in cloud account. The operation will be retried on next deployment"); } if (loadBalancer.isPresent() && ! hasCorrectVisibility(loadBalancer.get(), zoneEndpoint)) { - newLoadBalancer = newLoadBalancer.with(State.removable, now); + newLoadBalancer = loadBalancer.get().with(State.removable, nodeRepository.clock().instant()); throw new LoadBalancerServiceException("Could not (re)configure " + id + " due to change in load balancer visibility. The operation will be retried on next deployment"); } - LoadBalancerInstance instance = provisionInstance(id, loadBalancer, zoneEndpoint, requested); - newLoadBalancer = newLoadBalancer.with(instance); + newLoadBalancer = loadBalancer.orElseGet(() -> createNewLoadBalancer(id, zoneEndpoint, requested)); // Determine id-seed. + newLoadBalancer = newLoadBalancer.with(provisionInstance(newLoadBalancer, zoneEndpoint, requested)); // Update instance. } catch (LoadBalancerServiceException e) { log.log(Level.WARNING, "Failed to provision load balancer", e); throw e; @@ -229,55 +229,43 @@ public class LoadBalancerProvisioner { newLoadBalancer.instance().get().settings().isPublicEndpoint() == zoneEndpoint.isPublicEndpoint(); } - private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, ZoneEndpoint settings, NodeList nodes) { - Instant now = nodeRepository.clock().instant(); - LoadBalancerId id = new LoadBalancerId(transaction.application(), cluster); - Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); - if (loadBalancer.isEmpty()) throw new IllegalArgumentException("Could not activate load balancer that was never prepared: " + id); - if (loadBalancer.get().instance().isEmpty()) throw new IllegalArgumentException("Activating " + id + ", but prepare never provisioned a load balancer instance"); - - try { - LoadBalancerInstance instance = configureInstance(id, nodes, loadBalancer.get(), settings, loadBalancer.get().instance().get().cloudAccount()); - db.writeLoadBalancers(List.of(loadBalancer.get().with(instance).with(State.active, now)), - loadBalancer.get().state(), transaction.nested()); - } catch (LoadBalancerServiceException e) { - db.writeLoadBalancers(List.of(loadBalancer.get()), loadBalancer.get().state(), transaction.nested()); - throw e; - } + /** Creates a new load balancer, with an instance if one is taken from the pool, or without otherwise. */ + private LoadBalancer createNewLoadBalancer(LoadBalancerId id, ZoneEndpoint zoneEndpoint, NodeSpec requested) { + LoadBalancerSpec spec = new LoadBalancerSpec(id.application(), id.cluster(), Set.of(), zoneEndpoint, + requested.cloudAccount(), toSeed(id, requested.type())); + return provisionFromPool(spec, requested.type()) + .orElseGet(() -> new LoadBalancer(id, spec.idSeed(), Optional.empty(), State.reserved, nodeRepository.clock().instant())); } /** Provision a load balancer instance, if necessary */ - private LoadBalancerInstance provisionInstance(LoadBalancerId id, - Optional<LoadBalancer> currentLoadBalancer, + private LoadBalancerInstance provisionInstance(LoadBalancer currentLoadBalancer, ZoneEndpoint zoneEndpoint, NodeSpec requested) { - Set<Real> reals = currentLoadBalancer.flatMap(LoadBalancer::instance) + LoadBalancerId id = currentLoadBalancer.id(); + Set<Real> reals = currentLoadBalancer.instance() .map(LoadBalancerInstance::reals) .orElse(Set.of()); // Targeted reals are changed on activation. ZoneEndpoint settings = new ZoneEndpoint(zoneEndpoint.isPublicEndpoint(), zoneEndpoint.isPrivateEndpoint(), - currentLoadBalancer.flatMap(LoadBalancer::instance) + currentLoadBalancer.instance() .map(LoadBalancerInstance::settings) .map(ZoneEndpoint::allowedUrns) .orElse(List.of())); // Allowed URNs are changed on activation. - if ( currentLoadBalancer.isPresent() - && currentLoadBalancer.get().instance().isPresent() - && currentLoadBalancer.get().instance().get().settings().equals(settings)) - return currentLoadBalancer.get().instance().get(); + if (currentLoadBalancer.instance().map(instance -> settings.equals(instance.settings())).orElse(false)) + return currentLoadBalancer.instance().get(); log.log(Level.INFO, () -> "Provisioning instance for " + id); try { - LoadBalancerSpec spec = new LoadBalancerSpec(id.application(), id.cluster(), reals, settings, requested.cloudAccount()); - return provisionFromPool(spec, requested.type()).orElseGet(() -> service.provision(spec, toSeed(id, requested.type()))) - // Provisioning a private endpoint service requires hard resources to be ready, so we delay it until activation. - .withServiceIds(currentLoadBalancer.flatMap(LoadBalancer::instance).map(LoadBalancerInstance::serviceIds).orElse(List.of())); + return service.provision(new LoadBalancerSpec(id.application(), id.cluster(), reals, settings, requested.cloudAccount(), currentLoadBalancer.idSeed())) + // Provisioning a private endpoint service requires hard resources to be ready, so we delay it until activation. + .withServiceIds(currentLoadBalancer.instance().map(LoadBalancerInstance::serviceIds).orElse(List.of())); } catch (Exception e) { throw new LoadBalancerServiceException("Could not provision " + id + ". The operation will be retried on next deployment.", e); } } - private Optional<LoadBalancerInstance> provisionFromPool(LoadBalancerSpec spec, NodeType type) { + private Optional<LoadBalancer> provisionFromPool(LoadBalancerSpec spec, NodeType type) { if (type != NodeType.tenant) return Optional.empty(); if ( ! spec.settings().isDefault()) return Optional.empty(); if (preProvisionPoolSize.value() == 0) return Optional.empty(); @@ -293,9 +281,14 @@ public class LoadBalancerProvisioner { if (chosen.state() != State.active || chosen.instance().isEmpty()) throw new IllegalStateException("expected active load balancer in pre-provisioned pool, but got " + chosen); log.log(Level.INFO, "Using " + chosen + " from pre-provisioned pool"); - service.reallocate(chosen.instance().get(), spec); + service.reallocate(new LoadBalancerSpec(spec.application(), spec.cluster(), spec.reals(), spec.settings(), spec.cloudAccount(), chosen.idSeed())); db.removeLoadBalancer(chosen.id()); // Using a transaction to remove this, and write the instance, would be better, but much hassle. - return chosen.instance(); // Should be immediately written again outside of this! + // Should be immediately written again outside of this! + return Optional.of(new LoadBalancer(new LoadBalancerId(spec.application(), spec.cluster()), + chosen.idSeed(), + chosen.instance(), + State.reserved, + nodeRepository.clock().instant())); } catch (Exception e) { log.log(Level.WARNING, "Failed to provision load balancer from pool", e); @@ -327,10 +320,11 @@ public class LoadBalancerProvisioner { // No need for lock while we provision, since we'll write atomically only after we're done, and the job lock ensures single writer. while (head - tail < size) { ClusterSpec.Id slot = slotId(head); - LoadBalancerSpec spec = preProvisionSpec(slot, nodeRepository.zone().cloud().account()); LoadBalancerId id = new LoadBalancerId(preProvisionOwner, slot); + LoadBalancerSpec spec = preProvisionSpec(slot, nodeRepository.zone().cloud().account(), toSeed(id)); db.writeLoadBalancer(new LoadBalancer(id, - Optional.of(service.provision(spec, toSeed(id, NodeType.tenant))), + spec.idSeed(), + Optional.of(service.provision(spec)), State.active, // Keep the expirer away. nodeRepository.clock().instant()), null); @@ -343,7 +337,7 @@ public class LoadBalancerProvisioner { } public static String toSeed(LoadBalancerId id) { - return id.serializedForm(); + return ":" + id.serializedForm() + ":"; // ಠ_ಠ } public static String toLegacySeed(ApplicationId application, ClusterSpec.Id cluster) { @@ -353,6 +347,23 @@ public class LoadBalancerProvisioner { cluster.value(); // ಠ_ಠ } + private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, ZoneEndpoint settings, NodeList nodes) { + Instant now = nodeRepository.clock().instant(); + LoadBalancerId id = new LoadBalancerId(transaction.application(), cluster); + Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); + if (loadBalancer.isEmpty()) throw new IllegalArgumentException("Could not activate load balancer that was never prepared: " + id); + if (loadBalancer.get().instance().isEmpty()) throw new IllegalArgumentException("Activating " + id + ", but prepare never provisioned a load balancer instance"); + + try { + LoadBalancerInstance instance = configureInstance(id, nodes, loadBalancer.get(), settings, loadBalancer.get().instance().get().cloudAccount()); + db.writeLoadBalancers(List.of(loadBalancer.get().with(instance).with(State.active, now)), + loadBalancer.get().state(), transaction.nested()); + } catch (LoadBalancerServiceException e) { + db.writeLoadBalancers(List.of(loadBalancer.get()), loadBalancer.get().state(), transaction.nested()); + throw e; + } + } + /** Reconfigure a load balancer instance, if necessary */ private LoadBalancerInstance configureInstance(LoadBalancerId id, NodeList nodes, LoadBalancer currentLoadBalancer, @@ -365,7 +376,7 @@ public class LoadBalancerProvisioner { log.log(Level.FINE, () -> "Configuring instance for " + id + ", targeting: " + reals); try { return service.configure(currentLoadBalancer.instance().orElseThrow(() -> new IllegalArgumentException("expected existing instance for " + id)), - new LoadBalancerSpec(id.application(), id.cluster(), reals, zoneEndpoint, cloudAccount), + new LoadBalancerSpec(id.application(), id.cluster(), reals, zoneEndpoint, cloudAccount, currentLoadBalancer.idSeed()), shouldDeactivateRouting || currentLoadBalancer.state() != LoadBalancer.State.active); } catch (Exception e) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index dd63d18ea38..d47f5a4e4d6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -50,6 +50,7 @@ public class LoadBalancersResponse extends SlimeJsonResponse { loadBalancers.forEach(lb -> { Cursor lbObject = loadBalancerArray.addObject(); lbObject.setString("id", lb.id().serializedForm()); + lbObject.setString("idSeed", lb.idSeed()); lbObject.setString("state", lb.state().name()); lbObject.setLong("changedAt", lb.changedAt().toEpochMilli()); lbObject.setString("application", lb.id().application().application().value()); @@ -90,7 +91,6 @@ public class LoadBalancersResponse extends SlimeJsonResponse { } instance.serviceId().ifPresent(serviceId -> lbObject.setString("serviceId", serviceId.value())); lbObject.setBool("public", instance.settings().isPublicEndpoint()); - lbObject.setString("idSeed", instance.idSeed()); }); lb.instance() .map(LoadBalancerInstance::cloudAccount) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java index 91d04ba5ff9..198709ba4bb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java @@ -29,9 +29,9 @@ public class SharedLoadBalancerServiceTest { @Test public void test_create_lb() { LoadBalancerSpec spec = new LoadBalancerSpec(applicationId, clusterId, reals, - ZoneEndpoint.defaultEndpoint, CloudAccount.empty); + ZoneEndpoint.defaultEndpoint, CloudAccount.empty, "seed"); - var lb = loadBalancerService.configure(loadBalancerService.provision(spec, "seed"), spec, false); + var lb = loadBalancerService.configure(loadBalancerService.provision(spec), spec, false); assertEquals(Optional.of(HostName.of("vip.example.com")), lb.hostname()); assertEquals(Optional.empty(), lb.dnsZone()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index c4688f0b3dd..9ec63933921 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -38,8 +38,8 @@ public class LoadBalancerSerializerTest { var now = Instant.now(); { var loadBalancer = new LoadBalancer(loadBalancerId, + "1", Optional.of(new LoadBalancerInstance( - "1", Optional.of(DomainName.of("lb-host")), Optional.empty(), Optional.empty(), @@ -60,7 +60,7 @@ public class LoadBalancerSerializerTest { var serialized = LoadBalancerSerializer.fromJson(loadBalancer.id(), LoadBalancerSerializer.toJson(loadBalancer)); assertEquals(loadBalancer.id(), serialized.id()); - assertEquals(loadBalancer.instance().get().idSeed(), serialized.instance().get().idSeed()); + assertEquals(loadBalancer.idSeed(), serialized.idSeed()); assertEquals(loadBalancer.instance().get().hostname(), serialized.instance().get().hostname()); assertEquals(loadBalancer.instance().get().dnsZone(), serialized.instance().get().dnsZone()); assertEquals(loadBalancer.instance().get().ports(), serialized.instance().get().ports()); @@ -74,8 +74,8 @@ public class LoadBalancerSerializerTest { } { var loadBalancer = new LoadBalancer(loadBalancerId, + "", Optional.of(new LoadBalancerInstance( - "", Optional.empty(), Optional.of("1.2.3.4"), Optional.of("fd00::1"), @@ -91,7 +91,7 @@ public class LoadBalancerSerializerTest { var serialized = LoadBalancerSerializer.fromJson(loadBalancer.id(), LoadBalancerSerializer.toJson(loadBalancer)); assertEquals(loadBalancer.id(), serialized.id()); - assertEquals(loadBalancer.instance().get().idSeed(), serialized.instance().get().idSeed()); + assertEquals(loadBalancer.idSeed(), serialized.idSeed()); assertEquals(loadBalancer.instance().get().hostname(), serialized.instance().get().hostname()); assertEquals(loadBalancer.instance().get().ip4Address(), serialized.instance().get().ip4Address()); assertEquals(loadBalancer.instance().get().ip6Address(), serialized.instance().get().ip6Address()); @@ -110,10 +110,11 @@ public class LoadBalancerSerializerTest { @Test public void no_instance_serialization() { var now = Instant.now(); - var loadBalancer = new LoadBalancer(loadBalancerId, Optional.empty(), LoadBalancer.State.reserved, now); + var loadBalancer = new LoadBalancer(loadBalancerId, "seed", Optional.empty(), LoadBalancer.State.reserved, now); var serialized = LoadBalancerSerializer.fromJson(loadBalancerId, LoadBalancerSerializer.toJson(loadBalancer)); assertEquals(loadBalancer.id(), serialized.id()); + assertEquals(loadBalancer.idSeed(), serialized.idSeed()); assertEquals(loadBalancer.instance(), serialized.instance()); assertEquals(loadBalancer.state(), serialized.state()); assertEquals(loadBalancer.changedAt().truncatedTo(MILLIS), serialized.changedAt()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index fce788c7e32..7096ea3de46 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -199,7 +199,7 @@ public class LoadBalancerProvisionerTest { List<LoadBalancer> loadBalancers = tester.nodeRepository().loadBalancers().list(app1).asList(); assertEquals(1, loadBalancers.size()); assertEquals(1, tester.nodeRepository().loadBalancers().list(preProvisionOwner).asList().size()); - assertEquals(preProvisionOwner.serializedForm() + ":1", loadBalancers.get(0).instance().get().idSeed()); + assertEquals(":" + preProvisionOwner.serializedForm() + ":1:", loadBalancers.get(0).idSeed()); // Shrink pool to 0 entries. flagSource.withIntFlag(PermanentFlags.PRE_PROVISIONED_LB_COUNT.id(), 0); @@ -215,9 +215,9 @@ public class LoadBalancerProvisionerTest { assertThrows(IllegalStateException.class, provisioner::refreshPool).getMessage()); tester.loadBalancerService().throwOnCreate(false); provisioner.refreshPool(); - assertEquals(List.of(preProvisionOwner.serializedForm() + ":3"), + assertEquals(List.of(":" + preProvisionOwner.serializedForm() + ":3:"), tester.nodeRepository().loadBalancers().list(preProvisionOwner) - .mapToList(lb -> lb.instance().get().idSeed())); + .mapToList(LoadBalancer::idSeed)); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json index bc5a04edc2f..ee8a8588c5f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json @@ -7,7 +7,7 @@ "dnsZone": "zone-id-1", "hostname": "lb-hosted-vespa.pre-provision-1", "id": "tenant4:application4:instance4:id4", - "idSeed": "hosted-vespa:pre-provision:default:1", + "idSeed": ":hosted-vespa:pre-provision:default:1:", "instance": "instance4", "networks": [ "10.2.3.0/24", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json index 883820634fe..6de3b500a00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json @@ -7,7 +7,7 @@ "dnsZone": "zone-id-1", "hostname": "lb-tenant1.application1.instance1-id1", "id": "tenant1:application1:instance1:id1", - "idSeed": "tenant1:application1:instance1:id1", + "idSeed": ":tenant1:application1:instance1:id1:", "instance": "instance1", "networks": [ "10.2.3.0/24", @@ -80,7 +80,7 @@ "dnsZone": "zone-id-1", "hostname": "lb-hosted-vespa.pre-provision-1", "id": "tenant4:application4:instance4:id4", - "idSeed": "hosted-vespa:pre-provision:default:1", + "idSeed": ":hosted-vespa:pre-provision:default:1:", "instance": "instance4", "networks": [ "10.2.3.0/24", |