Merge pull request #24979 from vespa-engine/jonmv/relax-cloud-account-requirements

Configure VPC endpoint services for container clusters

10 files changed, 105 insertions, 34 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java
index aa7d388c78f..fde3c85bdab 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.lb;
 import ai.vespa.http.DomainName;
 import com.google.common.collect.ImmutableSortedSet;
 import com.yahoo.config.provision.CloudAccount;
+import com.yahoo.config.provision.LoadBalancerSettings;
 
 import java.util.Objects;
 import java.util.Optional;
@@ -23,16 +24,18 @@ public class LoadBalancerInstance {
     private final Set<Integer> ports;
     private final Set<String> networks;
     private final Set<Real> reals;
+    private final LoadBalancerSettings settings;
     private final CloudAccount cloudAccount;
 
     public LoadBalancerInstance(Optional<DomainName> hostname, Optional<String> ipAddress, Optional<DnsZone> dnsZone, Set<Integer> ports,
-                                Set<String> networks, Set<Real> reals, CloudAccount cloudAccount) {
+                                Set<String> networks, Set<Real> reals, LoadBalancerSettings settings, CloudAccount cloudAccount) {
         this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null");
         this.ipAddress = Objects.requireNonNull(ipAddress, "ip must be non-null");
         this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null");
         this.ports = ImmutableSortedSet.copyOf(requirePorts(ports));
         this.networks = ImmutableSortedSet.copyOf(Objects.requireNonNull(networks, "networks must be non-null"));
         this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals, "targets must be non-null"));
+        this.settings = Objects.requireNonNull(settings, "settings must be non-null");
         this.cloudAccount = Objects.requireNonNull(cloudAccount, "cloudAccount must be non-null");
 
         if (hostname.isEmpty() == ipAddress.isEmpty()) {
@@ -71,6 +74,11 @@ public class LoadBalancerInstance {
         return reals;
     }
 
+    /** Static user-configured settings of this load balancer */
+    public LoadBalancerSettings settings() {
+        return settings;
+    }
+
     /** Cloud account of this load balancer */
     public CloudAccount cloudAccount() {
         return cloudAccount;
@@ -78,7 +86,7 @@ public class LoadBalancerInstance {
 
     /** Returns a copy of this with reals set to given reals */
     public LoadBalancerInstance withReals(Set<Real> reals) {
-        return new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, cloudAccount);
+        return new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, settings, cloudAccount);
     }
 
     private static Set<Integer> requirePorts(Set<Integer> ports) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java
index ac5330dce12..2b6f64012b1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java
@@ -61,6 +61,7 @@ public class LoadBalancerServiceMock implements LoadBalancerService {
                 Collections.singleton(4443),
                 ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"),
                 spec.reals(),
+                spec.settings(),
                 spec.cloudAccount());
         instances.put(id, instance);
         return instance;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java
index 18c6ed52046..c4a6f6b2a37 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java
@@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSortedSet;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.CloudAccount;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.LoadBalancerSettings;
 
 import java.util.Objects;
 import java.util.Set;
@@ -19,13 +20,15 @@ public class LoadBalancerSpec {
     private final ApplicationId application;
     private final ClusterSpec.Id cluster;
     private final Set<Real> reals;
+    private final LoadBalancerSettings settings;
     private final CloudAccount cloudAccount;
 
     public LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals,
-                            CloudAccount cloudAccount) {
+                            LoadBalancerSettings settings, CloudAccount cloudAccount) {
         this.application = Objects.requireNonNull(application);
         this.cluster = Objects.requireNonNull(cluster);
         this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals));
+        this.settings = Objects.requireNonNull(settings);
         this.cloudAccount = Objects.requireNonNull(cloudAccount);
     }
 
@@ -44,6 +47,11 @@ public class LoadBalancerSpec {
         return reals;
     }
 
+    /** Static user-configured settings for this load balancer. */
+    public LoadBalancerSettings settings() {
+        return settings;
+    }
+
     /** Cloud account to use when satisfying this */
     public CloudAccount cloudAccount() {
         return cloudAccount;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java
index c126b3969fa..d9032bd0a5b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.lb;
 
 import ai.vespa.http.DomainName;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.LoadBalancerSettings;
 import com.yahoo.config.provision.NodeType;
 
 import java.util.Objects;
@@ -12,7 +13,7 @@ import java.util.Set;
 /**
  * This implementation of {@link LoadBalancerService} returns the load balancer(s) that exist by default in the shared
  * routing layer.
- *
+ * <p>
  * Since such load balancers always exist, we can return the hostname of the routing layer VIP directly. Nothing has to
  * be provisioned.
  *
@@ -28,12 +29,14 @@ public class SharedLoadBalancerService implements LoadBalancerService {
 
     @Override
     public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) {
+        if ( ! spec.settings().isEmpty()) throw new IllegalArgumentException("custom load balancer settings are not supported with " + getClass());
         return new LoadBalancerInstance(Optional.of(DomainName.of(vipHostname)),
                                         Optional.empty(),
                                         Optional.empty(),
                                         Set.of(4443),
                                         Set.of(),
                                         spec.reals(),
+                                        spec.settings(),
                                         spec.cloudAccount());
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
index f0abe184614..0264d0df837 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
@@ -111,7 +111,9 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer {
             try {
                 attempts.add(1);
                 LOG.log(Level.INFO, () -> "Removing reals from inactive load balancer " + lb.id() + ": " + Sets.difference(lb.instance().get().reals(), reals));
-                service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals, lb.instance().get().cloudAccount()), true);
+                service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals,
+                                                    lb.instance().get().settings(), lb.instance().get().cloudAccount()),
+                               true);
                 db.writeLoadBalancer(lb.with(lb.instance().map(instance -> instance.withReals(reals))), lb.state());
             } catch (Exception e) {
                 failed.add(lb.id());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
index c0bf9926ae1..9eccb21b756 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.persistence;
 
 import ai.vespa.http.DomainName;
 import com.yahoo.config.provision.CloudAccount;
+import com.yahoo.config.provision.LoadBalancerSettings;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
@@ -48,6 +49,8 @@ public class LoadBalancerSerializer {
     private static final String ipAddressField = "ipAddress";
     private static final String portField = "port";
     private static final String cloudAccountField = "cloudAccount";
+    private static final String settingsField = "settings";
+    private static final String allowedUrnsField = "allowedUrns";
 
     public static byte[] toJson(LoadBalancer loadBalancer) {
         Slime slime = new Slime();
@@ -71,6 +74,11 @@ public class LoadBalancerSerializer {
             realObject.setLong(portField, real.port());
         }));
         loadBalancer.instance()
+                    .map(LoadBalancerInstance::settings)
+                    .filter(settings -> ! settings.isEmpty())
+                    .ifPresent(settings -> settings.allowedUrns().forEach(root.setObject(settingsField)
+                                                                              .setArray(allowedUrnsField)::addString));
+        loadBalancer.instance()
                     .map(LoadBalancerInstance::cloudAccount)
                     .filter(cloudAccount -> !cloudAccount.isUnspecified())
                     .ifPresent(cloudAccount -> root.setString(cloudAccountField, cloudAccount.value()));
@@ -101,9 +109,10 @@ public class LoadBalancerSerializer {
         Optional<DomainName> hostname = optionalString(object.field(hostnameField), Function.identity()).filter(s -> !s.isEmpty()).map(DomainName::of);
         Optional<String> ipAddress = optionalString(object.field(lbIpAddressField), Function.identity()).filter(s -> !s.isEmpty());
         Optional<DnsZone> dnsZone = optionalString(object.field(dnsZoneField), DnsZone::new);
+        LoadBalancerSettings settings = loadBalancerSettings(object.field(settingsField));
         CloudAccount cloudAccount = optionalString(object.field(cloudAccountField), CloudAccount::from).orElse(CloudAccount.empty);
         Optional<LoadBalancerInstance> instance = hostname.isEmpty() && ipAddress.isEmpty() ? Optional.empty() :
-                Optional.of(new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, cloudAccount));
+                Optional.of(new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, settings, cloudAccount));
 
         return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()),
                                 instance,
@@ -111,6 +120,13 @@ public class LoadBalancerSerializer {
                                 Instant.ofEpochMilli(object.field(changedAtField).asLong()));
     }
 
+    private static LoadBalancerSettings loadBalancerSettings(Inspector settingsObject) {
+        if ( ! settingsObject.valid()) return LoadBalancerSettings.empty;
+        return new LoadBalancerSettings(SlimeUtils.entriesStream(settingsObject.field(allowedUrnsField))
+                                                  .map(Inspector::asString)
+                                                  .toList());
+    }
+
     private static <T> Optional<T> optionalValue(Inspector field, Function<Inspector, T> fieldMapper) {
         return Optional.of(field).filter(Inspector::valid).map(fieldMapper);
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
index 0763d19bae3..c9c2199a234 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
@@ -6,7 +6,9 @@ import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.ApplicationTransaction;
 import com.yahoo.config.provision.CloudAccount;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.ClusterSpec.Type;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.LoadBalancerSettings;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.exception.LoadBalancerServiceException;
@@ -39,6 +41,9 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.reducing;
+
 /**
  * Provisions and configures application load balancers.
  *
@@ -75,12 +80,12 @@ public class LoadBalancerProvisioner {
 
     /**
      * Prepare a load balancer for given application and cluster.
-     *
+     * <p>
      * If a load balancer for the cluster already exists, it will be reconfigured based on the currently allocated
      * nodes. It's state will remain unchanged.
-     *
+     * <p>
      * If no load balancer exists, a new one will be provisioned in {@link LoadBalancer.State#reserved}.
-     *
+     * <p>
      * Calling this for irrelevant node or cluster types is a no-op.
      */
     public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) {
@@ -89,33 +94,33 @@ public class LoadBalancerProvisioner {
             ClusterSpec.Id clusterId = effectiveId(cluster);
             LoadBalancerId loadBalancerId = requireNonClashing(new LoadBalancerId(application, clusterId));
             NodeList nodes = nodesOf(clusterId, application);
-            prepare(loadBalancerId, nodes, requestedNodes.cloudAccount());
+            prepare(loadBalancerId, nodes, cluster.loadBalancerSettings(), requestedNodes.cloudAccount());
         }
     }
 
     /**
      * Activate load balancer for given application and cluster.
-     *
+     * <p>
      * If a load balancer for the cluster already exists, it will be reconfigured based on the currently allocated
      * nodes and the load balancer itself will be moved to {@link LoadBalancer.State#active}.
-     *
+     * <p>
      * Load balancers for clusters that are no longer in given clusters are deactivated.
-     *
+     * <p>
      * Calling this when no load balancer has been prepared for given cluster is a no-op.
      */
     public void activate(Set<ClusterSpec> clusters, NodeList newActive, ApplicationTransaction transaction) {
-        Set<ClusterSpec.Id> activatingClusters = clusters.stream()
-                                                         .map(LoadBalancerProvisioner::effectiveId)
-                                                         .collect(Collectors.toSet());
+        Map<ClusterSpec.Id, ClusterSpec> activatingClusters = clusters.stream()
+                                                                      .collect(groupingBy(LoadBalancerProvisioner::effectiveId,
+                                                                                          reducing(null, (o, n) -> o == null || o.type() != Type.container ? n : o)));
         for (var cluster : loadBalancedClustersOf(newActive).entrySet()) {
-            if (!activatingClusters.contains(cluster.getKey())) continue;
+            if (!activatingClusters.containsKey(cluster.getKey())) continue;
 
             Node clusterNode = cluster.getValue().first().get();
             if (!shouldProvision(transaction.application(), clusterNode.type(), clusterNode.allocation().get().membership().cluster().type())) continue;
-            activate(transaction, cluster.getKey(), cluster.getValue());
+            activate(transaction, cluster.getKey(), activatingClusters.get(cluster.getKey()).loadBalancerSettings(), cluster.getValue());
         }
         // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed
-        var surplusLoadBalancers = surplusLoadBalancersOf(transaction.application(), activatingClusters);
+        var surplusLoadBalancers = surplusLoadBalancersOf(transaction.application(), activatingClusters.keySet());
         deactivate(surplusLoadBalancers, transaction.nested());
     }
 
@@ -180,10 +185,10 @@ public class LoadBalancerProvisioner {
         return loadBalancerId;
     }
 
-    private void prepare(LoadBalancerId id, NodeList nodes, CloudAccount cloudAccount) {
+    private void prepare(LoadBalancerId id, NodeList nodes, LoadBalancerSettings loadBalancerSettings, CloudAccount cloudAccount) {
         Instant now = nodeRepository.clock().instant();
         Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id);
-        Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, cloudAccount);
+        Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, loadBalancerSettings, cloudAccount);
         LoadBalancer newLoadBalancer;
         LoadBalancer.State fromState = null;
         if (loadBalancer.isEmpty()) {
@@ -202,14 +207,14 @@ public class LoadBalancerProvisioner {
         requireInstance(id, instance, cloudAccount);
     }
 
-    private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, NodeList nodes) {
+    private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, LoadBalancerSettings loadBalancerSettings, NodeList nodes) {
         Instant now = nodeRepository.clock().instant();
         LoadBalancerId id = new LoadBalancerId(transaction.application(), cluster);
         Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id);
-        if (loadBalancer.isEmpty()) throw new IllegalArgumentException("Could not active load balancer that was never prepared: " + id);
+        if (loadBalancer.isEmpty()) throw new IllegalArgumentException("Could not activate load balancer that was never prepared: " + id);
         if (loadBalancer.get().instance().isEmpty()) throw new IllegalArgumentException("Activating " + id + ", but prepare never provisioned a load balancer instance");
 
-        Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, loadBalancer.get().instance().get().cloudAccount());
+        Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, loadBalancerSettings, loadBalancer.get().instance().get().cloudAccount());
         LoadBalancer.State state = instance.isPresent() ? LoadBalancer.State.active : loadBalancer.get().state();
         LoadBalancer newLoadBalancer = loadBalancer.get().with(instance).with(state, now);
         db.writeLoadBalancers(List.of(newLoadBalancer), loadBalancer.get().state(), transaction.nested());
@@ -219,6 +224,7 @@ public class LoadBalancerProvisioner {
     /** Provision or reconfigure a load balancer instance, if necessary */
     private Optional<LoadBalancerInstance> provisionInstance(LoadBalancerId id, NodeList nodes,
                                                              Optional<LoadBalancer> currentLoadBalancer,
+                                                             LoadBalancerSettings loadBalancerSettings,
                                                              CloudAccount cloudAccount) {
         boolean shouldDeactivateRouting = deactivateRouting.with(FetchVector.Dimension.APPLICATION_ID,
                                                                  id.application().serializedForm())
@@ -229,10 +235,10 @@ public class LoadBalancerProvisioner {
         } else {
             reals = realsOf(nodes);
         }
-        if (hasReals(currentLoadBalancer, reals)) return currentLoadBalancer.get().instance();
+        if (isUpToDate(currentLoadBalancer, reals, loadBalancerSettings)) return currentLoadBalancer.get().instance();
         log.log(Level.INFO, () -> "Provisioning instance for " + id + ", targeting: " + reals);
         try {
-            return Optional.of(service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals, cloudAccount),
+            return Optional.of(service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals, loadBalancerSettings, cloudAccount),
                                               shouldDeactivateRouting || allowEmptyReals(currentLoadBalancer)));
         } catch (Exception e) {
             log.log(Level.WARNING, e, () -> "Could not (re)configure " + id + ", targeting: " +
@@ -288,11 +294,12 @@ public class LoadBalancerProvisioner {
         return loadBalancer.instance().isEmpty() || loadBalancer.instance().get().cloudAccount().equals(cloudAccount);
     }
 
-    /** Returns whether load balancer has given reals */
-    private static boolean hasReals(Optional<LoadBalancer> loadBalancer, Set<Real> reals) {
+    /** Returns whether load balancer has given reals and settings */
+    private static boolean isUpToDate(Optional<LoadBalancer> loadBalancer, Set<Real> reals, LoadBalancerSettings loadBalancerSettings) {
         if (loadBalancer.isEmpty()) return false;
         if (loadBalancer.get().instance().isEmpty()) return false;
-        return loadBalancer.get().instance().get().reals().equals(reals);
+        return    loadBalancer.get().instance().get().reals().equals(reals)
+               && loadBalancer.get().instance().get().settings().equals(loadBalancerSettings);
     }
 
     /** Returns whether to allow given load balancer to have no reals */
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java
index be3216c79a4..92c7ba7fe27 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java
@@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.CloudAccount;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.LoadBalancerSettings;
 import org.junit.Test;
 
 import java.util.Optional;
@@ -27,7 +28,9 @@ public class SharedLoadBalancerServiceTest {
 
     @Test
     public void test_create_lb() {
-        var lb = loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals, CloudAccount.empty), false);
+        var lb = loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals,
+                                                                 LoadBalancerSettings.empty, CloudAccount.empty),
+                                            false);
 
         assertEquals(Optional.of(HostName.of("vip.example.com")), lb.hostname());
         assertEquals(Optional.empty(), lb.dnsZone());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java
index d2d805a1bd5..1e0385d152a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java
@@ -6,6 +6,7 @@ import com.google.common.collect.ImmutableSet;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.CloudAccount;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.LoadBalancerSettings;
 import com.yahoo.vespa.hosted.provision.lb.DnsZone;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
@@ -14,6 +15,7 @@ import com.yahoo.vespa.hosted.provision.lb.Real;
 import org.junit.Test;
 
 import java.time.Instant;
+import java.util.List;
 import java.util.Optional;
 
 import static java.time.temporal.ChronoUnit.MILLIS;
@@ -43,6 +45,7 @@ public class LoadBalancerSerializerTest {
                                                                     new Real(DomainName.of("real-2"),
                                                                              "127.0.0.2",
                                                                              4080)),
+                                                    new LoadBalancerSettings(List.of("123")),
                                                     CloudAccount.from("012345678912"))),
                                             LoadBalancer.State.active,
                                             now);
@@ -56,6 +59,7 @@ public class LoadBalancerSerializerTest {
         assertEquals(loadBalancer.state(), serialized.state());
         assertEquals(loadBalancer.changedAt().truncatedTo(MILLIS), serialized.changedAt());
         assertEquals(loadBalancer.instance().get().reals(), serialized.instance().get().reals());
+        assertEquals(loadBalancer.instance().get().settings(), serialized.instance().get().settings());
         assertEquals(loadBalancer.instance().get().cloudAccount(), serialized.instance().get().cloudAccount());
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
index 325e4b58174..4635eb6bc7c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.CloudAccount;
 import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.LoadBalancerSettings;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.exception.LoadBalancerServiceException;
@@ -215,7 +216,7 @@ public class LoadBalancerProvisionerTest {
     public void provision_load_balancer_combined_cluster() {
         Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().list(app1).asList();
         var combinedId = ClusterSpec.Id.from("container1");
-        var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, ClusterSpec.Id.from("content1"), Optional.of(combinedId)));
+        var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, ClusterSpec.Id.from("content1"), Optional.of(combinedId), LoadBalancerSettings.empty));
         assertEquals(1, lbs.get().size());
         assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().get().reals().size());
         tester.activate(app1, nodes);
@@ -314,6 +315,24 @@ public class LoadBalancerProvisionerTest {
     }
 
     @Test
+    public void load_balancer_with_custom_settings() {
+        ClusterResources resources = new ClusterResources(3, 1, nodeResources);
+        Capacity capacity = Capacity.from(resources, resources, false, true, Optional.of(CloudAccount.empty));
+        tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1"))));
+        LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list();
+        assertEquals(1, loadBalancers.size());
+        assertEquals(LoadBalancerSettings.empty, loadBalancers.first().get().instance().get().settings());
+
+        // Next deployment contains new settings
+        LoadBalancerSettings settings = new LoadBalancerSettings(List.of("alice", "bob"));
+        tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1"), Optional.empty(), settings)));
+        loadBalancers = tester.nodeRepository().loadBalancers().list();
+        assertEquals(1, loadBalancers.size());
+        assertEquals(settings, loadBalancers.first().get().instance().get().settings());
+    }
+
+
+    @Test
     public void load_balancer_with_custom_cloud_account() {
         ClusterResources resources = new ClusterResources(3, 1, nodeResources);
         CloudAccount cloudAccount0 = CloudAccount.empty;
@@ -412,11 +431,11 @@ public class LoadBalancerProvisionerTest {
     }
 
     private static ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id) {
-        return clusterRequest(type,  id, Optional.empty());
+        return clusterRequest(type, id, Optional.empty(), LoadBalancerSettings.empty);
     }
 
-    private static ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id, Optional<ClusterSpec.Id> combinedId) {
-        return ClusterSpec.request(type, id).vespaVersion("6.42").combinedId(combinedId).build();
+    private static ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id, Optional<ClusterSpec.Id> combinedId, LoadBalancerSettings settings) {
+        return ClusterSpec.request(type, id).vespaVersion("6.42").combinedId(combinedId).loadBalancerSettings(settings).build();
     }
 
     private static <T> T get(Set<T> set, int position) {