Merge pull request #12178 from vespa-engine/mpolden/lb-lock-per-app

Take load balancer lock per application

9 files changed, 168 insertions, 121 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 7d875434e1c..4af68fa702c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision;
 
 import com.google.inject.Inject;
@@ -17,6 +17,7 @@ import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList;
 import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions;
@@ -50,6 +51,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.function.BiFunction;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -194,7 +196,16 @@ public class NodeRepository extends AbstractComponent {
 
     /** Returns a filterable list of all load balancers in this repository */
     public LoadBalancerList loadBalancers() {
-        return LoadBalancerList.copyOf(database().readLoadBalancers().values());
+        return loadBalancers((ignored) -> true);
+    }
+
+    /** Returns a filterable list of load balancers belonging to given application */
+    public LoadBalancerList loadBalancers(ApplicationId application) {
+        return loadBalancers((id) -> id.application().equals(application));
+    }
+
+    private LoadBalancerList loadBalancers(Predicate<LoadBalancerId> predicate) {
+        return LoadBalancerList.copyOf(db.readLoadBalancers(predicate).values());
     }
 
     public List<Node> getNodes(ApplicationId id, Node.State ... inState) { return db.getNodes(id, inState); }
@@ -204,7 +215,7 @@ public class NodeRepository extends AbstractComponent {
     /**
      * Returns the ACL for the node (trusted nodes, networks and ports)
      */
-    private NodeAcl getNodeAcl(Node node, NodeList candidates, LoadBalancerList loadBalancers) {
+    private NodeAcl getNodeAcl(Node node, NodeList candidates) {
         Set<Node> trustedNodes = new TreeSet<>(Comparator.comparing(Node::hostname));
         Set<Integer> trustedPorts = new LinkedHashSet<>();
         Set<String> trustedNetworks = new LinkedHashSet<>();
@@ -221,10 +232,10 @@ public class NodeRepository extends AbstractComponent {
         candidates.parentOf(node).ifPresent(trustedNodes::add);
         node.allocation().ifPresent(allocation -> {
             trustedNodes.addAll(candidates.owner(allocation.owner()).asList());
-            loadBalancers.owner(allocation.owner()).asList().stream()
-                         .map(LoadBalancer::instance)
-                         .map(LoadBalancerInstance::networks)
-                         .forEach(trustedNetworks::addAll);
+            loadBalancers(allocation.owner()).asList().stream()
+                                             .map(LoadBalancer::instance)
+                                             .map(LoadBalancerInstance::networks)
+                                             .forEach(trustedNetworks::addAll);
         });
 
         switch (node.type()) {
@@ -293,13 +304,12 @@ public class NodeRepository extends AbstractComponent {
      */
     public List<NodeAcl> getNodeAcls(Node node, boolean children) {
         NodeList candidates = list();
-        LoadBalancerList loadBalancers = loadBalancers();
         if (children) {
             return candidates.childrenOf(node).asList().stream()
-                             .map(childNode -> getNodeAcl(childNode, candidates, loadBalancers))
+                             .map(childNode -> getNodeAcl(childNode, candidates))
                              .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
         }
-        return Collections.singletonList(getNodeAcl(node, candidates, loadBalancers));
+        return Collections.singletonList(getNodeAcl(node, candidates));
     }
 
     public NodeFlavors getAvailableFlavors() {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java
index 014d3df8d9a..bad16bf7d12 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java
@@ -1,9 +1,6 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.lb;
 
-import com.yahoo.config.provision.ApplicationId;
-
-import java.time.Instant;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -24,21 +21,11 @@ public class LoadBalancerList implements Iterable<LoadBalancer> {
         this.loadBalancers = List.copyOf(Objects.requireNonNull(loadBalancers, "loadBalancers must be non-null"));
     }
 
-    /** Returns the subset of load balancers owned by given application */
-    public LoadBalancerList owner(ApplicationId application) {
-        return of(loadBalancers.stream().filter(lb -> lb.id().application().equals(application)));
-    }
-
     /** Returns the subset of load balancers that are in given state */
     public LoadBalancerList in(LoadBalancer.State state) {
         return of(loadBalancers.stream().filter(lb -> lb.state() == state));
     }
 
-    /** Returns the subset of load balancers that last changed before given instant */
-    public LoadBalancerList changedBefore(Instant instant) {
-        return of(loadBalancers.stream().filter(lb -> lb.changedAt().isBefore(instant)));
-    }
-
     public List<LoadBalancer> asList() {
         return loadBalancers;
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
index bcb0c901f14..0f363991310 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
@@ -15,6 +15,8 @@ import java.util.ArrayList;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 /**
@@ -51,37 +53,31 @@ public class LoadBalancerExpirer extends Maintainer {
 
     /** Move reserved load balancer that have expired to inactive */
     private void expireReserved() {
-        try (var lock = db.lockLoadBalancers()) {
-            var now = nodeRepository().clock().instant();
-            var expirationTime = now.minus(reservedExpiry);
-            var expired = nodeRepository().loadBalancers()
-                                          .in(State.reserved)
-                                          .changedBefore(expirationTime);
-            expired.forEach(lb -> db.writeLoadBalancer(lb.with(State.inactive, now)));
-        }
+        var now = nodeRepository().clock().instant();
+        withLoadBalancersIn(State.reserved, lb -> {
+            var gracePeriod = now.minus(reservedExpiry);
+            if (!lb.changedAt().isBefore(gracePeriod)) return; // Should not move to inactive yet
+            db.writeLoadBalancer(lb.with(State.inactive, now));
+        });
     }
 
     /** Deprovision inactive load balancers that have expired */
     private void removeInactive() {
         var failed = new ArrayList<LoadBalancerId>();
-        Exception lastException = null;
-        try (var lock = db.lockLoadBalancers()) {
-            var now = nodeRepository().clock().instant();
-            var expirationTime = now.minus(inactiveExpiry);
-            var expired = nodeRepository().loadBalancers()
-                                          .in(State.inactive)
-                                          .changedBefore(expirationTime);
-            for (var lb : expired) {
-                if (!allocatedNodes(lb.id()).isEmpty()) continue; // Defer removal if there are still nodes allocated to application
-                try {
-                    service.remove(lb.id().application(), lb.id().cluster());
-                    db.removeLoadBalancer(lb.id());
-                } catch (Exception e) {
-                    failed.add(lb.id());
-                    lastException = e;
-                }
+        var lastException = new AtomicReference<Exception>();
+        var now = nodeRepository().clock().instant();
+        withLoadBalancersIn(State.inactive, lb -> {
+            var gracePeriod = now.minus(inactiveExpiry);
+            if (!lb.changedAt().isBefore(gracePeriod)) return; // Should not be removed yet
+            if (!allocatedNodes(lb.id()).isEmpty()) return;    // Still has nodes, do not remove
+            try {
+                service.remove(lb.id().application(), lb.id().cluster());
+                db.removeLoadBalancer(lb.id());
+            } catch (Exception e){
+                failed.add(lb.id());
+                lastException.set(e);
             }
-        }
+        });
         if (!failed.isEmpty()) {
             log.log(LogLevel.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s",
                                                     failed.size(),
@@ -89,30 +85,27 @@ public class LoadBalancerExpirer extends Maintainer {
                                                           .map(LoadBalancerId::serializedForm)
                                                           .collect(Collectors.joining(", ")),
                                                     interval()),
-                    lastException);
+                    lastException.get());
         }
     }
 
     /** Remove reals from inactive load balancers */
     private void pruneReals() {
         var failed = new ArrayList<LoadBalancerId>();
-        Exception lastException = null;
-        try (var lock = db.lockLoadBalancers()) {
-            var deactivated = nodeRepository().loadBalancers().in(State.inactive);
-            for (var lb : deactivated) {
-                var allocatedNodes = allocatedNodes(lb.id()).stream().map(Node::hostname).collect(Collectors.toSet());
-                var reals = new LinkedHashSet<>(lb.instance().reals());
-                // Remove any real no longer allocated to this application
-                reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value()));
-                try {
-                    service.create(lb.id().application(), lb.id().cluster(), reals, true);
-                    db.writeLoadBalancer(lb.with(lb.instance().withReals(reals)));
-                } catch (Exception e) {
-                    failed.add(lb.id());
-                    lastException = e;
-                }
+        var lastException = new AtomicReference<Exception>();
+        withLoadBalancersIn(State.inactive, lb -> {
+            var allocatedNodes = allocatedNodes(lb.id()).stream().map(Node::hostname).collect(Collectors.toSet());
+            var reals = new LinkedHashSet<>(lb.instance().reals());
+            // Remove any real no longer allocated to this application
+            reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value()));
+            try {
+                service.create(lb.id().application(), lb.id().cluster(), reals, true);
+                db.writeLoadBalancer(lb.with(lb.instance().withReals(reals)));
+            } catch (Exception e) {
+                failed.add(lb.id());
+                lastException.set(e);
             }
-        }
+        });
         if (!failed.isEmpty()) {
             log.log(LogLevel.WARNING, String.format("Failed to remove reals from %d load balancers: %s, retrying in %s",
                                                     failed.size(),
@@ -120,7 +113,21 @@ public class LoadBalancerExpirer extends Maintainer {
                                                           .map(LoadBalancerId::serializedForm)
                                                           .collect(Collectors.joining(", ")),
                                                     interval()),
-                    lastException);
+                    lastException.get());
+        }
+    }
+
+    /** Apply operation to all load balancers that exist in given state, while holding lock */
+    private void withLoadBalancersIn(LoadBalancer.State state, Consumer<LoadBalancer> operation) {
+        try (var legacyLock = db.lockLoadBalancers()) {
+            for (var id : db.readLoadBalancerIds()) {
+                try (var lock = db.lockLoadBalancers(id.application())) {
+                    var loadBalancer = db.readLoadBalancer(id);
+                    if (loadBalancer.isEmpty()) continue;              // Load balancer was removed during loop
+                    if (loadBalancer.get().state() != state) continue; // Wrong state
+                    operation.accept(loadBalancer.get());
+                }
+            }
         }
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index ce0bcc2e337..0a8575578ce 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.persistence;
 
 import com.google.common.util.concurrent.UncheckedTimeoutException;
@@ -38,6 +38,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -483,18 +484,16 @@ public class CuratorDatabaseClient {
 
     // Load balancers
     public List<LoadBalancerId> readLoadBalancerIds() {
-        return curatorDatabase.getChildren(loadBalancersRoot).stream()
-                              .map(LoadBalancerId::fromSerializedForm)
-                              .collect(Collectors.toUnmodifiableList());
+        return readLoadBalancerIds((ignored) -> true);
     }
 
-    public Map<LoadBalancerId, LoadBalancer> readLoadBalancers() {
-        return readLoadBalancerIds().stream()
-                                    .map(this::readLoadBalancer)
-                                    .filter(Optional::isPresent)
-                                    .map(Optional::get)
-                                    .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()),
-                                                           Collections::unmodifiableMap));
+    public Map<LoadBalancerId, LoadBalancer> readLoadBalancers(Predicate<LoadBalancerId> filter) {
+        return readLoadBalancerIds(filter).stream()
+                                          .map(this::readLoadBalancer)
+                                          .filter(Optional::isPresent)
+                                          .map(Optional::get)
+                                          .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()),
+                                                                     Collections::unmodifiableMap));
     }
 
     public Optional<LoadBalancer> readLoadBalancer(LoadBalancerId id) {
@@ -522,14 +521,26 @@ public class CuratorDatabaseClient {
         transaction.commit();
     }
 
+    // TODO(mpolden): Remove this and all usages once migration to per-application lock is complete
     public Lock lockLoadBalancers() {
         return lock(lockRoot.append("loadBalancersLock"), defaultLockTimeout);
     }
 
+    public Lock lockLoadBalancers(ApplicationId application) {
+        return lock(lockRoot.append("loadBalancersLock2").append(application.serializedForm()), defaultLockTimeout);
+    }
+
     private Path loadBalancerPath(LoadBalancerId id) {
         return loadBalancersRoot.append(id.serializedForm());
     }
 
+    private List<LoadBalancerId> readLoadBalancerIds(Predicate<LoadBalancerId> predicate) {
+        return curatorDatabase.getChildren(loadBalancersRoot).stream()
+                              .map(LoadBalancerId::fromSerializedForm)
+                              .filter(predicate)
+                              .collect(Collectors.toUnmodifiableList());
+    }
+
     private Transaction.Operation createOrSet(Path path, byte[] data) {
         if (curatorDatabase.exists(path)) {
             return CuratorOperations.setData(path.getAbsolute(), data);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
index 1500154aa07..d4d5b46dfdf 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
 import com.yahoo.config.provision.ApplicationId;
@@ -51,10 +51,12 @@ public class LoadBalancerProvisioner {
         this.db = nodeRepository.database();
         this.service = service;
         // Read and write all load balancers to make sure they are stored in the latest version of the serialization format
-        try (var lock = db.lockLoadBalancers()) {
+        try (var legacyLock = db.lockLoadBalancers()) {
             for (var id : db.readLoadBalancerIds()) {
-                var loadBalancer = db.readLoadBalancer(id);
-                loadBalancer.ifPresent(db::writeLoadBalancer);
+                try (var lock = db.lockLoadBalancers(id.application())) {
+                    var loadBalancer = db.readLoadBalancer(id);
+                    loadBalancer.ifPresent(db::writeLoadBalancer);
+                }
             }
         }
     }
@@ -73,8 +75,10 @@ public class LoadBalancerProvisioner {
         if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type
         if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type
         if (application.instance().isTester()) return; // Do not provision for tester instances
-        try (var loadBalancersLock = db.lockLoadBalancers()) {
-            provision(application, cluster.id(), false, loadBalancersLock);
+        try (var legacyLock = db.lockLoadBalancers()) {
+            try (var lock = db.lockLoadBalancers(application)) {
+                provision(application, cluster.id(), false, lock);
+            }
         }
     }
 
@@ -90,15 +94,17 @@ public class LoadBalancerProvisioner {
      */
     public void activate(ApplicationId application, Set<ClusterSpec> clusters,
                          @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) {
-        try (var loadBalancersLock = db.lockLoadBalancers()) {
-            var containerClusters = containerClusterOf(clusters);
-            for (var clusterId : containerClusters) {
-                // Provision again to ensure that load balancer instance is re-configured with correct nodes
-                provision(application, clusterId, true, loadBalancersLock);
+        try (var legacyLock = db.lockLoadBalancers()) {
+            try (var lock = db.lockLoadBalancers(application)) {
+                var containerClusters = containerClusterOf(clusters);
+                for (var clusterId : containerClusters) {
+                    // Provision again to ensure that load balancer instance is re-configured with correct nodes
+                    provision(application, clusterId, true, lock);
+                }
+                // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed
+                var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters);
+                deactivate(surplusLoadBalancers, transaction);
             }
-            // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed
-            var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters);
-            deactivate(surplusLoadBalancers, transaction);
         }
     }
 
@@ -108,16 +114,17 @@ public class LoadBalancerProvisioner {
      */
     public void deactivate(ApplicationId application, NestedTransaction transaction) {
         try (var applicationLock = nodeRepository.lock(application)) {
-            try (Mutex loadBalancersLock = db.lockLoadBalancers()) {
-                deactivate(nodeRepository.loadBalancers().owner(application).asList(), transaction);
+            try (var legacyLock = db.lockLoadBalancers()) {
+                try (var lock = db.lockLoadBalancers(application)) {
+                    deactivate(nodeRepository.loadBalancers(application).asList(), transaction);
+                }
             }
         }
     }
 
     /** Returns load balancers of given application that are no longer referenced by given clusters */
     private List<LoadBalancer> surplusLoadBalancersOf(ApplicationId application, Set<ClusterSpec.Id> activeClusters) {
-        var activeLoadBalancersByCluster = nodeRepository.loadBalancers()
-                                                         .owner(application)
+        var activeLoadBalancersByCluster = nodeRepository.loadBalancers(application)
                                                          .in(LoadBalancer.State.active)
                                                          .asList()
                                                          .stream()
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java
index 9f8f4a804d1..3147d4caded 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.restapi.v2;
 
 import com.yahoo.config.provision.ApplicationId;
@@ -37,10 +37,14 @@ public class LoadBalancersResponse extends HttpResponse {
     }
 
     private List<LoadBalancer> loadBalancers() {
-        LoadBalancerList loadBalancers = nodeRepository.loadBalancers();
-        return application().map(loadBalancers::owner)
-                            .map(LoadBalancerList::asList)
-                            .orElseGet(loadBalancers::asList);
+        LoadBalancerList loadBalancers;
+        var application = application();
+        if (application.isPresent()) {
+            loadBalancers = nodeRepository.loadBalancers(application.get());
+        } else {
+            loadBalancers = nodeRepository.loadBalancers();
+        }
+        return loadBalancers.asList();
     }
 
     @Override
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
index 12b48fd7a35..4b7534b431e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.component.Vtag;
@@ -32,14 +32,14 @@ import static org.junit.Assert.assertTrue;
  */
 public class LoadBalancerExpirerTest {
 
-    private ProvisioningTester tester = new ProvisioningTester.Builder().build();
+    private final ProvisioningTester tester = new ProvisioningTester.Builder().build();
 
     @Test
     public void expire_inactive() {
         LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(),
                                                               Duration.ofDays(1),
                                                               tester.loadBalancerService());
-        Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers();
+        Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true);
 
         // Deploy two applications with a total of three load balancers
         ClusterSpec.Id cluster1 = ClusterSpec.Id.from("qrs");
@@ -67,10 +67,11 @@ public class LoadBalancerExpirerTest {
         // Expirer prunes reals before expiration time of load balancer itself
         expirer.maintain();
         assertEquals(Set.of(), tester.loadBalancerService().instances().get(lb1).reals());
-        assertEquals(Set.of(), tester.nodeRepository().loadBalancers().owner(lb1.application()).asList().get(0).instance().reals());
+        assertEquals(Set.of(), loadBalancers.get().get(lb1).instance().reals());
 
         // Expirer defers removal of load balancer until expiration time passes
         expirer.maintain();
+        assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb1).state());
         assertTrue("Inactive load balancer not removed", tester.loadBalancerService().instances().containsKey(lb1));
 
         // Expirer removes load balancers once expiration time passes
@@ -85,7 +86,7 @@ public class LoadBalancerExpirerTest {
         // A single cluster is removed
         deployApplication(app2, cluster1);
         expirer.maintain();
-        assertEquals(LoadBalancer.State.inactive, loadBalancers.get().get(lb3).state());
+        assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb3).state());
 
         // Expirer defers removal while nodes are still allocated to cluster
         expirer.maintain();
@@ -103,7 +104,7 @@ public class LoadBalancerExpirerTest {
         LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(),
                                                               Duration.ofDays(1),
                                                               tester.loadBalancerService());
-        Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers();
+        Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true);
 
 
         // Prepare application
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
index 92d066e5f16..8995897769c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
@@ -164,12 +164,33 @@ public class AclProvisioningTest {
 
     @Test
     public void trusted_nodes_for_application_with_load_balancer() {
-        // Populate repo
-        tester.makeReadyNodes(10, nodeResources);
+        // Provision hosts and containers
+        var hosts = tester.makeReadyNodes(2, "default", NodeType.host);
+        tester.deployZoneApp();
+        for (var host : hosts) {
+            tester.makeReadyVirtualDockerNodes(2, new NodeResources(2, 8, 50, 1),
+                                               host.hostname());
+        }
 
-        // Allocate 2 nodes
-        List<Node> activeNodes = deploy(2);
+        // Deploy application
+        var application = tester.makeApplicationId();
+        List<Node> activeNodes = deploy(application, 2);
         assertEquals(2, activeNodes.size());
+
+        // Load balancer is allocated to application
+        var loadBalancers = tester.nodeRepository().loadBalancers(application);
+        assertEquals(1, loadBalancers.asList().size());
+        var lbNetworks = loadBalancers.asList().get(0).instance().networks();
+        assertEquals(2, lbNetworks.size());
+
+        // ACL for nodes with allocation trust their respective load balancer networks, if any
+        for (var host : hosts) {
+            var acls = tester.nodeRepository().getNodeAcls(host, true);
+            assertEquals(2, acls.size());
+            assertEquals(Set.of(), acls.get(0).trustedNetworks());
+            assertEquals(application, acls.get(1).node().allocation().get().owner());
+            assertEquals(lbNetworks, acls.get(1).trustedNetworks());
+        }
     }
 
     @Test
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
index a26e802dfe8..ee9a582c4db 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
 import com.google.common.collect.Iterators;
@@ -40,15 +40,14 @@ public class LoadBalancerProvisionerTest {
 
     private final ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default");
     private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default");
-
     private final ApplicationId infraApp1 = ApplicationId.from("vespa", "tenant-host", "default");
 
-    private ProvisioningTester tester = new ProvisioningTester.Builder().build();
+    private final ProvisioningTester tester = new ProvisioningTester.Builder().build();
 
     @Test
     public void provision_load_balancer() {
-        Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers().owner(app1).asList();
-        Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers().owner(app2).asList();
+        Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers(app1).asList();
+        Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers(app2).asList();
         ClusterSpec.Id containerCluster1 = ClusterSpec.Id.from("qrs1");
         ClusterSpec.Id contentCluster = ClusterSpec.Id.from("content");
 
@@ -80,7 +79,7 @@ public class LoadBalancerProvisionerTest {
         tester.activate(app1, prepare(app1,
                                       clusterRequest(ClusterSpec.Type.container, containerCluster1),
                                       clusterRequest(ClusterSpec.Type.content, contentCluster)));
-        LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers().owner(app1).asList().get(0);
+        LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers(app1).asList().get(0);
         assertEquals(2, loadBalancer.instance().reals().size());
         assertTrue("Failed node is removed", loadBalancer.instance().reals().stream()
                                                          .map(Real::hostname)
@@ -158,7 +157,7 @@ public class LoadBalancerProvisionerTest {
         var nodes = prepare(app1, Capacity.fromCount(2, new NodeResources(1, 4, 10, 0.3), false, true),
                                            true,
                                            clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs")));
-        Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers().owner(app1).asList().get(0);
+        Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers(app1).asList().get(0);
         assertTrue("Load balancer provisioned with empty reals", tester.loadBalancerService().instances().get(lb.get().id()).reals().isEmpty());
         assignIps(tester.nodeRepository().getNodes(app1));
         tester.activate(app1, nodes);
@@ -189,7 +188,7 @@ public class LoadBalancerProvisionerTest {
                                            clusterRequest(ClusterSpec.Type.container,
                                                           ClusterSpec.Id.from("tenant-host"))));
         assertTrue("No load balancer provisioned", tester.loadBalancerService().instances().isEmpty());
-        assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(infraApp1).asList());
+        assertEquals(List.of(), tester.nodeRepository().loadBalancers(infraApp1).asList());
     }
 
     @Test
@@ -197,12 +196,12 @@ public class LoadBalancerProvisionerTest {
         tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.content,
                                                            ClusterSpec.Id.from("tenant-host"))));
         assertTrue("No load balancer provisioned", tester.loadBalancerService().instances().isEmpty());
-        assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(app1).asList());
+        assertEquals(List.of(), tester.nodeRepository().loadBalancers(app1).asList());
     }
 
     @Test
     public void provision_load_balancer_combined_cluster() {
-        Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().owner(app1).asList();
+        Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(app1).asList();
         ClusterSpec.Id cluster = ClusterSpec.Id.from("foo");
 
         var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, cluster));