Merge branch 'master' into jvenstad/remove-feature-flag-for-cache-invalidation-strategy

28 files changed, 588 insertions, 139 deletions

diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml
index 9276ce0e7c9..22ab615bfad 100644
--- a/node-repository/src/main/config/node-repository.xml
+++ b/node-repository/src/main/config/node-repository.xml
@@ -11,5 +11,10 @@
     <binding>https://*/nodes/v2/*</binding>
 </handler>
 
+<handler id="com.yahoo.vespa.hosted.provision.restapi.v2.LoadBalancersApiHandler" bundle="node-repository">
+    <binding>http://*/loadbalancers/v1/*</binding>
+    <binding>https://*/loadbalancers/v1/*</binding>
+</handler>
+
 <preprocess:include file="node-flavors.xml" required="false" />
 <preprocess:include file="node-repository-config.xml" required="false" />
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
index 7e518ee1728..442013b8a6a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
@@ -1,7 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision;
 
-import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.net.InetAddresses;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterMembership;
@@ -77,7 +77,7 @@ public final class Node {
         Objects.requireNonNull(history, "A null node history is not permitted");
         Objects.requireNonNull(type, "A null node type is not permitted");
 
-        this.ipAddresses = ImmutableSortedSet.copyOf(IP.naturalOrder, ipAddresses);
+        this.ipAddresses = ImmutableSet.copyOf(ipAddresses);
         this.ipAddressPool = new IP.AddressPool(this, ipAddressPool);
         this.hostname = hostname;
         this.parentHostname = parentHostname;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java
index effc5b1a41d..4ac3a839ae1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.lb;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Ordering;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.hosted.provision.maintenance.LoadBalancerExpirer;
 
 import java.util.List;
 import java.util.Objects;
@@ -50,8 +51,8 @@ public class LoadBalancer {
     }
 
     /**
-     * Returns whether this load balancer is inactive. Inactive load balancers cannot be reactivated, and are
-     * eventually deleted
+     * Returns whether this load balancer is inactive. Inactive load balancers cannot be re-activated, and are
+     * eventually removed by {@link LoadBalancerExpirer}.
      */
     public boolean inactive() {
         return inactive;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java
index b589e5aed2f..a5a0d8cb2f8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java
@@ -17,6 +17,10 @@ public class LoadBalancerServiceMock implements LoadBalancerService {
 
     private final Map<LoadBalancerId, LoadBalancer> loadBalancers = new HashMap<>();
 
+    public Map<LoadBalancerId, LoadBalancer> loadBalancers() {
+        return Collections.unmodifiableMap(loadBalancers);
+    }
+
     @Override
     public Protocol protocol() {
         return Protocol.ipv4;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
index 9d87a835960..68d597fb839 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
@@ -108,9 +108,12 @@ public class InfrastructureProvisioner extends Maintainer {
     }
 
     private void removeApplication(ApplicationId applicationId) {
-        NestedTransaction nestedTransaction = new NestedTransaction();
-        provisioner.remove(nestedTransaction, applicationId);
-        nestedTransaction.commit();
-        duperModel.infraApplicationRemoved(applicationId);
+        // Use the DuperModel as source-of-truth on whether it has also been activated (to avoid periodic removals)
+        if (duperModel.infraApplicationIsActive(applicationId)) {
+            NestedTransaction nestedTransaction = new NestedTransaction();
+            provisioner.remove(nestedTransaction, applicationId);
+            nestedTransaction.commit();
+            duperModel.infraApplicationRemoved(applicationId);
+        }
     }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
new file mode 100644
index 00000000000..4b66dff3032
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
@@ -0,0 +1,83 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * Periodically remove inactive load balancers permanently.
+ *
+ * When an application is removed, any associated load balancers are only deactivated. This maintainer ensures that
+ * such resources are eventually freed.
+ *
+ * @author mpolden
+ */
+public class LoadBalancerExpirer extends Maintainer {
+
+    private final LoadBalancerService service;
+    private final CuratorDatabaseClient db;
+
+    public LoadBalancerExpirer(NodeRepository nodeRepository, Duration interval, JobControl jobControl,
+                               LoadBalancerService service) {
+        super(nodeRepository, interval, jobControl);
+        this.service = Objects.requireNonNull(service, "service must be non-null");
+        this.db = nodeRepository.database();
+    }
+
+    @Override
+    protected void maintain() {
+        removeInactive();
+    }
+
+    private void removeInactive() {
+        List<LoadBalancerId> failed = new ArrayList<>();
+        Exception lastException = null;
+        try (Lock lock = db.lockLoadBalancers()) {
+            for (LoadBalancerId loadBalancer : inactiveLoadBlancers()) {
+                if (hasNodes(loadBalancer.application())) { // Defer removal if there are still nodes allocated to application
+                    continue;
+                }
+                try {
+                    service.remove(loadBalancer);
+                    db.removeLoadBalancer(loadBalancer);
+                } catch (Exception e) {
+                    failed.add(loadBalancer);
+                    lastException = e;
+                }
+            }
+        }
+        if (!failed.isEmpty()) {
+            log.log(LogLevel.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s",
+                                                    failed.size(),
+                                                    failed.stream()
+                                                          .map(LoadBalancerId::serializedForm)
+                                                          .collect(Collectors.joining(", ")),
+                                                    interval()),
+                    lastException);
+        }
+    }
+
+    private boolean hasNodes(ApplicationId application) {
+        return !nodeRepository().getNodes(application).isEmpty();
+    }
+
+    private List<LoadBalancerId> inactiveLoadBlancers() {
+        return db.readLoadBalancers().entrySet().stream()
+                 .filter(entry -> entry.getValue().inactive())
+                 .map(Map.Entry::getKey)
+                 .collect(Collectors.toList());
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
index f5576ae00fc..49ede9962eb 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
@@ -56,8 +56,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
         try {
             if (jobControl.isActive(name()))
                 maintain();
-        }
-        catch (RuntimeException e) {
+        } catch (Throwable e) {
             log.log(Level.WARNING, this + " failed. Will retry in " + interval.toMinutes() + " minutes", e);
         }
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 946c43ca8fc..2bc60de3c8d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -12,6 +12,7 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService;
 import com.yahoo.vespa.hosted.provision.maintenance.retire.RetireIPv4OnlyNodes;
 import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy;
 import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicyList;
@@ -50,6 +51,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
     private final NodeRetirer nodeRetirer;
     private final MetricsReporter metricsReporter;
     private final InfrastructureProvisioner infrastructureProvisioner;
+    private final LoadBalancerExpirer loadBalancerExpirer;
 
     private final JobControl jobControl;
     private final InfrastructureVersions infrastructureVersions;
@@ -59,15 +61,17 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Orchestrator orchestrator, Metric metric,
                                      ConfigserverConfig configserverConfig,
-                                     DuperModelInfraApi duperModelInfraApi) {
+                                     DuperModelInfraApi duperModelInfraApi,
+                                     LoadBalancerService loadBalancerService) {
         this(nodeRepository, deployer, provisioner, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(),
-                orchestrator, metric, configserverConfig, duperModelInfraApi);
+                orchestrator, metric, configserverConfig, duperModelInfraApi, loadBalancerService);
     }
 
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Clock clock, Orchestrator orchestrator, Metric metric,
-                                     ConfigserverConfig configserverConfig, DuperModelInfraApi duperModelInfraApi) {
+                                     ConfigserverConfig configserverConfig, DuperModelInfraApi duperModelInfraApi,
+                                     LoadBalancerService loadBalancerService) {
         DefaultTimes defaults = new DefaultTimes(zone);
         jobControl = new JobControl(nodeRepository.database());
         infrastructureVersions = new InfrastructureVersions(nodeRepository.database());
@@ -84,6 +88,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         nodeRebooter = new NodeRebooter(nodeRepository, clock, durationFromEnv("reboot_interval").orElse(defaults.rebootInterval), jobControl);
         metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, durationFromEnv("metrics_interval").orElse(defaults.metricsInterval), jobControl);
         infrastructureProvisioner = new InfrastructureProvisioner(provisioner, nodeRepository, infrastructureVersions, durationFromEnv("infrastructure_provision_interval").orElse(defaults.infrastructureProvisionInterval), jobControl, duperModelInfraApi);
+        loadBalancerExpirer = new LoadBalancerExpirer(nodeRepository, durationFromEnv("load_balancer_expiry").orElse(defaults.loadBalancerExpiry), jobControl, loadBalancerService);
 
         // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now
         infrastructureProvisioner.maintain();
@@ -109,6 +114,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         provisionedExpirer.deconstruct();
         metricsReporter.deconstruct();
         infrastructureProvisioner.deconstruct();
+        loadBalancerExpirer.deconstruct();
     }
 
     public JobControl jobControl() { return jobControl; }
@@ -156,6 +162,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         private final Duration metricsInterval;
         private final Duration retiredInterval;
         private final Duration infrastructureProvisionInterval;
+        private final Duration loadBalancerExpiry;
 
         private final NodeFailer.ThrottlePolicy throttlePolicy;
 
@@ -171,6 +178,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
             metricsInterval = Duration.ofMinutes(1);
             infrastructureProvisionInterval = Duration.ofMinutes(3);
             throttlePolicy = NodeFailer.ThrottlePolicy.hosted;
+            loadBalancerExpiry = Duration.ofHours(1);
 
             if (zone.environment().equals(Environment.prod) && zone.system() != SystemName.cd) {
                 inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java
index 074a20fc82d..fbc358893e1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java
@@ -1,7 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.node;
 
-import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.net.InetAddresses;
 import com.google.common.primitives.UnsignedBytes;
 import com.yahoo.vespa.hosted.provision.Node;
@@ -59,7 +59,7 @@ public class IP {
 
         public AddressPool(Node owner, Set<String> addresses) {
             this.owner = Objects.requireNonNull(owner, "owner must be non-null");
-            this.addresses = ImmutableSortedSet.copyOf(naturalOrder, requireAddresses(addresses));
+            this.addresses = ImmutableSet.copyOf(requireAddresses(addresses));
         }
 
         /**
@@ -200,9 +200,9 @@ public class IP {
 
         /** All IP addresses in this */
         public Set<String> addresses() {
-            ImmutableSortedSet.Builder<String> builder = ImmutableSortedSet.orderedBy(naturalOrder);
-            builder.add(ipv6Address);
+            ImmutableSet.Builder<String> builder = ImmutableSet.builder();
             ipv4Address.ifPresent(builder::add);
+            builder.add(ipv6Address);
             return builder.build();
         }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
index 2ceffc54dd5..2715b1131b3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
@@ -36,7 +36,7 @@ public class CuratorDatabase {
     private final CuratorCounter changeGenerationCounter;
 
     /** A partial cache of the Curator database, which is only valid if generations match */
-    private final AtomicReference<CuratorDatabaseCache> cache = new AtomicReference<>();
+    private final AtomicReference<Cache> cache = new AtomicReference<>();
 
     /** Whether we should return data from the cache or always read fro ZooKeeper */
     private final boolean useCache;
@@ -110,12 +110,12 @@ public class CuratorDatabase {
     // the data to read is protected by a lock which is held now, and during any writes of the data.
 
     /** Returns the immediate, local names of the children under this node in any order */
-    List<String> getChildren(Path path) { return getCache().getChildren(path); }
+    List<String> getChildren(Path path) { return getSession().getChildren(path); }
 
-    Optional<byte[]> getData(Path path) { return getCache().getData(path); }
+    Optional<byte[]> getData(Path path) { return getSession().getData(path); }
 
     /** Invalidates the current cache if outdated. */
-    private CuratorDatabaseCache getCache() {
+    Session getSession() {
         if (changeGenerationCounter.get() != cache.get().generation)
             synchronized (cacheCreationLock) {
                 while (changeGenerationCounter.get() != cache.get().generation)
@@ -126,8 +126,8 @@ public class CuratorDatabase {
     }
 
     /** Caches must only be instantiated using this method */
-    private CuratorDatabaseCache newCache(long generation) {
-        return useCache ? new CuratorDatabaseCache(generation, curator) : new DeactivatedCache(generation, curator);
+    private Cache newCache(long generation) {
+        return useCache ? new Cache(generation, curator) : new NoCache(generation, curator);
     }
 
     /**
@@ -135,10 +135,10 @@ public class CuratorDatabase {
      * This is merely a recording of what Curator returned at various points in time when 
      * it had the counter at this generation.
      */
-    private static class CuratorDatabaseCache {
+    private static class Cache implements Session {
 
         private final long generation;
-        
+
         /** The curator instance used to fetch missing data */
         protected final Curator curator;
 
@@ -149,23 +149,17 @@ public class CuratorDatabase {
         private final Map<Path, Optional<byte[]>> data = new ConcurrentHashMap<>();
 
         /** Create an empty snapshot at a given generation (as an empty snapshot is a valid partial snapshot) */
-        private CuratorDatabaseCache(long generation, Curator curator) {
+        private Cache(long generation, Curator curator) {
             this.generation = generation;
             this.curator = curator;
         }
 
-        public long generation() { return generation; }
-
-        /**
-         * Returns the children of this path, which may be empty.
-         */
+        @Override
         public List<String> getChildren(Path path) { 
             return children.computeIfAbsent(path, key -> ImmutableList.copyOf(curator.getChildren(path)));
         }
 
-        /**
-         * Returns the a copy of the content of this child - which may be empty.
-         */
+        @Override
         public Optional<byte[]> getData(Path path) {
             return data.computeIfAbsent(path, key -> curator.getData(path)).map(data -> Arrays.copyOf(data, data.length));
         }
@@ -173,9 +167,9 @@ public class CuratorDatabase {
     }
 
     /** An implementation of the curator database cache which does no caching */
-    private static class DeactivatedCache extends CuratorDatabaseCache {
-        
-        private DeactivatedCache(long generation, Curator curator) { super(generation, curator); }
+    private static class NoCache extends Cache {
+
+        private NoCache(long generation, Curator curator) { super(generation, curator); }
 
         @Override
         public List<String> getChildren(Path path) { return curator.getChildren(path); }
@@ -185,4 +179,17 @@ public class CuratorDatabase {
 
     }
 
+    interface Session {
+
+        /**
+         * Returns the children of this path, which may be empty.
+         */
+        List<String> getChildren(Path path);
+
+        /**
+         * Returns the a copy of the content of this child - which may be empty.
+         */
+        Optional<byte[]> getData(Path path);
+
+    }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index c4031f3ccba..da4d2a0afb2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -251,9 +251,10 @@ public class CuratorDatabaseClient {
         List<Node> nodes = new ArrayList<>();
         if (states.length == 0)
             states = Node.State.values();
+        CuratorDatabase.Session session = curatorDatabase.getSession();
         for (Node.State state : states) {
-            for (String hostname : curatorDatabase.getChildren(toPath(state))) {
-                Optional<Node> node = getNode(hostname, state);
+            for (String hostname : session.getChildren(toPath(state))) {
+                Optional<Node> node = getNode(session, hostname, state);
                 node.ifPresent(nodes::add); // node might disappear between getChildren and getNode
             }
         }
@@ -270,21 +271,29 @@ public class CuratorDatabaseClient {
         return nodes;
     }
 
-    /** 
+    /**
      * Returns a particular node, or empty if this noe is not in any of the given states.
      * If no states are given this returns the node if it is present in any state.
      */
-    public Optional<Node> getNode(String hostname, Node.State ... states) {
+    public Optional<Node> getNode(CuratorDatabase.Session session, String hostname, Node.State ... states) {
         if (states.length == 0)
             states = Node.State.values();
         for (Node.State state : states) {
-            Optional<byte[]> nodeData = curatorDatabase.getData(toPath(state, hostname));
+            Optional<byte[]> nodeData = session.getData(toPath(state, hostname));
             if (nodeData.isPresent())
                 return nodeData.map((data) -> nodeSerializer.fromJson(state, data));
         }
         return Optional.empty();
     }
 
+    /** 
+     * Returns a particular node, or empty if this noe is not in any of the given states.
+     * If no states are given this returns the node if it is present in any state.
+     */
+    public Optional<Node> getNode(String hostname, Node.State ... states) {
+        return getNode(curatorDatabase.getSession(), hostname, states);
+    }
+
     private Path toPath(Node.State nodeState) { return root.append(toDir(nodeState)); }
 
     private Path toPath(Node node) {
@@ -449,10 +458,10 @@ public class CuratorDatabaseClient {
         });
     }
 
-    public void removeLoadBalancer(LoadBalancer loadBalancer) {
+    public void removeLoadBalancer(LoadBalancerId loadBalancer) {
         NestedTransaction transaction = new NestedTransaction();
         CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction);
-        curatorTransaction.add(CuratorOperations.delete(loadBalancerPath(loadBalancer.id()).getAbsolute()));
+        curatorTransaction.add(CuratorOperations.delete(loadBalancerPath(loadBalancer).getAbsolute()));
         transaction.commit();
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index 1c640a6f074..f96ecc0431a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.Generation;
 import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.node.Status;
 
 import java.io.IOException;
@@ -141,7 +142,7 @@ public class NodeSerializer {
     }
 
     private void toSlime(Set<String> ipAddresses, Cursor array) {
-        ipAddresses.forEach(array::addString);
+        ipAddresses.stream().sorted(IP.naturalOrder).forEach(array::addString);
     }
 
     // ---------------- Deserialization --------------------------------------------------
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java
index 483f19ed5b0..f3d8f42f3b7 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java
@@ -9,7 +9,7 @@ import com.yahoo.vespa.hosted.provision.maintenance.JobControl;
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.net.URI;
+import java.util.TreeSet;
 
 /** A response containing maintenance job status */
 public class JobsResponse extends HttpResponse {
@@ -25,13 +25,12 @@ public class JobsResponse extends HttpResponse {
     public void render(OutputStream stream) throws IOException {
         Slime slime = new Slime();
         Cursor root = slime.setObject();
-
         Cursor jobArray = root.setArray("jobs");
-        for (String jobName : jobControl.jobs())
+        for (String jobName : new TreeSet<>(jobControl.jobs()))
             jobArray.addObject().setString("name", jobName);
 
         Cursor inactiveArray = root.setArray("inactive");
-        for (String jobName : jobControl.inactiveJobs())
+        for (String jobName : new TreeSet<>(jobControl.inactiveJobs()))
             inactiveArray.addString(jobName);
 
         new JsonFormat(true).encode(stream, slime);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java
new file mode 100644
index 00000000000..6ffac2c0fbc
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java
@@ -0,0 +1,52 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v2;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.vespa.hosted.provision.NoSuchNodeException;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.yolean.Exceptions;
+
+import javax.inject.Inject;
+import java.util.logging.Level;
+
+/**
+ * @author mpolden
+ */
+public class LoadBalancersApiHandler extends LoggingRequestHandler {
+
+    private final NodeRepository nodeRepository;
+
+    @Inject
+    public LoadBalancersApiHandler(LoggingRequestHandler.Context parentCtx, NodeRepository nodeRepository) {
+        super(parentCtx);
+        this.nodeRepository = nodeRepository;
+    }
+
+    @Override
+    public HttpResponse handle(HttpRequest request) {
+        try {
+            switch (request.getMethod()) {
+                case GET: return handleGET(request);
+                default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
+            }
+        }
+        catch (NotFoundException | NoSuchNodeException e) {
+            return ErrorResponse.notFoundError(Exceptions.toMessageString(e));
+        }
+        catch (IllegalArgumentException e) {
+            return ErrorResponse.badRequest(Exceptions.toMessageString(e));
+        }
+        catch (RuntimeException e) {
+            log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e);
+            return ErrorResponse.internalServerError(Exceptions.toMessageString(e));
+        }
+    }
+
+    private HttpResponse handleGET(HttpRequest request) {
+        String path = request.getUri().getPath();
+        if (path.equals("/loadbalancers/v1/")) return new LoadBalancersResponse(request, nodeRepository);
+        throw new NotFoundException("Nothing at path '" + path + "'");
+    }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java
new file mode 100644
index 00000000000..04a1cdaeeda
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java
@@ -0,0 +1,78 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v2;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
+import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * @author mpolden
+ */
+public class LoadBalancersResponse extends HttpResponse {
+
+    private final NodeRepository nodeRepository;
+    private final HttpRequest request;
+
+    public LoadBalancersResponse(HttpRequest request, NodeRepository nodeRepository) {
+        super(200);
+        this.request = request;
+        this.nodeRepository = nodeRepository;
+    }
+
+    private Optional<ApplicationId> application() {
+        return Optional.ofNullable(request.getProperty("application")).map(ApplicationFilter::toApplicationId);
+    }
+
+    private List<LoadBalancer> loadBalancers() {
+        return application().map(nodeRepository.database()::readLoadBalancers)
+                            .orElseGet(() -> new ArrayList<>(nodeRepository.database().readLoadBalancers().values()));
+    }
+
+    @Override
+    public String getContentType() { return "application/json"; }
+
+    @Override
+    public void render(OutputStream stream) throws IOException {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        Cursor loadBalancerArray = root.setArray("loadBalancers");
+
+        loadBalancers().forEach(lb -> {
+            Cursor lbObject = loadBalancerArray.addObject();
+            lbObject.setString("id", lb.id().serializedForm());
+            lbObject.setString("application", lb.id().application().application().value());
+            lbObject.setString("tenant", lb.id().application().tenant().value());
+            lbObject.setString("instance", lb.id().application().instance().value());
+            lbObject.setString("cluster", lb.id().cluster().value());
+            lbObject.setString("hostname", lb.hostname().value());
+
+            Cursor portArray = lbObject.setArray("ports");
+            lb.ports().forEach(portArray::addLong);
+
+            Cursor realArray = lbObject.setArray("reals");
+            lb.reals().forEach(real -> {
+                Cursor realObject = realArray.addObject();
+                realObject.setString("hostname", real.hostname().value());
+                realObject.setString("ipAddress", real.ipAddress());
+                realObject.setLong("port", real.port());
+            });
+
+            lbObject.setBool("inactive", lb.inactive());
+        });
+
+        new JsonFormat(true).encode(stream, slime);
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
index 733f5df7858..d2ab3c20080 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
@@ -6,10 +6,10 @@ import com.yahoo.config.provision.ClusterMembership;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.History;
@@ -81,7 +81,7 @@ class NodesResponse extends HttpResponse {
 
     @Override
     public void render(OutputStream stream) throws IOException {
-        stream.write(toJson());
+        new JsonFormat(true).encode(stream, slime);
     }
 
     @Override
@@ -89,10 +89,6 @@ class NodesResponse extends HttpResponse {
         return "application/json";
     }
 
-    private byte[] toJson() throws IOException {
-        return SlimeUtils.toJsonBytes(slime);
-    }
-
     private void statesToSlime(Cursor root) {
         Cursor states = root.setObject("states");
         for (Node.State state : Node.State.values())
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java
index 95f69dc1c2a..6225a8a4fc4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java
@@ -24,32 +24,52 @@ import java.util.stream.Collectors;
 
 /**
  * Authorizer for config server REST APIs. This contains the rules for all API paths where the authorization process
- * requires information from the node-repository to make a decision
+ * may require information from the node-repository to make a decision
  *
  * @author mpolden
  * @author bjorncs
  */
 public class Authorizer implements BiPredicate<NodePrincipal, URI> {
-
     private final NodeRepository nodeRepository;
     private final Set<String> whitelistedHostnames;
+    private final AthenzIdentity controllerIdentity;
+    private final AthenzIdentity configServerIdentity = new AthenzService("vespa.vespa", "configserver");
+    private final AthenzIdentity proxyIdentity = new AthenzService("vespa.vespa", "proxy");
+    private final AthenzIdentity tenantIdentity = new AthenzService("vespa.vespa", "tenant-host");
     private final Set<AthenzIdentity> trustedIdentities;
+    private final Set<AthenzIdentity> hostAdminIdentities;
 
     // TODO Remove whitelisted hostnames as these nodes should be included through 'trustedIdentities'
     public Authorizer(SystemName system, NodeRepository nodeRepository, Set<String> whitelistedHostnames) {
         this.nodeRepository = nodeRepository;
         this.whitelistedHostnames = whitelistedHostnames;
-        this.trustedIdentities = getTrustedIdentities(system);
+        controllerIdentity = system == SystemName.main
+                ? new AthenzService("vespa.vespa", "hosting")
+                : new AthenzService("vespa.vespa.cd", "hosting");
+        this.trustedIdentities = new HashSet<>(Arrays.asList(controllerIdentity, configServerIdentity));
+        this.hostAdminIdentities = new HashSet<>(Arrays.asList(controllerIdentity, configServerIdentity, proxyIdentity, tenantIdentity));
     }
 
     /** Returns whether principal is authorized to access given URI */
     @Override
     public boolean test(NodePrincipal principal, URI uri) {
-        // Trusted services can access everything
-        if (principal.getAthenzIdentityName().isPresent()
-                && trustedIdentities.contains(principal.getAthenzIdentityName().get())) {
-            return true;
+        if (principal.getAthenzIdentityName().isPresent()) {
+            // All host admins can retrieve flags data
+            if (uri.getPath().equals("/flags/v1/data") || uri.getPath().equals("/flags/v1/data/")) {
+                return hostAdminIdentities.contains(principal.getAthenzIdentityName().get());
+            }
+
+            // Only controller can access everything else in flags
+            if (uri.getPath().startsWith("/flags/v1/")) {
+                return principal.getAthenzIdentityName().get().equals(controllerIdentity);
+            }
+
+            // Trusted services can access everything
+            if (trustedIdentities.contains(principal.getAthenzIdentityName().get())) {
+                return true;
+            }
         }
+
         if (principal.getHostname().isPresent()) {
             String hostname = principal.getHostname().get();
             if (isAthenzProviderApi(uri)) {
@@ -108,18 +128,6 @@ public class Authorizer implements BiPredicate<NodePrincipal, URI> {
         return !resources.isEmpty() && resources.stream().anyMatch(resource -> predicate.test(resource, principal));
     }
 
-
-    private static Set<AthenzIdentity> getTrustedIdentities(SystemName system) {
-        Set<AthenzIdentity> trustedIdentities = new HashSet<>();
-        trustedIdentities.add(new AthenzService("vespa.vespa", "configserver"));
-        AthenzService controllerIdentity =
-                system == SystemName.main
-                        ? new AthenzService("vespa.vespa", "hosting")
-                        : new AthenzService("vespa.vespa.cd", "hosting");
-        trustedIdentities.add(controllerIdentity);
-        return trustedIdentities;
-    }
-
     private Optional<Node> getNode(String hostname) {
         // Ignore potential path traversal. Node repository happily passes arguments unsanitized all the way down to
         // curator...
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
index 110d0ca94d0..bc8772af952 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
@@ -9,31 +9,34 @@ package com.yahoo.vespa.hosted.provision.testutils;
  */
 public class ContainerConfig {
 
-        public static String servicesXmlV2(int port) {
-                return "<jdisc version='1.0'>\n" +
-                       "  <config name=\"container.handler.threadpool\">\n" +
-                       "    <maxthreads>10</maxthreads>\n" +
-                       "  </config>  \n" +
-                       "  <component id='com.yahoo.test.ManualClock'/>\n" +
-                       "  <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" +
-                       "  <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>\n" +
-                       "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockDeployer'/>\n" +
-                       "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisioner'/>\n" +
-                       "  <component id='com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker'/>\n" +
-                       "  <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>\n" +
-                       "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockDuperModel'/>\n" +
-                       "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>\n" +
-                       "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/>\n" +
-                       "  <component id='com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock'/>\n" +
-                       "  <component id='com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance'/>\n" +
-                       "  <component id='com.yahoo.config.provision.Zone'/>\n" +
-                       "  <handler id='com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler'>\n" +
-                       "    <binding>http://*/nodes/v2/*</binding>\n" +
-                       "  </handler>\n" +
-                       "  <http>\n" +
-                       "    <server id='myServer' port='" + port + "'/>\n" +
-                       "  </http>\n" +
-                       "</jdisc>";
-        }
+    public static String servicesXmlV2(int port) {
+        return "<jdisc version='1.0'>\n" +
+               "  <config name=\"container.handler.threadpool\">\n" +
+               "    <maxthreads>10</maxthreads>\n" +
+               "  </config>  \n" +
+               "  <component id='com.yahoo.test.ManualClock'/>\n" +
+               "  <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockDeployer'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisioner'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockDuperModel'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance'/>\n" +
+               "  <component id='com.yahoo.config.provision.Zone'/>\n" +
+               "  <handler id='com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler'>\n" +
+               "    <binding>http://*/nodes/v2/*</binding>\n" +
+               "  </handler>\n" +
+               "  <handler id='com.yahoo.vespa.hosted.provision.restapi.v2.LoadBalancersApiHandler'>\n" +
+               "    <binding>http://*/loadbalancers/v1/*</binding>\n" +
+               "  </handler>\n" +
+               "  <http>\n" +
+               "    <server id='myServer' port='" + port + "'/>\n" +
+               "  </http>\n" +
+               "</jdisc>";
+    }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 1b8ae58a97d..183255db06b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -18,6 +18,7 @@ import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.flag.FlagId;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.Status;
@@ -111,6 +112,8 @@ public class MockNodeRepository extends NodeRepository {
         dirtyRecursively("host55.yahoo.com", Agent.system, getClass().getSimpleName());
 
         ApplicationId zoneApp = ApplicationId.from(TenantName.from("zoneapp"), ApplicationName.from("zoneapp"), InstanceName.from("zoneapp"));
+        // TODO: Remove this once feature flag is removed
+        this.flags().setEnabled(FlagId.exclusiveLoadBalancer, zoneApp, true);
         ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container,
                                                       ClusterSpec.Id.from("node-admin"),
                                                       Version.fromString("6.42"),
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
index bc83e3525ad..4fd20d6991b 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
@@ -78,6 +78,7 @@ public class InfrastructureProvisionerTest {
     public void remove_application_if_without_target_version() {
         when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.empty());
         addNode(1, Node.State.active, Optional.of(target));
+        when(duperModelInfraApi.infraApplicationIsActive(eq(application.getApplicationId()))).thenReturn(true);
         infrastructureProvisioner.maintain();
         verify(duperModelInfraApi).infraApplicationRemoved(application.getApplicationId());
         verifyRemoved(1);
@@ -85,12 +86,26 @@ public class InfrastructureProvisionerTest {
 
     @Test
     public void remove_application_if_without_nodes() {
+        remove_application_without_nodes(true);
+    }
+
+    @Test
+    public void skip_remove_unless_active() {
+        remove_application_without_nodes(false);
+    }
+
+    private void remove_application_without_nodes(boolean applicationIsActive) {
         when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
         addNode(1, Node.State.failed, Optional.of(target));
         addNode(2, Node.State.parked, Optional.empty());
+        when(duperModelInfraApi.infraApplicationIsActive(eq(application.getApplicationId()))).thenReturn(applicationIsActive);
         infrastructureProvisioner.maintain();
-        verify(duperModelInfraApi).infraApplicationRemoved(application.getApplicationId());
-        verifyRemoved(1);
+        if (applicationIsActive) {
+            verify(duperModelInfraApi).infraApplicationRemoved(application.getApplicationId());
+            verifyRemoved(1);
+        } else {
+            verifyRemoved(0);
+        }
     }
 
     @Test
@@ -199,6 +214,7 @@ public class InfrastructureProvisionerTest {
     @Test
     public void avoid_provisioning_if_no_usable_nodes() {
         when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
+        when(duperModelInfraApi.infraApplicationIsActive(eq(application.getApplicationId()))).thenReturn(true);
 
         infrastructureProvisioner.maintain();
         verifyRemoved(1);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
new file mode 100644
index 00000000000..59323bfdeb5
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
@@ -0,0 +1,96 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.component.Vtag;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.hosted.provision.flag.FlagId;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class LoadBalancerExpirerTest {
+
+    private ProvisioningTester tester;
+
+    @Before
+    public void before() {
+        tester = new ProvisioningTester(Zone.defaultZone());
+    }
+
+    @Test
+    public void test_maintain() {
+        LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(),
+                                                              Duration.ofDays(1),
+                                                              new JobControl(tester.nodeRepository().database()),
+                                                              tester.loadBalancerService());
+        tester.nodeRepository().flags().setEnabled(FlagId.exclusiveLoadBalancer, true);
+        Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers();
+
+        // Deploy two applications with load balancers
+        ClusterSpec.Id cluster = ClusterSpec.Id.from("qrs");
+        ApplicationId app1 = tester.makeApplicationId();
+        ApplicationId app2 = tester.makeApplicationId();
+        LoadBalancerId lb1 = new LoadBalancerId(app1, cluster);
+        LoadBalancerId lb2 = new LoadBalancerId(app2, cluster);
+        deployApplication(app1, cluster);
+        deployApplication(app2, cluster);
+        assertEquals(2, loadBalancers.get().size());
+
+        // Remove one application deactivates load balancers for that application
+        removeApplication(app1);
+        assertTrue(loadBalancers.get().get(lb1).inactive());
+        assertFalse(loadBalancers.get().get(lb2).inactive());
+
+        // Expirer defers removal while nodes are still allocated to application
+        expirer.maintain();
+        assertEquals(2, tester.loadBalancerService().loadBalancers().size());
+
+        // Expirer removes load balancers once nodes are deallocated
+        dirtyNodesOf(app1);
+        expirer.maintain();
+        assertFalse("Inactive load balancer removed", tester.loadBalancerService().loadBalancers().containsKey(lb1));
+
+        // Active load balancer is left alone
+        assertFalse(loadBalancers.get().get(lb2).inactive());
+        assertTrue("Active load balancer is not removed", tester.loadBalancerService().loadBalancers().containsKey(lb2));
+    }
+
+    private void dirtyNodesOf(ApplicationId application) {
+        tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application), Agent.system, "unit-test");
+    }
+
+    private void removeApplication(ApplicationId application) {
+        NestedTransaction transaction = new NestedTransaction();
+        tester.provisioner().remove(transaction, application);
+        transaction.commit();
+    }
+
+    private void deployApplication(ApplicationId application, ClusterSpec.Id cluster) {
+        tester.makeReadyNodes(10, "default");
+        List<HostSpec> hosts = tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster,
+                                                                               Vtag.currentVersion, false),
+                                              2, 1,
+                                              "default");
+        tester.activate(application, hosts);
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index 5612c8dc665..2c63b9fd62c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -101,7 +101,7 @@ public class ProvisioningTester {
         }
     }
 
-    static FlavorsConfig createConfig() {
+    public static FlavorsConfig createConfig() {
         FlavorConfigBuilder b = new FlavorConfigBuilder();
         b.addFlavor("default", 2., 4., 100, Flavor.Type.BARE_METAL).cost(3);
         b.addFlavor("small", 1., 2., 50, Flavor.Type.BARE_METAL).cost(2);
@@ -170,11 +170,11 @@ public class ProvisioningTester {
         deactivateTransaction.commit();
     }
 
-    Collection<String> toHostNames(Collection<HostSpec> hosts) {
+    public Collection<String> toHostNames(Collection<HostSpec> hosts) {
         return hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet());
     }
 
-    Set<String> toHostNames(List<Node> nodes) {
+    public Set<String> toHostNames(List<Node> nodes) {
         return nodes.stream().map(Node::hostname).collect(Collectors.toSet());
     }
 
@@ -182,7 +182,7 @@ public class ProvisioningTester {
      * Asserts that each active node in this application has a restart count equaling the
      * number of matches to the given filters
      */
-    void assertRestartCount(ApplicationId application, HostFilter... filters) {
+    public void assertRestartCount(ApplicationId application, HostFilter... filters) {
         for (Node node : nodeRepository.getNodes(application, Node.State.active)) {
             int expectedRestarts = 0;
             for (HostFilter filter : filters)
@@ -199,7 +199,7 @@ public class ProvisioningTester {
         assertEquals(beforeFailCount + 1, failedNode.status().failCount());
     }
 
-    void assertMembersOf(ClusterSpec requestedCluster, Collection<HostSpec> hosts) {
+    public void assertMembersOf(ClusterSpec requestedCluster, Collection<HostSpec> hosts) {
         Set<Integer> indices = new HashSet<>();
         for (HostSpec host : hosts) {
             ClusterSpec nodeCluster = host.membership().get().cluster();
@@ -214,14 +214,14 @@ public class ProvisioningTester {
         assertEquals("Indexes in " + requestedCluster + " are disjunct", hosts.size(), indices.size());
     }
 
-    HostSpec removeOne(Set<HostSpec> hosts) {
+    public HostSpec removeOne(Set<HostSpec> hosts) {
         Iterator<HostSpec> i = hosts.iterator();
         HostSpec removed = i.next();
         i.remove();
         return removed;
     }
 
-    ApplicationId makeApplicationId() {
+    public ApplicationId makeApplicationId() {
         return ApplicationId.from(
                 TenantName.from(UUID.randomUUID().toString()),
                 ApplicationName.from(UUID.randomUUID().toString()),
@@ -232,15 +232,15 @@ public class ProvisioningTester {
         return makeReadyNodes(n, flavor, NodeType.tenant);
     }
 
-    List<Node> makeReadyNodes(int n, String flavor, NodeType type) {
+    public List<Node> makeReadyNodes(int n, String flavor, NodeType type) {
         return makeReadyNodes(n, flavor, type, 0);
     }
 
-    List<Node> makeProvisionedNodes(int count, String flavor, NodeType type, int ipAddressPoolSize) {
+    public List<Node> makeProvisionedNodes(int count, String flavor, NodeType type, int ipAddressPoolSize) {
         return makeProvisionedNodes(count, flavor, type, ipAddressPoolSize, false);
     }
 
-    List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
+    public List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
         List<Node> nodes = new ArrayList<>(n);
 
         for (int i = 0; i < n; i++) {
@@ -290,7 +290,7 @@ public class ProvisioningTester {
         return nodes;
     }
 
-    List<Node> makeConfigServers(int n, String flavor, Version configServersVersion) {
+    public List<Node> makeConfigServers(int n, String flavor, Version configServersVersion) {
         List<Node> nodes = new ArrayList<>(n);
         MockNameResolver nameResolver = (MockNameResolver)nodeRepository().nameResolver();
 
@@ -322,33 +322,33 @@ public class ProvisioningTester {
         return nodeRepository.getNodes(application.getApplicationId(), Node.State.active);
     }
 
-    List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize) {
+    public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize) {
         return makeReadyNodes(n, flavor, type, ipAddressPoolSize, false);
     }
 
-    List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
+    public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
         List<Node> nodes = makeProvisionedNodes(n, flavor, type, ipAddressPoolSize, dualStack);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
 
     /** Creates a set of virtual docker nodes on a single docker host */
-    List<Node> makeReadyDockerNodes(int n, String flavor, String dockerHostId) {
+    public List<Node> makeReadyDockerNodes(int n, String flavor, String dockerHostId) {
         return makeReadyVirtualNodes(n, flavor, Optional.of(dockerHostId));
     }
 
     /** Creates a set of virtual nodes on a single parent host */
-    List<Node> makeReadyVirtualNodes(int n, String flavor, Optional<String> parentHostId) {
+    public List<Node> makeReadyVirtualNodes(int n, String flavor, Optional<String> parentHostId) {
         return makeReadyVirtualNodes(n, 0, flavor, parentHostId, index -> UUID.randomUUID().toString());
     }
 
     /** Creates a set of virtual nodes on a single parent host */
-    List<Node> makeReadyVirtualNode(int index, String flavor, String parentHostId) {
+    public List<Node> makeReadyVirtualNode(int index, String flavor, String parentHostId) {
         return makeReadyVirtualNodes(1, index, flavor, Optional.of(parentHostId), i -> String.format("node%03d", i));
     }
 
     /** Creates a set of virtual nodes on a single parent host */
-    List<Node> makeReadyVirtualNodes(int count, int startIndex, String flavor, Optional<String> parentHostId,
+    public List<Node> makeReadyVirtualNodes(int count, int startIndex, String flavor, Optional<String> parentHostId,
                                      Function<Integer, String> nodeNamer) {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = startIndex; i < count + startIndex; i++) {
@@ -362,16 +362,16 @@ public class ProvisioningTester {
         return nodes;
     }
 
-    List<Node> makeReadyVirtualNodes(int n, String flavor, String parentHostId) {
+    public List<Node> makeReadyVirtualNodes(int n, String flavor, String parentHostId) {
         return makeReadyVirtualNodes(n, flavor, Optional.of(parentHostId));
     }
 
     /** Returns the hosts from the input list which are not retired */
-    List<HostSpec> nonRetired(Collection<HostSpec> hosts) {
+    public List<HostSpec> nonRetired(Collection<HostSpec> hosts) {
         return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList());
     }
 
-    void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) {
+    public void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) {
         long actualNodesWithFlavor = hostSpecs.stream()
                 .map(HostSpec::hostname)
                 .map(this::getNodeFlavor)
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index ae7f3f14975..ec09805ff5d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -706,6 +706,13 @@ public class RestApiTest {
         }
     }
 
+    @Test
+    public void test_load_balancers() throws Exception {
+        assertFile(new Request("http://localhost:8080/loadbalancers/v1/"), "load-balancers.json");
+        assertFile(new Request("http://localhost:8080/loadbalancers/v1/?application=zoneapp.zoneapp.zoneapp"), "load-balancers.json");
+        assertResponse(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant.nonexistent.default"), "{\"loadBalancers\":[]}");
+    }
+
     private String asDockerNodeJson(String hostname, String parentHostname, int additionalIpCount, String... ipAddress) {
         return "{\"hostname\":\"" + hostname + "\", \"parentHostname\":\"" + parentHostname + "\"," +
                 createIpAddresses(ipAddress) +
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java
index 38128e66861..5e643bd09ab 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java
@@ -139,6 +139,30 @@ public class AuthorizerTest {
     }
 
     @Test
+    public void flags_authorization() {
+        // Tenant nodes cannot access flags resources
+        assertFalse(authorizedTenantNode("node1", "/flags/v1/data"));
+        assertFalse(authorizedTenantNode("node1", "/flags/v1/data/flagid"));
+        assertFalse(authorizedTenantNode("node1", "/flags/v1/foo"));
+
+        // Host node can access data
+        assertTrue(authorizedTenantHostNode("host1", "/flags/v1/data"));
+        assertFalse(authorizedTenantHostNode("host1", "/flags/v1/data/flagid"));
+        assertFalse(authorizedTenantHostNode("host1", "/flags/v1/foo"));
+        assertTrue(authorizedTenantHostNode("proxy1-host", "/flags/v1/data"));
+        assertFalse(authorizedTenantHostNode("proxy1-host", "/flags/v1/data/flagid"));
+        assertFalse(authorizedTenantHostNode("proxy1-host", "/flags/v1/foo"));
+        assertTrue(authorizedController("vespa.vespa.configserver", "/flags/v1/data"));
+        assertFalse(authorizedController("vespa.vespa.configserver", "/flags/v1/data/flagid"));
+        assertFalse(authorizedController("vespa.vespa.configserver", "/flags/v1/foo"));
+
+        // Controller can access everything
+        assertTrue(authorizedController("vespa.vespa.hosting", "/flags/v1/data"));
+        assertTrue(authorizedController("vespa.vespa.hosting", "/flags/v1/data/flagid"));
+        assertTrue(authorizedController("vespa.vespa.hosting", "/flags/v1/foo"));
+    }
+
+    @Test
     public void routing_authorization() {
         // Node of proxy or proxyhost type can access routing resource
         assertFalse(authorizedTenantNode("node1", "/routing/v1/status"));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json
index 8fd09b4a274..a606777e9fd 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json
@@ -4,7 +4,7 @@
       "id": "exclusive-load-balancer",
       "enabled": false,
       "enabledHostnames": [],
-      "enabledApplications": []
+      "enabledApplications": ["zoneapp:zoneapp:zoneapp"]
     }
   ]
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json
index 78de52e4e85..4baf75f2169 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json
@@ -7,6 +7,7 @@
         "host1"
       ],
       "enabledApplications": [
+        "zoneapp:zoneapp:zoneapp",
         "foo:bar:default"
       ]
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json
new file mode 100644
index 00000000000..c882f7652d8
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json
@@ -0,0 +1,43 @@
+{
+  "loadBalancers": [
+    {
+      "id": "zoneapp:zoneapp:zoneapp:node-admin",
+      "application": "zoneapp",
+      "tenant": "zoneapp",
+      "instance": "zoneapp",
+      "cluster": "node-admin",
+      "hostname": "lb-zoneapp.zoneapp.zoneapp-node-admin",
+      "ports": [
+        4443
+      ],
+      "reals": [
+        {
+          "hostname": "dockerhost4.yahoo.com",
+          "ipAddress": "127.0.0.1",
+          "port": 4443
+        },
+        {
+          "hostname": "dockerhost5.yahoo.com",
+          "ipAddress": "127.0.0.1",
+          "port": 4443
+        },
+        {
+          "hostname": "dockerhost2.yahoo.com",
+          "ipAddress": "127.0.0.1",
+          "port": 4443
+        },
+        {
+          "hostname": "dockerhost3.yahoo.com",
+          "ipAddress": "127.0.0.1",
+          "port": 4443
+        },
+        {
+          "hostname": "dockerhost1.yahoo.com",
+          "ipAddress": "127.0.0.1",
+          "port": 4443
+        }
+      ],
+      "inactive": false
+    }
+  ]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json
index 99cb9fd91f5..1432d2f4ea5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json
@@ -1,46 +1,49 @@
 {
-  "jobs":[
+  "jobs": [
     {
-      "name":"PeriodicApplicationMaintainer"
+      "name": "DirtyExpirer"
     },
     {
-      "name":"FailedExpirer"
+      "name": "FailedExpirer"
     },
     {
-      "name":"ReservationExpirer"
+      "name": "InactiveExpirer"
     },
     {
-      "name":"RetiredExpirer"
+      "name": "InfrastructureProvisioner"
     },
     {
-      "name":"NodeRebooter"
+      "name": "LoadBalancerExpirer"
     },
     {
-      "name":"InactiveExpirer"
+      "name": "MetricsReporter"
     },
     {
-      "name":"DirtyExpirer"
+      "name": "NodeFailer"
     },
     {
-      "name":"NodeRetirer"
+      "name": "NodeRebooter"
     },
     {
-      "name":"OperatorChangeApplicationMaintainer"
+      "name": "NodeRetirer"
     },
     {
-      "name":"ProvisionedExpirer"
+      "name": "OperatorChangeApplicationMaintainer"
     },
     {
-      "name":"MetricsReporter"
+      "name": "PeriodicApplicationMaintainer"
     },
     {
-      "name":"InfrastructureProvisioner"
+      "name": "ProvisionedExpirer"
     },
     {
-      "name":"NodeFailer"
+      "name": "ReservationExpirer"
+    },
+    {
+      "name": "RetiredExpirer"
     }
   ],
-  "inactive":[
+  "inactive": [
     "NodeFailer"
   ]
 }