Support for removing load balancer names

9 files changed, 312 insertions, 21 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java
index e8300262a89..114ae4dbfe7 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java
@@ -1,4 +1,4 @@
-// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.api.integration.configserver;
 
 import com.yahoo.config.provision.ClusterSpec;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/loadbalancer/LoadBalancerName.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/loadbalancer/LoadBalancerName.java
new file mode 100644
index 00000000000..b2963a62eba
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/loadbalancer/LoadBalancerName.java
@@ -0,0 +1,52 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.loadbalancer;
+
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
+
+import java.util.Objects;
+
+/**
+ * Represents a pair of RecordId and RecordName
+ *
+ * @author mortent
+ */
+public class LoadBalancerName {
+    private final RecordId recordId;
+    private final RecordName recordName;
+
+    public LoadBalancerName(RecordId recordId, RecordName recordName) {
+        this.recordId = recordId;
+        this.recordName = recordName;
+    }
+
+    public RecordId recordId() {
+        return recordId;
+    }
+
+    public RecordName recordName() {
+        return recordName;
+    }
+
+    @Override
+    public String toString() {
+        return "LoadBalancerName{" +
+               "recordId=" + recordId +
+               ", recordName=" + recordName +
+               '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        LoadBalancerName that = (LoadBalancerName) o;
+        return recordId.equals(that.recordId) &&
+               recordName.equals(that.recordName);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(recordId, recordName);
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 8b9c8fd4fb2..132b4a28514 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -82,7 +82,7 @@ public class ControllerMaintenance extends AbstractComponent {
         osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl);
         contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, contactRetriever);
         costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig);
-        loadbalancerMaintainer = new LoadBalancerMaintainer(controller, Duration.ofMinutes(5), jobControl, nameService);
+        loadbalancerMaintainer = new LoadBalancerMaintainer(controller, Duration.ofMinutes(5), jobControl, nameService, curator);
     }
 
     public Upgrader upgrader() { return upgrader; }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java
index 48e22b6baf2..71b2482a650 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java
@@ -1,4 +1,4 @@
-// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.google.common.base.Strings;
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.LockedApplication;
@@ -14,18 +15,25 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalanc
 import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Maintains loadbalancer endpoints.
@@ -39,13 +47,16 @@ public class LoadBalancerMaintainer extends Maintainer {
     private static final String IGNORE_ENDPOINT_VALUE = "default";
 
     private final NameService nameService;
+    private final CuratorDb curatorDb;
 
     public LoadBalancerMaintainer(Controller controller,
                                   Duration interval,
                                   JobControl jobControl,
-                                  NameService nameService) {
+                                  NameService nameService,
+                                  CuratorDb curatorDb) {
         super(controller, interval, jobControl);
         this.nameService = nameService;
+        this.curatorDb = curatorDb;
     }
 
     @Override
@@ -55,10 +66,69 @@ public class LoadBalancerMaintainer extends Maintainer {
 
         // Create or update cnames
         List<Application> applications = controller().applications().asList();
-        applications.forEach(this::registerLoadBalancerEndpoint);
+        Map<ApplicationId, List<LoadBalancerName>> applicationEndpointMap = applications.stream()
+                .collect(Collectors.toMap(Application::id, this::registerLoadBalancerEndpoint));
 
-        // Delete removed application rotations
-        // TODO
+        try (Lock lock = curatorDb.lockLoadBalancerNames()) {
+            updatePersistedLoadBalancerNames(applicationEndpointMap);
+            removeObsoleteLoadBalancerNames();
+        }
+    }
+
+
+    private void removeObsoleteLoadBalancerNames() {
+        Map<ApplicationId, List<LoadBalancerName>> persistedLoadBalancerNames = new HashMap<>(curatorDb.readLoadBalancerNames());
+        Map<ApplicationId, List<LoadBalancerName>> result = new HashMap<>();
+
+        Map<ApplicationId, List<String>> wantedLoadBalancers = new HashMap<>();
+
+        for (Application application : controller().applications().asList()) {
+            for (Map.Entry<ZoneId, Deployment> entry : application.deployments().entrySet()) {
+                Map<ClusterSpec.Id, HostName> loadBalancers = entry.getValue().loadBalancers();
+                List<String> loadBalancerNames = loadBalancers.keySet().stream()
+                        .map(cluster -> getEndpointName(cluster, application.id(), entry.getKey()))
+                        .collect(Collectors.toList());
+                wantedLoadBalancers.merge(application.id(), loadBalancerNames, (v1, v2) ->
+                        Stream.concat(v1.stream(), v2.stream()).collect(Collectors.toList()));
+            }
+        }
+
+        for (Map.Entry<ApplicationId, List<LoadBalancerName>> loadbalancerEntry : persistedLoadBalancerNames.entrySet()) {
+            List<String> wanted = wantedLoadBalancers.getOrDefault(loadbalancerEntry.getKey(), Collections.emptyList());
+            List<LoadBalancerName> current = loadbalancerEntry.getValue();
+            List<LoadBalancerName> toBeRemoved = current.stream()
+                    .filter(lbname -> !wanted.contains(lbname.recordName().asString()))
+                    .collect(Collectors.toList());
+            removeLoadBalancerNames(toBeRemoved);
+            List<LoadBalancerName> resultingLoadBalancers = current.stream().filter(lb -> !toBeRemoved.contains(lb)).collect(Collectors.toList());
+            result.put(loadbalancerEntry.getKey(), resultingLoadBalancers);
+        }
+
+        curatorDb.writeLoadBalancerNames(result);
+    }
+
+    private void updatePersistedLoadBalancerNames(Map<ApplicationId, List<LoadBalancerName>> applicationEndpointMap) {
+        // Read current list of maintained load balancer endpoints
+        Map<ApplicationId, List<LoadBalancerName>> existingApplicationEndpointList = curatorDb.readLoadBalancerNames();
+
+        // Update ZK with new load balancer endpoints
+        Map<ApplicationId, List<LoadBalancerName>> allCreated = new HashMap<>(applicationEndpointMap);
+        existingApplicationEndpointList.forEach((k, v) -> allCreated.merge(k, v, (v1, v2) ->
+                // Merge the two lists, removing duplicates
+                List.copyOf(new LinkedHashSet<LoadBalancerName>(
+                        Stream.concat(v1.stream(), v2.stream()).collect(Collectors.toList()))
+                )));
+
+        curatorDb.writeLoadBalancerNames(allCreated);
+
+    }
+
+    private void removeLoadBalancerNames(List<LoadBalancerName> loadBalancerNames) {
+        if(loadBalancerNames.isEmpty()) return;
+        log.log(LogLevel.INFO, String.format("Removing %d Load Balancer names", loadBalancerNames.size()));
+        for (LoadBalancerName loadBalancerName : loadBalancerNames) {
+            nameService.removeRecord(loadBalancerName.recordId());
+        }
     }
 
     private void updateApplicationLoadBalancers(Application application) {
@@ -89,7 +159,8 @@ public class LoadBalancerMaintainer extends Maintainer {
         controller().applications().store(lockedApplication);
     }
 
-    private void registerLoadBalancerEndpoint(Application application) {
+    private List<LoadBalancerName> registerLoadBalancerEndpoint(Application application) {
+        List<LoadBalancerName> hostNamesRegistered = new ArrayList<>();
         for (Map.Entry<ZoneId, Deployment> deploymentEntry : application.deployments().entrySet()) {
             ZoneId zone = deploymentEntry.getKey();
             Deployment deployment = deploymentEntry.getValue();
@@ -98,11 +169,14 @@ public class LoadBalancerMaintainer extends Maintainer {
                     RecordName recordName = RecordName.from(getEndpointName(loadBalancers.getKey(), application.id(), zone));
                     RecordData recordData = RecordData.fqdn(loadBalancers.getValue().value());
                     Optional<Record> existingRecord = nameService.findRecord(Record.Type.CNAME, recordName);
+                    final RecordId recordId;
                     if(existingRecord.isPresent()) {
+                        recordId = existingRecord.get().id();
                         nameService.updateRecord(existingRecord.get().id(), recordData);
                     } else {
-                        nameService.createCname(recordName, recordData);
+                        recordId = nameService.createCname(recordName, recordData);
                     }
+                    hostNamesRegistered.add(new LoadBalancerName(recordId, recordName));
                 } catch (Exception e) {
                     // Catching any exception, will be retried on next run
                     log.log(LogLevel.WARNING,
@@ -112,6 +186,7 @@ public class LoadBalancerMaintainer extends Maintainer {
                 }
             }
         }
+        return hostNamesRegistered;
     }
 
     static String getEndpointName(ClusterSpec.Id clusterId, ApplicationId applicationId, ZoneId zoneId) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index 74e60a40b4c..5fac0896246 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
+import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
@@ -81,6 +82,7 @@ public class CuratorDb {
     private final RunSerializer runSerializer = new RunSerializer();
     private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer();
     private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer);
+    private final LoadBalancerNameSerializer loadBalancerNameSerializer = new LoadBalancerNameSerializer();
 
     private final Curator curator;
     private final Duration tryLockTimeout;
@@ -174,6 +176,9 @@ public class CuratorDb {
         return lock(lockRoot.append("osVersionStatus"), defaultLockTimeout);
     }
 
+    public Lock lockLoadBalancerNames() {
+        return lock(lockRoot.append("loadBalancerNames"), defaultLockTimeout);
+    }
     // -------------- Helpers ------------------------------------------
 
     /** Try locking with a low timeout, meaning it is OK to fail lock acquisition.
@@ -455,6 +460,16 @@ public class CuratorDb {
         curator.set(openStackServerPoolPath(), data);
     }
 
+    // -------------- Load balancer names -------------------------------------
+
+    public void writeLoadBalancerNames(Map<ApplicationId, List<LoadBalancerName>> loadBalancerNames) {
+        curator.set(loadBalancerNamePath(), asJson(loadBalancerNameSerializer.toSlime(loadBalancerNames)));
+    }
+
+    public Map<ApplicationId, List<LoadBalancerName>> readLoadBalancerNames() {
+        return readSlime(loadBalancerNamePath()).map(loadBalancerNameSerializer::fromSlime).orElseGet(Collections::emptyMap);
+    }
+
     // -------------- Paths ---------------------------------------------------
 
     private Path lockPath(TenantName tenant) {
@@ -530,6 +545,10 @@ public class CuratorDb {
         return root.append("versionStatus");
     }
 
+    private static Path loadBalancerNamePath() {
+        return root.append("loadBalancerNames");
+    }
+
     private static Path provisionStatePath() {
         return root.append("provisioning").append("states");
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializer.java
new file mode 100644
index 00000000000..a76e1189294
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializer.java
@@ -0,0 +1,56 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.ObjectTraverser;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
+import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Serializer and deserializer for LoadBalancerName
+ *
+ * @author mortent
+ */
+public class LoadBalancerNameSerializer {
+
+    private static final String loadBalancerNamesField = "loadBalancerNames";
+
+    public Slime toSlime(Map<ApplicationId, List<LoadBalancerName>> loadBalancerNames) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        Cursor applicationEndpoints = root.setObject(loadBalancerNamesField);
+
+        for (Map.Entry<ApplicationId, List<LoadBalancerName>> entry : loadBalancerNames.entrySet()) {
+            ApplicationId applicationId = entry.getKey();
+            Cursor cursor = applicationEndpoints.setArray(applicationId.serializedForm());
+            entry.getValue().forEach( lb -> cursor.addObject().setString(lb.recordId().asString(), lb.recordName().asString()));
+        }
+        return slime;
+    }
+
+    public Map<ApplicationId, List<LoadBalancerName>> fromSlime(Slime slime) {
+
+        Inspector object = slime.get().field(loadBalancerNamesField);
+        Map<ApplicationId, List<LoadBalancerName>> loadBalancerNames = new HashMap<>();
+        object.traverse((ObjectTraverser) (appId, inspector) ->
+                loadBalancerNames.put(ApplicationId.fromSerializedForm(appId), loadBalancerNamesFromSlime(inspector)));
+
+        return loadBalancerNames;
+    }
+
+    private List<LoadBalancerName> loadBalancerNamesFromSlime(Inspector root) {
+        List<LoadBalancerName> names = new ArrayList<>();
+        root.traverse((ArrayTraverser) (i, inspector) -> inspector.traverse((ObjectTraverser)(x,y)-> names.add(new LoadBalancerName(new RecordId(x), RecordName.from(y.asString())))));
+        return names;
+    }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index 5beb95493c3..45e87859576 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -180,6 +180,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         this.loadBalancers.put(new DeploymentId(applicationId, zoneId), loadBalancers);
     }
 
+    public void removeLoadBalancers(DeploymentId deployment) {
+        this.loadBalancers.remove(deployment);
+    }
+
     @Override
     public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames,
                                        Set<String> rotationNames, byte[] content) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java
index 380281a273b..05c3de79023 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java
@@ -1,4 +1,4 @@
-// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.config.provision.ApplicationId;
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
@@ -15,11 +16,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import org.junit.Test;
 
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -37,7 +40,8 @@ public class LoadBalancerMaintainerTest {
 
         LoadBalancerMaintainer loadbalancerMaintainer = new LoadBalancerMaintainer(tester.controller(), Duration.ofHours(12),
                                                                                    new JobControl(new MockCuratorDb()),
-                                                                                   tester.controllerTester().nameService());
+                                                                                   tester.controllerTester().nameService(),
+                                                                                   tester.controllerTester().curator());
 
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .environment(Environment.prod)
@@ -45,21 +49,22 @@ public class LoadBalancerMaintainerTest {
                 .region("us-central-1")
                 .build();
 
-        int numberOfClusters = 2;
+        int numberOfClustersPerZone = 2;
 
         // Deploy application
         tester.deployCompletely(application, applicationPackage);
-        tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet()
-                .forEach(zone -> tester.configServer()
-                        .addLoadBalancers(zone, application.id(), getLoadBalancers(zone, application.id(), numberOfClusters)));
-
+        setupClustersWithLoadBalancers(tester, application, numberOfClustersPerZone);
 
         loadbalancerMaintainer.maintain();
         Map<RecordId, Record> records = tester.controllerTester().nameService().records();
         long recordCount = records.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count();
-        records.entrySet().stream().forEach(entry -> System.out.println("entry = " + entry));
         assertEquals(4,recordCount);
 
+        Map<ApplicationId, List<LoadBalancerName>> loadBalancerNames = tester.controller().curator().readLoadBalancerNames();
+        List<LoadBalancerName> names = loadBalancerNames.get(application.id());
+        assertEquals(4, names.size());
+
+
         // no update
         loadbalancerMaintainer.maintain();
         Map<RecordId, Record> records2 = tester.controllerTester().nameService().records();
@@ -68,17 +73,64 @@ public class LoadBalancerMaintainerTest {
         assertEquals(records, records2);
 
 
-        // add cluster
-        tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet()
-                .forEach(zone -> tester.configServer()
-                        .addLoadBalancers(zone, application.id(), getLoadBalancers(zone, application.id(), numberOfClusters + 1)));
+        // add 1 cluster per zone
+        setupClustersWithLoadBalancers(tester, application, numberOfClustersPerZone + 1);
 
         loadbalancerMaintainer.maintain();
         Map<RecordId, Record> records3 = tester.controllerTester().nameService().records();
         long recordCount3 = records3.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count();
         assertEquals(6,recordCount3);
+
+        Map<ApplicationId, List<LoadBalancerName>> loadBalancerNames3 = tester.controller().curator().readLoadBalancerNames();
+        List<LoadBalancerName> names3 = loadBalancerNames3.get(application.id());
+        assertEquals(6, names3.size());
+
+
+        // Add application
+        Application application2 = tester.createApplication("app2", "tenant1", 1, 1L);
+        tester.deployCompletely(application2, applicationPackage);
+        setupClustersWithLoadBalancers(tester, application2, numberOfClustersPerZone);
+
+        loadbalancerMaintainer.maintain();
+        Map<RecordId, Record> records4 = tester.controllerTester().nameService().records();
+        long recordCount4 = records4.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count();
+        assertEquals(10,recordCount4);
+
+        Map<ApplicationId, List<LoadBalancerName>> loadBalancerNames4 = tester.controller().curator().readLoadBalancerNames();
+        List<LoadBalancerName> names4 = loadBalancerNames4.get(application2.id());
+        assertEquals(4, names4.size());
+
+
+        // Remove cluster in app1
+        setupClustersWithLoadBalancers(tester, application, numberOfClustersPerZone);
+
+        loadbalancerMaintainer.maintain();
+        Map<RecordId, Record> records5 = tester.controllerTester().nameService().records();
+        long recordCount5 = records5.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count();
+        assertEquals(8,recordCount5);
+
+        // Remove application app2
+        tester.controller().applications().get(application2.id())
+                .map(app -> app.deployments().keySet())
+                .orElse(Collections.emptySet())
+                .forEach(zone -> tester.controller().applications().deactivate(application2.id(), zone));
+
+        loadbalancerMaintainer.maintain();
+        Map<RecordId, Record> records6 = tester.controllerTester().nameService().records();
+        long recordCount6 = records6.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count();
+        assertEquals(4,recordCount6);
+
     }
 
+    private void setupClustersWithLoadBalancers(DeploymentTester tester, Application application, int numberOfClustersPerZone) {
+        tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet()
+                .forEach(zone -> tester.configServer()
+                        .removeLoadBalancers(new DeploymentId(application.id(), zone)));
+        tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet()
+                .forEach(zone -> tester.configServer()
+                        .addLoadBalancers(zone, application.id(), getLoadBalancers(zone, application.id(), numberOfClustersPerZone)));
+
+    }
 
 
     @Test
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializerTest.java
new file mode 100644
index 00000000000..a66fe127cd5
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializerTest.java
@@ -0,0 +1,33 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
+import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author mortent
+ */
+public class LoadBalancerNameSerializerTest {
+
+    @Test
+    public void test_serialization() throws IOException {
+        LoadBalancerNameSerializer serializer = new LoadBalancerNameSerializer();
+        ImmutableMap<ApplicationId, List<LoadBalancerName>> lbnames = ImmutableMap.of(
+                ApplicationId.from("foo", "bar", "default"), Lists.newArrayList(new LoadBalancerName(new RecordId("123.4123.:124123"), RecordName.from("foo.bar"))));
+        Map<ApplicationId, List<LoadBalancerName>> deserialized = serializer.fromSlime(serializer.toSlime(lbnames));
+
+        assertThat(deserialized, equalTo(lbnames));
+    }
+}
\ No newline at end of file