Add load balancer provisioner

14 files changed, 816 insertions, 5 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
index acf62ae91b9..398f3fceca8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision;
 
 import com.google.common.collect.ImmutableList;
@@ -52,18 +52,25 @@ public class NodeList {
         return new NodeList(nodes.stream().filter(node -> node.allocation().get().membership().cluster().type().equals(type)).collect(Collectors.toList()));
     }
 
+    /** Returns the subset of nodes that are in the given state */
+    public NodeList in(Node.State state) {
+        return nodes.stream()
+                    .filter(node -> node.state() == state)
+                    .collect(collectingAndThen(Collectors.toList(), NodeList::new));
+    }
+
     /** Returns the subset of nodes owned by the given application */
     public NodeList owner(ApplicationId application) {
         return nodes.stream()
-                .filter(node -> node.allocation().map(a -> a.owner().equals(application)).orElse(false))
-                .collect(collectingAndThen(Collectors.toList(), NodeList::new));
+                    .filter(node -> node.allocation().map(a -> a.owner().equals(application)).orElse(false))
+                    .collect(collectingAndThen(Collectors.toList(), NodeList::new));
     }
 
     /** Returns the subset of nodes matching the given node type */
     public NodeList nodeType(NodeType nodeType) {
         return nodes.stream()
-                .filter(node -> node.type() == nodeType)
-                .collect(collectingAndThen(Collectors.toList(), NodeList::new));
+                    .filter(node -> node.type() == nodeType)
+                    .collect(collectingAndThen(Collectors.toList(), NodeList::new));
     }
 
     /** Returns the parent nodes of the given child nodes */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java
new file mode 100644
index 00000000000..07bd1ea06ed
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java
@@ -0,0 +1,73 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.lb;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Ordering;
+import com.yahoo.config.provision.HostName;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Represents a load balancer for an application.
+ *
+ * @author mpolden
+ */
+public class LoadBalancer {
+
+    private final LoadBalancerId id;
+    private final HostName hostname;
+    private final List<Integer> ports;
+    private final List<Real> reals;
+    private final boolean deleted;
+
+    public LoadBalancer(LoadBalancerId id, HostName hostname, List<Integer> ports, List<Real> reals, boolean deleted) {
+        this.id = Objects.requireNonNull(id, "id must be non-null");
+        this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null");
+        this.ports = Ordering.natural().immutableSortedCopy(requirePorts(ports));
+        this.reals = ImmutableList.copyOf(Objects.requireNonNull(reals, "targets must be non-null"));
+        this.deleted = deleted;
+    }
+
+    /** An identifier for this load balancer. The ID is unique inside the zone */
+    public LoadBalancerId id() {
+        return id;
+    }
+
+    /** Fully-qualified domain name of this load balancer. This hostname can be used for query and feed. */
+    public HostName hostname() {
+        return hostname;
+    }
+
+    /** Listening port(s) of this load balancer */
+    public List<Integer> ports() {
+        return ports;
+    }
+
+    /** Real servers behind this load balancer */
+    public List<Real> reals() {
+        return reals;
+    }
+
+    /** Whether this load balancer has been deleted */
+    public boolean deleted() {
+        return deleted;
+    }
+
+    /** Return a copy of this that is marked as deleted */
+    public LoadBalancer delete() {
+        return new LoadBalancer(id, hostname, ports, reals, true);
+    }
+
+    private static List<Integer> requirePorts(List<Integer> ports) {
+        Objects.requireNonNull(ports, "ports must be non-null");
+        if (ports.isEmpty()) {
+            throw new IllegalArgumentException("ports must be non-empty");
+        }
+        if (!ports.stream().allMatch(port -> port >= 1 && port <= 65535)) {
+            throw new IllegalArgumentException("all ports must be >= 1 and <= 65535");
+        }
+        return ports;
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java
new file mode 100644
index 00000000000..1431f21de47
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java
@@ -0,0 +1,65 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.lb;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+
+import java.util.Objects;
+
+/**
+ * Uniquely identifies a load balancer for an application's container cluster.
+ *
+ * @author mpolden
+ */
+public class LoadBalancerId {
+
+    private final ApplicationId application;
+    private final ClusterSpec.Id cluster;
+    private final String serializedForm;
+
+    public LoadBalancerId(ApplicationId application, ClusterSpec.Id cluster) {
+        this.application = Objects.requireNonNull(application, "application must be non-null");
+        this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null");
+        this.serializedForm = serializedForm(application, cluster);
+    }
+
+    public ApplicationId application() {
+        return application;
+    }
+
+    public ClusterSpec.Id cluster() {
+        return cluster;
+    }
+
+    /** Serialized form of this */
+    public String serializedForm() {
+        return serializedForm;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        LoadBalancerId that = (LoadBalancerId) o;
+        return Objects.equals(application, that.application) &&
+               Objects.equals(cluster, that.cluster);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(application, cluster);
+    }
+
+    /** Create an instance from a serialized value on the form tenant:application:instance:cluster-id */
+    public static LoadBalancerId fromSerializedForm(String value) {
+        int lastSeparator = value.lastIndexOf(":");
+        ApplicationId application = ApplicationId.fromSerializedForm(value.substring(0, lastSeparator));
+        ClusterSpec.Id cluster = ClusterSpec.Id.from(value.substring(lastSeparator + 1));
+        return new LoadBalancerId(application, cluster);
+    }
+
+    private static String serializedForm(ApplicationId application, ClusterSpec.Id cluster) {
+        return application.serializedForm() + ":" + cluster.value();
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java
new file mode 100644
index 00000000000..b5f59414c65
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java
@@ -0,0 +1,32 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.lb;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+
+import java.util.List;
+
+/**
+ * A managed load balance service.
+ *
+ * @author mpolden
+ */
+public interface LoadBalancerService {
+
+    /** Create a load balancer for given application cluster. Implementations are expected to be idempotent */
+    LoadBalancer create(ApplicationId application, ClusterSpec.Id cluster, List<Real> reals);
+
+    /** Permanently remove load balancer with given ID */
+    void remove(LoadBalancerId loadBalancer);
+
+    /** Returns the protocol supported by this load balancer service */
+    Protocol protocol();
+
+    /** Load balancer protocols */
+    enum Protocol {
+        ipv4,
+        ipv6,
+        dualstack
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceProvider.java
new file mode 100644
index 00000000000..7f37798b977
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceProvider.java
@@ -0,0 +1,45 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.lb;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.container.di.componentgraph.Provider;
+
+import java.util.List;
+
+/**
+ * A provider for a {@link LoadBalancerService}. This provides a default instance for cases where a component has not
+ * been explicitly configured.
+ *
+ * @author mpolden
+ */
+public class LoadBalancerServiceProvider implements Provider<LoadBalancerService> {
+
+    private static final LoadBalancerService instance = new LoadBalancerService() {
+
+        @Override
+        public LoadBalancer create(ApplicationId application, ClusterSpec.Id cluster, List<Real> reals) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void remove(LoadBalancerId loadBalancer) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Protocol protocol() {
+            throw new UnsupportedOperationException();
+        }
+
+    };
+
+    @Override
+    public LoadBalancerService get() {
+        return instance;
+    }
+
+    @Override
+    public void deconstruct() {}
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/Real.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/Real.java
new file mode 100644
index 00000000000..784d58f103e
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/Real.java
@@ -0,0 +1,80 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.lb;
+
+import com.google.common.net.InetAddresses;
+import com.yahoo.config.provision.HostName;
+
+import java.util.Objects;
+
+/**
+ * Represents a server behind a load balancer.
+ *
+ * @author mpolden
+ */
+public class Real {
+
+    private static int defaultPort = 4443;
+
+    private final HostName hostname;
+    private final String ipAddress;
+    private final int port;
+
+    public Real(HostName hostname, String ipAddress) {
+        this(hostname, ipAddress, defaultPort);
+    }
+
+    public Real(HostName hostname, String ipAddress, int port) {
+        this.hostname = hostname;
+        this.ipAddress = requireIpAddress(ipAddress);
+        if (port < 1 || port > 65535) {
+            throw new IllegalArgumentException("port number must be >= 1 and <= 65535");
+        }
+        this.port = port;
+    }
+
+    /** The hostname of this real */
+    public HostName hostname() {
+        return hostname;
+    }
+
+    /** Target IP address for this real */
+    public String ipAddress() {
+        return ipAddress;
+    }
+
+    /** Target port for this real */
+    public int port() {
+        return port;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Real real = (Real) o;
+        return port == real.port &&
+               Objects.equals(hostname, real.hostname) &&
+               Objects.equals(ipAddress, real.ipAddress);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(hostname, ipAddress, port);
+    }
+
+    @Override
+    public String toString() {
+        return "real server " + hostname + " (" + ipAddress + ":" + port + ")";
+    }
+
+    private static String requireIpAddress(String ipAddress) {
+        Objects.requireNonNull(ipAddress, "ipAddress must be non-null");
+        try {
+            InetAddresses.forString(ipAddress);
+            return ipAddress;
+        } catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("ipAddress must be a valid IP address", e);
+        }
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/package-info.java
new file mode 100644
index 00000000000..cdcf58b81e3
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/package-info.java
@@ -0,0 +1,8 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author mpolden
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.provision.lb;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index 632244cec69..d824f9fa53b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -16,6 +16,8 @@ import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.curator.transaction.CuratorOperations;
 import com.yahoo.vespa.curator.transaction.CuratorTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.Status;
 
@@ -34,6 +36,9 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
+import static java.util.stream.Collectors.collectingAndThen;
+import static java.util.stream.Collectors.toMap;
+
 /**
  * Client which reads and writes nodes to a curator database.
  * Nodes are stored in files named <code>/provision/v1/[nodestate]/[hostname]</code>.
@@ -49,6 +54,7 @@ public class CuratorDatabaseClient {
 
     private static final Path root = Path.fromString("/provision/v1");
     private static final Path lockRoot = root.append("locks");
+    private static final Path loadBalancersRoot = root.append("loadBalancers");
     private static final Duration defaultLockTimeout = Duration.ofMinutes(2);
 
     private final NodeSerializer nodeSerializer;
@@ -76,6 +82,7 @@ public class CuratorDatabaseClient {
         curatorDatabase.create(inactiveJobsPath());
         curatorDatabase.create(infrastructureVersionsPath());
         curatorDatabase.create(osVersionsPath());
+        curatorDatabase.create(loadBalancersRoot);
     }
 
     /**
@@ -400,4 +407,50 @@ public class CuratorDatabaseClient {
     private Path osVersionsPath() {
         return root.append("osVersions");
     }
+
+    public Map<LoadBalancerId, LoadBalancer> readLoadBalancers() {
+        return curatorDatabase.getChildren(loadBalancersRoot).stream()
+                              .map(LoadBalancerId::fromSerializedForm)
+                              .map(this::readLoadBalancer)
+                              .filter(Optional::isPresent)
+                              .map(Optional::get)
+                              .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()),
+                                                         Collections::unmodifiableMap));
+    }
+
+    public List<LoadBalancer> readLoadBalancers(ApplicationId application) {
+        return readLoadBalancers().values().stream()
+                                  .filter(lb -> lb.id().application().equals(application))
+                                  .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
+    }
+
+    public Optional<LoadBalancer> readLoadBalancer(LoadBalancerId id) {
+        return read(loadBalancerPath(id), LoadBalancerSerializer::fromJson);
+    }
+
+    public void writeLoadBalancer(LoadBalancer loadBalancer) {
+        Path path = loadBalancerPath(loadBalancer.id());
+        curatorDatabase.create(path);
+        NestedTransaction transaction = new NestedTransaction();
+        CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction);
+        curatorTransaction.add(CuratorOperations.setData(path.getAbsolute(),
+                                                         LoadBalancerSerializer.toJson(loadBalancer)));
+        transaction.commit();
+    }
+
+    public void removeLoadBalancer(LoadBalancer loadBalancer) {
+        NestedTransaction transaction = new NestedTransaction();
+        CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction);
+        curatorTransaction.add(CuratorOperations.delete(loadBalancerPath(loadBalancer.id()).getAbsolute()));
+        transaction.commit();
+    }
+
+    public Lock lockLoadBalancers() {
+        return lock(lockRoot.append("loadBalancersLock"), defaultLockTimeout);
+    }
+
+    private Path loadBalancerPath(LoadBalancerId id) {
+        return loadBalancersRoot.append(id.serializedForm());
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
new file mode 100644
index 00000000000..d3ee1100d12
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
@@ -0,0 +1,78 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.yahoo.config.provision.HostName;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
+import com.yahoo.vespa.hosted.provision.lb.Real;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Serializer for load balancers.
+ *
+ * @author mpolden
+ */
+public class LoadBalancerSerializer {
+
+    private static final String idField = "id";
+    private static final String hostnameField = "hostname";
+    private static final String deletedField = "deleted";
+    private static final String portsField = "ports";
+    private static final String realsField = "reals";
+    private static final String ipAddressField = "ipAddress";
+    private static final String portField = "port";
+
+    public static byte[] toJson(LoadBalancer loadBalancer) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+
+        root.setString(idField, loadBalancer.id().serializedForm());
+        root.setString(hostnameField, loadBalancer.hostname().toString());
+        Cursor portArray = root.setArray(portsField);
+        loadBalancer.ports().forEach(portArray::addLong);
+        Cursor realArray = root.setArray(realsField);
+        loadBalancer.reals().forEach(real -> {
+            Cursor realObject = realArray.addObject();
+            realObject.setString(hostnameField, real.hostname().value());
+            realObject.setString(ipAddressField, real.ipAddress());
+            realObject.setLong(portField, real.port());
+        });
+        root.setBool(deletedField, loadBalancer.deleted());
+
+        try {
+            return SlimeUtils.toJsonBytes(slime);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public static LoadBalancer fromJson(byte[] data) {
+        Cursor object = SlimeUtils.jsonToSlime(data).get();
+
+        List<Real> reals = new ArrayList<>();
+        object.field(realsField).traverse((ArrayTraverser) (i, realObject) -> {
+            reals.add(new Real(HostName.from(realObject.field(hostnameField).asString()),
+                               realObject.field(ipAddressField).asString(),
+                               (int) realObject.field(portField).asLong()));
+
+        });
+
+        List<Integer> ports = new ArrayList<>();
+        object.field(portsField).traverse((ArrayTraverser) (i, port) -> ports.add((int) port.asLong()));
+
+        return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()),
+                                HostName.from(object.field(hostnameField).asString()),
+                                ports,
+                                reals,
+                                object.field(deletedField).asBool());
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
new file mode 100644
index 00000000000..847d95de8bd
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
@@ -0,0 +1,125 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.google.common.net.InetAddresses;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.transaction.Mutex;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService;
+import com.yahoo.vespa.hosted.provision.lb.Real;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+
+import java.net.Inet4Address;
+import java.net.Inet6Address;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Provides provisioning of load balancers for applications.
+ *
+ * @author mpolden
+ */
+public class LoadBalancerProvisioner {
+
+    private final NodeRepository nodeRepository;
+    private final CuratorDatabaseClient db;
+    private final LoadBalancerService service;
+
+    public LoadBalancerProvisioner(NodeRepository nodeRepository, LoadBalancerService service) {
+        this.nodeRepository = nodeRepository;
+        this.db = nodeRepository.database();
+        this.service = service;
+    }
+
+    /**
+     * Provision load balancer(s) for given application.
+     *
+     * If the application has multiple container clusters, one load balancer will be provisioned for each cluster.
+     */
+    public Map<LoadBalancerId, LoadBalancer> provision(ApplicationId application) {
+        try (Mutex applicationLock = nodeRepository.lock(application)) {
+            try (Mutex loadBalancersLock = db.lockLoadBalancers()) {
+                Map<LoadBalancerId, LoadBalancer> loadBalancers = new LinkedHashMap<>();
+                for (Map.Entry<ClusterSpec.Id, List<Node>> kv : activeContainers(application).entrySet()) {
+                    LoadBalancer loadBalancer = create(application, kv.getKey(), kv.getValue());
+                    loadBalancers.put(loadBalancer.id(), loadBalancer);
+                    db.writeLoadBalancer(loadBalancer);
+                }
+                return Collections.unmodifiableMap(loadBalancers);
+            }
+        }
+    }
+
+    /** Mark all load balancers assigned to given application as deleted */
+    public void remove(ApplicationId application) {
+        try (Mutex applicationLock = nodeRepository.lock(application)) {
+            try (Mutex loadBalancersLock = db.lockLoadBalancers()) {
+                if (!activeContainers(application).isEmpty()) {
+                    throw new IllegalArgumentException(application + " has active containers, refusing to remove load balancers");
+                }
+                db.readLoadBalancers(application)
+                  .stream()
+                  .map(LoadBalancer::delete)
+                  .forEach(db::writeLoadBalancer);
+            }
+        }
+    }
+
+    private LoadBalancer create(ApplicationId application, ClusterSpec.Id cluster, List<Node> nodes) {
+        Map<HostName, Set<String>> hostnameToIpAdresses = nodes.stream()
+                                                               .collect(Collectors.toMap(node -> HostName.from(node.hostname()),
+                                                                                         this::reachableIpAddresses));
+        List<Real> reals = new ArrayList<>();
+        hostnameToIpAdresses.forEach((hostname, ipAddresses) -> {
+            ipAddresses.forEach(ipAddress -> reals.add(new Real(hostname, ipAddress)));
+        });
+        return service.create(application, cluster, reals);
+    }
+
+    /** Returns a list of active containers for given application, grouped by cluster ID */
+    private Map<ClusterSpec.Id, List<Node>> activeContainers(ApplicationId application) {
+        return new NodeList(nodeRepository.getNodes())
+                .owner(application)
+                .in(Node.State.active)
+                .type(ClusterSpec.Type.container)
+                .asList()
+                .stream()
+                .collect(Collectors.groupingBy(n -> n.allocation().get().membership().cluster().id()));
+    }
+
+    /** Find IP addresses reachable by the load balancer service */
+    private Set<String> reachableIpAddresses(Node node) {
+        Set<String> reachable = new LinkedHashSet<>(node.ipAddresses());
+        // Remove addresses unreachable by the load balancer service
+        switch (service.protocol()) {
+            case ipv4:
+                reachable.removeIf(this::isIpv6);
+                break;
+            case ipv6:
+                reachable.removeIf(this::isIpv4);
+                break;
+        }
+        return reachable;
+    }
+
+    private boolean isIpv4(String ipAddress) {
+        return InetAddresses.forString(ipAddress) instanceof Inet4Address;
+    }
+
+    private boolean isIpv6(String ipAddress) {
+        return InetAddresses.forString(ipAddress) instanceof Inet6Address;
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/LoadBalancerServiceMock.java
new file mode 100644
index 00000000000..c4e595a3fcf
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/LoadBalancerServiceMock.java
@@ -0,0 +1,46 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.testutils;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService;
+import com.yahoo.vespa.hosted.provision.lb.Real;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author mpolden
+ */
+public class LoadBalancerServiceMock implements LoadBalancerService {
+
+    private final Map<LoadBalancerId, LoadBalancer> loadBalancers = new HashMap<>();
+
+    @Override
+    public Protocol protocol() {
+        return Protocol.ipv4;
+    }
+
+    @Override
+    public LoadBalancer create(ApplicationId application, ClusterSpec.Id cluster, List<Real> reals) {
+        LoadBalancer loadBalancer = new LoadBalancer(
+                new LoadBalancerId(application, cluster),
+                HostName.from("lb-" + application.toShortString() + "-" + cluster.value()),
+                Collections.singletonList(4443),
+                reals,
+                false);
+        loadBalancers.put(loadBalancer.id(), loadBalancer);
+        return loadBalancer;
+    }
+
+    @Override
+    public void remove(LoadBalancerId loadBalancer) {
+        loadBalancers.remove(loadBalancer);
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java
new file mode 100644
index 00000000000..2dc3288194b
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java
@@ -0,0 +1,45 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
+import com.yahoo.vespa.hosted.provision.lb.Real;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mpolden
+ */
+public class LoadBalancerSerializerTest {
+
+    @Test
+    public void test_serialization() {
+        LoadBalancer loadBalancer = new LoadBalancer(new LoadBalancerId(ApplicationId.from("tenant1",
+                                                                                           "application1",
+                                                                                           "default"),
+                                                                        ClusterSpec.Id.from("qrs")),
+                                                     HostName.from("lb-host"),
+                                                     Arrays.asList(4080, 4443),
+                                                     Arrays.asList(new Real(HostName.from("real-1"),
+                                                                            "127.0.0.1",
+                                                                            4080),
+                                                                   new Real(HostName.from("real-2"),
+                                                                            "127.0.0.2",
+                                                                            4080)),
+                                                     false);
+
+        LoadBalancer serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer));
+        assertEquals(loadBalancer.id(), serialized.id());
+        assertEquals(loadBalancer.hostname(), serialized.hostname());
+        assertEquals(loadBalancer.ports(), serialized.ports());
+        assertEquals(loadBalancer.deleted(), serialized.deleted());
+        assertEquals(loadBalancer.reals(), serialized.reals());
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
new file mode 100644
index 00000000000..e4b3d0be526
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
@@ -0,0 +1,150 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService;
+import com.yahoo.vespa.hosted.provision.lb.Real;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author mpolden
+ */
+public class LoadBalancerProvisionerTest {
+
+    private final ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default");
+    private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default");
+
+    private ProvisioningTester tester;
+    private LoadBalancerService service;
+    private LoadBalancerProvisioner loadBalancerProvisioner;
+
+    @Before
+    public void before() {
+        tester = new ProvisioningTester(Zone.defaultZone());
+        service = tester.loadBalancerService();
+        loadBalancerProvisioner = new LoadBalancerProvisioner(tester.nodeRepository(), service);
+    }
+
+    @Test
+    public void provision_load_balancer() {
+        ClusterSpec.Id containerCluster1 = ClusterSpec.Id.from("qrs1");
+        ClusterSpec.Id contentCluster = ClusterSpec.Id.from("content");
+        tester.activate(app1, prepare(app1,
+                                      clusterRequest(ClusterSpec.Type.container, containerCluster1),
+                                      clusterRequest(ClusterSpec.Type.content, contentCluster)));
+        tester.activate(app2, prepare(app2,
+                                      clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs"))));
+
+        // Provision a load balancer for each application
+        Map<LoadBalancerId, LoadBalancer> loadBalancers = loadBalancerProvisioner.provision(app1);
+        loadBalancerProvisioner.provision(app2);
+        assertEquals(1, loadBalancers.size());
+
+        LoadBalancer loadBalancer = loadBalancers.values().iterator().next();
+        assertEquals(loadBalancer.id().application(), app1);
+        assertEquals(loadBalancer.id().cluster(), containerCluster1);
+        assertEquals(loadBalancer.ports(), Collections.singletonList(4443));
+        assertEquals(loadBalancer.reals().get(0).ipAddress(), "127.0.0.1");
+        assertEquals(loadBalancer.reals().get(0).port(), 4443);
+        assertEquals(loadBalancer.reals().get(1).ipAddress(), "127.0.0.2");
+        assertEquals(loadBalancer.reals().get(1).port(), 4443);
+
+        // A container is failed
+        List<Node> containers = tester.getNodes(app1).type(ClusterSpec.Type.container).asList();
+        Node container1 = containers.get(0);
+        Node container2 = containers.get(1);
+        tester.nodeRepository().fail(container1.hostname(), Agent.system, "Failed by unit test");
+
+        // Reprovisioning load balancer removes failed container
+        loadBalancer = loadBalancerProvisioner.provision(app1).values().iterator().next();
+        assertEquals(1, loadBalancer.reals().size());
+        assertEquals(container2.hostname(), loadBalancer.reals().get(0).hostname().value());
+
+        // Redeploying replaces failed node
+        tester.activate(app1, prepare(app1,
+                                      clusterRequest(ClusterSpec.Type.container, containerCluster1),
+                                      clusterRequest(ClusterSpec.Type.content, contentCluster)));
+
+        // Reprovisioning load balancer adds the new node
+        Node container3 = tester.getNodes(app1).type(ClusterSpec.Type.container).asList().get(1);
+        loadBalancer = loadBalancerProvisioner.provision(app1).values().iterator().next();
+        assertEquals(2, loadBalancer.reals().size());
+        assertEquals(container3.hostname(), loadBalancer.reals().get(1).hostname().value());
+
+        // Add another container cluster
+        ClusterSpec.Id containerCluster2 = ClusterSpec.Id.from("qrs2");
+        tester.activate(app1, prepare(app1,
+                                      clusterRequest(ClusterSpec.Type.container, containerCluster1),
+                                      clusterRequest(ClusterSpec.Type.container, containerCluster2),
+                                      clusterRequest(ClusterSpec.Type.content, contentCluster)));
+
+        // Load balancer is provisioned for second container cluster
+        loadBalancers = loadBalancerProvisioner.provision(app1);
+        assertEquals(2, loadBalancers.size());
+        List<HostName> activeContainers = tester.getNodes(app1, Node.State.active)
+                                                .type(ClusterSpec.Type.container).asList()
+                                                .stream()
+                                                .map(Node::hostname)
+                                                .map(HostName::from)
+                                                .sorted()
+                                                .collect(Collectors.toList());
+        List<HostName> reals = loadBalancers.values().stream()
+                                            .flatMap(lb -> lb.reals().stream())
+                                            .map(Real::hostname)
+                                            .sorted()
+                                            .collect(Collectors.toList());
+        assertEquals(activeContainers, reals);
+
+        // Removing load balancer with active containers fails
+        try {
+            loadBalancerProvisioner.remove(app1);
+            fail("Expected exception");
+        } catch (IllegalArgumentException ignored) {}
+
+        // Application and load balancer is removed
+        NestedTransaction removeTransaction = new NestedTransaction();
+        tester.provisioner().remove(removeTransaction, app1);
+        removeTransaction.commit();
+
+        loadBalancerProvisioner.remove(app1);
+        List<LoadBalancer> assignedLoadBalancer = tester.nodeRepository().database().readLoadBalancers(app1);
+        assertEquals(2, loadBalancers.size());
+        assertTrue("Load balancers marked for deletion", assignedLoadBalancer.stream().allMatch(LoadBalancer::deleted));
+    }
+
+    private ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id) {
+        return ClusterSpec.request(type, id, Version.fromString("6.42"), false);
+    }
+
+    private Set<HostSpec> prepare(ApplicationId application, ClusterSpec... specs) {
+        tester.makeReadyNodes(specs.length * 2, "default");
+        Set<HostSpec> allNodes = new LinkedHashSet<>();
+        for (ClusterSpec spec : specs) {
+            allNodes.addAll(tester.prepare(application, spec, 2, 1, "default"));
+        }
+        return allNodes;
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index 81414c0ac2d..8c0937d34b8 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter;
 import com.yahoo.vespa.hosted.provision.persistence.NameResolver;
+import com.yahoo.vespa.hosted.provision.testutils.LoadBalancerServiceMock;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.service.monitor.application.ConfigServerApplication;
@@ -68,6 +69,7 @@ public class ProvisioningTester {
     private final NodeRepositoryProvisioner provisioner;
     private final CapacityPolicies capacityPolicies;
     private final ProvisionLogger provisionLogger;
+    private final LoadBalancerServiceMock loadBalancerService;
 
     private int nextHost = 0;
     private int nextIP = 0;
@@ -90,6 +92,7 @@ public class ProvisioningTester {
             this.orchestrator = mock(Orchestrator.class);
             doThrow(new RuntimeException()).when(orchestrator).acquirePermissionToRemove(any());
             this.provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone);
+            this.loadBalancerService = new LoadBalancerServiceMock();
             this.capacityPolicies = new CapacityPolicies(zone, nodeFlavors);
             this.provisionLogger = new NullProvisionLogger();
         }
@@ -126,6 +129,7 @@ public class ProvisioningTester {
     public Orchestrator orchestrator() { return orchestrator; }
     public ManualClock clock() { return clock; }
     public NodeRepositoryProvisioner provisioner() { return provisioner; }
+    public LoadBalancerServiceMock loadBalancerService() { return loadBalancerService; }
     public CapacityPolicies capacityPolicies() { return capacityPolicies; }
     public NodeList getNodes(ApplicationId id, Node.State ... inState) { return new NodeList(nodeRepository.getNodes(id, inState)); }