diff options
author | Martin Polden <mpolden@mpolden.no> | 2018-11-22 14:07:40 +0100 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2018-11-23 11:16:57 +0100 |
commit | 6f2dc7361f36eeb661da6fc8195be502855e4ec5 (patch) | |
tree | 1198c915dc43cda26e023d877de39332a119342a /node-repository | |
parent | 0713e24de6a12c3584112c9af60dc4105d2056d8 (diff) |
Add load balancer provisioner
Diffstat (limited to 'node-repository')
14 files changed, 816 insertions, 5 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index acf62ae91b9..398f3fceca8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; import com.google.common.collect.ImmutableList; @@ -52,18 +52,25 @@ public class NodeList { return new NodeList(nodes.stream().filter(node -> node.allocation().get().membership().cluster().type().equals(type)).collect(Collectors.toList())); } + /** Returns the subset of nodes that are in the given state */ + public NodeList in(Node.State state) { + return nodes.stream() + .filter(node -> node.state() == state) + .collect(collectingAndThen(Collectors.toList(), NodeList::new)); + } + /** Returns the subset of nodes owned by the given application */ public NodeList owner(ApplicationId application) { return nodes.stream() - .filter(node -> node.allocation().map(a -> a.owner().equals(application)).orElse(false)) - .collect(collectingAndThen(Collectors.toList(), NodeList::new)); + .filter(node -> node.allocation().map(a -> a.owner().equals(application)).orElse(false)) + .collect(collectingAndThen(Collectors.toList(), NodeList::new)); } /** Returns the subset of nodes matching the given node type */ public NodeList nodeType(NodeType nodeType) { return nodes.stream() - .filter(node -> node.type() == nodeType) - .collect(collectingAndThen(Collectors.toList(), NodeList::new)); + .filter(node -> node.type() == nodeType) + .collect(collectingAndThen(Collectors.toList(), NodeList::new)); } /** Returns the parent nodes of the given child nodes */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java new file mode 100644 index 00000000000..07bd1ea06ed --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java @@ -0,0 +1,73 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.lb; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Ordering; +import com.yahoo.config.provision.HostName; + +import java.util.List; +import java.util.Objects; + +/** + * Represents a load balancer for an application. + * + * @author mpolden + */ +public class LoadBalancer { + + private final LoadBalancerId id; + private final HostName hostname; + private final List<Integer> ports; + private final List<Real> reals; + private final boolean deleted; + + public LoadBalancer(LoadBalancerId id, HostName hostname, List<Integer> ports, List<Real> reals, boolean deleted) { + this.id = Objects.requireNonNull(id, "id must be non-null"); + this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); + this.ports = Ordering.natural().immutableSortedCopy(requirePorts(ports)); + this.reals = ImmutableList.copyOf(Objects.requireNonNull(reals, "targets must be non-null")); + this.deleted = deleted; + } + + /** An identifier for this load balancer. The ID is unique inside the zone */ + public LoadBalancerId id() { + return id; + } + + /** Fully-qualified domain name of this load balancer. This hostname can be used for query and feed. */ + public HostName hostname() { + return hostname; + } + + /** Listening port(s) of this load balancer */ + public List<Integer> ports() { + return ports; + } + + /** Real servers behind this load balancer */ + public List<Real> reals() { + return reals; + } + + /** Whether this load balancer has been deleted */ + public boolean deleted() { + return deleted; + } + + /** Return a copy of this that is marked as deleted */ + public LoadBalancer delete() { + return new LoadBalancer(id, hostname, ports, reals, true); + } + + private static List<Integer> requirePorts(List<Integer> ports) { + Objects.requireNonNull(ports, "ports must be non-null"); + if (ports.isEmpty()) { + throw new IllegalArgumentException("ports must be non-empty"); + } + if (!ports.stream().allMatch(port -> port >= 1 && port <= 65535)) { + throw new IllegalArgumentException("all ports must be >= 1 and <= 65535"); + } + return ports; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java new file mode 100644 index 00000000000..1431f21de47 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java @@ -0,0 +1,65 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.lb; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; + +import java.util.Objects; + +/** + * Uniquely identifies a load balancer for an application's container cluster. + * + * @author mpolden + */ +public class LoadBalancerId { + + private final ApplicationId application; + private final ClusterSpec.Id cluster; + private final String serializedForm; + + public LoadBalancerId(ApplicationId application, ClusterSpec.Id cluster) { + this.application = Objects.requireNonNull(application, "application must be non-null"); + this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); + this.serializedForm = serializedForm(application, cluster); + } + + public ApplicationId application() { + return application; + } + + public ClusterSpec.Id cluster() { + return cluster; + } + + /** Serialized form of this */ + public String serializedForm() { + return serializedForm; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LoadBalancerId that = (LoadBalancerId) o; + return Objects.equals(application, that.application) && + Objects.equals(cluster, that.cluster); + } + + @Override + public int hashCode() { + return Objects.hash(application, cluster); + } + + /** Create an instance from a serialized value on the form tenant:application:instance:cluster-id */ + public static LoadBalancerId fromSerializedForm(String value) { + int lastSeparator = value.lastIndexOf(":"); + ApplicationId application = ApplicationId.fromSerializedForm(value.substring(0, lastSeparator)); + ClusterSpec.Id cluster = ClusterSpec.Id.from(value.substring(lastSeparator + 1)); + return new LoadBalancerId(application, cluster); + } + + private static String serializedForm(ApplicationId application, ClusterSpec.Id cluster) { + return application.serializedForm() + ":" + cluster.value(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java new file mode 100644 index 00000000000..b5f59414c65 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.lb; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; + +import java.util.List; + +/** + * A managed load balance service. + * + * @author mpolden + */ +public interface LoadBalancerService { + + /** Create a load balancer for given application cluster. Implementations are expected to be idempotent */ + LoadBalancer create(ApplicationId application, ClusterSpec.Id cluster, List<Real> reals); + + /** Permanently remove load balancer with given ID */ + void remove(LoadBalancerId loadBalancer); + + /** Returns the protocol supported by this load balancer service */ + Protocol protocol(); + + /** Load balancer protocols */ + enum Protocol { + ipv4, + ipv6, + dualstack + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceProvider.java new file mode 100644 index 00000000000..7f37798b977 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceProvider.java @@ -0,0 +1,45 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.lb; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.container.di.componentgraph.Provider; + +import java.util.List; + +/** + * A provider for a {@link LoadBalancerService}. This provides a default instance for cases where a component has not + * been explicitly configured. + * + * @author mpolden + */ +public class LoadBalancerServiceProvider implements Provider<LoadBalancerService> { + + private static final LoadBalancerService instance = new LoadBalancerService() { + + @Override + public LoadBalancer create(ApplicationId application, ClusterSpec.Id cluster, List<Real> reals) { + throw new UnsupportedOperationException(); + } + + @Override + public void remove(LoadBalancerId loadBalancer) { + throw new UnsupportedOperationException(); + } + + @Override + public Protocol protocol() { + throw new UnsupportedOperationException(); + } + + }; + + @Override + public LoadBalancerService get() { + return instance; + } + + @Override + public void deconstruct() {} + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/Real.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/Real.java new file mode 100644 index 00000000000..784d58f103e --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/Real.java @@ -0,0 +1,80 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.lb; + +import com.google.common.net.InetAddresses; +import com.yahoo.config.provision.HostName; + +import java.util.Objects; + +/** + * Represents a server behind a load balancer. + * + * @author mpolden + */ +public class Real { + + private static int defaultPort = 4443; + + private final HostName hostname; + private final String ipAddress; + private final int port; + + public Real(HostName hostname, String ipAddress) { + this(hostname, ipAddress, defaultPort); + } + + public Real(HostName hostname, String ipAddress, int port) { + this.hostname = hostname; + this.ipAddress = requireIpAddress(ipAddress); + if (port < 1 || port > 65535) { + throw new IllegalArgumentException("port number must be >= 1 and <= 65535"); + } + this.port = port; + } + + /** The hostname of this real */ + public HostName hostname() { + return hostname; + } + + /** Target IP address for this real */ + public String ipAddress() { + return ipAddress; + } + + /** Target port for this real */ + public int port() { + return port; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Real real = (Real) o; + return port == real.port && + Objects.equals(hostname, real.hostname) && + Objects.equals(ipAddress, real.ipAddress); + } + + @Override + public int hashCode() { + return Objects.hash(hostname, ipAddress, port); + } + + @Override + public String toString() { + return "real server " + hostname + " (" + ipAddress + ":" + port + ")"; + } + + private static String requireIpAddress(String ipAddress) { + Objects.requireNonNull(ipAddress, "ipAddress must be non-null"); + try { + InetAddresses.forString(ipAddress); + return ipAddress; + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("ipAddress must be a valid IP address", e); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/package-info.java new file mode 100644 index 00000000000..cdcf58b81e3 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author mpolden + */ +@ExportPackage +package com.yahoo.vespa.hosted.provision.lb; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 632244cec69..d824f9fa53b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -16,6 +16,8 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Status; @@ -34,6 +36,9 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static java.util.stream.Collectors.collectingAndThen; +import static java.util.stream.Collectors.toMap; + /** * Client which reads and writes nodes to a curator database. * Nodes are stored in files named <code>/provision/v1/[nodestate]/[hostname]</code>. @@ -49,6 +54,7 @@ public class CuratorDatabaseClient { private static final Path root = Path.fromString("/provision/v1"); private static final Path lockRoot = root.append("locks"); + private static final Path loadBalancersRoot = root.append("loadBalancers"); private static final Duration defaultLockTimeout = Duration.ofMinutes(2); private final NodeSerializer nodeSerializer; @@ -76,6 +82,7 @@ public class CuratorDatabaseClient { curatorDatabase.create(inactiveJobsPath()); curatorDatabase.create(infrastructureVersionsPath()); curatorDatabase.create(osVersionsPath()); + curatorDatabase.create(loadBalancersRoot); } /** @@ -400,4 +407,50 @@ public class CuratorDatabaseClient { private Path osVersionsPath() { return root.append("osVersions"); } + + public Map<LoadBalancerId, LoadBalancer> readLoadBalancers() { + return curatorDatabase.getChildren(loadBalancersRoot).stream() + .map(LoadBalancerId::fromSerializedForm) + .map(this::readLoadBalancer) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()), + Collections::unmodifiableMap)); + } + + public List<LoadBalancer> readLoadBalancers(ApplicationId application) { + return readLoadBalancers().values().stream() + .filter(lb -> lb.id().application().equals(application)) + .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + + public Optional<LoadBalancer> readLoadBalancer(LoadBalancerId id) { + return read(loadBalancerPath(id), LoadBalancerSerializer::fromJson); + } + + public void writeLoadBalancer(LoadBalancer loadBalancer) { + Path path = loadBalancerPath(loadBalancer.id()); + curatorDatabase.create(path); + NestedTransaction transaction = new NestedTransaction(); + CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + curatorTransaction.add(CuratorOperations.setData(path.getAbsolute(), + LoadBalancerSerializer.toJson(loadBalancer))); + transaction.commit(); + } + + public void removeLoadBalancer(LoadBalancer loadBalancer) { + NestedTransaction transaction = new NestedTransaction(); + CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + curatorTransaction.add(CuratorOperations.delete(loadBalancerPath(loadBalancer.id()).getAbsolute())); + transaction.commit(); + } + + public Lock lockLoadBalancers() { + return lock(lockRoot.append("loadBalancersLock"), defaultLockTimeout); + } + + private Path loadBalancerPath(LoadBalancerId id) { + return loadBalancersRoot.append(id.serializedForm()); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java new file mode 100644 index 00000000000..d3ee1100d12 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -0,0 +1,78 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.HostName; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; +import com.yahoo.vespa.hosted.provision.lb.Real; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Serializer for load balancers. + * + * @author mpolden + */ +public class LoadBalancerSerializer { + + private static final String idField = "id"; + private static final String hostnameField = "hostname"; + private static final String deletedField = "deleted"; + private static final String portsField = "ports"; + private static final String realsField = "reals"; + private static final String ipAddressField = "ipAddress"; + private static final String portField = "port"; + + public static byte[] toJson(LoadBalancer loadBalancer) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + root.setString(idField, loadBalancer.id().serializedForm()); + root.setString(hostnameField, loadBalancer.hostname().toString()); + Cursor portArray = root.setArray(portsField); + loadBalancer.ports().forEach(portArray::addLong); + Cursor realArray = root.setArray(realsField); + loadBalancer.reals().forEach(real -> { + Cursor realObject = realArray.addObject(); + realObject.setString(hostnameField, real.hostname().value()); + realObject.setString(ipAddressField, real.ipAddress()); + realObject.setLong(portField, real.port()); + }); + root.setBool(deletedField, loadBalancer.deleted()); + + try { + return SlimeUtils.toJsonBytes(slime); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static LoadBalancer fromJson(byte[] data) { + Cursor object = SlimeUtils.jsonToSlime(data).get(); + + List<Real> reals = new ArrayList<>(); + object.field(realsField).traverse((ArrayTraverser) (i, realObject) -> { + reals.add(new Real(HostName.from(realObject.field(hostnameField).asString()), + realObject.field(ipAddressField).asString(), + (int) realObject.field(portField).asLong())); + + }); + + List<Integer> ports = new ArrayList<>(); + object.field(portsField).traverse((ArrayTraverser) (i, port) -> ports.add((int) port.asLong())); + + return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()), + HostName.from(object.field(hostnameField).asString()), + ports, + reals, + object.field(deletedField).asBool()); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java new file mode 100644 index 00000000000..847d95de8bd --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -0,0 +1,125 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.google.common.net.InetAddresses; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; +import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; +import com.yahoo.vespa.hosted.provision.lb.Real; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; + +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Provides provisioning of load balancers for applications. + * + * @author mpolden + */ +public class LoadBalancerProvisioner { + + private final NodeRepository nodeRepository; + private final CuratorDatabaseClient db; + private final LoadBalancerService service; + + public LoadBalancerProvisioner(NodeRepository nodeRepository, LoadBalancerService service) { + this.nodeRepository = nodeRepository; + this.db = nodeRepository.database(); + this.service = service; + } + + /** + * Provision load balancer(s) for given application. + * + * If the application has multiple container clusters, one load balancer will be provisioned for each cluster. + */ + public Map<LoadBalancerId, LoadBalancer> provision(ApplicationId application) { + try (Mutex applicationLock = nodeRepository.lock(application)) { + try (Mutex loadBalancersLock = db.lockLoadBalancers()) { + Map<LoadBalancerId, LoadBalancer> loadBalancers = new LinkedHashMap<>(); + for (Map.Entry<ClusterSpec.Id, List<Node>> kv : activeContainers(application).entrySet()) { + LoadBalancer loadBalancer = create(application, kv.getKey(), kv.getValue()); + loadBalancers.put(loadBalancer.id(), loadBalancer); + db.writeLoadBalancer(loadBalancer); + } + return Collections.unmodifiableMap(loadBalancers); + } + } + } + + /** Mark all load balancers assigned to given application as deleted */ + public void remove(ApplicationId application) { + try (Mutex applicationLock = nodeRepository.lock(application)) { + try (Mutex loadBalancersLock = db.lockLoadBalancers()) { + if (!activeContainers(application).isEmpty()) { + throw new IllegalArgumentException(application + " has active containers, refusing to remove load balancers"); + } + db.readLoadBalancers(application) + .stream() + .map(LoadBalancer::delete) + .forEach(db::writeLoadBalancer); + } + } + } + + private LoadBalancer create(ApplicationId application, ClusterSpec.Id cluster, List<Node> nodes) { + Map<HostName, Set<String>> hostnameToIpAdresses = nodes.stream() + .collect(Collectors.toMap(node -> HostName.from(node.hostname()), + this::reachableIpAddresses)); + List<Real> reals = new ArrayList<>(); + hostnameToIpAdresses.forEach((hostname, ipAddresses) -> { + ipAddresses.forEach(ipAddress -> reals.add(new Real(hostname, ipAddress))); + }); + return service.create(application, cluster, reals); + } + + /** Returns a list of active containers for given application, grouped by cluster ID */ + private Map<ClusterSpec.Id, List<Node>> activeContainers(ApplicationId application) { + return new NodeList(nodeRepository.getNodes()) + .owner(application) + .in(Node.State.active) + .type(ClusterSpec.Type.container) + .asList() + .stream() + .collect(Collectors.groupingBy(n -> n.allocation().get().membership().cluster().id())); + } + + /** Find IP addresses reachable by the load balancer service */ + private Set<String> reachableIpAddresses(Node node) { + Set<String> reachable = new LinkedHashSet<>(node.ipAddresses()); + // Remove addresses unreachable by the load balancer service + switch (service.protocol()) { + case ipv4: + reachable.removeIf(this::isIpv6); + break; + case ipv6: + reachable.removeIf(this::isIpv4); + break; + } + return reachable; + } + + private boolean isIpv4(String ipAddress) { + return InetAddresses.forString(ipAddress) instanceof Inet4Address; + } + + private boolean isIpv6(String ipAddress) { + return InetAddresses.forString(ipAddress) instanceof Inet6Address; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/LoadBalancerServiceMock.java new file mode 100644 index 00000000000..c4e595a3fcf --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/LoadBalancerServiceMock.java @@ -0,0 +1,46 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.testutils; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; +import com.yahoo.vespa.hosted.provision.lb.Real; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author mpolden + */ +public class LoadBalancerServiceMock implements LoadBalancerService { + + private final Map<LoadBalancerId, LoadBalancer> loadBalancers = new HashMap<>(); + + @Override + public Protocol protocol() { + return Protocol.ipv4; + } + + @Override + public LoadBalancer create(ApplicationId application, ClusterSpec.Id cluster, List<Real> reals) { + LoadBalancer loadBalancer = new LoadBalancer( + new LoadBalancerId(application, cluster), + HostName.from("lb-" + application.toShortString() + "-" + cluster.value()), + Collections.singletonList(4443), + reals, + false); + loadBalancers.put(loadBalancer.id(), loadBalancer); + return loadBalancer; + } + + @Override + public void remove(LoadBalancerId loadBalancer) { + loadBalancers.remove(loadBalancer); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java new file mode 100644 index 00000000000..2dc3288194b --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -0,0 +1,45 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; +import com.yahoo.vespa.hosted.provision.lb.Real; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class LoadBalancerSerializerTest { + + @Test + public void test_serialization() { + LoadBalancer loadBalancer = new LoadBalancer(new LoadBalancerId(ApplicationId.from("tenant1", + "application1", + "default"), + ClusterSpec.Id.from("qrs")), + HostName.from("lb-host"), + Arrays.asList(4080, 4443), + Arrays.asList(new Real(HostName.from("real-1"), + "127.0.0.1", + 4080), + new Real(HostName.from("real-2"), + "127.0.0.2", + 4080)), + false); + + LoadBalancer serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer)); + assertEquals(loadBalancer.id(), serialized.id()); + assertEquals(loadBalancer.hostname(), serialized.hostname()); + assertEquals(loadBalancer.ports(), serialized.ports()); + assertEquals(loadBalancer.deleted(), serialized.deleted()); + assertEquals(loadBalancer.reals(), serialized.reals()); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java new file mode 100644 index 00000000000..e4b3d0be526 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -0,0 +1,150 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.Zone; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; +import com.yahoo.vespa.hosted.provision.lb.Real; +import com.yahoo.vespa.hosted.provision.node.Agent; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author mpolden + */ +public class LoadBalancerProvisionerTest { + + private final ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default"); + private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default"); + + private ProvisioningTester tester; + private LoadBalancerService service; + private LoadBalancerProvisioner loadBalancerProvisioner; + + @Before + public void before() { + tester = new ProvisioningTester(Zone.defaultZone()); + service = tester.loadBalancerService(); + loadBalancerProvisioner = new LoadBalancerProvisioner(tester.nodeRepository(), service); + } + + @Test + public void provision_load_balancer() { + ClusterSpec.Id containerCluster1 = ClusterSpec.Id.from("qrs1"); + ClusterSpec.Id contentCluster = ClusterSpec.Id.from("content"); + tester.activate(app1, prepare(app1, + clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.content, contentCluster))); + tester.activate(app2, prepare(app2, + clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs")))); + + // Provision a load balancer for each application + Map<LoadBalancerId, LoadBalancer> loadBalancers = loadBalancerProvisioner.provision(app1); + loadBalancerProvisioner.provision(app2); + assertEquals(1, loadBalancers.size()); + + LoadBalancer loadBalancer = loadBalancers.values().iterator().next(); + assertEquals(loadBalancer.id().application(), app1); + assertEquals(loadBalancer.id().cluster(), containerCluster1); + assertEquals(loadBalancer.ports(), Collections.singletonList(4443)); + assertEquals(loadBalancer.reals().get(0).ipAddress(), "127.0.0.1"); + assertEquals(loadBalancer.reals().get(0).port(), 4443); + assertEquals(loadBalancer.reals().get(1).ipAddress(), "127.0.0.2"); + assertEquals(loadBalancer.reals().get(1).port(), 4443); + + // A container is failed + List<Node> containers = tester.getNodes(app1).type(ClusterSpec.Type.container).asList(); + Node container1 = containers.get(0); + Node container2 = containers.get(1); + tester.nodeRepository().fail(container1.hostname(), Agent.system, "Failed by unit test"); + + // Reprovisioning load balancer removes failed container + loadBalancer = loadBalancerProvisioner.provision(app1).values().iterator().next(); + assertEquals(1, loadBalancer.reals().size()); + assertEquals(container2.hostname(), loadBalancer.reals().get(0).hostname().value()); + + // Redeploying replaces failed node + tester.activate(app1, prepare(app1, + clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.content, contentCluster))); + + // Reprovisioning load balancer adds the new node + Node container3 = tester.getNodes(app1).type(ClusterSpec.Type.container).asList().get(1); + loadBalancer = loadBalancerProvisioner.provision(app1).values().iterator().next(); + assertEquals(2, loadBalancer.reals().size()); + assertEquals(container3.hostname(), loadBalancer.reals().get(1).hostname().value()); + + // Add another container cluster + ClusterSpec.Id containerCluster2 = ClusterSpec.Id.from("qrs2"); + tester.activate(app1, prepare(app1, + clusterRequest(ClusterSpec.Type.container, containerCluster1), + clusterRequest(ClusterSpec.Type.container, containerCluster2), + clusterRequest(ClusterSpec.Type.content, contentCluster))); + + // Load balancer is provisioned for second container cluster + loadBalancers = loadBalancerProvisioner.provision(app1); + assertEquals(2, loadBalancers.size()); + List<HostName> activeContainers = tester.getNodes(app1, Node.State.active) + .type(ClusterSpec.Type.container).asList() + .stream() + .map(Node::hostname) + .map(HostName::from) + .sorted() + .collect(Collectors.toList()); + List<HostName> reals = loadBalancers.values().stream() + .flatMap(lb -> lb.reals().stream()) + .map(Real::hostname) + .sorted() + .collect(Collectors.toList()); + assertEquals(activeContainers, reals); + + // Removing load balancer with active containers fails + try { + loadBalancerProvisioner.remove(app1); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + + // Application and load balancer is removed + NestedTransaction removeTransaction = new NestedTransaction(); + tester.provisioner().remove(removeTransaction, app1); + removeTransaction.commit(); + + loadBalancerProvisioner.remove(app1); + List<LoadBalancer> assignedLoadBalancer = tester.nodeRepository().database().readLoadBalancers(app1); + assertEquals(2, loadBalancers.size()); + assertTrue("Load balancers marked for deletion", assignedLoadBalancer.stream().allMatch(LoadBalancer::deleted)); + } + + private ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id) { + return ClusterSpec.request(type, id, Version.fromString("6.42"), false); + } + + private Set<HostSpec> prepare(ApplicationId application, ClusterSpec... specs) { + tester.makeReadyNodes(specs.length * 2, "default"); + Set<HostSpec> allNodes = new LinkedHashSet<>(); + for (ClusterSpec spec : specs) { + allNodes.addAll(tester.prepare(application, spec, 2, 1, "default")); + } + return allNodes; + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 81414c0ac2d..8c0937d34b8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; +import com.yahoo.vespa.hosted.provision.testutils.LoadBalancerServiceMock; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; @@ -68,6 +69,7 @@ public class ProvisioningTester { private final NodeRepositoryProvisioner provisioner; private final CapacityPolicies capacityPolicies; private final ProvisionLogger provisionLogger; + private final LoadBalancerServiceMock loadBalancerService; private int nextHost = 0; private int nextIP = 0; @@ -90,6 +92,7 @@ public class ProvisioningTester { this.orchestrator = mock(Orchestrator.class); doThrow(new RuntimeException()).when(orchestrator).acquirePermissionToRemove(any()); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone); + this.loadBalancerService = new LoadBalancerServiceMock(); this.capacityPolicies = new CapacityPolicies(zone, nodeFlavors); this.provisionLogger = new NullProvisionLogger(); } @@ -126,6 +129,7 @@ public class ProvisioningTester { public Orchestrator orchestrator() { return orchestrator; } public ManualClock clock() { return clock; } public NodeRepositoryProvisioner provisioner() { return provisioner; } + public LoadBalancerServiceMock loadBalancerService() { return loadBalancerService; } public CapacityPolicies capacityPolicies() { return capacityPolicies; } public NodeList getNodes(ApplicationId id, Node.State ... inState) { return new NodeList(nodeRepository.getNodes(id, inState)); } |