diff options
Diffstat (limited to 'node-repository')
16 files changed, 157 insertions, 74 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index 09723d83e3e..edf2932ad6e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -3,9 +3,7 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.provision.NodeRepository; - -import java.util.Set; +import com.yahoo.config.provision.NodeType; /** * A managed load balance service. @@ -15,17 +13,14 @@ import java.util.Set; public interface LoadBalancerService { /** - * Create a load balancer for given application cluster. Implementations are expected to be idempotent + * Create a load balancer from the given specification. Implementations are expected to be idempotent * - * @param application Application owning the LB - * @param cluster Target cluster of the LB - * @param reals Reals that should be configured on the LB + * @param spec Load balancer specification * @param force Whether reconfiguration should be forced (e.g. allow configuring an empty set of reals on a * pre-existing load balancer). * @return The provisioned load balancer instance */ - LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, boolean force, - NodeRepository nodeRepository); + LoadBalancerInstance create(LoadBalancerSpec spec, boolean force); /** Permanently remove load balancer for given application cluster */ void remove(ApplicationId application, ClusterSpec.Id cluster); @@ -33,6 +28,12 @@ public interface LoadBalancerService { /** Returns the protocol supported by this load balancer service */ Protocol protocol(); + /** Returns whether load balancers created by this service can forward traffic to given node and cluster type */ + default boolean canForwardTo(NodeType nodeType, ClusterSpec.Type clusterType) { + return (nodeType == NodeType.tenant && clusterType.isContainer()) || + (nodeType == NodeType.config && clusterType == ClusterSpec.Type.admin); + } + /** Load balancer protocols */ enum Protocol { ipv4, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index 9bd1189420a..f4d689056c3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -5,13 +5,11 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Set; /** * @author mpolden @@ -30,19 +28,18 @@ public class LoadBalancerServiceMock implements LoadBalancerService { } @Override - public LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, boolean force, - NodeRepository nodeRepository) { - var id = new LoadBalancerId(application, cluster); + public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { + var id = new LoadBalancerId(spec.application(), spec.cluster()); var oldInstance = instances.get(id); - if (!force && oldInstance != null && !oldInstance.reals().isEmpty() && reals.isEmpty()) { + if (!force && oldInstance != null && !oldInstance.reals().isEmpty() && spec.reals().isEmpty()) { throw new IllegalArgumentException("Refusing to remove all reals from load balancer " + id); } var instance = new LoadBalancerInstance( - HostName.from("lb-" + application.toShortString() + "-" + cluster.value()), + HostName.from("lb-" + spec.application().toShortString() + "-" + spec.cluster().value()), Optional.of(new DnsZone("zone-id-1")), Collections.singleton(4443), ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"), - reals); + spec.reals()); instances.put(id, instance); return instance; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java new file mode 100644 index 00000000000..b76f5ba2bcf --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java @@ -0,0 +1,43 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.lb; + +import com.google.common.collect.ImmutableSortedSet; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; + +import java.util.Objects; +import java.util.Set; + +/** + * A specification for a load balancer. + * + * @author mpolden + */ +public class LoadBalancerSpec { + + private final ApplicationId application; + private final ClusterSpec.Id cluster; + private final Set<Real> reals; + + public LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals) { + this.application = Objects.requireNonNull(application); + this.cluster = Objects.requireNonNull(cluster); + this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals)); + } + + /** Owner of the load balancer */ + public ApplicationId application() { + return application; + } + + /** The target cluster of this load balancer */ + public ClusterSpec.Id cluster() { + return cluster; + } + + /** Real servers to attach to this load balancer */ + public Set<Real> reals() { + return reals; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java index 07074bc45af..7667672e470 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.Comparator; import java.util.Optional; @@ -18,11 +17,10 @@ import java.util.Set; public class PassthroughLoadBalancerService implements LoadBalancerService { @Override - public LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, boolean force, - NodeRepository nodeRepository) { - var real = reals.stream() - .min(Comparator.naturalOrder()) - .orElseThrow(() -> new IllegalArgumentException("No reals given")); + public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { + var real = spec.reals().stream() + .min(Comparator.naturalOrder()) + .orElseThrow(() -> new IllegalArgumentException("No reals given")); return new LoadBalancerInstance(real.hostname(), Optional.empty(), Set.of(real.port()), Set.of(real.ipAddress() + "/32"), Set.of()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index a8faafc0bad..bc4381573c6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; -import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; @@ -27,13 +26,14 @@ public class SharedLoadBalancerService implements LoadBalancerService { private static final Comparator<Node> hostnameComparator = Comparator.comparing(Node::hostname); - @Inject - public SharedLoadBalancerService() { + private final NodeRepository nodeRepository; + + public SharedLoadBalancerService(NodeRepository nodeRepository) { + this.nodeRepository = Objects.requireNonNull(nodeRepository); } @Override - public LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, boolean force, - NodeRepository nodeRepository) { + public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { var proxyNodes = new ArrayList<>(nodeRepository.getNodes(NodeType.proxy)); proxyNodes.sort(hostnameComparator); @@ -52,7 +52,7 @@ public class SharedLoadBalancerService implements LoadBalancerService { Optional.empty(), Set.of(4080, 4443), networkNames, - reals + spec.reals() ); } @@ -66,6 +66,12 @@ public class SharedLoadBalancerService implements LoadBalancerService { return Protocol.dualstack; } + @Override + public boolean canForwardTo(NodeType nodeType, ClusterSpec.Type clusterType) { + // Shared routing layer only supports routing to tenant nodes + return nodeType == NodeType.tenant && clusterType.isContainer(); + } + private static String withPrefixLength(String address) { if (IP.isV6(address)) { return address + "/128"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 483b4dc8f84..6edd57de1c1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -1,13 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer.State; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; import java.time.Duration; @@ -17,6 +17,7 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.logging.Level; import java.util.stream.Collectors; /** @@ -99,7 +100,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { // Remove any real no longer allocated to this application reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value())); try { - service.create(lb.id().application(), lb.id().cluster(), reals, true, nodeRepository()); + service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals), true); db.writeLoadBalancer(lb.with(lb.instance().withReals(reals))); } catch (Exception e) { failed.add(lb.id()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 7875afeb28c..4323622df8b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -84,7 +84,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { nodeRebooter = new NodeRebooter(nodeRepository, clock, flagSource); metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval, clock); infrastructureProvisioner = new InfrastructureProvisioner(nodeRepository, infraDeployer, defaults.infrastructureProvisionInterval); - loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService().map(lbService -> + loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService(nodeRepository).map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService)); dynamicProvisioningMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner -> new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index 7ad69d673e7..7158ccc57e3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -111,10 +111,10 @@ class Activator { /** Activate load balancers */ private void activateLoadBalancers(ApplicationId application, Collection<HostSpec> hosts, NestedTransaction transaction, @SuppressWarnings("unused") Mutex applicationLock) { - loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(application, clustersOf(hosts), applicationLock, transaction)); + loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(application, allClustersOf(hosts), applicationLock, transaction)); } - private static Set<ClusterSpec> clustersOf(Collection<HostSpec> hosts) { + private static Set<ClusterSpec> allClustersOf(Collection<HostSpec> hosts) { return hosts.stream() .map(HostSpec::membership) .flatMap(Optional::stream) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index 004c74b5f70..38dd9f29873 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -17,7 +17,7 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { private final HostResourcesCalculator hostResourcesCalculator = new IdentityHostResourcesCalculator(); @Override - public Optional<LoadBalancerService> getLoadBalancerService() { + public Optional<LoadBalancerService> getLoadBalancerService(NodeRepository nodeRepository) { return Optional.empty(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index c6945e1779b..460e1e71e65 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -8,6 +8,9 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -15,6 +18,7 @@ import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec; import com.yahoo.vespa.hosted.provision.lb.Real; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; @@ -23,6 +27,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.logging.Level; @@ -45,11 +50,13 @@ public class LoadBalancerProvisioner { private final NodeRepository nodeRepository; private final CuratorDatabaseClient db; private final LoadBalancerService service; + private final BooleanFlag provisionConfigServerLoadBalancer; - public LoadBalancerProvisioner(NodeRepository nodeRepository, LoadBalancerService service) { + public LoadBalancerProvisioner(NodeRepository nodeRepository, LoadBalancerService service, FlagSource flagSource) { this.nodeRepository = nodeRepository; this.db = nodeRepository.database(); this.service = service; + this.provisionConfigServerLoadBalancer = Flags.CONFIGSERVER_PROVISION_LB.bindTo(flagSource); // Read and write all load balancers to make sure they are stored in the latest version of the serialization format for (var id : db.readLoadBalancerIds()) { try (var lock = db.lock(id.application())) { @@ -70,11 +77,12 @@ public class LoadBalancerProvisioner { * Calling this for irrelevant node or cluster types is a no-op. */ public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { - if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type - if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type + if (!canForwardTo(requestedNodes.type(), cluster)) return; // Nothing to provision for this node and cluster type if (application.instance().isTester()) return; // Do not provision for tester instances try (var lock = db.lock(application)) { - provision(application, effectiveId(cluster), false, lock); + ClusterSpec.Id clusterId = effectiveId(cluster); + List<Node> nodes = nodesOf(clusterId, application); + provision(application, clusterId, nodes,false, lock); } } @@ -91,13 +99,14 @@ public class LoadBalancerProvisioner { public void activate(ApplicationId application, Set<ClusterSpec> clusters, @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { try (var lock = db.lock(application)) { - var containerClusters = containerClustersOf(clusters); - for (var clusterId : containerClusters) { + for (var cluster : loadBalancedClustersOf(application).entrySet()) { // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(application, clusterId, true, lock); + provision(application, cluster.getKey(), cluster.getValue(), true, lock); } // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed - var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); + var surplusLoadBalancers = surplusLoadBalancersOf(application, clusters.stream() + .map(LoadBalancerProvisioner::effectiveId) + .collect(Collectors.toSet())); deactivate(surplusLoadBalancers, transaction); } } @@ -138,9 +147,17 @@ public class LoadBalancerProvisioner { db.writeLoadBalancers(deactivatedLoadBalancers, transaction); } + // TODO(mpolden): Inline when feature flag is removed + private boolean canForwardTo(NodeType type, ClusterSpec cluster) { + boolean canForwardTo = service.canForwardTo(type, cluster.type()); + if (canForwardTo && type == NodeType.config) { + return provisionConfigServerLoadBalancer.value(); + } + return canForwardTo; + } /** Idempotently provision a load balancer for given application and cluster */ - private void provision(ApplicationId application, ClusterSpec.Id clusterId, boolean activate, + private void provision(ApplicationId application, ClusterSpec.Id clusterId, List<Node> nodes, boolean activate, @SuppressWarnings("unused") Mutex loadBalancersLock) { var id = new LoadBalancerId(application, clusterId); var now = nodeRepository.clock().instant(); @@ -148,7 +165,7 @@ public class LoadBalancerProvisioner { if (loadBalancer.isEmpty() && activate) return; // Nothing to activate as this load balancer was never prepared var force = loadBalancer.isPresent() && loadBalancer.get().state() != LoadBalancer.State.active; - var instance = create(application, clusterId, allocatedContainers(application, clusterId), force); + var instance = provisionInstance(application, clusterId, nodes, force); LoadBalancer newLoadBalancer; if (loadBalancer.isEmpty()) { newLoadBalancer = new LoadBalancer(id, instance, LoadBalancer.State.reserved, now); @@ -159,7 +176,8 @@ public class LoadBalancerProvisioner { db.writeLoadBalancer(newLoadBalancer); } - private LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, List<Node> nodes, boolean force) { + private LoadBalancerInstance provisionInstance(ApplicationId application, ClusterSpec.Id cluster, List<Node> nodes, + boolean force) { var reals = new LinkedHashSet<Real>(); for (var node : nodes) { for (var ip : reachableIpAddresses(node)) { @@ -169,7 +187,7 @@ public class LoadBalancerProvisioner { log.log(Level.FINE, "Creating load balancer for " + cluster + " in " + application.toShortString() + ", targeting: " + reals); try { - return service.create(application, cluster, reals, force, nodeRepository); + return service.create(new LoadBalancerSpec(application, cluster, reals), force); } catch (Exception e) { throw new LoadBalancerServiceException("Failed to (re)configure load balancer for " + cluster + " in " + application + ", targeting: " + reals + ". The operation will be " + @@ -177,14 +195,21 @@ public class LoadBalancerProvisioner { } } - /** Returns a list of active and reserved nodes of type container in given cluster */ - private List<Node> allocatedContainers(ApplicationId application, ClusterSpec.Id clusterId) { - return NodeList.copyOf(nodeRepository.getNodes(NodeType.tenant, Node.State.reserved, Node.State.active)) - .owner(application) - .matching(node -> node.state().isAllocated()) - .container() - .matching(node -> effectiveId(node.allocation().get().membership().cluster()).equals(clusterId)) - .asList(); + /** Returns the nodes allocated to the given load balanced cluster */ + private List<Node> nodesOf(ClusterSpec.Id loadBalancedCluster, ApplicationId application) { + return loadBalancedClustersOf(application).getOrDefault(loadBalancedCluster, List.of()); + } + + /** Returns the load balanced clusters of given application and their nodes */ + private Map<ClusterSpec.Id, List<Node>> loadBalancedClustersOf(ApplicationId application) { + NodeList nodes = NodeList.copyOf(nodeRepository.getNodes(Node.State.reserved, Node.State.active)) + .owner(application); + if (nodes.stream().anyMatch(node -> node.type() == NodeType.config)) { + nodes = nodes.nodeType(NodeType.config).type(ClusterSpec.Type.admin); + } else { + nodes = nodes.nodeType(NodeType.tenant).container(); + } + return nodes.stream().collect(Collectors.groupingBy(node -> effectiveId(node.allocation().get().membership().cluster()))); } /** Find IP addresses reachable by the load balancer service */ @@ -202,14 +227,6 @@ public class LoadBalancerProvisioner { return reachable; } - /** Returns the container cluster IDs of the given clusters */ - private static Set<ClusterSpec.Id> containerClustersOf(Set<ClusterSpec> clusters) { - return clusters.stream() - .filter(c -> c.type().isContainer()) - .map(LoadBalancerProvisioner::effectiveId) - .collect(Collectors.toUnmodifiableSet()); - } - private static ClusterSpec.Id effectiveId(ClusterSpec cluster) { return cluster.combinedId().orElse(cluster.id()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 59fca955a68..f9d8213072e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; @@ -15,7 +14,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; -import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.FlagSource; @@ -36,6 +34,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -66,7 +65,8 @@ public class NodeRepositoryProvisioner implements Provisioner { this.allocationOptimizer = new AllocationOptimizer(nodeRepository); this.capacityPolicies = new CapacityPolicies(nodeRepository); this.zone = zone; - this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService().map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService)); + this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService(nodeRepository) + .map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService, flagSource)); this.nodeResourceLimits = new NodeResourceLimits(nodeRepository); this.preparer = new Preparer(nodeRepository, zone.environment() == Environment.prod ? SPARE_CAPACITY_PROD : SPARE_CAPACITY_NONPROD, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java index a86bd581516..563fc50e697 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import java.util.Optional; @@ -12,7 +13,7 @@ import java.util.Optional; */ public interface ProvisionServiceProvider { - Optional<LoadBalancerService> getLoadBalancerService(); + Optional<LoadBalancerService> getLoadBalancerService(NodeRepository nodeRepository); Optional<HostProvisioner> getHostProvisioner(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java index 9a02f65daf9..20538732c7a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.testutils; import com.google.inject.Inject; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; @@ -37,7 +38,7 @@ public class MockProvisionServiceProvider implements ProvisionServiceProvider { } @Override - public Optional<LoadBalancerService> getLoadBalancerService() { + public Optional<LoadBalancerService> getLoadBalancerService(NodeRepository nodeRepository) { return loadBalancerService; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java index e70fc184b87..997aec8a156 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java @@ -20,8 +20,8 @@ public class PassthroughLoadBalancerServiceTest { var lbService = new PassthroughLoadBalancerService(); var real = new Real(HostName.from("host1.example.com"), "192.0.2.10"); var reals = Set.of(real, new Real(HostName.from("host2.example.com"), "192.0.2.11")); - var instance = lbService.create(ApplicationId.from("tenant1", "app1", "default"), - ClusterSpec.Id.from("c1"), reals, false, null); + var instance = lbService.create(new LoadBalancerSpec(ApplicationId.from("tenant1", "app1", "default"), + ClusterSpec.Id.from("c1"), reals), false); assertEquals(real.hostname(), instance.hostname()); assertEquals(Set.of(real.port()), instance.ports()); assertEquals(Set.of(real.ipAddress() + "/32"), instance.networks()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java index 64d189b9111..06f18d94c5f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals; public class SharedLoadBalancerServiceTest { private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); - private final SharedLoadBalancerService loadBalancerService = new SharedLoadBalancerService(); + private final SharedLoadBalancerService loadBalancerService = new SharedLoadBalancerService(tester.nodeRepository()); private final ApplicationId applicationId = ApplicationId.from("tenant1", "application1", "default"); private final ClusterSpec.Id clusterId = ClusterSpec.Id.from("qrs1"); private final Set<Real> reals = Set.of( @@ -30,7 +30,7 @@ public class SharedLoadBalancerServiceTest { @Test public void test_create_lb() { tester.makeReadyNodes(2, "default", NodeType.proxy); - var lb = loadBalancerService.create(applicationId, clusterId, reals, false, tester.nodeRepository()); + var lb = loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals), false); assertEquals(HostName.from("host-1.yahoo.com"), lb.hostname()); assertEquals(Optional.empty(), lb.dnsZone()); @@ -40,7 +40,7 @@ public class SharedLoadBalancerServiceTest { @Test(expected = IllegalStateException.class) public void test_exception_on_missing_proxies() { - loadBalancerService.create(applicationId, clusterId, reals, false, tester.nodeRepository()); + loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals), false); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 26039c29ae8..f48127f650d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -11,6 +11,8 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; @@ -43,7 +45,8 @@ public class LoadBalancerProvisionerTest { private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default"); private final ApplicationId infraApp1 = ApplicationId.from("vespa", "tenant-host", "default"); - private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private final ProvisioningTester tester = new ProvisioningTester.Builder().flagSource(flagSource).build(); @Test public void provision_load_balancer() { @@ -212,6 +215,21 @@ public class LoadBalancerProvisionerTest { assertEquals(combinedId, lbs.get().get(0).id().cluster()); } + @Test + public void provision_load_balancer_config_server_cluster() { + flagSource.withBooleanFlag(Flags.CONFIGSERVER_PROVISION_LB.id(), true); + ApplicationId configServerApp = ApplicationId.from("hosted-vespa", "zone-config-servers", "default"); + Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(configServerApp).asList(); + var cluster = ClusterSpec.Id.from("zone-config-servers"); + var nodes = prepare(configServerApp, Capacity.fromRequiredNodeType(NodeType.config), false, + clusterRequest(ClusterSpec.Type.admin, cluster)); + assertEquals(1, lbs.get().size()); + assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().reals().size()); + tester.activate(configServerApp, nodes); + assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); + assertEquals(cluster, lbs.get().get(0).id().cluster()); + } + private void dirtyNodesOf(ApplicationId application) { tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application), Agent.system, this.getClass().getSimpleName()); } |