diff options
author | Jon Bratseth <bratseth@verizonmedia.com> | 2020-02-20 09:42:19 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@verizonmedia.com> | 2020-02-20 09:42:19 +0100 |
commit | 5acf4c47e98674cdf73289a782dfda9da7041ead (patch) | |
tree | 9a2720a3326326cc2a0b69d29b6877e4039d5f18 /node-repository | |
parent | d2449a3e66075e7d680263a204302e83b5ba0148 (diff) | |
parent | 1cc70ca6f328e7e88e8b4e279cac7544624f055b (diff) |
Merge branch 'master' into bratseth/node-metrics
Diffstat (limited to 'node-repository')
77 files changed, 894 insertions, 515 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index e66fef9ac96..321c5632302 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; @@ -41,6 +42,7 @@ public final class Node { private final NodeType type; private final Reports reports; private final Optional<String> modelName; + private final Optional<TenantName> reservedTo; /** Record of the last event of each type happening to this node */ private final History history; @@ -50,52 +52,46 @@ public final class Node { /** Creates a node in the initial state (reserved) */ public static Node createDockerNode(Set<String> ipAddresses, String hostname, String parentHostname, NodeResources resources, NodeType type) { - return new Node("fake-" + hostname, new IP.Config(ipAddresses, Set.of()), hostname, Optional.of(parentHostname), new Flavor(resources), Status.initial(), State.reserved, - Optional.empty(), History.empty(), type, new Reports(), Optional.empty()); + return new Node("fake-" + hostname, new IP.Config(ipAddresses, Set.of()), hostname, Optional.of(parentHostname), + new Flavor(resources), Status.initial(), State.reserved, + Optional.empty(), History.empty(), type, new Reports(), Optional.empty(), Optional.empty()); } /** Creates a node in the initial state (provisioned) */ - public static Node create(String openStackId, IP.Config ipConfig, String hostname, Optional<String> parentHostname, Optional<String> modelName, Flavor flavor, NodeType type) { + public static Node create(String openStackId, IP.Config ipConfig, String hostname, Optional<String> parentHostname, + Optional<String> modelName, Flavor flavor, Optional<TenantName> reservedTo, NodeType type) { return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, Status.initial(), State.provisioned, - Optional.empty(), History.empty(), type, new Reports(), modelName); + Optional.empty(), History.empty(), type, new Reports(), modelName, reservedTo); } /** Creates a node. See also the {@code create} helper methods. */ public Node(String id, IP.Config ipConfig, String hostname, Optional<String> parentHostname, Flavor flavor, Status status, State state, Optional<Allocation> allocation, History history, NodeType type, - Reports reports, Optional<String> modelName) { - Objects.requireNonNull(id, "A node must have an ID"); - requireNonEmptyString(hostname, "A node must have a hostname"); - Objects.requireNonNull(ipConfig, "A node must a have an IP config"); - requireNonEmptyString(parentHostname, "A parent host name must be a proper value"); - Objects.requireNonNull(flavor, "A node must have a flavor"); - Objects.requireNonNull(status, "A node must have a status"); - Objects.requireNonNull(state, "A null node state is not permitted"); - Objects.requireNonNull(allocation, "A null node allocation is not permitted"); - Objects.requireNonNull(history, "A null node history is not permitted"); - Objects.requireNonNull(type, "A null node type is not permitted"); - Objects.requireNonNull(reports, "A null reports is not permitted"); - Objects.requireNonNull(modelName, "A null modelName is not permitted"); + Reports reports, Optional<String> modelName, Optional<TenantName> reservedTo) { + this.id = Objects.requireNonNull(id, "A node must have an ID"); + this.hostname = requireNonEmptyString(hostname, "A node must have a hostname"); + this.ipConfig = Objects.requireNonNull(ipConfig, "A node must a have an IP config"); + this.parentHostname = requireNonEmptyString(parentHostname, "A parent host name must be a proper value"); + this.flavor = Objects.requireNonNull(flavor, "A node must have a flavor"); + this.status = Objects.requireNonNull(status, "A node must have a status"); + this.state = Objects.requireNonNull(state, "A null node state is not permitted"); + this.allocation = Objects.requireNonNull(allocation, "A null node allocation is not permitted"); + this.history = Objects.requireNonNull(history, "A null node history is not permitted"); + this.type = Objects.requireNonNull(type, "A null node type is not permitted"); + this.reports = Objects.requireNonNull(reports, "A null reports is not permitted"); + this.modelName = Objects.requireNonNull(modelName, "A null modelName is not permitted"); + this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null"); if (state == State.active) requireNonEmpty(ipConfig.primary(), "An active node must have at least one valid IP address"); + if (parentHostname.isPresent()) { if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); if (modelName.isPresent()) throw new IllegalArgumentException("A child node cannot have model name set"); } - this.hostname = hostname; - this.ipConfig = ipConfig; - this.parentHostname = parentHostname; - this.id = id; - this.flavor = flavor; - this.status = status; - this.state = state; - this.allocation = allocation; - this.history = history; - this.type = type; - this.reports = reports; - this.modelName = modelName; + if (type != NodeType.host && reservedTo.isPresent()) + throw new IllegalArgumentException("Only hosts can be reserved to a tenant"); } /** Returns the IP addresses of this node */ @@ -166,6 +162,12 @@ public final class Node { public Optional<String> modelName() { return modelName; } /** + * Returns the tenant this node is reserved to, if any. Only hosts can be reserved to a tenant. + * If this is set, resources on this host cannot be allocated to any other tenant + */ + public Optional<TenantName> reservedTo() { return reservedTo; } + + /** * Returns a copy of this node with wantToRetire set to the given value and updated history. * If given wantToRetire is equal to the current, the method is no-op. */ @@ -209,42 +211,47 @@ public final class Node { /** Returns a node with the status assigned to the given value */ public Node with(Status status) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo); } /** Returns a node with the type assigned to the given value */ public Node with(NodeType type) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo); } /** Returns a node with the flavor assigned to the given value */ public Node with(Flavor flavor) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, reservedTo); } /** Returns a copy of this with the reboot generation set to generation */ public Node withReboot(Generation generation) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, allocation, history, type, reports, modelName); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, + allocation, history, type, reports, modelName, reservedTo); } /** Returns a copy of this with the openStackId set */ public Node withOpenStackId(String openStackId) { - return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); + return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, reservedTo); } /** Returns a copy of this with model name set to given value */ public Node withModelName(String modelName) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, Optional.of(modelName)); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, Optional.of(modelName), reservedTo); } /** Returns a copy of this with model name cleared */ public Node withoutModelName() { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, Optional.empty()); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, Optional.empty(), reservedTo); } /** Returns a copy of this with a history record saying it was detected to be down at this instant */ - public Node downAt(Instant instant) { - return with(history.with(new History.Event(History.Event.Type.down, Agent.system, instant))); + public Node downAt(Instant instant, Agent agent) { + return with(history.with(new History.Event(History.Event.Type.down, agent, instant))); } /** Returns a copy of this with any history record saying it has been detected down removed */ @@ -265,26 +272,39 @@ public final class Node { */ public Node with(Allocation allocation) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - Optional.of(allocation), history, type, reports, modelName); + Optional.of(allocation), history, type, reports, modelName, reservedTo); } /** Returns a new Node without an allocation. */ public Node withoutAllocation() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - Optional.empty(), history, type, reports, modelName); + Optional.empty(), history, type, reports, modelName, reservedTo); } /** Returns a copy of this node with IP config set to the given value. */ public Node with(IP.Config ipConfig) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName); + allocation, history, type, reports, modelName, reservedTo); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node withParentHostname(String parentHostname) { return new Node(id, ipConfig, hostname, Optional.of(parentHostname), flavor, status, state, - allocation, history, type, reports, modelName); + allocation, history, type, reports, modelName, reservedTo); + } + + public Node withReservedTo(TenantName tenant) { + if (type != NodeType.host) + throw new IllegalArgumentException("Only host nodes can be reserved, " + hostname + " has type " + type); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, Optional.of(tenant)); + } + + /** Returns a copy of this node which is not reserved to a tenant */ + public Node withoutReservedTo() { + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, Optional.empty()); } /** Returns a copy of this node with the current reboot generation set to the given number at the given instant */ @@ -316,28 +336,32 @@ public final class Node { /** Returns a copy of this node with the given history. */ public Node with(History history) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, reservedTo); } public Node with(Reports reports) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, reservedTo); } - private static void requireNonEmptyString(Optional<String> value, String message) { + private static Optional<String> requireNonEmptyString(Optional<String> value, String message) { Objects.requireNonNull(value, message); value.ifPresent(v -> requireNonEmptyString(v, message)); + return value; } - private static void requireNonEmptyString(String value, String message) { + private static String requireNonEmptyString(String value, String message) { Objects.requireNonNull(value, message); if (value.trim().isEmpty()) throw new IllegalArgumentException(message + ", but was '" + value + "'"); + return value; } - private static void requireNonEmpty(Set<String> values, String message) { - if (values == null || values.isEmpty()) { + private static Set<String> requireNonEmpty(Set<String> values, String message) { + if (values == null || values.isEmpty()) throw new IllegalArgumentException(message); - } + return values; } /** Computes the allocation skew of a host node */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index 37987f0512d..bdae658f76e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeType; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.EnumSet; import java.util.Iterator; import java.util.List; @@ -74,11 +75,26 @@ public class NodeList implements Iterable<Node> { !node.status().vespaVersion().get().equals(node.allocation().get().membership().cluster().vespaVersion())); } + /** Returns the subset of nodes that are currently changing their OS version to given version */ + public NodeList changingOsVersionTo(Version version) { + return filter(node -> node.status().osVersion().changingTo(version)); + } + /** Returns the subset of nodes that are currently changing their OS version */ public NodeList changingOsVersion() { return filter(node -> node.status().osVersion().changing()); } + /** Returns a copy of this sorted by current OS version (lowest to highest) */ + public NodeList byIncreasingOsVersion() { + return nodes.stream() + .sorted(Comparator.comparing(node -> node.status() + .osVersion() + .current() + .orElse(Version.emptyVersion))) + .collect(collectingAndThen(Collectors.toList(), NodeList::wrap)); + } + /** Returns the subset of nodes that are currently on the given OS version */ public NodeList onOsVersion(Version version) { return filter(node -> node.status().osVersion().matches(version)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 1c9e1445447..7d925b2a4aa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; import com.google.inject.Inject; @@ -10,12 +10,16 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.NodeRepositoryConfig; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; @@ -49,6 +53,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.function.BiFunction; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -98,8 +103,8 @@ public class NodeRepository extends AbstractComponent { * This will use the system time to make time-sensitive decisions */ @Inject - public NodeRepository(NodeRepositoryConfig config, NodeFlavors flavors, Curator curator, Zone zone) { - this(flavors, curator, Clock.systemUTC(), zone, new DnsNameResolver(), DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache()); + public NodeRepository(NodeRepositoryConfig config, NodeFlavors flavors, Curator curator, Zone zone, FlagSource flagSource) { + this(flavors, curator, Clock.systemUTC(), zone, new DnsNameResolver(), DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache(), flagSource); } /** @@ -107,7 +112,7 @@ public class NodeRepository extends AbstractComponent { * which will be used for time-sensitive decisions. */ public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, NameResolver nameResolver, - DockerImage dockerImage, boolean useCuratorClientCache) { + DockerImage dockerImage, boolean useCuratorClientCache, FlagSource flagSource) { this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache); this.zone = zone; this.clock = clock; @@ -116,7 +121,7 @@ public class NodeRepository extends AbstractComponent { this.osVersions = new OsVersions(this); this.infrastructureVersions = new InfrastructureVersions(db); this.firmwareChecks = new FirmwareChecks(db, clock); - this.dockerImages = new DockerImages(db, dockerImage); + this.dockerImages = new DockerImages(db, dockerImage, Flags.DOCKER_IMAGE_OVERRIDE.bindTo(flagSource)); this.jobControl = new JobControl(db); // read and write all nodes to make sure they are stored in the latest version of the serialized format @@ -127,8 +132,8 @@ public class NodeRepository extends AbstractComponent { /** Returns the curator database client used by this */ public CuratorDatabaseClient database() { return db; } - /** Returns the Docker image to use for nodes in this */ - public DockerImage dockerImage(NodeType nodeType) { return dockerImages.dockerImageFor(nodeType); } + /** Returns the Docker image to use for given node */ + public DockerImage dockerImage(Node node) { return dockerImages.dockerImageFor(node); } /** @return The name resolver used to resolve hostname and ip addresses */ public NameResolver nameResolver() { return nameResolver; } @@ -193,7 +198,16 @@ public class NodeRepository extends AbstractComponent { /** Returns a filterable list of all load balancers in this repository */ public LoadBalancerList loadBalancers() { - return LoadBalancerList.copyOf(database().readLoadBalancers().values()); + return loadBalancers((ignored) -> true); + } + + /** Returns a filterable list of load balancers belonging to given application */ + public LoadBalancerList loadBalancers(ApplicationId application) { + return loadBalancers((id) -> id.application().equals(application)); + } + + private LoadBalancerList loadBalancers(Predicate<LoadBalancerId> predicate) { + return LoadBalancerList.copyOf(db.readLoadBalancers(predicate).values()); } public List<Node> getNodes(ApplicationId id, Node.State ... inState) { return db.getNodes(id, inState); } @@ -203,7 +217,7 @@ public class NodeRepository extends AbstractComponent { /** * Returns the ACL for the node (trusted nodes, networks and ports) */ - private NodeAcl getNodeAcl(Node node, NodeList candidates, LoadBalancerList loadBalancers) { + private NodeAcl getNodeAcl(Node node, NodeList candidates) { Set<Node> trustedNodes = new TreeSet<>(Comparator.comparing(Node::hostname)); Set<Integer> trustedPorts = new LinkedHashSet<>(); Set<String> trustedNetworks = new LinkedHashSet<>(); @@ -220,10 +234,10 @@ public class NodeRepository extends AbstractComponent { candidates.parentOf(node).ifPresent(trustedNodes::add); node.allocation().ifPresent(allocation -> { trustedNodes.addAll(candidates.owner(allocation.owner()).asList()); - loadBalancers.owner(allocation.owner()).asList().stream() - .map(LoadBalancer::instance) - .map(LoadBalancerInstance::networks) - .forEach(trustedNetworks::addAll); + loadBalancers(allocation.owner()).asList().stream() + .map(LoadBalancer::instance) + .map(LoadBalancerInstance::networks) + .forEach(trustedNetworks::addAll); }); switch (node.type()) { @@ -292,13 +306,12 @@ public class NodeRepository extends AbstractComponent { */ public List<NodeAcl> getNodeAcls(Node node, boolean children) { NodeList candidates = list(); - LoadBalancerList loadBalancers = loadBalancers(); if (children) { return candidates.childrenOf(node).asList().stream() - .map(childNode -> getNodeAcl(childNode, candidates, loadBalancers)) + .map(childNode -> getNodeAcl(childNode, candidates)) .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); } - return Collections.singletonList(getNodeAcl(node, candidates, loadBalancers)); + return Collections.singletonList(getNodeAcl(node, candidates)); } public NodeFlavors getAvailableFlavors() { @@ -309,16 +322,15 @@ public class NodeRepository extends AbstractComponent { /** Creates a new node object, without adding it to the node repo. If no IP address is given, it will be resolved */ public Node createNode(String openStackId, String hostname, IP.Config ipConfig, Optional<String> parentHostname, - Flavor flavor, NodeType type) { + Flavor flavor, Optional<TenantName> reservedTo, NodeType type) { if (ipConfig.primary().isEmpty()) { // TODO: Remove this. Only test code hits this path ipConfig = ipConfig.with(nameResolver.getAllByNameOrThrow(hostname)); } - return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, type); + return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, reservedTo, type); } - public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Flavor flavor, - NodeType type) { - return createNode(openStackId, hostname, IP.Config.EMPTY, parentHostname, flavor, type); + public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Flavor flavor, NodeType type) { + return createNode(openStackId, hostname, IP.Config.EMPTY, parentHostname, flavor, Optional.empty(), type); } /** Adds a list of newly created docker container nodes to the node repository as <i>reserved</i> nodes */ @@ -341,7 +353,7 @@ public class NodeRepository extends AbstractComponent { /** Adds a list of (newly created) nodes to the node repository as <i>provisioned</i> nodes */ public List<Node> addNodes(List<Node> nodes) { - try (Mutex lock = lockAllocation()) { + try (Mutex lock = lockUnallocated()) { for (int i = 0; i < nodes.size(); i++) { var node = nodes.get(i); var message = "Cannot add " + node.hostname() + ": A node with this name already exists"; @@ -361,7 +373,7 @@ public class NodeRepository extends AbstractComponent { /** Sets a list of nodes ready and returns the nodes in the ready state */ public List<Node> setReady(List<Node> nodes, Agent agent, String reason) { - try (Mutex lock = lockAllocation()) { + try (Mutex lock = lockUnallocated()) { List<Node> nodesWithResetFields = nodes.stream() .map(node -> { if (node.state() != Node.State.provisioned && node.state() != Node.State.dirty) @@ -585,7 +597,7 @@ public class NodeRepository extends AbstractComponent { } public List<Node> removeRecursively(Node node, boolean force) { - try (Mutex lock = lockAllocation()) { + try (Mutex lock = lockUnallocated()) { List<Node> removed = new ArrayList<>(); if (node.type().isDockerHost()) { @@ -684,7 +696,7 @@ public class NodeRepository extends AbstractComponent { * Writes these nodes after they have changed some internal state but NOT changed their state field. * This does NOT lock the node repository implicitly, but callers are expected to already hold the lock. * - * @param lock Already acquired lock + * @param lock already acquired lock * @return the written nodes for convenience */ public List<Node> write(List<Node> nodes, @SuppressWarnings("unused") Mutex lock) { @@ -713,7 +725,7 @@ public class NodeRepository extends AbstractComponent { // perform operation while holding locks List<Node> resultingNodes = new ArrayList<>(); - try (Mutex lock = lockAllocation()) { + try (Mutex lock = lockUnallocated()) { for (Node node : unallocatedNodes) resultingNodes.add(action.apply(node, lock)); } @@ -738,12 +750,12 @@ public class NodeRepository extends AbstractComponent { /** Create a lock with a timeout which provides exclusive rights to making changes to the given application */ public Mutex lock(ApplicationId application, Duration timeout) { return db.lock(application, timeout); } - /** Create a lock which provides exclusive rights to allocating nodes */ - public Mutex lockAllocation() { return db.lockInactive(); } + /** Create a lock which provides exclusive rights to modifying unallocated nodes */ + public Mutex lockUnallocated() { return db.lockInactive(); } /** Acquires the appropriate lock for this node */ public Mutex lock(Node node) { - return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockAllocation(); + return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockUnallocated(); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java index 014d3df8d9a..bad16bf7d12 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java @@ -1,9 +1,6 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; -import com.yahoo.config.provision.ApplicationId; - -import java.time.Instant; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -24,21 +21,11 @@ public class LoadBalancerList implements Iterable<LoadBalancer> { this.loadBalancers = List.copyOf(Objects.requireNonNull(loadBalancers, "loadBalancers must be non-null")); } - /** Returns the subset of load balancers owned by given application */ - public LoadBalancerList owner(ApplicationId application) { - return of(loadBalancers.stream().filter(lb -> lb.id().application().equals(application))); - } - /** Returns the subset of load balancers that are in given state */ public LoadBalancerList in(LoadBalancer.State state) { return of(loadBalancers.stream().filter(lb -> lb.state() == state)); } - /** Returns the subset of load balancers that last changed before given instant */ - public LoadBalancerList changedBefore(Instant instant) { - return of(loadBalancers.stream().filter(lb -> lb.changedAt().isBefore(instant))); - } - public List<LoadBalancer> asList() { return loadBalancers; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java index ae2f68a3143..f428e276df8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java @@ -33,7 +33,7 @@ public class DirtyExpirer extends Expirer { @Override protected void expire(List<Node> expired) { for (Node expiredNode : expired) - nodeRepository.fail(expiredNode.hostname(), Agent.system, "Node is stuck in dirty"); + nodeRepository.fail(expiredNode.hostname(), Agent.DirtyExpirer, "Node is stuck in dirty"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 1c27f9d1713..f75621e18a0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.NodePrioritizer; import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceComparator; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; @@ -44,13 +45,15 @@ public class DynamicProvisioningMaintainer extends Maintainer { private static final ApplicationId preprovisionAppId = ApplicationId.from("hosted-vespa", "tenant-host", "preprovision"); private final HostProvisioner hostProvisioner; + private final HostResourcesCalculator hostResourcesCalculator; private final BooleanFlag dynamicProvisioningEnabled; private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag; - DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, - HostProvisioner hostProvisioner, FlagSource flagSource) { + DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, HostProvisioner hostProvisioner, + HostResourcesCalculator hostResourcesCalculator, FlagSource flagSource) { super(nodeRepository, interval); this.hostProvisioner = hostProvisioner; + this.hostResourcesCalculator = hostResourcesCalculator; this.dynamicProvisioningEnabled = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource); this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource); } @@ -59,7 +62,7 @@ public class DynamicProvisioningMaintainer extends Maintainer { protected void maintain() { if (! dynamicProvisioningEnabled.value()) return; - try (Mutex lock = nodeRepository().lockAllocation()) { + try (Mutex lock = nodeRepository().lockUnallocated()) { NodeList nodes = nodeRepository().list(); updateProvisioningNodes(nodes, lock); @@ -68,17 +71,14 @@ public class DynamicProvisioningMaintainer extends Maintainer { } void updateProvisioningNodes(NodeList nodes, Mutex lock) { - Map<String, Node> provisionedHostsByHostname = nodes.state(Node.State.provisioned).nodeType(NodeType.host) - .asList().stream() - .collect(Collectors.toMap(Node::hostname, Function.identity())); - - Map<Node, Set<Node>> nodesByProvisionedParent = nodes.asList().stream() - .filter(node -> node.parentHostname().map(provisionedHostsByHostname::containsKey).orElse(false)) + Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant).asList().stream() + .filter(node -> node.parentHostname().isPresent()) .collect(Collectors.groupingBy( - node -> provisionedHostsByHostname.get(node.parentHostname().get()), + node -> node.parentHostname().get(), Collectors.toSet())); - nodesByProvisionedParent.forEach((host, children) -> { + nodes.state(Node.State.provisioned).nodeType(NodeType.host).forEach(host -> { + Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of()); try { List<Node> updatedNodes = hostProvisioner.provision(host, children); nodeRepository().write(updatedNodes, lock); @@ -112,7 +112,7 @@ public class DynamicProvisioningMaintainer extends Maintainer { NodeResources resources = it.next(); removableHosts.stream() .filter(host -> NodePrioritizer.ALLOCATABLE_HOST_STATES.contains(host.state())) - .filter(host -> host.flavor().resources().satisfies(resources)) + .filter(host -> hostResourcesCalculator.availableCapacityOf(host.flavor().name(), host.flavor().resources()).satisfies(resources)) .min(Comparator.comparingInt(n -> n.flavor().cost())) .ifPresent(host -> { removableHosts.remove(host); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index 438732ad4a8..264df716fa8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -107,7 +107,7 @@ public class FailedExpirer extends Maintainer { .collect(Collectors.toList()); if (unparkedChildren.isEmpty()) { - nodeRepository.park(candidate.hostname(), false, Agent.system, + nodeRepository.park(candidate.hostname(), false, Agent.FailedExpirer, "Parked by FailedExpirer due to hardware issue"); } else { log.info(String.format("Expired failed node %s with hardware issue was not parked because of " + @@ -118,7 +118,7 @@ public class FailedExpirer extends Maintainer { nodesToRecycle.add(candidate); } } - nodeRepository.setDirty(nodesToRecycle, Agent.system, "Expired by FailedExpirer"); + nodeRepository.setDirty(nodesToRecycle, Agent.FailedExpirer, "Expired by FailedExpirer"); } /** Returns whether the current node fail count should be used as an indicator of hardware issue */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java index a6b88d50acb..21746c96411 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java @@ -41,9 +41,9 @@ public class InactiveExpirer extends Expirer { expired.forEach(node -> { if (node.status().wantToRetire() && node.history().event(History.Event.Type.wantToRetire).get().agent() == Agent.operator) { - nodeRepository.park(node.hostname(), false, Agent.system, "Expired by InactiveExpirer"); + nodeRepository.park(node.hostname(), false, Agent.InactiveExpirer, "Expired by InactiveExpirer"); } else { - nodeRepository.setDirty(node, Agent.system, "Expired by InactiveExpirer"); + nodeRepository.setDirty(node, Agent.InactiveExpirer, "Expired by InactiveExpirer"); } }); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index bcb0c901f14..e2b70608d58 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -15,6 +15,8 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.stream.Collectors; /** @@ -51,37 +53,31 @@ public class LoadBalancerExpirer extends Maintainer { /** Move reserved load balancer that have expired to inactive */ private void expireReserved() { - try (var lock = db.lockLoadBalancers()) { - var now = nodeRepository().clock().instant(); - var expirationTime = now.minus(reservedExpiry); - var expired = nodeRepository().loadBalancers() - .in(State.reserved) - .changedBefore(expirationTime); - expired.forEach(lb -> db.writeLoadBalancer(lb.with(State.inactive, now))); - } + var now = nodeRepository().clock().instant(); + withLoadBalancersIn(State.reserved, lb -> { + var gracePeriod = now.minus(reservedExpiry); + if (!lb.changedAt().isBefore(gracePeriod)) return; // Should not move to inactive yet + db.writeLoadBalancer(lb.with(State.inactive, now)); + }); } /** Deprovision inactive load balancers that have expired */ private void removeInactive() { var failed = new ArrayList<LoadBalancerId>(); - Exception lastException = null; - try (var lock = db.lockLoadBalancers()) { - var now = nodeRepository().clock().instant(); - var expirationTime = now.minus(inactiveExpiry); - var expired = nodeRepository().loadBalancers() - .in(State.inactive) - .changedBefore(expirationTime); - for (var lb : expired) { - if (!allocatedNodes(lb.id()).isEmpty()) continue; // Defer removal if there are still nodes allocated to application - try { - service.remove(lb.id().application(), lb.id().cluster()); - db.removeLoadBalancer(lb.id()); - } catch (Exception e) { - failed.add(lb.id()); - lastException = e; - } + var lastException = new AtomicReference<Exception>(); + var now = nodeRepository().clock().instant(); + withLoadBalancersIn(State.inactive, lb -> { + var gracePeriod = now.minus(inactiveExpiry); + if (!lb.changedAt().isBefore(gracePeriod)) return; // Should not be removed yet + if (!allocatedNodes(lb.id()).isEmpty()) return; // Still has nodes, do not remove + try { + service.remove(lb.id().application(), lb.id().cluster()); + db.removeLoadBalancer(lb.id()); + } catch (Exception e){ + failed.add(lb.id()); + lastException.set(e); } - } + }); if (!failed.isEmpty()) { log.log(LogLevel.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s", failed.size(), @@ -89,30 +85,27 @@ public class LoadBalancerExpirer extends Maintainer { .map(LoadBalancerId::serializedForm) .collect(Collectors.joining(", ")), interval()), - lastException); + lastException.get()); } } /** Remove reals from inactive load balancers */ private void pruneReals() { var failed = new ArrayList<LoadBalancerId>(); - Exception lastException = null; - try (var lock = db.lockLoadBalancers()) { - var deactivated = nodeRepository().loadBalancers().in(State.inactive); - for (var lb : deactivated) { - var allocatedNodes = allocatedNodes(lb.id()).stream().map(Node::hostname).collect(Collectors.toSet()); - var reals = new LinkedHashSet<>(lb.instance().reals()); - // Remove any real no longer allocated to this application - reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value())); - try { - service.create(lb.id().application(), lb.id().cluster(), reals, true); - db.writeLoadBalancer(lb.with(lb.instance().withReals(reals))); - } catch (Exception e) { - failed.add(lb.id()); - lastException = e; - } + var lastException = new AtomicReference<Exception>(); + withLoadBalancersIn(State.inactive, lb -> { + var allocatedNodes = allocatedNodes(lb.id()).stream().map(Node::hostname).collect(Collectors.toSet()); + var reals = new LinkedHashSet<>(lb.instance().reals()); + // Remove any real no longer allocated to this application + reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value())); + try { + service.create(lb.id().application(), lb.id().cluster(), reals, true); + db.writeLoadBalancer(lb.with(lb.instance().withReals(reals))); + } catch (Exception e) { + failed.add(lb.id()); + lastException.set(e); } - } + }); if (!failed.isEmpty()) { log.log(LogLevel.WARNING, String.format("Failed to remove reals from %d load balancers: %s, retrying in %s", failed.size(), @@ -120,7 +113,19 @@ public class LoadBalancerExpirer extends Maintainer { .map(LoadBalancerId::serializedForm) .collect(Collectors.joining(", ")), interval()), - lastException); + lastException.get()); + } + } + + /** Apply operation to all load balancers that exist in given state, while holding lock */ + private void withLoadBalancersIn(LoadBalancer.State state, Consumer<LoadBalancer> operation) { + for (var id : db.readLoadBalancerIds()) { + try (var lock = db.lockLoadBalancers(id.application())) { + var loadBalancer = db.readLoadBalancer(id); + if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop + if (loadBalancer.get().state() != state) continue; // Wrong state + operation.accept(loadBalancer.get()); + } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index da0db1f1896..ed6e7cc71ef 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -15,9 +15,10 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.orchestrator.Orchestrator; -import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.service.monitor.ServiceMonitor; +import java.time.Clock; import java.time.Duration; import java.util.HashMap; import java.util.List; @@ -35,22 +36,25 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed.any; public class MetricsReporter extends Maintainer { private final Metric metric; - private final Function<HostName, Optional<HostStatus>> orchestrator; + private final Function<HostName, Optional<HostInfo>> orchestrator; private final ServiceMonitor serviceMonitor; private final Map<Map<String, String>, Metric.Context> contextMap = new HashMap<>(); private final Supplier<Integer> pendingRedeploymentsSupplier; + private final Clock clock; MetricsReporter(NodeRepository nodeRepository, Metric metric, Orchestrator orchestrator, ServiceMonitor serviceMonitor, Supplier<Integer> pendingRedeploymentsSupplier, - Duration interval) { + Duration interval, + Clock clock) { super(nodeRepository, interval); this.metric = metric; - this.orchestrator = orchestrator.getNodeStatuses(); + this.orchestrator = orchestrator.getHostResolver(); this.serviceMonitor = serviceMonitor; this.pendingRedeploymentsSupplier = pendingRedeploymentsSupplier; + this.clock = clock; } @Override @@ -125,9 +129,15 @@ public class MetricsReporter extends Maintainer { metric.set("wantToDeprovision", node.status().wantToDeprovision() ? 1 : 0, context); metric.set("failReport", NodeFailer.reasonsToFailParentHost(node).isEmpty() ? 0 : 1, context); - orchestrator.apply(new HostName(node.hostname())) - .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN ? 1 : 0) - .ifPresent(allowedToBeDown -> metric.set("allowedToBeDown", allowedToBeDown, context)); + orchestrator.apply(new HostName(node.hostname())).ifPresent(info -> { + int suspended = info.status().isSuspended() ? 1 : 0; + metric.set("suspended", suspended, context); + metric.set("allowedToBeDown", suspended, context); // remove summer 2020. + long suspendedSeconds = info.suspendedSince() + .map(suspendedSince -> Duration.between(suspendedSince, clock.instant()).getSeconds()) + .orElse(0L); + metric.set("suspendedSeconds", suspendedSeconds, context); + }); long numberOfServices; HostName hostName = new HostName(node.hostname()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index dca5c092ad2..ca53b215237 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -97,7 +97,7 @@ public class NodeFailer extends Maintainer { int throttledNodeFailures = 0; // Ready nodes - try (Mutex lock = nodeRepository().lockAllocation()) { + try (Mutex lock = nodeRepository().lockUnallocated()) { updateNodeLivenessEventsForReadyNodes(lock); for (Map.Entry<Node, String> entry : getReadyNodesByFailureReason().entrySet()) { @@ -265,7 +265,7 @@ public class NodeFailer extends Maintainer { private boolean nodeSuspended(Node node) { try { - return orchestrator.getNodeStatus(new HostName(node.hostname())) == HostStatus.ALLOWED_TO_BE_DOWN; + return orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended(); } catch (HostNameNotFoundException e) { // Treat it as not suspended return false; @@ -323,7 +323,7 @@ public class NodeFailer extends Maintainer { try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) { node = nodeRepository().getNode(node.hostname(), Node.State.active).get(); // re-get inside lock - return nodeRepository().write(node.downAt(clock.instant()), lock); + return nodeRepository().write(node.downAt(clock.instant(), Agent.NodeFailer), lock); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index db466043d0c..a49049f8b04 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -81,12 +81,12 @@ public class NodeRepositoryMaintenance extends AbstractComponent { dirtyExpirer = new DirtyExpirer(nodeRepository, clock, defaults.dirtyExpiry); provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, defaults.provisionedExpiry); nodeRebooter = new NodeRebooter(nodeRepository, clock, flagSource); - metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval); + metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval, clock); infrastructureProvisioner = new InfrastructureProvisioner(nodeRepository, infraDeployer, defaults.infrastructureProvisionInterval); loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService().map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService)); dynamicProvisioningMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner -> - new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource)); + new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, provisionServiceProvider.getHostResourcesCalculator(), flagSource)); capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval); osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval); rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java index 3109e55df5c..e1407f2a41d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java @@ -27,7 +27,7 @@ public class ProvisionedExpirer extends Expirer { @Override protected void expire(List<Node> expired) { for (Node expiredNode : expired) - nodeRepository.parkRecursively(expiredNode.hostname(), Agent.system, "Node is stuck in provisioned"); + nodeRepository.parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, "Node is stuck in provisioned"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index d7dd93522e4..675e3400722 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -115,7 +115,7 @@ public class Rebalancer extends Maintainer { if (nodeToMove.get().status().wantToRetire() == wantToRetire) return false; - nodeRepository().write(nodeToMove.get().withWantToRetire(wantToRetire, Agent.system, clock.instant()), lock); + nodeRepository().write(nodeToMove.get().withWantToRetire(wantToRetire, Agent.Rebalancer, clock.instant()), lock); return true; } } @@ -154,7 +154,7 @@ public class Rebalancer extends Maintainer { // Immediately clean up if we reserved the node but could not activate or reserved a node on the wrong host expectedNewNode.flatMap(node -> nodeRepository().getNode(node.hostname(), Node.State.reserved)) - .ifPresent(node -> nodeRepository().setDirty(node, Agent.system, "Expired by Rebalancer")); + .ifPresent(node -> nodeRepository().setDirty(node, Agent.Rebalancer, "Expired by Rebalancer")); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java index b0aa389fe7d..03d466dbf09 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java @@ -28,6 +28,6 @@ public class ReservationExpirer extends Expirer { } @Override - protected void expire(List<Node> expired) { nodeRepository.setDirty(expired, Agent.system, "Expired by ReservationExpirer"); } + protected void expire(List<Node> expired) { nodeRepository.setDirty(expired, Agent.ReservationExpirer, "Expired by ReservationExpirer"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java index f46e2f501bc..7522e411e42 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java @@ -7,5 +7,15 @@ package com.yahoo.vespa.hosted.provision.node; * @author bratseth */ public enum Agent { - system, application, operator, NodeFailer + operator, // A hosted Vespa operator. Some logic recognizes these events. + application, // An application package change depoyment + system, // An unspecified system agent + // Specific system agents: + NodeFailer, + Rebalancer, + DirtyExpirer, + FailedExpirer, + InactiveExpirer, + ProvisionedExpirer, + ReservationExpirer } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java index b06bbbb54b5..0622862f5ab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java @@ -33,6 +33,11 @@ public class OsVersion { return wanted; } + /** Returns whether this node is currently changing its version to the given version */ + public boolean changingTo(Version version) { + return changing() && wanted.get().equals(version); + } + /** Returns whether this node is currently changing its version */ public boolean changing() { return wanted.isPresent() && !current.equals(wanted); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java index af32530a156..65cbc17aff5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.provision.node; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.time.Instant; import java.util.Arrays; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java index fd6094ae111..7885cec6b65 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java @@ -71,6 +71,6 @@ public class Reports { return this; } - public Reports build() { return new Reports(Collections.unmodifiableMap(reportMap)); } + public Reports build() { return new Reports(reportMap); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java index 106595fbd47..e10ff3d24cd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java @@ -120,9 +120,10 @@ public class OsVersions { /** Trigger upgrade of nodes of given type*/ private void upgrade(NodeType type, Version version) { var nodes = nodeRepository.list().nodeType(type); - var numberToUpgrade = Math.max(0, maxActiveUpgrades - nodes.changingOsVersion().size()); - var nodesToUpgrade = nodes.not().changingOsVersion() + var numberToUpgrade = Math.max(0, maxActiveUpgrades - nodes.changingOsVersionTo(version).size()); + var nodesToUpgrade = nodes.not().changingOsVersionTo(version) .not().onOsVersion(version) + .byIncreasingOsVersion() .first(numberToUpgrade); if (nodesToUpgrade.size() == 0) return; log.info("Upgrading " + nodesToUpgrade.size() + " nodes of type " + type + " to OS version " + version); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index a28845109dc..f211ea9eac5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.google.common.util.concurrent.UncheckedTimeoutException; @@ -38,6 +38,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.function.Function; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -206,7 +207,7 @@ public class CuratorDatabaseClient { toState, toState.isAllocated() ? node.allocation() : Optional.empty(), node.history().recordStateTransition(node.state(), toState, agent, clock.instant()), - node.type(), node.reports(), node.modelName()); + node.type(), node.reports(), node.modelName(), node.reservedTo()); writeNode(toState, curatorTransaction, node, newNode); writtenNodes.add(newNode); } @@ -483,18 +484,16 @@ public class CuratorDatabaseClient { // Load balancers public List<LoadBalancerId> readLoadBalancerIds() { - return curatorDatabase.getChildren(loadBalancersRoot).stream() - .map(LoadBalancerId::fromSerializedForm) - .collect(Collectors.toUnmodifiableList()); + return readLoadBalancerIds((ignored) -> true); } - public Map<LoadBalancerId, LoadBalancer> readLoadBalancers() { - return readLoadBalancerIds().stream() - .map(this::readLoadBalancer) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()), - Collections::unmodifiableMap)); + public Map<LoadBalancerId, LoadBalancer> readLoadBalancers(Predicate<LoadBalancerId> filter) { + return readLoadBalancerIds(filter).stream() + .map(this::readLoadBalancer) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()), + Collections::unmodifiableMap)); } public Optional<LoadBalancer> readLoadBalancer(LoadBalancerId id) { @@ -522,14 +521,21 @@ public class CuratorDatabaseClient { transaction.commit(); } - public Lock lockLoadBalancers() { - return lock(lockRoot.append("loadBalancersLock"), defaultLockTimeout); + public Lock lockLoadBalancers(ApplicationId application) { + return lock(lockRoot.append("loadBalancersLock2").append(application.serializedForm()), defaultLockTimeout); } private Path loadBalancerPath(LoadBalancerId id) { return loadBalancersRoot.append(id.serializedForm()); } + private List<LoadBalancerId> readLoadBalancerIds(Predicate<LoadBalancerId> predicate) { + return curatorDatabase.getChildren(loadBalancersRoot).stream() + .map(LoadBalancerId::fromSerializedForm) + .filter(predicate) + .collect(Collectors.toUnmodifiableList()); + } + private Transaction.Operation createOrSet(Path path, byte[] data) { if (curatorDatabase.exists(path)) { return CuratorOperations.setData(path.getAbsolute(), data); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index ae4c93621e5..66172521d4c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -6,7 +6,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.lb.DnsZone; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 2cbfbc349a6..9614c1aa5c7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -19,7 +19,8 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.Type; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -77,6 +78,7 @@ public class NodeSerializer { private static final String firmwareCheckKey = "firmwareCheck"; private static final String reportsKey = "reports"; private static final String modelNameKey = "modelName"; + private static final String reservedToKey = "reservedTo"; // Node resource fields // ...for hosts and nodes allocated by legacy flavor specs @@ -149,6 +151,7 @@ public class NodeSerializer { node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong(firmwareCheckKey, instant.toEpochMilli())); node.reports().toSlime(object, reportsKey); node.modelName().ifPresent(modelName -> object.setString(modelNameKey, modelName)); + node.reservedTo().ifPresent(tenant -> object.setString(reservedToKey, tenant.value())); } private void toSlime(Flavor flavor, Cursor object) { @@ -222,7 +225,8 @@ public class NodeSerializer { historyFromSlime(object.field(historyKey)), nodeTypeFromString(object.field(nodeTypeKey).asString()), Reports.fromSlime(object.field(reportsKey)), - modelNameFromSlime(object)); + modelNameFromSlime(object), + reservedToFromSlime(object.field(reservedToKey))); } private Status statusFromSlime(Inspector object) { @@ -341,6 +345,13 @@ public class NodeSerializer { return Optional.empty(); } + private Optional<TenantName> reservedToFromSlime(Inspector object) { + if (! object.valid()) return Optional.empty(); + if (object.type() != Type.STRING) + throw new IllegalArgumentException("Expected 'reservedTo' to be a string but is " + object); + return Optional.of(TenantName.from(object.asString())); + } + // ----------------- Enum <-> string mappings ---------------------------------------- /** Returns the event type, or null if this event type should be ignored */ @@ -388,19 +399,31 @@ public class NodeSerializer { private Agent eventAgentFromSlime(Inspector eventAgentField) { switch (eventAgentField.asString()) { + case "operator" : return Agent.operator; case "application" : return Agent.application; case "system" : return Agent.system; - case "operator" : return Agent.operator; case "NodeFailer" : return Agent.NodeFailer; + case "Rebalancer" : return Agent.Rebalancer; + case "DirtyExpirer" : return Agent.DirtyExpirer; + case "FailedExpirer" : return Agent.FailedExpirer; + case "InactiveExpirer" : return Agent.InactiveExpirer; + case "ProvisionedExpirer" : return Agent.ProvisionedExpirer; + case "ReservationExpirer" : return Agent.ReservationExpirer; } throw new IllegalArgumentException("Unknown node event agent '" + eventAgentField.asString() + "'"); } private String toString(Agent agent) { switch (agent) { + case operator : return "operator"; case application : return "application"; case system : return "system"; - case operator : return "operator"; case NodeFailer : return "NodeFailer"; + case Rebalancer : return "Rebalancer"; + case DirtyExpirer : return "DirtyExpirer"; + case FailedExpirer : return "FailedExpirer"; + case InactiveExpirer : return "InactiveExpirer"; + case ProvisionedExpirer : return "ProvisionedExpirer"; + case ReservationExpirer : return "ReservationExpirer"; } throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java index 37dc8a8a1ad..6615dff24e5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java @@ -7,7 +7,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java index dfa79a4fd9a..e27cdcd1842 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java @@ -7,7 +7,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java index 14c8c7fa18e..915b1acedaa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java @@ -5,7 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.NodeType; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.node.OsVersion; import java.io.IOException; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java index 1839f7187fa..1149b15a2b0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java @@ -5,7 +5,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.util.HashSet; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index a7519de3776..36034b62cfb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ParentHostUnavailableException; -import com.yahoo.log.LogLevel; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; @@ -90,6 +89,22 @@ class Activator { nodeRepository.deactivate(activeToRemove, transaction); nodeRepository.activate(updateFrom(hosts, continuedActive), transaction); // update active with any changes nodeRepository.activate(updatePortsFrom(hosts, reservedToActivate), transaction); + unreserveParentsOf(reservedToActivate); + } + + /** When a tenant node is activated on a host, we can open up that host for use by others */ + private void unreserveParentsOf(List<Node> nodes) { + for (Node node : nodes) { + if ( node.parentHostname().isEmpty()) continue; + Optional<Node> parent = nodeRepository.getNode(node.parentHostname().get()); + if (parent.isEmpty()) continue; + if (parent.get().reservedTo().isEmpty()) continue; + try (Mutex lock = nodeRepository.lock(parent.get())) { + Optional<Node> lockedParent = nodeRepository.getNode(parent.get().hostname()); + if (lockedParent.isEmpty()) continue; + nodeRepository.write(lockedParent.get().withoutReservedTo(), lock); + } + } } /** Activate load balancers */ @@ -117,9 +132,9 @@ class Activator { .filter(node -> node.state() != Node.State.active) .map(Node::hostname) .collect(Collectors.toSet()); - long numNonActive = nonActiveHosts.size(); - if (numNonActive > 0) { - long numActive = parentHostnames.size() - numNonActive; + + if (nonActiveHosts.size() > 0) { + long numActive = parentHostnames.size() - nonActiveHosts.size(); var messageBuilder = new StringBuilder() .append(numActive).append("/").append(parentHostnames.size()) .append(" hosts for ") diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java index a609103ac89..fb76dc54d1a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -26,12 +26,6 @@ public class DockerHostCapacity { this.hostResourcesCalculator = Objects.requireNonNull(hostResourcesCalculator, "hostResourcesCalculator must be non-null"); } - /** Returns the allocation skew of this host */ - public double skew(Node host) { - NodeResources free = freeCapacityOf(host, false); - return Node.skew(host.flavor().resources(), free); - } - int compareWithoutInactive(Node hostA, Node hostB) { int result = compare(freeCapacityOf(hostB, true), freeCapacityOf(hostA, true)); if (result != 0) return result; @@ -72,7 +66,7 @@ public class DockerHostCapacity { NodeResources freeCapacityOf(Node host, boolean excludeInactive) { // Only hosts have free capacity if (!host.type().canRun(NodeType.tenant)) return new NodeResources(0, 0, 0, 0); - NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(host.flavor().resources()); + NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(host.flavor().name(), host.flavor().resources()); return allNodes.childrenOf(host).asList().stream() .filter(node -> !(excludeInactive && isInactiveOrRetired(node))) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java index 9a06f2a980a..4416106f23e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java @@ -3,9 +3,14 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.StringFlag; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; import java.time.Duration; @@ -13,6 +18,7 @@ import java.util.Collections; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; import java.util.logging.Logger; /** @@ -28,6 +34,7 @@ public class DockerImages { private final CuratorDatabaseClient db; private final DockerImage defaultImage; private final Duration cacheTtl; + private final StringFlag imageOverride; /** * Docker image is read on every request to /nodes/v2/node/[fqdn]. Cache current getDockerImages to avoid @@ -36,20 +43,41 @@ public class DockerImages { */ private volatile Supplier<Map<NodeType, DockerImage>> dockerImages; - public DockerImages(CuratorDatabaseClient db, DockerImage defaultImage) { - this(db, defaultImage, defaultCacheTtl); + public DockerImages(CuratorDatabaseClient db, DockerImage defaultImage, StringFlag imageOverride) { + this(db, defaultImage, defaultCacheTtl, imageOverride); } - DockerImages(CuratorDatabaseClient db, DockerImage defaultImage, Duration cacheTtl) { + DockerImages(CuratorDatabaseClient db, DockerImage defaultImage, Duration cacheTtl, StringFlag imageOverride) { this.db = db; this.defaultImage = defaultImage; this.cacheTtl = cacheTtl; + this.imageOverride = imageOverride; createCache(); } private void createCache() { this.dockerImages = Suppliers.memoizeWithExpiration(() -> Collections.unmodifiableMap(db.readDockerImages()), - cacheTtl.toMillis(), TimeUnit.MILLISECONDS); + cacheTtl.toMillis(), TimeUnit.MILLISECONDS); + } + + /** Returns the image to use for given node and zone */ + public DockerImage dockerImageFor(Node node) { + if (node.type().isDockerHost()) { + // Docker hosts do not run in containers, and thus has no image. Return the image of the child node type + // instead as this allows the host to pre-download the (likely) image its node will run. + // + // Note that if the Docker image has been overridden through feature flag, the preloaded image won't match. + return dockerImageFor(node.type().childNodeType()); + } + return node.allocation() + .map(Allocation::owner) + .map(ApplicationId::serializedForm) + // Return overridden image for this application + .map(application -> imageOverride.with(FetchVector.Dimension.APPLICATION_ID, application).value()) + .filter(Predicate.not(String::isEmpty)) + .map(DockerImage::fromString) + // ... or default Docker image for this node type + .orElseGet(() -> dockerImageFor(node.type())); } /** Returns the current docker images for each node type */ @@ -58,7 +86,7 @@ public class DockerImages { } /** Returns the current docker image for given node type, or default */ - public DockerImage dockerImageFor(NodeType type) { + private DockerImage dockerImageFor(NodeType type) { return getDockerImages().getOrDefault(type, defaultImage); } @@ -69,8 +97,8 @@ public class DockerImages { } try (Lock lock = db.lockDockerImages()) { Map<NodeType, DockerImage> dockerImages = db.readDockerImages(); - - dockerImage.ifPresentOrElse(image -> dockerImages.put(nodeType, image), () -> dockerImages.remove(nodeType)); + dockerImage.ifPresentOrElse(image -> dockerImages.put(nodeType, image), + () -> dockerImages.remove(nodeType)); db.writeDockerImages(dockerImages); createCache(); // Throw away current cache log.info("Set docker image for " + nodeType + " nodes to " + dockerImage.map(DockerImage::asString).orElse(null)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index 05915b82bae..b5c4478cd5a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -30,7 +30,7 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { public static class NoopHostResourcesCalculator implements HostResourcesCalculator { @Override - public NodeResources availableCapacityOf(NodeResources hostResources) { + public NodeResources availableCapacityOf(String flavorName, NodeResources hostResources) { return hostResources; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index eab8bb68863..5753bbb3c5a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -31,7 +31,6 @@ public class GroupPreparer { private final Optional<HostProvisioner> hostProvisioner; private final HostResourcesCalculator hostResourcesCalculator; private final BooleanFlag dynamicProvisioningEnabledFlag; - private final BooleanFlag enableInPlaceResize; private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag; public GroupPreparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, @@ -40,7 +39,6 @@ public class GroupPreparer { this.hostProvisioner = hostProvisioner; this.hostResourcesCalculator = hostResourcesCalculator; this.dynamicProvisioningEnabledFlag = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource); - this.enableInPlaceResize = Flags.ENABLE_IN_PLACE_RESIZE.bindTo(flagSource); this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource); } @@ -65,25 +63,23 @@ public class GroupPreparer { boolean dynamicProvisioningEnabled = hostProvisioner.isPresent() && dynamicProvisioningEnabledFlag .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) .value(); - boolean inPlaceResizeEnabled = enableInPlaceResize - .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) - .value(); + boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty(); try (Mutex lock = nodeRepository.lock(application)) { // Lock ready pool to ensure that the same nodes are not simultaneously allocated by others - try (Mutex allocationLock = nodeRepository.lockAllocation()) { + try (Mutex allocationLock = nodeRepository.lockUnallocated()) { // Create a prioritized set of nodes LockedNodeList nodeList = nodeRepository.list(allocationLock); NodePrioritizer prioritizer = new NodePrioritizer(nodeList, application, cluster, requestedNodes, spareCount, wantedGroups, nodeRepository.nameResolver(), - hostResourcesCalculator, inPlaceResizeEnabled); + hostResourcesCalculator, allocateFully); prioritizer.addApplicationNodes(); prioritizer.addSurplusNodes(surplusActiveNodes); prioritizer.addReadyNodes(); - prioritizer.addNewDockerNodes(dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty()); + prioritizer.addNewDockerNodes(); // Allocate from the prioritized list NodeAllocation allocation = new NodeAllocation(nodeList, application, cluster, requestedNodes, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java index c5808a53837..a5570dbf169 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java @@ -9,6 +9,6 @@ import com.yahoo.config.provision.NodeResources; public interface HostResourcesCalculator { /** Calculates the resources that are reserved for host level processes and returns the remainder. */ - NodeResources availableCapacityOf(NodeResources hostResources); + NodeResources availableCapacityOf(String flavorName, NodeResources hostResources); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index f0caa358cb8..26f8cffa519 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; @@ -51,8 +51,8 @@ public class LoadBalancerProvisioner { this.db = nodeRepository.database(); this.service = service; // Read and write all load balancers to make sure they are stored in the latest version of the serialization format - try (var lock = db.lockLoadBalancers()) { - for (var id : db.readLoadBalancerIds()) { + for (var id : db.readLoadBalancerIds()) { + try (var lock = db.lockLoadBalancers(id.application())) { var loadBalancer = db.readLoadBalancer(id); loadBalancer.ifPresent(db::writeLoadBalancer); } @@ -72,8 +72,9 @@ public class LoadBalancerProvisioner { public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type - try (var loadBalancersLock = db.lockLoadBalancers()) { - provision(application, cluster.id(), false, loadBalancersLock); + if (application.instance().isTester()) return; // Do not provision for tester instances + try (var lock = db.lockLoadBalancers(application)) { + provision(application, cluster.id(), false, lock); } } @@ -89,11 +90,11 @@ public class LoadBalancerProvisioner { */ public void activate(ApplicationId application, Set<ClusterSpec> clusters, @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { - try (var loadBalancersLock = db.lockLoadBalancers()) { + try (var lock = db.lockLoadBalancers(application)) { var containerClusters = containerClusterOf(clusters); for (var clusterId : containerClusters) { // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(application, clusterId, true, loadBalancersLock); + provision(application, clusterId, true, lock); } // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); @@ -107,16 +108,15 @@ public class LoadBalancerProvisioner { */ public void deactivate(ApplicationId application, NestedTransaction transaction) { try (var applicationLock = nodeRepository.lock(application)) { - try (Mutex loadBalancersLock = db.lockLoadBalancers()) { - deactivate(nodeRepository.loadBalancers().owner(application).asList(), transaction); + try (var lock = db.lockLoadBalancers(application)) { + deactivate(nodeRepository.loadBalancers(application).asList(), transaction); } } } /** Returns load balancers of given application that are no longer referenced by given clusters */ private List<LoadBalancer> surplusLoadBalancersOf(ApplicationId application, Set<ClusterSpec.Id> activeClusters) { - var activeLoadBalancersByCluster = nodeRepository.loadBalancers() - .owner(application) + var activeLoadBalancersByCluster = nodeRepository.loadBalancers(application) .in(LoadBalancer.State.active) .asList() .stream() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 21e2c322594..c92f7889496 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -140,9 +140,9 @@ class NodeAllocation { continue; } node.node = offered.allocate(application, - ClusterMembership.from(cluster, highestIndex.add(1)), - requestedNodes.resources().orElse(node.node.flavor().resources()), - clock.instant()); + ClusterMembership.from(cluster, highestIndex.add(1)), + requestedNodes.resources().orElse(node.node.flavor().resources()), + clock.instant()); accepted.add(acceptNode(node, false, false)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index a17d92117fb..1ef23209369 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -40,29 +40,30 @@ public class NodePrioritizer { private final LockedNodeList allNodes; private final DockerHostCapacity capacity; private final NodeSpec requestedNodes; - private final ApplicationId appId; + private final ApplicationId application; private final ClusterSpec clusterSpec; private final NameResolver nameResolver; private final boolean isDocker; private final boolean isAllocatingForReplacement; private final boolean isTopologyChange; - private final boolean inPlaceResizeEnabled; + /** If set, a host can only have nodes by single tenant and does not allow in-place resizing. */ + private final boolean allocateFully; private final int currentClusterSize; private final Set<Node> spareHosts; - NodePrioritizer(LockedNodeList allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec, + NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, int spares, int wantedGroups, NameResolver nameResolver, HostResourcesCalculator hostResourcesCalculator, - boolean inPlaceResizeEnabled) { + boolean allocateFully) { this.allNodes = allNodes; this.capacity = new DockerHostCapacity(allNodes, hostResourcesCalculator); this.requestedNodes = nodeSpec; this.clusterSpec = clusterSpec; - this.appId = appId; + this.application = application; this.nameResolver = nameResolver; this.spareHosts = findSpareHosts(allNodes, capacity, spares); - this.inPlaceResizeEnabled = inPlaceResizeEnabled; + this.allocateFully = allocateFully; - NodeList nodesInCluster = allNodes.owner(appId).type(clusterSpec.type()).cluster(clusterSpec.id()); + NodeList nodesInCluster = allNodes.owner(application).type(clusterSpec.type()).cluster(clusterSpec.id()); NodeList nonRetiredNodesInCluster = nodesInCluster.not().retired(); long currentGroups = nonRetiredNodesInCluster.state(Node.State.active).stream() .flatMap(node -> node.allocation() @@ -117,23 +118,19 @@ public class NodePrioritizer { } } - /** - * Add a node on each docker host with enough capacity for the requested flavor - * - * @param exclusively whether the ready docker nodes should only be added on hosts that - * already have nodes allocated to this tenant - */ - void addNewDockerNodes(boolean exclusively) { + /** Add a node on each docker host with enough capacity for the requested flavor */ + void addNewDockerNodes() { if ( ! isDocker) return; LockedNodeList candidates = allNodes - .filter(node -> node.type() != NodeType.host || ALLOCATABLE_HOST_STATES.contains(node.state())); + .filter(node -> node.type() != NodeType.host || ALLOCATABLE_HOST_STATES.contains(node.state())) + .filter(node -> node.reservedTo().isEmpty() || node.reservedTo().get().equals(application.tenant())); - if (exclusively) { + if (allocateFully) { Set<String> candidateHostnames = candidates.asList().stream() .filter(node -> node.type() == NodeType.tenant) .filter(node -> node.allocation() - .map(a -> a.owner().tenant().equals(appId.tenant())) + .map(a -> a.owner().tenant().equals(this.application.tenant())) .orElse(false)) .flatMap(node -> node.parentHostname().stream()) .collect(Collectors.toSet()); @@ -152,7 +149,7 @@ public class NodePrioritizer { if (host.status().wantToRetire()) continue; boolean hostHasCapacityForWantedFlavor = capacity.hasCapacity(host, wantedResources); - boolean conflictingCluster = allNodes.childrenOf(host).owner(appId).asList().stream() + boolean conflictingCluster = allNodes.childrenOf(host).owner(application).asList().stream() .anyMatch(child -> child.allocation().get().membership().cluster().id().equals(clusterSpec.id())); if (!hostHasCapacityForWantedFlavor || conflictingCluster) continue; @@ -188,7 +185,7 @@ public class NodePrioritizer { .filter(node -> node.type() == requestedNodes.type()) .filter(node -> legalStates.contains(node.state())) .filter(node -> node.allocation().isPresent()) - .filter(node -> node.allocation().get().owner().equals(appId)) + .filter(node -> node.allocation().get().owner().equals(application)) .map(node -> toPrioritizable(node, false, false)) .forEach(prioritizableNode -> nodes.put(prioritizableNode.node, prioritizableNode)); } @@ -217,7 +214,7 @@ public class NodePrioritizer { builder.parent(parent).freeParentCapacity(parentCapacity); if (!isNewNode) - builder.resizable(inPlaceResizeEnabled && requestedNodes.canResize( + builder.resizable(!allocateFully && requestedNodes.canResize( node.flavor().resources(), parentCapacity, isTopologyChange, currentClusterSize)); if (spareHosts.contains(parent)) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 7cf55adc776..07cf297a314 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -131,7 +131,8 @@ public class NodeRepositoryProvisioner implements Provisioner { private boolean hasQuota(ApplicationId application, int requestedNodes) { if ( ! this.zone.system().isPublic()) return true; // no quota management - if (application.tenant().value().hashCode() == 3857) return requestedNodes <= 60; + if (application.tenant().value().hashCode() == 3857) return requestedNodes <= 60; + if (application.tenant().value().hashCode() == -1271827001) return requestedNodes <= 75; return requestedNodes <= 5; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java index c7732695069..6183bffe5ba 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java @@ -87,6 +87,10 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { throw new IllegalStateException("Nodes " + this.node + " and " + other.node + " have different states"); if (this.parent.isPresent() && other.parent.isPresent()) { + // Prefer reserved hosts (that they are reserved to the right tenant is ensured elsewhere) + if ( this.parent.get().reservedTo().isPresent() && ! other.parent.get().reservedTo().isPresent()) return -1; + if ( ! this.parent.get().reservedTo().isPresent() && other.parent.get().reservedTo().isPresent()) return 1; + int diskCostDifference = NodeResources.DiskSpeed.compare(this.parent.get().flavor().resources().diskSpeed(), other.parent.get().flavor().resources().diskSpeed()); if (diskCostDifference != 0) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 451ab089a8d..b979ccda740 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -34,7 +34,7 @@ public class ProvisionedHost { /** Generate {@link Node} instance representing the provisioned physical host */ public Node generateHost() { - return Node.create(id, IP.Config.EMPTY, hostHostname, Optional.empty(), Optional.empty(), hostFlavor, NodeType.host); + return Node.create(id, IP.Config.EMPTY, hostHostname, Optional.empty(), Optional.empty(), hostFlavor, Optional.empty(), NodeType.host); } /** Generate {@link Node} instance representing the node running on this physical host */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java index 9f8f4a804d1..3147d4caded 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.config.provision.ApplicationId; @@ -37,10 +37,14 @@ public class LoadBalancersResponse extends HttpResponse { } private List<LoadBalancer> loadBalancers() { - LoadBalancerList loadBalancers = nodeRepository.loadBalancers(); - return application().map(loadBalancers::owner) - .map(LoadBalancerList::asList) - .orElseGet(loadBalancers::asList); + LoadBalancerList loadBalancers; + var application = application(); + if (application.isPresent()) { + loadBalancers = nodeRepository.loadBalancers(application.get()); + } else { + loadBalancers = nodeRepository.loadBalancers(); + } + return loadBalancers.asList(); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java index 81cf401d358..11be95604c8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java @@ -6,11 +6,13 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Type; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -33,8 +35,8 @@ import java.util.stream.Collectors; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; -import static com.yahoo.config.provision.NodeResources.StorageType.remote; import static com.yahoo.config.provision.NodeResources.StorageType.local; +import static com.yahoo.config.provision.NodeResources.StorageType.remote; /** * A class which can take a partial JSON node/v2 node JSON structure and apply it to a node object. @@ -45,7 +47,6 @@ import static com.yahoo.config.provision.NodeResources.StorageType.local; public class NodePatcher { private static final String WANT_TO_RETIRE = "wantToRetire"; - private static final String WANT_TO_DEPROVISION = "wantToDeprovision"; private final NodeFlavors nodeFlavors; private final Inspector inspector; @@ -99,7 +100,6 @@ public class NodePatcher { private List<Node> applyFieldRecursive(List<Node> childNodes, String name, Inspector value) { switch (name) { case WANT_TO_RETIRE: - case WANT_TO_DEPROVISION: return childNodes.stream() .map(child -> applyField(child, name, value)) .collect(Collectors.toList()); @@ -138,7 +138,9 @@ public class NodePatcher { return IP.Config.verify(node.with(node.ipConfig().with(IP.Pool.of(asStringSet(value)))), nodes); case WANT_TO_RETIRE : return node.withWantToRetire(asBoolean(value), Agent.operator, clock.instant()); - case WANT_TO_DEPROVISION : + case "wantToDeprovision" : + if (node.type() != NodeType.host && asBoolean(value)) + throw new IllegalArgumentException("wantToDeprovision can only be set for hosts"); return node.with(node.status().withWantToDeprovision(asBoolean(value))); case "reports" : return nodeWithPatchedReports(node, value); @@ -160,34 +162,53 @@ public class NodePatcher { case "bandwidthGbps": return node.with(node.flavor().with(node.flavor().resources().withBandwidthGbps(value.asDouble()))); case "modelName": - if (value.type() == Type.NIX) { - return node.withoutModelName(); - } - return node.withModelName(asString(value)); + return value.type() == Type.NIX ? node.withoutModelName() : node.withModelName(asString(value)); case "requiredDiskSpeed": return patchRequiredDiskSpeed(asString(value)); + case "reservedTo": + return value.type() == Type.NIX ? node.withoutReservedTo() : node.withReservedTo(TenantName.from(value.asString())); default : throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); } } private Node nodeWithPatchedReports(Node node, Inspector reportsInspector) { + Node patchedNode; // "reports": null clears the reports - if (reportsInspector.type() == Type.NIX) return node.with(new Reports()); + if (reportsInspector.type() == Type.NIX) { + patchedNode = node.with(new Reports()); + } else { + var reportsBuilder = new Reports.Builder(node.reports()); + reportsInspector.traverse((ObjectTraverser) (reportId, reportInspector) -> { + if (reportInspector.type() == Type.NIX) { + // ... "reports": { "reportId": null } clears the report "reportId" + reportsBuilder.clearReport(reportId); + } else { + // ... "reports": { "reportId": {...} } overrides the whole report "reportId" + reportsBuilder.setReport(Report.fromSlime(reportId, reportInspector)); + } + }); + patchedNode = node.with(reportsBuilder.build()); + } - var reportsBuilder = new Reports.Builder(node.reports()); + boolean hadHardFailReports = node.reports().getReports().stream() + .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL); + boolean hasHardFailReports = patchedNode.reports().getReports().stream() + .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL); - reportsInspector.traverse((ObjectTraverser) (reportId, reportInspector) -> { - if (reportInspector.type() == Type.NIX) { - // ... "reports": { "reportId": null } clears the report "reportId" - reportsBuilder.clearReport(reportId); - } else { - // ... "reports": { "reportId": {...} } overrides the whole report "reportId" - reportsBuilder.setReport(Report.fromSlime(reportId, reportInspector)); - } - }); + // If this patch resulted in going from not having HARD_FAIL report to having one, or vice versa + if (hadHardFailReports != hasHardFailReports) { + // Do not automatically change wantToDeprovision when + // 1. Transitioning to having a HARD_FAIL report and being in state failed: + // To allow operators manually unset before the host is parked and deleted. + // 2. When in parked state: Deletion is imminent, possibly already underway + if ((hasHardFailReports && node.state() == Node.State.failed) || node.state() == Node.State.parked) + return patchedNode; + + patchedNode = patchedNode.with(patchedNode.status().withWantToDeprovision(hasHardFailReports)); + } - return node.with(reportsBuilder.build()); + return patchedNode; } private Set<String> asStringSet(Inspector field) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 692402ea914..f0d996ef595 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; @@ -19,7 +20,8 @@ import com.yahoo.restapi.ResourceResponse; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.Type; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -49,7 +51,7 @@ import java.util.function.Function; import java.util.logging.Level; import java.util.stream.Collectors; -import static com.yahoo.vespa.config.SlimeUtils.optionalString; +import static com.yahoo.slime.SlimeUtils.optionalString; /** * The implementation of the /nodes/v2 API. @@ -225,14 +227,15 @@ public class NodesApiHandler extends LoggingRequestHandler { Set<String> ipAddressPool = new HashSet<>(); inspector.field("additionalIpAddresses").traverse((ArrayTraverser) (i, item) -> ipAddressPool.add(item.asString())); - return Node.create( - inspector.field("openStackId").asString(), - new IP.Config(ipAddresses, ipAddressPool), - inspector.field("hostname").asString(), - parentHostname, - modelName, - flavorFromSlime(inspector), - nodeTypeFromSlime(inspector.field("type"))); + return Node.create(inspector.field("openStackId").asString(), + new IP.Config(ipAddresses, ipAddressPool), + inspector.field("hostname").asString(), + parentHostname, + modelName, + flavorFromSlime(inspector), + reservedToFromSlime(inspector.field("reservedTo")), + nodeTypeFromSlime(inspector.field("type")) + ); } private Flavor flavorFromSlime(Inspector inspector) { @@ -277,6 +280,13 @@ public class NodesApiHandler extends LoggingRequestHandler { return serializer.typeFrom(object.asString()); } + private Optional<TenantName> reservedToFromSlime(Inspector object) { + if (! object.valid()) return Optional.empty(); + if ( object.type() != Type.STRING) + throw new IllegalArgumentException("Expected 'reservedTo' to be a string but is " + object); + return Optional.of(TenantName.from(object.asString())); + } + public static NodeFilter toNodeFilter(HttpRequest request) { NodeFilter filter = NodeHostFilter.from(HostFilter.from(request.getProperty("hostname"), request.getProperty("flavor"), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 7ed05432e01..7f283452538 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -16,11 +16,10 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.orchestrator.Orchestrator; -import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; import java.io.IOException; import java.io.OutputStream; @@ -46,7 +45,7 @@ class NodesResponse extends HttpResponse { private final NodeFilter filter; private final boolean recursive; - private final Function<HostName, Optional<HostStatus>> orchestrator; + private final Function<HostName, Optional<HostInfo>> orchestrator; private final NodeRepository nodeRepository; private final Slime slime; private final NodeSerializer serializer = new NodeSerializer(); @@ -58,7 +57,7 @@ class NodesResponse extends HttpResponse { this.nodeParentUrl = toNodeParentUrl(request); filter = NodesApiHandler.toNodeFilter(request); this.recursive = request.getBooleanProperty("recursive"); - this.orchestrator = orchestrator.getNodeStatuses(); + this.orchestrator = orchestrator.getHostResolver(); this.nodeRepository = nodeRepository; slime = new Slime(); @@ -146,6 +145,7 @@ class NodesResponse extends HttpResponse { } object.setString("openStackId", node.id()); object.setString("flavor", node.flavor().name()); + node.reservedTo().ifPresent(reservedTo -> object.setString("reservedTo", reservedTo.value())); if (node.flavor().isConfigured()) object.setDouble("cpuCores", node.flavor().getMinCpuCores()); toSlime(node.flavor().resources(), object.setObject("resources")); @@ -157,13 +157,15 @@ class NodesResponse extends HttpResponse { toSlime(allocation.membership(), object.setObject("membership")); object.setLong("restartGeneration", allocation.restartGeneration().wanted()); object.setLong("currentRestartGeneration", allocation.restartGeneration().current()); - object.setString("wantedDockerImage", dockerImageFor(node.type()).withTag(allocation.membership().cluster().vespaVersion()).asString()); + object.setString("wantedDockerImage", nodeRepository.dockerImage(node).withTag(allocation.membership().cluster().vespaVersion()).asString()); object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString()); toSlime(allocation.requestedResources(), object.setObject("requestedResources")); allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts"))); orchestrator.apply(new HostName(node.hostname())) - .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN) - .ifPresent(allowedToBeDown -> object.setBool("allowedToBeDown", allowedToBeDown)); + .ifPresent(info -> { + object.setBool("allowedToBeDown", info.status().isSuspended()); + info.suspendedSince().ifPresent(since -> object.setLong("suspendedSinceMillis", since.toEpochMilli())); + }); }); object.setLong("rebootGeneration", node.status().reboot().wanted()); object.setLong("currentRebootGeneration", node.status().reboot().current()); @@ -220,16 +222,10 @@ class NodesResponse extends HttpResponse { // TODO: Remove current + wanted docker image from response for non-docker types private Optional<DockerImage> currentDockerImage(Node node) { return node.status().dockerImage() - .or(() -> Optional.of(node) - .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) - .flatMap(n -> n.status().vespaVersion() - .map(version -> dockerImageFor(n.type()).withTag(version)))); - } - - // Docker hosts are not running in an image, but return the image of the node type running on it anyway, - // this allows the docker host to pre-download the (likely) image its node will run - private DockerImage dockerImageFor(NodeType nodeType) { - return nodeRepository.dockerImage(nodeType.isDockerHost() ? nodeType.childNodeType() : nodeType); + .or(() -> Optional.of(node) + .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) + .flatMap(n -> n.status().vespaVersion() + .map(version -> nodeRepository.dockerImage(n).withTag(version)))); } private void ipAddressesToSlime(Set<String> ipAddresses, Cursor array) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index 4abf6d77268..d26accd7a84 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -13,7 +13,8 @@ public class ContainerConfig { return "<container version='1.0'>\n" + " <config name=\"container.handler.threadpool\">\n" + " <maxthreads>20</maxthreads>\n" + - " </config> \n" + + " </config>\n" + + " <accesslog type='disabled'/>\n" + " <component id='com.yahoo.test.ManualClock'/>\n" + " <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>\n" + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 0d331ac0f3f..a2579bee0a1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -54,7 +54,7 @@ public class MockNodeRepository extends NodeRepository { super(flavors, curator, Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); this.flavors = flavors; curator.setZooKeeperEnsembleConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234"); @@ -70,34 +70,34 @@ public class MockNodeRepository extends NodeRepository { // Regular nodes nodes.add(createNode("node1", "host1.yahoo.com", ipConfig(1), Optional.empty(), - new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant)); + new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant)); nodes.add(createNode("node2", "host2.yahoo.com", ipConfig(2), Optional.empty(), - new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant)); + new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant)); nodes.add(createNode("node3", "host3.yahoo.com", ipConfig(3), Optional.empty(), - new Flavor(new NodeResources(0.5, 48, 500, 1, fast, local)), NodeType.tenant)); + new Flavor(new NodeResources(0.5, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant)); Node node4 = createNode("node4", "host4.yahoo.com", ipConfig(4), Optional.of("dockerhost1.yahoo.com"), - new Flavor(new NodeResources(1, 4, 100, 1, fast, local)), NodeType.tenant); + new Flavor(new NodeResources(1, 4, 100, 1, fast, local)), Optional.empty(), NodeType.tenant); node4 = node4.with(node4.status() .withVespaVersion(new Version("6.41.0")) .withDockerImage(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.41.0"))); nodes.add(node4); Node node5 = createNode("node5", "host5.yahoo.com", ipConfig(5), Optional.of("dockerhost2.yahoo.com"), - new Flavor(new NodeResources(1, 8, 100, 1, slow, remote)), NodeType.tenant); + new Flavor(new NodeResources(1, 8, 100, 1, slow, remote)), Optional.empty(), NodeType.tenant); nodes.add(node5.with(node5.status() .withVespaVersion(new Version("1.2.3")) .withDockerImage(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:1.2.3")))); nodes.add(createNode("node6", "host6.yahoo.com", ipConfig(6), Optional.empty(), - new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant)); + new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant)); Node node7 = createNode("node7", "host7.yahoo.com", ipConfig(7), Optional.empty(), - new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant); + new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant); nodes.add(node7); // 8, 9, 11 and 12 are added by web service calls Node node10 = createNode("node10", "host10.yahoo.com", ipConfig(10), Optional.of("parent1.yahoo.com"), - new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant); + new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant); Status node10newStatus = node10.status(); node10newStatus = node10newStatus .withVespaVersion(Version.fromString("5.104.142")) @@ -106,26 +106,26 @@ public class MockNodeRepository extends NodeRepository { nodes.add(node10); Node node55 = createNode("node55", "host55.yahoo.com", ipConfig(55), Optional.empty(), - new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant); + new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant); nodes.add(node55.with(node55.status().withWantToRetire(true).withWantToDeprovision(true))); /* Setup docker hosts (two of these will be reserved for spares */ nodes.add(createNode("dockerhost1", "dockerhost1.yahoo.com", ipConfig(100, 1, 3), Optional.empty(), - flavors.getFlavorOrThrow("large"), NodeType.host)); + flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); nodes.add(createNode("dockerhost2", "dockerhost2.yahoo.com", ipConfig(101, 1, 3), Optional.empty(), - flavors.getFlavorOrThrow("large"), NodeType.host)); + flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); nodes.add(createNode("dockerhost3", "dockerhost3.yahoo.com", ipConfig(102, 1, 3), Optional.empty(), - flavors.getFlavorOrThrow("large"), NodeType.host)); + flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); nodes.add(createNode("dockerhost4", "dockerhost4.yahoo.com", ipConfig(103, 1, 3), Optional.empty(), - flavors.getFlavorOrThrow("large"), NodeType.host)); + flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); nodes.add(createNode("dockerhost5", "dockerhost5.yahoo.com", ipConfig(104, 1, 3), Optional.empty(), - flavors.getFlavorOrThrow("large"), NodeType.host)); + flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); // Config servers nodes.add(createNode("cfg1", "cfg1.yahoo.com", ipConfig(201), Optional.empty(), - flavors.getFlavorOrThrow("default"), NodeType.config)); + flavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.config)); nodes.add(createNode("cfg2", "cfg2.yahoo.com", ipConfig(202), Optional.empty(), - flavors.getFlavorOrThrow("default"), NodeType.config)); + flavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.config)); // Ready all nodes, except 7 and 55 nodes = addNodes(nodes); @@ -167,9 +167,9 @@ public class MockNodeRepository extends NodeRepository { List<Node> largeNodes = new ArrayList<>(); largeNodes.add(createNode("node13", "host13.yahoo.com", ipConfig(13), Optional.empty(), - new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), NodeType.tenant)); + new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant)); largeNodes.add(createNode("node14", "host14.yahoo.com", ipConfig(14), Optional.empty(), - new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), NodeType.tenant)); + new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant)); addNodes(largeNodes); setReady(largeNodes, Agent.system, getClass().getSimpleName()); ApplicationId app4 = ApplicationId.from(TenantName.from("tenant4"), ApplicationName.from("application4"), InstanceName.from("instance4")); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java index 96ec0349fb2..d183e62c96c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java @@ -1,16 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; +import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.Host; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; +import java.time.Instant; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -18,9 +23,9 @@ import java.util.function.Function; /** * @author bratseth */ -public class OrchestratorMock implements Orchestrator { +public class OrchestratorMock extends AbstractComponent implements Orchestrator { - private final Set<HostName> suspendedHosts = new HashSet<>(); + private final Map<HostName, HostInfo> suspendedHosts = new HashMap<>(); private final Set<ApplicationId> suspendedApplications = new HashSet<>(); @Override @@ -30,12 +35,13 @@ public class OrchestratorMock implements Orchestrator { @Override public HostStatus getNodeStatus(HostName hostName) { - return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS; + HostInfo hostInfo = suspendedHosts.get(hostName); + return hostInfo == null ? HostStatus.NO_REMARKS : hostInfo.status(); } @Override - public Function<HostName, Optional<HostStatus>> getNodeStatuses() { - return hostName -> Optional.of(getNodeStatus(hostName)); + public Function<HostName, Optional<HostInfo>> getHostResolver() { + return hostName -> Optional.of(suspendedHosts.getOrDefault(hostName, HostInfo.createNoRemarks())); } @Override @@ -48,7 +54,7 @@ public class OrchestratorMock implements Orchestrator { @Override public void suspend(HostName hostName) { - suspendedHosts.add(hostName); + suspendedHosts.put(hostName, HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH)); } @Override @@ -73,7 +79,7 @@ public class OrchestratorMock implements Orchestrator { } @Override - public void acquirePermissionToRemove(HostName hostName) {} + public void acquirePermissionToRemove(HostName hostName) { } @Override public void suspendAll(HostName parentHostname, List<HostName> hostNames) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index f14b6d59ee0..95555185292 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; @@ -27,9 +28,12 @@ public class NodeRepositoryTester { private final NodeRepository nodeRepository; private final Clock clock; private final MockCurator curator; - - + public NodeRepositoryTester() { + this(new InMemoryFlagSource()); + } + + public NodeRepositoryTester(InMemoryFlagSource flagSource) { nodeFlavors = new NodeFlavors(createConfig()); clock = new ManualClock(); curator = new MockCurator(); @@ -37,7 +41,7 @@ public class NodeRepositoryTester { nodeRepository = new NodeRepository(nodeFlavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, flagSource); } public NodeRepository nodeRepository() { return nodeRepository; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index da174a8d38c..96236b5fb84 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -22,6 +22,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.IP; @@ -54,7 +55,8 @@ public class CapacityCheckerTester { Curator curator = new MockCurator(); NodeFlavors f = new NodeFlavors(new FlavorConfigBuilder().build()); nodeRepository = new NodeRepository(f, curator, clock, zone, new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, + new InMemoryFlagSource()); } private void updateCapacityChecker() { @@ -133,8 +135,8 @@ public class CapacityCheckerTester { NodeResources nr = containingNodeResources(childResources, excessCapacity); Node node = nodeRepository.createNode(hostname, hostname, - new IP.Config(Set.of("::"), availableIps), Optional.empty(), - new Flavor(nr), NodeType.host); + new IP.Config(Set.of("::"), availableIps), Optional.empty(), + new Flavor(nr), Optional.empty(), NodeType.host); hosts.add(node); } return hosts; @@ -152,8 +154,8 @@ public class CapacityCheckerTester { .mapToObj(n -> String.format("%04X::%04X", hostid, n)) .collect(Collectors.toSet()); Node node = nodeRepository.createNode(hostname, hostname, - new IP.Config(Set.of("::"), availableIps), Optional.empty(), - new Flavor(capacity), NodeType.host); + new IP.Config(Set.of("::"), availableIps), Optional.empty(), + new Flavor(capacity), Optional.empty(), NodeType.host); hosts.add(node); } return hosts; @@ -263,8 +265,8 @@ public class CapacityCheckerTester { Flavor f = new Flavor(nr); Node node = nodeRepository.createNode(nodeModel.id, nodeModel.hostname, - new IP.Config(nodeModel.ipAddresses, nodeModel.additionalIpAddresses), - nodeModel.parentHostname, f, nodeModel.type); + new IP.Config(nodeModel.ipAddresses, nodeModel.additionalIpAddresses), + nodeModel.parentHostname, f, Optional.empty(), nodeModel.type); if (membership != null) { return node.allocate(owner, membership, node.flavor().resources(), Instant.now()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 71788fb1a30..36f876fb6bb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -26,9 +26,11 @@ import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; +import org.junit.Before; import org.junit.Test; import java.time.Duration; @@ -49,6 +51,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -64,18 +67,23 @@ public class DynamicProvisioningMaintainerTest { private final HostProvisionerTester tester = new HostProvisionerTester(); private final HostProvisioner hostProvisioner = mock(HostProvisioner.class); + private final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource() .withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true) .withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(), PreprovisionCapacity.class); private final DynamicProvisioningMaintainer maintainer = new DynamicProvisioningMaintainer( - tester.nodeRepository, Duration.ofDays(1), hostProvisioner, flagSource); + tester.nodeRepository, Duration.ofDays(1), hostProvisioner, hostResourcesCalculator, flagSource); @Test public void delegates_to_host_provisioner_and_writes_back_result() { addNodes(); + Node host3 = tester.nodeRepository.getNode("host3").orElseThrow(); Node host4 = tester.nodeRepository.getNode("host4").orElseThrow(); Node host41 = tester.nodeRepository.getNode("host4-1").orElseThrow(); - assertTrue(Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); + assertTrue(Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); + + Node host3new = host3.with(host3.ipConfig().with(Set.of("::5"))); + when(hostProvisioner.provision(eq(host3), eq(Set.of()))).thenReturn(List.of(host3new)); Node host4new = host4.with(host4.ipConfig().with(Set.of("::2"))); Node host41new = host41.with(host4.ipConfig().with(Set.of("::4", "10.0.0.1"))); @@ -83,8 +91,10 @@ public class DynamicProvisioningMaintainerTest { maintainer.updateProvisioningNodes(tester.nodeRepository.list(), () -> {}); verify(hostProvisioner).provision(eq(host4), eq(Set.of(host41))); + verify(hostProvisioner).provision(eq(host3), eq(Set.of())); verifyNoMoreInteractions(hostProvisioner); + assertEquals(Optional.of(host3new), tester.nodeRepository.getNode("host3")); assertEquals(Optional.of(host4new), tester.nodeRepository.getNode("host4")); assertEquals(Optional.of(host41new), tester.nodeRepository.getNode("host4-1")); } @@ -127,7 +137,7 @@ public class DynamicProvisioningMaintainerTest { @Test public void provision_deficit_and_deprovision_excess() { - flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(1, 3, 2, 1), new PreprovisionCapacity(2, 3, 2, 2)), PreprovisionCapacity.class); + flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(2, 4, 8, 1), new PreprovisionCapacity(2, 3, 2, 2)), PreprovisionCapacity.class); addNodes(); maintainer.convergeToCapacity(tester.nodeRepository.list()); @@ -150,6 +160,15 @@ public class DynamicProvisioningMaintainerTest { verifyNoMoreInteractions(hostProvisioner); } + @Before + public void setup() { + doAnswer(invocation -> { + String flavorName = invocation.getArgument(0, String.class); + if ("default".equals(flavorName)) return new NodeResources(2, 4, 8, 1); + return invocation.getArguments()[1]; + }).when(hostResourcesCalculator).availableCapacityOf(any(), any()); + } + public void addNodes() { List.of(createNode("host1", Optional.empty(), NodeType.host, Node.State.active, Optional.of(tenantHostApp)), createNode("host1-1", Optional.of("host1"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)), @@ -157,6 +176,7 @@ public class DynamicProvisioningMaintainerTest { createNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)), createNode("host2-1", Optional.of("host2"), NodeType.tenant, Node.State.failed, Optional.empty()), + createNode("host3", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), createNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), @@ -186,7 +206,8 @@ public class DynamicProvisioningMaintainerTest { private final ManualClock clock = new ManualClock(); private final NodeRepository nodeRepository = new NodeRepository( - nodeFlavors, new MockCurator(), clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), true); + nodeFlavors, new MockCurator(), clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), + DockerImage.fromString("docker-image"), true, new InMemoryFlagSource()); Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) { Node node = createNode(hostname, parentHostname, nodeType, state, application); @@ -204,7 +225,7 @@ public class DynamicProvisioningMaintainerTest { false)); var ipConfig = new IP.Config(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of()); return new Node("fake-id-" + hostname, ipConfig, hostname, parentHostname, flavor, Status.initial(), - state, allocation, History.empty(), nodeType, new Reports(), Optional.empty()); + state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty()); } } -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 8509722b016..c293a3436b8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -256,7 +256,7 @@ public class FailedExpirerTest { this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), - true); + true, new InMemoryFlagSource()); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index 12b48fd7a35..7e8fcddb1ae 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Vtag; @@ -32,14 +32,14 @@ import static org.junit.Assert.assertTrue; */ public class LoadBalancerExpirerTest { - private ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); @Test public void expire_inactive() { LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(), Duration.ofDays(1), tester.loadBalancerService()); - Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers(); + Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true); // Deploy two applications with a total of three load balancers ClusterSpec.Id cluster1 = ClusterSpec.Id.from("qrs"); @@ -67,14 +67,15 @@ public class LoadBalancerExpirerTest { // Expirer prunes reals before expiration time of load balancer itself expirer.maintain(); assertEquals(Set.of(), tester.loadBalancerService().instances().get(lb1).reals()); - assertEquals(Set.of(), tester.nodeRepository().loadBalancers().owner(lb1.application()).asList().get(0).instance().reals()); + assertEquals(Set.of(), loadBalancers.get().get(lb1).instance().reals()); // Expirer defers removal of load balancer until expiration time passes expirer.maintain(); + assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb1).state()); assertTrue("Inactive load balancer not removed", tester.loadBalancerService().instances().containsKey(lb1)); // Expirer removes load balancers once expiration time passes - tester.clock().advance(Duration.ofHours(1)); + tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1))); expirer.maintain(); assertFalse("Inactive load balancer removed", tester.loadBalancerService().instances().containsKey(lb1)); @@ -85,7 +86,7 @@ public class LoadBalancerExpirerTest { // A single cluster is removed deployApplication(app2, cluster1); expirer.maintain(); - assertEquals(LoadBalancer.State.inactive, loadBalancers.get().get(lb3).state()); + assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb3).state()); // Expirer defers removal while nodes are still allocated to cluster expirer.maintain(); @@ -93,7 +94,7 @@ public class LoadBalancerExpirerTest { dirtyNodesOf(app2, cluster2); // Expirer removes load balancer for removed cluster - tester.clock().advance(Duration.ofHours(1)); + tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1))); expirer.maintain(); assertFalse("Inactive load balancer removed", tester.loadBalancerService().instances().containsKey(lb3)); } @@ -103,7 +104,7 @@ public class LoadBalancerExpirerTest { LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(), Duration.ofDays(1), tester.loadBalancerService()); - Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers(); + Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true); // Prepare application @@ -121,7 +122,7 @@ public class LoadBalancerExpirerTest { // Application never activates and nodes are dirtied. Expirer moves load balancer to inactive after timeout dirtyNodesOf(app, cluster); - tester.clock().advance(Duration.ofHours(1)); + tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1))); expirer.maintain(); assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb).state()); @@ -130,7 +131,7 @@ public class LoadBalancerExpirerTest { assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb).state()); // Expirer removes inactive load balancer - tester.clock().advance(Duration.ofHours(1)); + tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1))); expirer.maintain(); assertFalse("Inactive load balancer removed", loadBalancers.get().containsKey(lb)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java index a4b66d3cf9e..246f2509397 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -35,7 +36,7 @@ public class MaintenanceTester { public final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); public MaintenanceTester() { curator.setZooKeeperEnsembleConnectionSpec("zk1.host:1,zk2.host:2,zk3.host:3"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 539d8c7cff2..321812497bd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -10,8 +10,10 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; +import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -22,6 +24,7 @@ import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; @@ -29,6 +32,7 @@ import org.junit.Test; import java.time.Clock; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -53,7 +57,7 @@ public class MetricsReporterTest { NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); nodeRepository.addNodes(List.of(node)); Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy); @@ -82,12 +86,17 @@ public class MetricsReporterTest { expectedMetrics.put("wantToRetire", 0); expectedMetrics.put("wantToDeprovision", 0); expectedMetrics.put("failReport", 0); - expectedMetrics.put("allowedToBeDown", 0); + expectedMetrics.put("allowedToBeDown", 1); + expectedMetrics.put("suspended", 1); + expectedMetrics.put("suspendedSeconds", 123L); expectedMetrics.put("numberOfServices", 0L); + ManualClock clock = new ManualClock(Instant.ofEpochSecond(124)); Orchestrator orchestrator = mock(Orchestrator.class); ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); - when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS)); + when(orchestrator.getHostResolver()).thenReturn(hostName -> + Optional.of(HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.ofEpochSecond(1))) + ); ServiceModel serviceModel = mock(ServiceModel.class); when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); @@ -99,8 +108,8 @@ public class MetricsReporterTest { orchestrator, serviceMonitor, () -> 42, - Duration.ofMinutes(1) - ); + Duration.ofMinutes(1), + clock); metricsReporter.maintain(); assertEquals(expectedMetrics, metric.values); @@ -113,13 +122,13 @@ public class MetricsReporterTest { NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); // Allow 4 containers Set<String> ipAddressPool = Set.of("::2", "::3", "::4", "::5"); Node dockerHost = Node.create("openStackId1", new IP.Config(Set.of("::1"), ipAddressPool), "dockerHost", - Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); nodeRepository.addNodes(List.of(dockerHost)); nodeRepository.dirtyRecursively("dockerHost", Agent.system, getClass().getSimpleName()); nodeRepository.setReady("dockerHost", Agent.system, getClass().getSimpleName()); @@ -127,29 +136,30 @@ public class MetricsReporterTest { Node container1 = Node.createDockerNode(Set.of("::2"), "container1", "dockerHost", new NodeResources(1, 3, 2, 1), NodeType.tenant); container1 = container1.with(allocation(Optional.of("app1"), container1).get()); - nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockAllocation())); + nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockUnallocated())); Node container2 = Node.createDockerNode(Set.of("::3"), "container2", "dockerHost", new NodeResources(2, 4, 4, 1), NodeType.tenant); container2 = container2.with(allocation(Optional.of("app2"), container2).get()); - nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockAllocation())); + nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockUnallocated())); Orchestrator orchestrator = mock(Orchestrator.class); ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); - when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS)); + when(orchestrator.getHostResolver()).thenReturn(hostName -> Optional.of(HostInfo.createNoRemarks())); ServiceModel serviceModel = mock(ServiceModel.class); when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); TestMetric metric = new TestMetric(); + ManualClock clock = new ManualClock(); MetricsReporter metricsReporter = new MetricsReporter( nodeRepository, metric, orchestrator, serviceMonitor, () -> 42, - Duration.ofMinutes(1) - ); + Duration.ofMinutes(1), + clock); metricsReporter.maintain(); assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 5872a78e1e2..033ddcd827e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -75,7 +75,8 @@ public class NodeFailTester { clock = new ManualClock(); curator = new MockCurator(); nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, + new InMemoryFlagSource()); provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); hostLivenessTracker = new TestHostLivenessTracker(clock); orchestrator = new OrchestratorMock(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java index 50c00c730bb..22d7f03c449 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java @@ -56,7 +56,7 @@ public class OperatorChangeApplicationMaintainerTest { this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); this.fixture = new Fixture(zone, nodeRepository); createReadyNodes(15, this.fixture.nodeResources, nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index b1c3b23016c..913b8b53c46 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -62,7 +62,7 @@ public class PeriodicApplicationMaintainerTest { this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); this.fixture = new Fixture(zone, nodeRepository); createReadyNodes(15, fixture.nodeResources, nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index d1a330a3bd6..d0c678bdf45 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -152,7 +152,7 @@ public class RebalancerTest { private static class IdentityHostResourcesCalculator implements HostResourcesCalculator { @Override - public NodeResources availableCapacityOf(NodeResources hostResources) { + public NodeResources availableCapacityOf(String flavorName, NodeResources hostResources) { return hostResources; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index 11ee6637720..96c3cc09b6b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -47,7 +47,7 @@ public class ReservationExpirerTest { NodeRepository nodeRepository = new NodeRepository(flavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); List<Node> nodes = new ArrayList<>(2); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 67af2df36e7..bdbe046fbdf 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -64,8 +64,8 @@ public class RetiredExpirerTest { private final Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + new MockNameResolver().mockAnyLookup(), + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, new InMemoryFlagSource()); private final NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); private final Orchestrator orchestrator = mock(Orchestrator.class); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java index 5e44b2e903e..fcf2ab5a52d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java @@ -179,7 +179,7 @@ public class IPTest { private static Node createNode(Set<String> ipAddresses) { return Node.create("id1", new IP.Config(Set.of("127.0.0.1"), ipAddresses), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), - NodeType.host); + Optional.empty(), NodeType.host); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index ebb64d650f1..5e859cd3d25 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -5,13 +5,18 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.node.OsVersion; +import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import org.junit.Test; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.function.Function; import java.util.function.Supplier; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -75,8 +80,11 @@ public class OsVersionsTest { tester.makeReadyNodes(totalNodes, "default", NodeType.host); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().list().nodeType(NodeType.host); - // Some nodes have reported current version - setCurrentVersion(hostNodes.get().asList().subList(0, 2), Version.fromString("7.0")); + // 5 nodes have no version. The other 15 are spread across different versions + var hostNodesList = hostNodes.get().asList(); + for (int i = totalNodes - maxActiveUpgrades - 1; i >= 0; i--) { + setCurrentVersion(List.of(hostNodesList.get(i)), new Version(7, 0, i)); + } // Set target var version1 = Version.fromString("7.1"); @@ -86,28 +94,68 @@ public class OsVersionsTest { // Activate target for (int i = 0; i < totalNodes; i += maxActiveUpgrades) { versions.setActive(NodeType.host, true); - var nodesUpgrading = hostNodes.get().changingOsVersion(); + var nodes = hostNodes.get(); + var nodesUpgrading = nodes.changingOsVersion(); assertEquals("Target is changed for a subset of nodes", maxActiveUpgrades, nodesUpgrading.size()); assertEquals("Wanted version is set for nodes upgrading", version1, - nodesUpgrading.stream() - .map(node -> node.status().osVersion().wanted().get()) - .min(Comparator.naturalOrder()).get()); + minVersion(nodesUpgrading, OsVersion::wanted)); + var nodesOnLowestVersion = nodes.asList().stream() + .sorted(Comparator.comparing(node -> node.status().osVersion().current().orElse(Version.emptyVersion))) + .collect(Collectors.toList()) + .subList(0, maxActiveUpgrades); + assertEquals("Nodes on lowest version are told to upgrade", + nodesUpgrading.asList(), nodesOnLowestVersion); completeUpgradeOf(nodesUpgrading.asList()); } // Activating again after all nodes have upgraded does nothing versions.setActive(NodeType.host, true); - assertEquals(version1, hostNodes.get().stream() - .map(n -> n.status().osVersion().current().get()) - .min(Comparator.naturalOrder()).get()); + assertEquals("All nodes upgraded", version1, minVersion(hostNodes.get(), OsVersion::current)); + } + + @Test + public void test_newer_upgrade_aborts_upgrade_to_stale_version() { + var versions = new OsVersions(tester.nodeRepository(), Integer.MAX_VALUE); + tester.makeReadyNodes(10, "default", NodeType.host); + Supplier<NodeList> hostNodes = () -> tester.nodeRepository().list().nodeType(NodeType.host); + + // Some nodes are targeting an older version + var version1 = Version.fromString("7.1"); + setWantedVersion(hostNodes.get().asList().subList(0, 5), version1); + + // Trigger upgrade to next version + var version2 = Version.fromString("7.2"); + versions.setTarget(NodeType.host, version2, false); + versions.setActive(NodeType.host, true); + + // Wanted version is changed to newest target for all nodes + assertEquals(version2, minVersion(hostNodes.get(), OsVersion::wanted)); + } + + private Version minVersion(NodeList nodes, Function<OsVersion, Optional<Version>> versionField) { + return nodes.asList().stream() + .map(Node::status) + .map(Status::osVersion) + .map(versionField) + .flatMap(Optional::stream) + .min(Comparator.naturalOrder()) + .orElse(Version.emptyVersion); + + } + + private void setWantedVersion(List<Node> nodes, Version wantedVersion) { + writeNode(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withWanted(Optional.of(wantedVersion))))); } private void setCurrentVersion(List<Node> nodes, Version currentVersion) { + writeNode(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(Optional.of(currentVersion))))); + } + + private void writeNode(List<Node> nodes, UnaryOperator<Node> updateFunc) { for (var node : nodes) { try (var lock = tester.nodeRepository().lock(node)) { node = tester.nodeRepository().getNode(node.hostname()).get(); - node = node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(Optional.of(currentVersion)))); - tester.nodeRepository().write(node, lock); + tester.nodeRepository().write(updateFunc.apply(node), lock); } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java index 08e7772b5ba..8e048001b98 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -229,7 +229,7 @@ public class SerializationTest { @Test public void serialize_parentHostname() { final String parentHostname = "parent.yahoo.com"; - Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", Optional.of(parentHostname), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); + Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", Optional.of(parentHostname), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.tenant); Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node)); assertEquals(parentHostname, deserializedNode.parentHostname().get()); @@ -441,7 +441,8 @@ public class SerializationTest { Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), - NodeType.tenant); + Optional.empty(), NodeType.tenant + ); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index 92d066e5f16..1d028f13340 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; @@ -40,14 +38,14 @@ public class AclProvisioningTest { tester.makeReadyNodes(10, new NodeResources(1, 4, 10, 1)); List<Node> dockerHost = tester.makeReadyNodes(1, new NodeResources(1, 4, 10, 1), NodeType.host); ApplicationId zoneApplication = tester.makeApplicationId(); - deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.host)); + tester.deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.host)); tester.makeReadyVirtualDockerNodes(1,new NodeResources(1, 4, 10, 1), dockerHost.get(0).hostname()); List<Node> proxyNodes = tester.makeReadyNodes(3, new NodeResources(1, 4, 10, 1), NodeType.proxy); // Allocate 2 nodes ApplicationId application = tester.makeApplicationId(); - List<Node> activeNodes = deploy(application, Capacity.fromCount(2, new NodeResources(1, 4, 10, 1), false, true)); + List<Node> activeNodes = tester.deploy(application, Capacity.fromCount(2, new NodeResources(1, 4, 10, 1), false, true)); assertEquals(2, activeNodes.size()); // Get trusted nodes for the first active node @@ -112,7 +110,7 @@ public class AclProvisioningTest { // Deploy zone application ApplicationId zoneApplication = tester.makeApplicationId(); - deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.proxy)); + tester.deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.proxy)); // Get trusted nodes for first proxy node List<Node> proxyNodes = tester.nodeRepository().getNodes(zoneApplication); @@ -154,7 +152,7 @@ public class AclProvisioningTest { // Allocate ApplicationId controllerApplication = tester.makeApplicationId(); - List<Node> controllers = deploy(controllerApplication, Capacity.fromRequiredNodeType(NodeType.controller)); + List<Node> controllers = tester.deploy(controllerApplication, Capacity.fromRequiredNodeType(NodeType.controller)); // Controllers and hosts all trust each other List<NodeAcl> controllerAcls = tester.nodeRepository().getNodeAcls(controllers.get(0), false); @@ -164,12 +162,33 @@ public class AclProvisioningTest { @Test public void trusted_nodes_for_application_with_load_balancer() { - // Populate repo - tester.makeReadyNodes(10, nodeResources); + // Provision hosts and containers + var hosts = tester.makeReadyNodes(2, "default", NodeType.host); + tester.deployZoneApp(); + for (var host : hosts) { + tester.makeReadyVirtualDockerNodes(2, new NodeResources(2, 8, 50, 1), + host.hostname()); + } - // Allocate 2 nodes - List<Node> activeNodes = deploy(2); + // Deploy application + var application = tester.makeApplicationId(); + List<Node> activeNodes = deploy(application, 2); assertEquals(2, activeNodes.size()); + + // Load balancer is allocated to application + var loadBalancers = tester.nodeRepository().loadBalancers(application); + assertEquals(1, loadBalancers.asList().size()); + var lbNetworks = loadBalancers.asList().get(0).instance().networks(); + assertEquals(2, lbNetworks.size()); + + // ACL for nodes with allocation trust their respective load balancer networks, if any + for (var host : hosts) { + var acls = tester.nodeRepository().getNodeAcls(host, true); + assertEquals(2, acls.size()); + assertEquals(Set.of(), acls.get(0).trustedNetworks()); + assertEquals(application, acls.get(1).node().allocation().get().owner()); + assertEquals(lbNetworks, acls.get(1).trustedNetworks()); + } } @Test @@ -191,15 +210,7 @@ public class AclProvisioningTest { } private List<Node> deploy(ApplicationId application, int nodeCount) { - return deploy(application, Capacity.fromCount(nodeCount, nodeResources)); - } - - private List<Node> deploy(ApplicationId application, Capacity capacity) { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), - Version.fromString("6.42"), false); - List<HostSpec> prepared = tester.prepare(application, cluster, capacity, 1); - tester.activate(application, Set.copyOf(prepared)); - return tester.getNodes(application, Node.State.active).asList(); + return tester.deploy(application, Capacity.fromCount(nodeCount, nodeResources)); } private static void assertAcls(List<List<Node>> expected, NodeAcl actual) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java index 7e7338ab9ae..2c01cdde932 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -81,7 +81,7 @@ public class AllocationSimulator { var ipConfig = new IP.Config(Set.of("127.0.0.1"), parent.isPresent() ? Set.of() : getAdditionalIP()); return new Node("fake", ipConfig, hostname, parent, flavor, Status.initial(), parent.isPresent() ? Node.State.ready : Node.State.active, allocation(tenant, flavor), History.empty(), - parent.isPresent() ? NodeType.tenant : NodeType.host, new Reports(), Optional.empty()); + parent.isPresent() ? NodeType.tenant : NodeType.host, new Reports(), Optional.empty(), Optional.empty()); } private Set<String> getAdditionalIP() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java index e2015cfd30a..ba9a04573e1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java @@ -37,15 +37,15 @@ public class DockerHostCapacityTest { @Before public void setup() { - doAnswer(invocation -> invocation.getArguments()[0]).when(hostResourcesCalculator).availableCapacityOf(any()); + doAnswer(invocation -> invocation.getArguments()[1]).when(hostResourcesCalculator).availableCapacityOf(any(), any()); // Create flavors NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); // Create three docker hosts - host1 = Node.create("host1", new IP.Config(Set.of("::1"), generateIPs(2, 4)), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); - host2 = Node.create("host2", new IP.Config(Set.of("::11"), generateIPs(12, 3)), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); - host3 = Node.create("host3", new IP.Config(Set.of("::21"), generateIPs(22, 1)), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + host1 = Node.create("host1", new IP.Config(Set.of("::1"), generateIPs(2, 4)), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); + host2 = Node.create("host2", new IP.Config(Set.of("::11"), generateIPs(12, 3)), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); + host3 = Node.create("host3", new IP.Config(Set.of("::21"), generateIPs(22, 1)), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); // Add two containers to host1 var nodeA = Node.createDockerNode(Set.of("::2"), "nodeA", "host1", resources1, NodeType.tenant); @@ -95,9 +95,9 @@ public class DockerHostCapacityTest { capacity.freeCapacityOf(host3, false)); doAnswer(invocation -> { - NodeResources totalHostResources = (NodeResources) invocation.getArguments()[0]; + NodeResources totalHostResources = (NodeResources) invocation.getArguments()[1]; return totalHostResources.subtract(new NodeResources(1, 2, 3, 0.5, NodeResources.DiskSpeed.any)); - }).when(hostResourcesCalculator).availableCapacityOf(any()); + }).when(hostResourcesCalculator).availableCapacityOf(any(), any()); assertEquals(new NodeResources(4, 2, 5, 1.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), capacity.freeCapacityOf(host1, false)); @@ -110,7 +110,7 @@ public class DockerHostCapacityTest { // Dev host can assign both configserver and tenant containers. var nodeFlavors = FlavorConfigBuilder.createDummies("devhost", "container"); - var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), generateIPs(2, 10)), "devhost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("devhost"), NodeType.devhost); + var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), generateIPs(2, 10)), "devhost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("devhost"), Optional.empty(), NodeType.devhost); var cfg = Node.createDockerNode(Set.of("::2"), "cfg", "devhost", resources1, NodeType.config); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java new file mode 100644 index 00000000000..70a57715d13 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java @@ -0,0 +1,51 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class DockerImagesTest { + + @Test + public void image_selection() { + var flagSource = new InMemoryFlagSource(); + var tester = new ProvisioningTester.Builder().flagSource(flagSource).build(); + + // Host uses tenant default image (for preload purposes) + var defaultImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"); + var hosts = tester.makeReadyNodes(2, "default", NodeType.host); + tester.deployZoneApp(); + for (var host : hosts) { + assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(host)); + } + + // Tenant node uses tenant default image + var resources = new NodeResources(2, 8, 50, 1); + for (var host : hosts) { + var nodes = tester.makeReadyVirtualDockerNodes(2, resources, host.hostname()); + for (var node : nodes) { + assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(node)); + } + } + + // Allocated containers uses overridden image when feature flag is set + var app = tester.makeApplicationId(); + var nodes = tester.deploy(app, Capacity.fromCount(2, resources)); + var customImage = DockerImage.fromString("docker.example.com/vespa/hosted"); + flagSource.withStringFlag(Flags.DOCKER_IMAGE_OVERRIDE.id(), customImage.asString()); + for (var node : nodes) { + assertEquals(customImage, tester.nodeRepository().dockerImages().dockerImageFor(node)); + } + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index b731ab309a6..1e59add8304 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -14,6 +14,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -107,6 +108,45 @@ public class DockerProvisioningTest { assertEquals(nodeCount, activeNodes.size()); } + @Test + public void reservations_are_respected() { + NodeResources resources = new NodeResources(10, 10, 10, 10); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + TenantName tenant1 = TenantName.from("tenant1"); + TenantName tenant2 = TenantName.from("tenant2"); + ApplicationId application1_1 = ApplicationId.from(tenant1, ApplicationName.from("application1"), InstanceName.defaultName()); + ApplicationId application2_1 = ApplicationId.from(tenant2, ApplicationName.from("application1"), InstanceName.defaultName()); + ApplicationId application2_2 = ApplicationId.from(tenant2, ApplicationName.from("application2"), InstanceName.defaultName()); + + tester.makeReadyNodes(10, resources, Optional.of(tenant1), NodeType.host, 1); + tester.makeReadyNodes(10, resources, Optional.empty(), NodeType.host, 1); + tester.deployZoneApp(); + + Version wantedVespaVersion = Version.fromString("6.39"); + List<HostSpec> nodes = tester.prepare(application2_1, + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), + 6, 1, resources); + assertHostSpecParentReservation(nodes, Optional.empty(), tester); // We do not get nodes on hosts reserved to tenant1 + tester.activate(application2_1, nodes); + + try { + tester.prepare(application2_2, + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), + 5, 1, resources); + fail("Expected exception"); + } + catch (OutOfCapacityException e) { + // Success: Not enough nonreserved hosts left + } + + nodes = tester.prepare(application1_1, + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), + 10, 1, resources); + assertHostSpecParentReservation(nodes, Optional.of(tenant1), tester); + tester.activate(application1_1, nodes); + assertNodeParentReservation(tester.getNodes(application1_1).asList(), Optional.empty(), tester); // Reservation is cleared after activation + } + /** Exclusive app first, then non-exclusive: Should give the same result as below */ @Test public void docker_application_deployment_with_exclusive_app_first() { @@ -260,4 +300,16 @@ public class DockerProvisioningTest { tester.activate(application, hosts); } + private void assertNodeParentReservation(List<Node> nodes, Optional<TenantName> reservation, ProvisioningTester tester) { + for (Node node : nodes) + assertEquals(reservation, tester.nodeRepository().getNode(node.parentHostname().get()).get().reservedTo()); + } + + private void assertHostSpecParentReservation(List<HostSpec> hostSpecs, Optional<TenantName> reservation, ProvisioningTester tester) { + for (HostSpec hostSpec : hostSpecs) { + Node node = tester.nodeRepository().getNode(hostSpec.hostname()).get(); + assertEquals(reservation, tester.nodeRepository().getNode(node.parentHostname().get()).get().reservedTo()); + } + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index d1498507a7c..53fecbf6095 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -423,7 +423,9 @@ public class DynamicDockerAllocationTest { } private void addAndAssignNode(ApplicationId id, String hostname, String parentHostname, ClusterSpec clusterSpec, NodeResources flavor, int index, ProvisioningTester tester) { - Node node1a = Node.create("open1", new IP.Config(Set.of("127.0.233." + index), Set.of()), hostname, Optional.of(parentHostname), Optional.empty(), new Flavor(flavor), NodeType.tenant); + Node node1a = Node.create("open1", new IP.Config(Set.of("127.0.233." + index), Set.of()), hostname, + Optional.of(parentHostname), Optional.empty(), new Flavor(flavor), Optional.empty(), NodeType.tenant + ); ClusterMembership clusterMembership1 = ClusterMembership.from( clusterSpec.with(Optional.of(ClusterSpec.Group.from(0))), index); // Need to add group here so that group is serialized in node allocation Node node1aAllocation = node1a.allocate(id, clusterMembership1, node1a.flavor().resources(), Instant.now()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java index c0e06ef3dff..0ad7d37d13b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -47,6 +46,7 @@ import static org.junit.Assert.fail; * @author freva */ public class InPlaceResizeProvisionTest { + private static final NodeResources smallResources = new NodeResources(2, 4, 8, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); private static final NodeResources mediumResources = new NodeResources(4, 8, 16, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); private static final NodeResources largeResources = new NodeResources(8, 16, 32, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); @@ -55,8 +55,7 @@ public class InPlaceResizeProvisionTest { private static final ClusterSpec container2 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container2"), Version.fromString("7.157.9"), false); private static final ClusterSpec content1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("content1"), Version.fromString("7.157.9"), false); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource() - .withBooleanFlag(Flags.ENABLE_IN_PLACE_RESIZE.id(), true); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final ProvisioningTester tester = new ProvisioningTester.Builder() .flagSource(flagSource) .zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); @@ -158,17 +157,6 @@ public class InPlaceResizeProvisionTest { assertTrue("All initial nodes should still be allocated to the application", initialHostnames.isEmpty()); } - @Test(expected = OutOfCapacityException.class) - public void no_in_place_resize_if_flag_not_set() { - flagSource.withBooleanFlag(Flags.ENABLE_IN_PLACE_RESIZE.id(), false); - addParentHosts(4, mediumResources.with(fast).with(local)); - - new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate(); - assertClusterSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local)); - - new PrepareHelper(tester, app).prepare(container1, 4, 1, smallResources); - } - @Test(expected = OutOfCapacityException.class) public void cannot_inplace_decrease_resources_while_increasing_cluster_size() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 6dfca4d2c04..ee9a582c4db 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.google.common.collect.Iterators; @@ -40,15 +40,14 @@ public class LoadBalancerProvisionerTest { private final ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default"); private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default"); - private final ApplicationId infraApp1 = ApplicationId.from("vespa", "tenant-host", "default"); - private ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); @Test public void provision_load_balancer() { - Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers().owner(app1).asList(); - Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers().owner(app2).asList(); + Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers(app1).asList(); + Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers(app2).asList(); ClusterSpec.Id containerCluster1 = ClusterSpec.Id.from("qrs1"); ClusterSpec.Id contentCluster = ClusterSpec.Id.from("content"); @@ -80,7 +79,7 @@ public class LoadBalancerProvisionerTest { tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.container, containerCluster1), clusterRequest(ClusterSpec.Type.content, contentCluster))); - LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers().owner(app1).asList().get(0); + LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers(app1).asList().get(0); assertEquals(2, loadBalancer.instance().reals().size()); assertTrue("Failed node is removed", loadBalancer.instance().reals().stream() .map(Real::hostname) @@ -158,7 +157,7 @@ public class LoadBalancerProvisionerTest { var nodes = prepare(app1, Capacity.fromCount(2, new NodeResources(1, 4, 10, 0.3), false, true), true, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs"))); - Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers().owner(app1).asList().get(0); + Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers(app1).asList().get(0); assertTrue("Load balancer provisioned with empty reals", tester.loadBalancerService().instances().get(lb.get().id()).reals().isEmpty()); assignIps(tester.nodeRepository().getNodes(app1)); tester.activate(app1, nodes); @@ -189,7 +188,7 @@ public class LoadBalancerProvisionerTest { clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("tenant-host")))); assertTrue("No load balancer provisioned", tester.loadBalancerService().instances().isEmpty()); - assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(infraApp1).asList()); + assertEquals(List.of(), tester.nodeRepository().loadBalancers(infraApp1).asList()); } @Test @@ -197,12 +196,12 @@ public class LoadBalancerProvisionerTest { tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.content, ClusterSpec.Id.from("tenant-host")))); assertTrue("No load balancer provisioned", tester.loadBalancerService().instances().isEmpty()); - assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(app1).asList()); + assertEquals(List.of(), tester.nodeRepository().loadBalancers(app1).asList()); } @Test public void provision_load_balancer_combined_cluster() { - Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().owner(app1).asList(); + Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(app1).asList(); ClusterSpec.Id cluster = ClusterSpec.Id.from("foo"); var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, cluster)); @@ -247,7 +246,7 @@ public class LoadBalancerProvisionerTest { } private void assignIps(List<Node> nodes) { - try (var lock = tester.nodeRepository().lockAllocation()) { + try (var lock = tester.nodeRepository().lockUnallocated()) { for (int i = 0; i < nodes.size(); i++) { tester.nodeRepository().write(nodes.get(i).with(IP.Config.EMPTY.with(Set.of("127.0.0." + i))), lock); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java index 718abf7d73d..1d9c135b4b5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java @@ -124,18 +124,26 @@ public class PrioritizableNodeTest { } private static Node node(String hostname, Node.State state) { - return new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), new Flavor(new NodeResources(2, 2, 2, 2)), - Status.initial(), state, Optional.empty(), History.empty(), NodeType.tenant, new Reports(), Optional.empty()); + return new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), + new Flavor(new NodeResources(2, 2, 2, 2)), + Status.initial(), state, Optional.empty(), History.empty(), NodeType.tenant, new Reports(), + Optional.empty(), Optional.empty()); } private static PrioritizableNode node(String hostname, NodeResources nodeResources, NodeResources allocatedHostResources, // allocated before adding nodeResources NodeResources totalHostResources) { - Node node = new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.of(hostname + "parent"), new Flavor(nodeResources), - Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.tenant, new Reports(), Optional.empty()); - Node parent = new Node(hostname + "parent", new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), new Flavor(totalHostResources), - Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.host, new Reports(), Optional.empty()); - return new PrioritizableNode(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), false, false, true, false); + Node node = new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.of(hostname + "parent"), + new Flavor(nodeResources), + Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.tenant, + new Reports(), Optional.empty(), Optional.empty()); + Node parent = new Node(hostname + "parent", new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), + new Flavor(totalHostResources), + Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.host, + new Reports(), Optional.empty(), Optional.empty()); + return new PrioritizableNode(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), + false, false, true, false); } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 679c22e4df2..85a6ed31073 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -87,7 +87,7 @@ public class ProvisioningTester { this.nodeFlavors = nodeFlavors; this.clock = new ManualClock(); this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, nameResolver, - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, flagSource); this.orchestrator = orchestrator; ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource); @@ -241,13 +241,16 @@ public class ProvisioningTester { } public List<Node> makeReadyNodes(int n, String flavor, NodeType type) { - return makeReadyNodes(n, asFlavor(flavor, type), type, 0); + return makeReadyNodes(n, asFlavor(flavor, type), Optional.empty(), type, 0); } public List<Node> makeReadyNodes(int n, NodeResources resources, NodeType type) { - return makeReadyNodes(n, new Flavor(resources), type, 0); + return makeReadyNodes(n, new Flavor(resources), Optional.empty(), type, 0); } public List<Node> makeReadyNodes(int n, NodeResources resources, NodeType type, int ipAddressPoolSize) { - return makeReadyNodes(n, new Flavor(resources), type, ipAddressPoolSize); + return makeReadyNodes(n, resources, Optional.empty(), type, ipAddressPoolSize); + } + public List<Node> makeReadyNodes(int n, NodeResources resources, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize) { + return makeReadyNodes(n, new Flavor(resources), reservedTo, type, ipAddressPoolSize); } public List<Node> makeProvisionedNodes(int count, String flavor, NodeType type, int ipAddressPoolSize) { @@ -255,9 +258,9 @@ public class ProvisioningTester { } public List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { - return makeProvisionedNodes(n, asFlavor(flavor, type), type, ipAddressPoolSize, dualStack); + return makeProvisionedNodes(n, asFlavor(flavor, type), Optional.empty(), type, ipAddressPoolSize, dualStack); } - public List<Node> makeProvisionedNodes(int n, Flavor flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { + public List<Node> makeProvisionedNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize, boolean dualStack) { List<Node> nodes = new ArrayList<>(n); for (int i = 0; i < n; i++) { @@ -299,6 +302,7 @@ public class ProvisioningTester { new IP.Config(hostIps, ipAddressPool), Optional.empty(), flavor, + reservedTo, type)); } nodes = nodeRepository.addNodes(nodes); @@ -315,11 +319,12 @@ public class ProvisioningTester { nameResolver.addRecord(hostname, ipv4); Node node = nodeRepository.createNode(hostname, - hostname, - new IP.Config(Set.of(ipv4), Set.of()), - Optional.empty(), - nodeFlavors.getFlavorOrThrow(flavor), - NodeType.config); + hostname, + new IP.Config(Set.of(ipv4), Set.of()), + Optional.empty(), + nodeFlavors.getFlavorOrThrow(flavor), + Optional.empty(), + NodeType.config); nodes.add(node); } @@ -338,17 +343,20 @@ public class ProvisioningTester { } public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize) { - return makeReadyNodes(n, asFlavor(flavor, type), type, ipAddressPoolSize); + return makeReadyNodes(n, asFlavor(flavor, type), Optional.empty(), type, ipAddressPoolSize); } - public List<Node> makeReadyNodes(int n, Flavor flavor, NodeType type, int ipAddressPoolSize) { - return makeReadyNodes(n, flavor, type, ipAddressPoolSize, false); + public List<Node> makeReadyNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize) { + return makeReadyNodes(n, flavor, reservedTo, type, ipAddressPoolSize, false); } public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { return makeReadyNodes(n, asFlavor(flavor, type), type, ipAddressPoolSize, dualStack); } public List<Node> makeReadyNodes(int n, Flavor flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { - List<Node> nodes = makeProvisionedNodes(n, flavor, type, ipAddressPoolSize, dualStack); + return makeReadyNodes(n, flavor, Optional.empty(), type, ipAddressPoolSize, dualStack); + } + public List<Node> makeReadyNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize, boolean dualStack) { + List<Node> nodes = makeProvisionedNodes(n, flavor, reservedTo, type, ipAddressPoolSize, dualStack); nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); } @@ -410,21 +418,20 @@ public class ProvisioningTester { activate(applicationId, Set.copyOf(list)); } + public List<Node> deploy(ApplicationId application, Capacity capacity) { + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), + Version.fromString("6.42"), false); + List<HostSpec> prepared = prepare(application, cluster, capacity, 1); + activate(application, Set.copyOf(prepared)); + return getNodes(application, Node.State.active).asList(); + } + + /** Returns the hosts from the input list which are not retired */ public List<HostSpec> nonRetired(Collection<HostSpec> hosts) { return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList()); } - public void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) { - long actualNodesWithFlavor = hostSpecs.stream() - .map(HostSpec::hostname) - .map(this::getNodeFlavor) - .map(Flavor::name) - .filter(name -> name.equals(flavor)) - .count(); - assertEquals(expectedCount, actualNodesWithFlavor); - } - public void assertAllocatedOn(String explanation, String hostFlavor, ApplicationId app) { for (Node node : nodeRepository.getNodes(app)) { Node parent = nodeRepository.getNode(node.parentHostname().get()).get(); @@ -432,17 +439,6 @@ public class ProvisioningTester { } } - public void printFreeResources() { - for (Node host : nodeRepository().getNodes(NodeType.host)) { - NodeResources free = host.flavor().resources(); - for (Node child : nodeRepository().getNodes(NodeType.tenant)) { - if (child.parentHostname().get().equals(host.hostname())) - free = free.subtract(child.flavor().resources()); - } - System.out.println(host.flavor().name() + " node. Free resources: " + free); - } - } - public int hostFlavorCount(String hostFlavor, ApplicationId app) { return (int)nodeRepository().getNodes(app).stream() .map(n -> nodeRepository().getNode(n.parentHostname().get()).get()) @@ -450,10 +446,6 @@ public class ProvisioningTester { .count(); } - private Flavor getNodeFlavor(String hostname) { - return nodeRepository.getNode(hostname).map(Node::flavor).orElseThrow(() -> new RuntimeException("No flavor for host " + hostname)); - } - public static final class Builder { private Curator curator; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 18f0d989900..c26614c630c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -6,12 +6,15 @@ import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.text.Utf8; +import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.OsUpgradeActivator; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; +import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import org.junit.After; import org.junit.Before; import org.junit.ComparisonFailure; @@ -23,6 +26,7 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -106,7 +110,7 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.8.1") + "," + // test with only 1 ip address asNodeJson("host9.yahoo.com", "large-variant", "127.0.9.1", "::9:1") + "," + - asHostJson("parent2.yahoo.com", "large-variant", "127.0.127.1", "::127:1") + "," + + asHostJson("parent2.yahoo.com", "large-variant", Optional.of(TenantName.from("myTenant")), "127.0.127.1", "::127:1") + "," + asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com", "::11") + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST), @@ -214,9 +218,6 @@ public class RestApiTest { Utf8.toBytes("{\"wantToRetire\": true}"), Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", Utf8.toBytes("{\"currentVespaVersion\": \"6.43.0\",\"currentDockerImage\": \"docker-registry.domain.tld:8080/dist/vespa:6.45.0\"}"), Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", @@ -225,6 +226,9 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\""); assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"modelName\": null}"), Request.Method.PATCH), @@ -232,6 +236,9 @@ public class RestApiTest { assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "modelName", false); container.handleRequest((new Request("http://localhost:8080/nodes/v2/upgrade/tenant", Utf8.toBytes("{\"dockerImage\": \"docker.domain.tld/my/image\"}"), Request.Method.PATCH))); + ((OrchestratorMock) container.components().getComponent(OrchestratorMock.class.getName())) + .suspend(new HostName("host4.yahoo.com")); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json"); } @@ -292,7 +299,7 @@ public class RestApiTest { // Attempt to POST host node with already assigned IP assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asHostJson("host200.yahoo.com", "default", "127.0.2.1") + "]", + "[" + asHostJson("host200.yahoo.com", "default", Optional.empty(), "127.0.2.1") + "]", Request.Method.POST), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.2.1] to host200.yahoo.com: [127.0.2.1] already assigned to host2.yahoo.com\"}"); @@ -304,7 +311,7 @@ public class RestApiTest { // Node types running a single container can share their IP address with child node assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", "127.0.42.1") + "]", + "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.42.1") + "]", Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node", @@ -320,7 +327,7 @@ public class RestApiTest { // ... nor with child node on different host assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", "127.0.43.1") + "]", + "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.43.1") + "]", Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com", @@ -534,11 +541,13 @@ public class RestApiTest { " \"actualCpuCores\": {" + " \"createdMillis\": 1, " + " \"description\": \"Actual number of CPU cores (2) differs from spec (4)\"," + + " \"type\": \"HARD_FAIL\"," + " \"value\":2" + " }," + " \"diskSpace\": {" + " \"createdMillis\": 2, " + " \"description\": \"Actual disk space (2TB) differs from spec (3TB)\"," + + " \"type\": \"HARD_FAIL\"," + " \"details\": {" + " \"inGib\": 3," + " \"disks\": [\"/dev/sda1\", \"/dev/sdb3\"]" + @@ -921,14 +930,15 @@ public class RestApiTest { "\"flavor\":\"" + flavor + "\"}"; } - private static String asHostJson(String hostname, String flavor, String... ipAddress) { - return asNodeJson(hostname, NodeType.host, flavor, ipAddress); + private static String asHostJson(String hostname, String flavor, Optional<TenantName> reservedTo, String... ipAddress) { + return asNodeJson(hostname, NodeType.host, flavor, reservedTo, ipAddress); } - private static String asNodeJson(String hostname, NodeType nodeType, String flavor, String... ipAddress) { + private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional<TenantName> reservedTo, String... ipAddress) { return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\"," + createIpAddresses(ipAddress) + "\"flavor\":\"" + flavor + "\"" + + (reservedTo.isPresent() ? ", \"reservedTo\":\"" + reservedTo.get().value() + "\"" : "") + ", \"type\":\"" + nodeType + "\"}"; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index f663f4adb8d..af3552945d9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -27,14 +27,15 @@ "wantedDockerImage": "docker.domain.tld/my/image:6.42.0", "wantedVespaVersion": "6.42.0", "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, + "allowedToBeDown": true, + "suspendedSinceMillis": 0, "rebootGeneration": 3, "currentRebootGeneration": 1, "vespaVersion": "6.43.0", "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.45.0", "failCount": 1, "wantToRetire": true, - "wantToDeprovision": true, + "wantToDeprovision": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json index d33c1c9e743..a3d53798d7c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json @@ -30,7 +30,7 @@ "currentRebootGeneration": 0, "failCount": 0, "wantToRetire": false, - "wantToDeprovision": false, + "wantToDeprovision": true, "history": [ { "event": "provisioned", @@ -65,6 +65,7 @@ "diskSpace": { "createdMillis": 2, "description": "Actual disk space (2TB) differs from spec (3TB)", + "type":"HARD_FAIL", "details": { "inGib": 3, "disks": [ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json index 4119e46e225..67b8d67c7f1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json @@ -30,7 +30,7 @@ "currentRebootGeneration": 0, "failCount": 0, "wantToRetire": false, - "wantToDeprovision": false, + "wantToDeprovision": true, "history": [ { "event": "provisioned", @@ -62,11 +62,13 @@ "actualCpuCores": { "createdMillis": 1, "description": "Actual number of CPU cores (2) differs from spec (4)", + "type":"HARD_FAIL", "value": 2 }, "diskSpace": { "createdMillis": 2, "description": "Actual disk space (2TB) differs from spec (3TB)", + "type":"HARD_FAIL", "details": { "inGib": 3, "disks": [ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json index 8be034cb036..ecc497172c7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json @@ -6,6 +6,7 @@ "hostname": "parent2.yahoo.com", "openStackId": "parent2.yahoo.com", "flavor": "large-variant", + "reservedTo": "myTenant", "cpuCores": 64.0, "resources": {"vcpu":64.0,"memoryGb":128.0,"diskGb":2000.0,"bandwidthGbps":15.0,"diskSpeed":"fast","storageType":"remote"}, "environment": "BARE_METAL", |