aboutsummaryrefslogtreecommitdiffstats
path: root/node-repository
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@verizonmedia.com>2020-02-20 09:42:19 +0100
committerJon Bratseth <bratseth@verizonmedia.com>2020-02-20 09:42:19 +0100
commit5acf4c47e98674cdf73289a782dfda9da7041ead (patch)
tree9a2720a3326326cc2a0b69d29b6877e4039d5f18 /node-repository
parentd2449a3e66075e7d680263a204302e83b5ba0148 (diff)
parent1cc70ca6f328e7e88e8b4e279cac7544624f055b (diff)
Merge branch 'master' into bratseth/node-metrics
Diffstat (limited to 'node-repository')
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java122
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java16
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java70
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java15
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java24
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java4
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java4
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java91
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java24
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java6
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java4
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java4
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java12
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java5
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java5
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java34
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java31
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java23
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java8
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java42
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java12
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java22
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java6
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java37
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java3
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java4
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java14
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java63
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java30
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java30
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java3
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java38
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java20
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java10
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java16
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java33
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java2
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java21
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java3
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java34
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java3
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java2
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java2
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java2
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java2
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java4
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java2
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java70
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java5
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java49
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java2
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java14
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java51
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java52
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java4
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java16
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java21
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java22
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java72
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java30
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json5
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json3
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json4
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json1
77 files changed, 894 insertions, 515 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
index e66fef9ac96..321c5632302 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
@@ -7,6 +7,7 @@ import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.TenantName;
import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.node.Allocation;
import com.yahoo.vespa.hosted.provision.node.Generation;
@@ -41,6 +42,7 @@ public final class Node {
private final NodeType type;
private final Reports reports;
private final Optional<String> modelName;
+ private final Optional<TenantName> reservedTo;
/** Record of the last event of each type happening to this node */
private final History history;
@@ -50,52 +52,46 @@ public final class Node {
/** Creates a node in the initial state (reserved) */
public static Node createDockerNode(Set<String> ipAddresses, String hostname, String parentHostname, NodeResources resources, NodeType type) {
- return new Node("fake-" + hostname, new IP.Config(ipAddresses, Set.of()), hostname, Optional.of(parentHostname), new Flavor(resources), Status.initial(), State.reserved,
- Optional.empty(), History.empty(), type, new Reports(), Optional.empty());
+ return new Node("fake-" + hostname, new IP.Config(ipAddresses, Set.of()), hostname, Optional.of(parentHostname),
+ new Flavor(resources), Status.initial(), State.reserved,
+ Optional.empty(), History.empty(), type, new Reports(), Optional.empty(), Optional.empty());
}
/** Creates a node in the initial state (provisioned) */
- public static Node create(String openStackId, IP.Config ipConfig, String hostname, Optional<String> parentHostname, Optional<String> modelName, Flavor flavor, NodeType type) {
+ public static Node create(String openStackId, IP.Config ipConfig, String hostname, Optional<String> parentHostname,
+ Optional<String> modelName, Flavor flavor, Optional<TenantName> reservedTo, NodeType type) {
return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, Status.initial(), State.provisioned,
- Optional.empty(), History.empty(), type, new Reports(), modelName);
+ Optional.empty(), History.empty(), type, new Reports(), modelName, reservedTo);
}
/** Creates a node. See also the {@code create} helper methods. */
public Node(String id, IP.Config ipConfig, String hostname, Optional<String> parentHostname,
Flavor flavor, Status status, State state, Optional<Allocation> allocation, History history, NodeType type,
- Reports reports, Optional<String> modelName) {
- Objects.requireNonNull(id, "A node must have an ID");
- requireNonEmptyString(hostname, "A node must have a hostname");
- Objects.requireNonNull(ipConfig, "A node must a have an IP config");
- requireNonEmptyString(parentHostname, "A parent host name must be a proper value");
- Objects.requireNonNull(flavor, "A node must have a flavor");
- Objects.requireNonNull(status, "A node must have a status");
- Objects.requireNonNull(state, "A null node state is not permitted");
- Objects.requireNonNull(allocation, "A null node allocation is not permitted");
- Objects.requireNonNull(history, "A null node history is not permitted");
- Objects.requireNonNull(type, "A null node type is not permitted");
- Objects.requireNonNull(reports, "A null reports is not permitted");
- Objects.requireNonNull(modelName, "A null modelName is not permitted");
+ Reports reports, Optional<String> modelName, Optional<TenantName> reservedTo) {
+ this.id = Objects.requireNonNull(id, "A node must have an ID");
+ this.hostname = requireNonEmptyString(hostname, "A node must have a hostname");
+ this.ipConfig = Objects.requireNonNull(ipConfig, "A node must a have an IP config");
+ this.parentHostname = requireNonEmptyString(parentHostname, "A parent host name must be a proper value");
+ this.flavor = Objects.requireNonNull(flavor, "A node must have a flavor");
+ this.status = Objects.requireNonNull(status, "A node must have a status");
+ this.state = Objects.requireNonNull(state, "A null node state is not permitted");
+ this.allocation = Objects.requireNonNull(allocation, "A null node allocation is not permitted");
+ this.history = Objects.requireNonNull(history, "A null node history is not permitted");
+ this.type = Objects.requireNonNull(type, "A null node type is not permitted");
+ this.reports = Objects.requireNonNull(reports, "A null reports is not permitted");
+ this.modelName = Objects.requireNonNull(modelName, "A null modelName is not permitted");
+ this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null");
if (state == State.active)
requireNonEmpty(ipConfig.primary(), "An active node must have at least one valid IP address");
+
if (parentHostname.isPresent()) {
if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool");
if (modelName.isPresent()) throw new IllegalArgumentException("A child node cannot have model name set");
}
- this.hostname = hostname;
- this.ipConfig = ipConfig;
- this.parentHostname = parentHostname;
- this.id = id;
- this.flavor = flavor;
- this.status = status;
- this.state = state;
- this.allocation = allocation;
- this.history = history;
- this.type = type;
- this.reports = reports;
- this.modelName = modelName;
+ if (type != NodeType.host && reservedTo.isPresent())
+ throw new IllegalArgumentException("Only hosts can be reserved to a tenant");
}
/** Returns the IP addresses of this node */
@@ -166,6 +162,12 @@ public final class Node {
public Optional<String> modelName() { return modelName; }
/**
+ * Returns the tenant this node is reserved to, if any. Only hosts can be reserved to a tenant.
+ * If this is set, resources on this host cannot be allocated to any other tenant
+ */
+ public Optional<TenantName> reservedTo() { return reservedTo; }
+
+ /**
* Returns a copy of this node with wantToRetire set to the given value and updated history.
* If given wantToRetire is equal to the current, the method is no-op.
*/
@@ -209,42 +211,47 @@ public final class Node {
/** Returns a node with the status assigned to the given value */
public Node with(Status status) {
- return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName);
+ return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo);
}
/** Returns a node with the type assigned to the given value */
public Node with(NodeType type) {
- return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName);
+ return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo);
}
/** Returns a node with the flavor assigned to the given value */
public Node with(Flavor flavor) {
- return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName);
+ return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state,
+ allocation, history, type, reports, modelName, reservedTo);
}
/** Returns a copy of this with the reboot generation set to generation */
public Node withReboot(Generation generation) {
- return new Node(id, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, allocation, history, type, reports, modelName);
+ return new Node(id, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state,
+ allocation, history, type, reports, modelName, reservedTo);
}
/** Returns a copy of this with the openStackId set */
public Node withOpenStackId(String openStackId) {
- return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName);
+ return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, status, state,
+ allocation, history, type, reports, modelName, reservedTo);
}
/** Returns a copy of this with model name set to given value */
public Node withModelName(String modelName) {
- return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, Optional.of(modelName));
+ return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state,
+ allocation, history, type, reports, Optional.of(modelName), reservedTo);
}
/** Returns a copy of this with model name cleared */
public Node withoutModelName() {
- return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, Optional.empty());
+ return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state,
+ allocation, history, type, reports, Optional.empty(), reservedTo);
}
/** Returns a copy of this with a history record saying it was detected to be down at this instant */
- public Node downAt(Instant instant) {
- return with(history.with(new History.Event(History.Event.Type.down, Agent.system, instant)));
+ public Node downAt(Instant instant, Agent agent) {
+ return with(history.with(new History.Event(History.Event.Type.down, agent, instant)));
}
/** Returns a copy of this with any history record saying it has been detected down removed */
@@ -265,26 +272,39 @@ public final class Node {
*/
public Node with(Allocation allocation) {
return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state,
- Optional.of(allocation), history, type, reports, modelName);
+ Optional.of(allocation), history, type, reports, modelName, reservedTo);
}
/** Returns a new Node without an allocation. */
public Node withoutAllocation() {
return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state,
- Optional.empty(), history, type, reports, modelName);
+ Optional.empty(), history, type, reports, modelName, reservedTo);
}
/** Returns a copy of this node with IP config set to the given value. */
public Node with(IP.Config ipConfig) {
return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state,
- allocation, history, type, reports, modelName);
+ allocation, history, type, reports, modelName, reservedTo);
}
/** Returns a copy of this node with the parent hostname assigned to the given value. */
public Node withParentHostname(String parentHostname) {
return new Node(id, ipConfig, hostname, Optional.of(parentHostname), flavor, status, state,
- allocation, history, type, reports, modelName);
+ allocation, history, type, reports, modelName, reservedTo);
+ }
+
+ public Node withReservedTo(TenantName tenant) {
+ if (type != NodeType.host)
+ throw new IllegalArgumentException("Only host nodes can be reserved, " + hostname + " has type " + type);
+ return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state,
+ allocation, history, type, reports, modelName, Optional.of(tenant));
+ }
+
+ /** Returns a copy of this node which is not reserved to a tenant */
+ public Node withoutReservedTo() {
+ return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state,
+ allocation, history, type, reports, modelName, Optional.empty());
}
/** Returns a copy of this node with the current reboot generation set to the given number at the given instant */
@@ -316,28 +336,32 @@ public final class Node {
/** Returns a copy of this node with the given history. */
public Node with(History history) {
- return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName);
+ return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state,
+ allocation, history, type, reports, modelName, reservedTo);
}
public Node with(Reports reports) {
- return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName);
+ return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state,
+ allocation, history, type, reports, modelName, reservedTo);
}
- private static void requireNonEmptyString(Optional<String> value, String message) {
+ private static Optional<String> requireNonEmptyString(Optional<String> value, String message) {
Objects.requireNonNull(value, message);
value.ifPresent(v -> requireNonEmptyString(v, message));
+ return value;
}
- private static void requireNonEmptyString(String value, String message) {
+ private static String requireNonEmptyString(String value, String message) {
Objects.requireNonNull(value, message);
if (value.trim().isEmpty())
throw new IllegalArgumentException(message + ", but was '" + value + "'");
+ return value;
}
- private static void requireNonEmpty(Set<String> values, String message) {
- if (values == null || values.isEmpty()) {
+ private static Set<String> requireNonEmpty(Set<String> values, String message) {
+ if (values == null || values.isEmpty())
throw new IllegalArgumentException(message);
- }
+ return values;
}
/** Computes the allocation skew of a host node */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
index 37987f0512d..bdae658f76e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeType;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
@@ -74,11 +75,26 @@ public class NodeList implements Iterable<Node> {
!node.status().vespaVersion().get().equals(node.allocation().get().membership().cluster().vespaVersion()));
}
+ /** Returns the subset of nodes that are currently changing their OS version to given version */
+ public NodeList changingOsVersionTo(Version version) {
+ return filter(node -> node.status().osVersion().changingTo(version));
+ }
+
/** Returns the subset of nodes that are currently changing their OS version */
public NodeList changingOsVersion() {
return filter(node -> node.status().osVersion().changing());
}
+ /** Returns a copy of this sorted by current OS version (lowest to highest) */
+ public NodeList byIncreasingOsVersion() {
+ return nodes.stream()
+ .sorted(Comparator.comparing(node -> node.status()
+ .osVersion()
+ .current()
+ .orElse(Version.emptyVersion)))
+ .collect(collectingAndThen(Collectors.toList(), NodeList::wrap));
+ }
+
/** Returns the subset of nodes that are currently on the given OS version */
public NodeList onOsVersion(Version version) {
return filter(node -> node.status().osVersion().matches(version));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 1c9e1445447..7d925b2a4aa 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision;
import com.google.inject.Inject;
@@ -10,12 +10,16 @@ import com.yahoo.config.provision.DockerImage;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.NodeFlavors;
import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.TenantName;
import com.yahoo.config.provision.Zone;
import com.yahoo.config.provisioning.NodeRepositoryConfig;
import com.yahoo.transaction.Mutex;
import com.yahoo.transaction.NestedTransaction;
import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance;
import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList;
import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions;
@@ -49,6 +53,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
import java.util.function.BiFunction;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -98,8 +103,8 @@ public class NodeRepository extends AbstractComponent {
* This will use the system time to make time-sensitive decisions
*/
@Inject
- public NodeRepository(NodeRepositoryConfig config, NodeFlavors flavors, Curator curator, Zone zone) {
- this(flavors, curator, Clock.systemUTC(), zone, new DnsNameResolver(), DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache());
+ public NodeRepository(NodeRepositoryConfig config, NodeFlavors flavors, Curator curator, Zone zone, FlagSource flagSource) {
+ this(flavors, curator, Clock.systemUTC(), zone, new DnsNameResolver(), DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache(), flagSource);
}
/**
@@ -107,7 +112,7 @@ public class NodeRepository extends AbstractComponent {
* which will be used for time-sensitive decisions.
*/
public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, NameResolver nameResolver,
- DockerImage dockerImage, boolean useCuratorClientCache) {
+ DockerImage dockerImage, boolean useCuratorClientCache, FlagSource flagSource) {
this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache);
this.zone = zone;
this.clock = clock;
@@ -116,7 +121,7 @@ public class NodeRepository extends AbstractComponent {
this.osVersions = new OsVersions(this);
this.infrastructureVersions = new InfrastructureVersions(db);
this.firmwareChecks = new FirmwareChecks(db, clock);
- this.dockerImages = new DockerImages(db, dockerImage);
+ this.dockerImages = new DockerImages(db, dockerImage, Flags.DOCKER_IMAGE_OVERRIDE.bindTo(flagSource));
this.jobControl = new JobControl(db);
// read and write all nodes to make sure they are stored in the latest version of the serialized format
@@ -127,8 +132,8 @@ public class NodeRepository extends AbstractComponent {
/** Returns the curator database client used by this */
public CuratorDatabaseClient database() { return db; }
- /** Returns the Docker image to use for nodes in this */
- public DockerImage dockerImage(NodeType nodeType) { return dockerImages.dockerImageFor(nodeType); }
+ /** Returns the Docker image to use for given node */
+ public DockerImage dockerImage(Node node) { return dockerImages.dockerImageFor(node); }
/** @return The name resolver used to resolve hostname and ip addresses */
public NameResolver nameResolver() { return nameResolver; }
@@ -193,7 +198,16 @@ public class NodeRepository extends AbstractComponent {
/** Returns a filterable list of all load balancers in this repository */
public LoadBalancerList loadBalancers() {
- return LoadBalancerList.copyOf(database().readLoadBalancers().values());
+ return loadBalancers((ignored) -> true);
+ }
+
+ /** Returns a filterable list of load balancers belonging to given application */
+ public LoadBalancerList loadBalancers(ApplicationId application) {
+ return loadBalancers((id) -> id.application().equals(application));
+ }
+
+ private LoadBalancerList loadBalancers(Predicate<LoadBalancerId> predicate) {
+ return LoadBalancerList.copyOf(db.readLoadBalancers(predicate).values());
}
public List<Node> getNodes(ApplicationId id, Node.State ... inState) { return db.getNodes(id, inState); }
@@ -203,7 +217,7 @@ public class NodeRepository extends AbstractComponent {
/**
* Returns the ACL for the node (trusted nodes, networks and ports)
*/
- private NodeAcl getNodeAcl(Node node, NodeList candidates, LoadBalancerList loadBalancers) {
+ private NodeAcl getNodeAcl(Node node, NodeList candidates) {
Set<Node> trustedNodes = new TreeSet<>(Comparator.comparing(Node::hostname));
Set<Integer> trustedPorts = new LinkedHashSet<>();
Set<String> trustedNetworks = new LinkedHashSet<>();
@@ -220,10 +234,10 @@ public class NodeRepository extends AbstractComponent {
candidates.parentOf(node).ifPresent(trustedNodes::add);
node.allocation().ifPresent(allocation -> {
trustedNodes.addAll(candidates.owner(allocation.owner()).asList());
- loadBalancers.owner(allocation.owner()).asList().stream()
- .map(LoadBalancer::instance)
- .map(LoadBalancerInstance::networks)
- .forEach(trustedNetworks::addAll);
+ loadBalancers(allocation.owner()).asList().stream()
+ .map(LoadBalancer::instance)
+ .map(LoadBalancerInstance::networks)
+ .forEach(trustedNetworks::addAll);
});
switch (node.type()) {
@@ -292,13 +306,12 @@ public class NodeRepository extends AbstractComponent {
*/
public List<NodeAcl> getNodeAcls(Node node, boolean children) {
NodeList candidates = list();
- LoadBalancerList loadBalancers = loadBalancers();
if (children) {
return candidates.childrenOf(node).asList().stream()
- .map(childNode -> getNodeAcl(childNode, candidates, loadBalancers))
+ .map(childNode -> getNodeAcl(childNode, candidates))
.collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
}
- return Collections.singletonList(getNodeAcl(node, candidates, loadBalancers));
+ return Collections.singletonList(getNodeAcl(node, candidates));
}
public NodeFlavors getAvailableFlavors() {
@@ -309,16 +322,15 @@ public class NodeRepository extends AbstractComponent {
/** Creates a new node object, without adding it to the node repo. If no IP address is given, it will be resolved */
public Node createNode(String openStackId, String hostname, IP.Config ipConfig, Optional<String> parentHostname,
- Flavor flavor, NodeType type) {
+ Flavor flavor, Optional<TenantName> reservedTo, NodeType type) {
if (ipConfig.primary().isEmpty()) { // TODO: Remove this. Only test code hits this path
ipConfig = ipConfig.with(nameResolver.getAllByNameOrThrow(hostname));
}
- return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, type);
+ return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, reservedTo, type);
}
- public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Flavor flavor,
- NodeType type) {
- return createNode(openStackId, hostname, IP.Config.EMPTY, parentHostname, flavor, type);
+ public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Flavor flavor, NodeType type) {
+ return createNode(openStackId, hostname, IP.Config.EMPTY, parentHostname, flavor, Optional.empty(), type);
}
/** Adds a list of newly created docker container nodes to the node repository as <i>reserved</i> nodes */
@@ -341,7 +353,7 @@ public class NodeRepository extends AbstractComponent {
/** Adds a list of (newly created) nodes to the node repository as <i>provisioned</i> nodes */
public List<Node> addNodes(List<Node> nodes) {
- try (Mutex lock = lockAllocation()) {
+ try (Mutex lock = lockUnallocated()) {
for (int i = 0; i < nodes.size(); i++) {
var node = nodes.get(i);
var message = "Cannot add " + node.hostname() + ": A node with this name already exists";
@@ -361,7 +373,7 @@ public class NodeRepository extends AbstractComponent {
/** Sets a list of nodes ready and returns the nodes in the ready state */
public List<Node> setReady(List<Node> nodes, Agent agent, String reason) {
- try (Mutex lock = lockAllocation()) {
+ try (Mutex lock = lockUnallocated()) {
List<Node> nodesWithResetFields = nodes.stream()
.map(node -> {
if (node.state() != Node.State.provisioned && node.state() != Node.State.dirty)
@@ -585,7 +597,7 @@ public class NodeRepository extends AbstractComponent {
}
public List<Node> removeRecursively(Node node, boolean force) {
- try (Mutex lock = lockAllocation()) {
+ try (Mutex lock = lockUnallocated()) {
List<Node> removed = new ArrayList<>();
if (node.type().isDockerHost()) {
@@ -684,7 +696,7 @@ public class NodeRepository extends AbstractComponent {
* Writes these nodes after they have changed some internal state but NOT changed their state field.
* This does NOT lock the node repository implicitly, but callers are expected to already hold the lock.
*
- * @param lock Already acquired lock
+ * @param lock already acquired lock
* @return the written nodes for convenience
*/
public List<Node> write(List<Node> nodes, @SuppressWarnings("unused") Mutex lock) {
@@ -713,7 +725,7 @@ public class NodeRepository extends AbstractComponent {
// perform operation while holding locks
List<Node> resultingNodes = new ArrayList<>();
- try (Mutex lock = lockAllocation()) {
+ try (Mutex lock = lockUnallocated()) {
for (Node node : unallocatedNodes)
resultingNodes.add(action.apply(node, lock));
}
@@ -738,12 +750,12 @@ public class NodeRepository extends AbstractComponent {
/** Create a lock with a timeout which provides exclusive rights to making changes to the given application */
public Mutex lock(ApplicationId application, Duration timeout) { return db.lock(application, timeout); }
- /** Create a lock which provides exclusive rights to allocating nodes */
- public Mutex lockAllocation() { return db.lockInactive(); }
+ /** Create a lock which provides exclusive rights to modifying unallocated nodes */
+ public Mutex lockUnallocated() { return db.lockInactive(); }
/** Acquires the appropriate lock for this node */
public Mutex lock(Node node) {
- return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockAllocation();
+ return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockUnallocated();
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java
index 014d3df8d9a..bad16bf7d12 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java
@@ -1,9 +1,6 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.lb;
-import com.yahoo.config.provision.ApplicationId;
-
-import java.time.Instant;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@@ -24,21 +21,11 @@ public class LoadBalancerList implements Iterable<LoadBalancer> {
this.loadBalancers = List.copyOf(Objects.requireNonNull(loadBalancers, "loadBalancers must be non-null"));
}
- /** Returns the subset of load balancers owned by given application */
- public LoadBalancerList owner(ApplicationId application) {
- return of(loadBalancers.stream().filter(lb -> lb.id().application().equals(application)));
- }
-
/** Returns the subset of load balancers that are in given state */
public LoadBalancerList in(LoadBalancer.State state) {
return of(loadBalancers.stream().filter(lb -> lb.state() == state));
}
- /** Returns the subset of load balancers that last changed before given instant */
- public LoadBalancerList changedBefore(Instant instant) {
- return of(loadBalancers.stream().filter(lb -> lb.changedAt().isBefore(instant)));
- }
-
public List<LoadBalancer> asList() {
return loadBalancers;
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java
index ae2f68a3143..f428e276df8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java
@@ -33,7 +33,7 @@ public class DirtyExpirer extends Expirer {
@Override
protected void expire(List<Node> expired) {
for (Node expiredNode : expired)
- nodeRepository.fail(expiredNode.hostname(), Agent.system, "Node is stuck in dirty");
+ nodeRepository.fail(expiredNode.hostname(), Agent.DirtyExpirer, "Node is stuck in dirty");
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
index 1c27f9d1713..f75621e18a0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
import com.yahoo.vespa.hosted.provision.provisioning.NodePrioritizer;
import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceComparator;
import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost;
@@ -44,13 +45,15 @@ public class DynamicProvisioningMaintainer extends Maintainer {
private static final ApplicationId preprovisionAppId = ApplicationId.from("hosted-vespa", "tenant-host", "preprovision");
private final HostProvisioner hostProvisioner;
+ private final HostResourcesCalculator hostResourcesCalculator;
private final BooleanFlag dynamicProvisioningEnabled;
private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag;
- DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval,
- HostProvisioner hostProvisioner, FlagSource flagSource) {
+ DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, HostProvisioner hostProvisioner,
+ HostResourcesCalculator hostResourcesCalculator, FlagSource flagSource) {
super(nodeRepository, interval);
this.hostProvisioner = hostProvisioner;
+ this.hostResourcesCalculator = hostResourcesCalculator;
this.dynamicProvisioningEnabled = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource);
this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource);
}
@@ -59,7 +62,7 @@ public class DynamicProvisioningMaintainer extends Maintainer {
protected void maintain() {
if (! dynamicProvisioningEnabled.value()) return;
- try (Mutex lock = nodeRepository().lockAllocation()) {
+ try (Mutex lock = nodeRepository().lockUnallocated()) {
NodeList nodes = nodeRepository().list();
updateProvisioningNodes(nodes, lock);
@@ -68,17 +71,14 @@ public class DynamicProvisioningMaintainer extends Maintainer {
}
void updateProvisioningNodes(NodeList nodes, Mutex lock) {
- Map<String, Node> provisionedHostsByHostname = nodes.state(Node.State.provisioned).nodeType(NodeType.host)
- .asList().stream()
- .collect(Collectors.toMap(Node::hostname, Function.identity()));
-
- Map<Node, Set<Node>> nodesByProvisionedParent = nodes.asList().stream()
- .filter(node -> node.parentHostname().map(provisionedHostsByHostname::containsKey).orElse(false))
+ Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant).asList().stream()
+ .filter(node -> node.parentHostname().isPresent())
.collect(Collectors.groupingBy(
- node -> provisionedHostsByHostname.get(node.parentHostname().get()),
+ node -> node.parentHostname().get(),
Collectors.toSet()));
- nodesByProvisionedParent.forEach((host, children) -> {
+ nodes.state(Node.State.provisioned).nodeType(NodeType.host).forEach(host -> {
+ Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of());
try {
List<Node> updatedNodes = hostProvisioner.provision(host, children);
nodeRepository().write(updatedNodes, lock);
@@ -112,7 +112,7 @@ public class DynamicProvisioningMaintainer extends Maintainer {
NodeResources resources = it.next();
removableHosts.stream()
.filter(host -> NodePrioritizer.ALLOCATABLE_HOST_STATES.contains(host.state()))
- .filter(host -> host.flavor().resources().satisfies(resources))
+ .filter(host -> hostResourcesCalculator.availableCapacityOf(host.flavor().name(), host.flavor().resources()).satisfies(resources))
.min(Comparator.comparingInt(n -> n.flavor().cost()))
.ifPresent(host -> {
removableHosts.remove(host);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
index 438732ad4a8..264df716fa8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
@@ -107,7 +107,7 @@ public class FailedExpirer extends Maintainer {
.collect(Collectors.toList());
if (unparkedChildren.isEmpty()) {
- nodeRepository.park(candidate.hostname(), false, Agent.system,
+ nodeRepository.park(candidate.hostname(), false, Agent.FailedExpirer,
"Parked by FailedExpirer due to hardware issue");
} else {
log.info(String.format("Expired failed node %s with hardware issue was not parked because of " +
@@ -118,7 +118,7 @@ public class FailedExpirer extends Maintainer {
nodesToRecycle.add(candidate);
}
}
- nodeRepository.setDirty(nodesToRecycle, Agent.system, "Expired by FailedExpirer");
+ nodeRepository.setDirty(nodesToRecycle, Agent.FailedExpirer, "Expired by FailedExpirer");
}
/** Returns whether the current node fail count should be used as an indicator of hardware issue */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
index a6b88d50acb..21746c96411 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
@@ -41,9 +41,9 @@ public class InactiveExpirer extends Expirer {
expired.forEach(node -> {
if (node.status().wantToRetire() &&
node.history().event(History.Event.Type.wantToRetire).get().agent() == Agent.operator) {
- nodeRepository.park(node.hostname(), false, Agent.system, "Expired by InactiveExpirer");
+ nodeRepository.park(node.hostname(), false, Agent.InactiveExpirer, "Expired by InactiveExpirer");
} else {
- nodeRepository.setDirty(node, Agent.system, "Expired by InactiveExpirer");
+ nodeRepository.setDirty(node, Agent.InactiveExpirer, "Expired by InactiveExpirer");
}
});
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
index bcb0c901f14..e2b70608d58 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
@@ -15,6 +15,8 @@ import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
/**
@@ -51,37 +53,31 @@ public class LoadBalancerExpirer extends Maintainer {
/** Move reserved load balancer that have expired to inactive */
private void expireReserved() {
- try (var lock = db.lockLoadBalancers()) {
- var now = nodeRepository().clock().instant();
- var expirationTime = now.minus(reservedExpiry);
- var expired = nodeRepository().loadBalancers()
- .in(State.reserved)
- .changedBefore(expirationTime);
- expired.forEach(lb -> db.writeLoadBalancer(lb.with(State.inactive, now)));
- }
+ var now = nodeRepository().clock().instant();
+ withLoadBalancersIn(State.reserved, lb -> {
+ var gracePeriod = now.minus(reservedExpiry);
+ if (!lb.changedAt().isBefore(gracePeriod)) return; // Should not move to inactive yet
+ db.writeLoadBalancer(lb.with(State.inactive, now));
+ });
}
/** Deprovision inactive load balancers that have expired */
private void removeInactive() {
var failed = new ArrayList<LoadBalancerId>();
- Exception lastException = null;
- try (var lock = db.lockLoadBalancers()) {
- var now = nodeRepository().clock().instant();
- var expirationTime = now.minus(inactiveExpiry);
- var expired = nodeRepository().loadBalancers()
- .in(State.inactive)
- .changedBefore(expirationTime);
- for (var lb : expired) {
- if (!allocatedNodes(lb.id()).isEmpty()) continue; // Defer removal if there are still nodes allocated to application
- try {
- service.remove(lb.id().application(), lb.id().cluster());
- db.removeLoadBalancer(lb.id());
- } catch (Exception e) {
- failed.add(lb.id());
- lastException = e;
- }
+ var lastException = new AtomicReference<Exception>();
+ var now = nodeRepository().clock().instant();
+ withLoadBalancersIn(State.inactive, lb -> {
+ var gracePeriod = now.minus(inactiveExpiry);
+ if (!lb.changedAt().isBefore(gracePeriod)) return; // Should not be removed yet
+ if (!allocatedNodes(lb.id()).isEmpty()) return; // Still has nodes, do not remove
+ try {
+ service.remove(lb.id().application(), lb.id().cluster());
+ db.removeLoadBalancer(lb.id());
+ } catch (Exception e){
+ failed.add(lb.id());
+ lastException.set(e);
}
- }
+ });
if (!failed.isEmpty()) {
log.log(LogLevel.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s",
failed.size(),
@@ -89,30 +85,27 @@ public class LoadBalancerExpirer extends Maintainer {
.map(LoadBalancerId::serializedForm)
.collect(Collectors.joining(", ")),
interval()),
- lastException);
+ lastException.get());
}
}
/** Remove reals from inactive load balancers */
private void pruneReals() {
var failed = new ArrayList<LoadBalancerId>();
- Exception lastException = null;
- try (var lock = db.lockLoadBalancers()) {
- var deactivated = nodeRepository().loadBalancers().in(State.inactive);
- for (var lb : deactivated) {
- var allocatedNodes = allocatedNodes(lb.id()).stream().map(Node::hostname).collect(Collectors.toSet());
- var reals = new LinkedHashSet<>(lb.instance().reals());
- // Remove any real no longer allocated to this application
- reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value()));
- try {
- service.create(lb.id().application(), lb.id().cluster(), reals, true);
- db.writeLoadBalancer(lb.with(lb.instance().withReals(reals)));
- } catch (Exception e) {
- failed.add(lb.id());
- lastException = e;
- }
+ var lastException = new AtomicReference<Exception>();
+ withLoadBalancersIn(State.inactive, lb -> {
+ var allocatedNodes = allocatedNodes(lb.id()).stream().map(Node::hostname).collect(Collectors.toSet());
+ var reals = new LinkedHashSet<>(lb.instance().reals());
+ // Remove any real no longer allocated to this application
+ reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value()));
+ try {
+ service.create(lb.id().application(), lb.id().cluster(), reals, true);
+ db.writeLoadBalancer(lb.with(lb.instance().withReals(reals)));
+ } catch (Exception e) {
+ failed.add(lb.id());
+ lastException.set(e);
}
- }
+ });
if (!failed.isEmpty()) {
log.log(LogLevel.WARNING, String.format("Failed to remove reals from %d load balancers: %s, retrying in %s",
failed.size(),
@@ -120,7 +113,19 @@ public class LoadBalancerExpirer extends Maintainer {
.map(LoadBalancerId::serializedForm)
.collect(Collectors.joining(", ")),
interval()),
- lastException);
+ lastException.get());
+ }
+ }
+
+ /** Apply operation to all load balancers that exist in given state, while holding lock */
+ private void withLoadBalancersIn(LoadBalancer.State state, Consumer<LoadBalancer> operation) {
+ for (var id : db.readLoadBalancerIds()) {
+ try (var lock = db.lockLoadBalancers(id.application())) {
+ var loadBalancer = db.readLoadBalancer(id);
+ if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop
+ if (loadBalancer.get().state() != state) continue; // Wrong state
+ operation.accept(loadBalancer.get());
+ }
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
index da0db1f1896..ed6e7cc71ef 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
@@ -15,9 +15,10 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.node.Allocation;
import com.yahoo.vespa.hosted.provision.node.History;
import com.yahoo.vespa.orchestrator.Orchestrator;
-import com.yahoo.vespa.orchestrator.status.HostStatus;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
import com.yahoo.vespa.service.monitor.ServiceMonitor;
+import java.time.Clock;
import java.time.Duration;
import java.util.HashMap;
import java.util.List;
@@ -35,22 +36,25 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed.any;
public class MetricsReporter extends Maintainer {
private final Metric metric;
- private final Function<HostName, Optional<HostStatus>> orchestrator;
+ private final Function<HostName, Optional<HostInfo>> orchestrator;
private final ServiceMonitor serviceMonitor;
private final Map<Map<String, String>, Metric.Context> contextMap = new HashMap<>();
private final Supplier<Integer> pendingRedeploymentsSupplier;
+ private final Clock clock;
MetricsReporter(NodeRepository nodeRepository,
Metric metric,
Orchestrator orchestrator,
ServiceMonitor serviceMonitor,
Supplier<Integer> pendingRedeploymentsSupplier,
- Duration interval) {
+ Duration interval,
+ Clock clock) {
super(nodeRepository, interval);
this.metric = metric;
- this.orchestrator = orchestrator.getNodeStatuses();
+ this.orchestrator = orchestrator.getHostResolver();
this.serviceMonitor = serviceMonitor;
this.pendingRedeploymentsSupplier = pendingRedeploymentsSupplier;
+ this.clock = clock;
}
@Override
@@ -125,9 +129,15 @@ public class MetricsReporter extends Maintainer {
metric.set("wantToDeprovision", node.status().wantToDeprovision() ? 1 : 0, context);
metric.set("failReport", NodeFailer.reasonsToFailParentHost(node).isEmpty() ? 0 : 1, context);
- orchestrator.apply(new HostName(node.hostname()))
- .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN ? 1 : 0)
- .ifPresent(allowedToBeDown -> metric.set("allowedToBeDown", allowedToBeDown, context));
+ orchestrator.apply(new HostName(node.hostname())).ifPresent(info -> {
+ int suspended = info.status().isSuspended() ? 1 : 0;
+ metric.set("suspended", suspended, context);
+ metric.set("allowedToBeDown", suspended, context); // remove summer 2020.
+ long suspendedSeconds = info.suspendedSince()
+ .map(suspendedSince -> Duration.between(suspendedSince, clock.instant()).getSeconds())
+ .orElse(0L);
+ metric.set("suspendedSeconds", suspendedSeconds, context);
+ });
long numberOfServices;
HostName hostName = new HostName(node.hostname());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index dca5c092ad2..ca53b215237 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -97,7 +97,7 @@ public class NodeFailer extends Maintainer {
int throttledNodeFailures = 0;
// Ready nodes
- try (Mutex lock = nodeRepository().lockAllocation()) {
+ try (Mutex lock = nodeRepository().lockUnallocated()) {
updateNodeLivenessEventsForReadyNodes(lock);
for (Map.Entry<Node, String> entry : getReadyNodesByFailureReason().entrySet()) {
@@ -265,7 +265,7 @@ public class NodeFailer extends Maintainer {
private boolean nodeSuspended(Node node) {
try {
- return orchestrator.getNodeStatus(new HostName(node.hostname())) == HostStatus.ALLOWED_TO_BE_DOWN;
+ return orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended();
} catch (HostNameNotFoundException e) {
// Treat it as not suspended
return false;
@@ -323,7 +323,7 @@ public class NodeFailer extends Maintainer {
try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) {
node = nodeRepository().getNode(node.hostname(), Node.State.active).get(); // re-get inside lock
- return nodeRepository().write(node.downAt(clock.instant()), lock);
+ return nodeRepository().write(node.downAt(clock.instant(), Agent.NodeFailer), lock);
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index db466043d0c..a49049f8b04 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -81,12 +81,12 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
dirtyExpirer = new DirtyExpirer(nodeRepository, clock, defaults.dirtyExpiry);
provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, defaults.provisionedExpiry);
nodeRebooter = new NodeRebooter(nodeRepository, clock, flagSource);
- metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval);
+ metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval, clock);
infrastructureProvisioner = new InfrastructureProvisioner(nodeRepository, infraDeployer, defaults.infrastructureProvisionInterval);
loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService().map(lbService ->
new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService));
dynamicProvisioningMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner ->
- new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource));
+ new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, provisionServiceProvider.getHostResourcesCalculator(), flagSource));
capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval);
osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval);
rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java
index 3109e55df5c..e1407f2a41d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java
@@ -27,7 +27,7 @@ public class ProvisionedExpirer extends Expirer {
@Override
protected void expire(List<Node> expired) {
for (Node expiredNode : expired)
- nodeRepository.parkRecursively(expiredNode.hostname(), Agent.system, "Node is stuck in provisioned");
+ nodeRepository.parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, "Node is stuck in provisioned");
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java
index d7dd93522e4..675e3400722 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java
@@ -115,7 +115,7 @@ public class Rebalancer extends Maintainer {
if (nodeToMove.get().status().wantToRetire() == wantToRetire) return false;
- nodeRepository().write(nodeToMove.get().withWantToRetire(wantToRetire, Agent.system, clock.instant()), lock);
+ nodeRepository().write(nodeToMove.get().withWantToRetire(wantToRetire, Agent.Rebalancer, clock.instant()), lock);
return true;
}
}
@@ -154,7 +154,7 @@ public class Rebalancer extends Maintainer {
// Immediately clean up if we reserved the node but could not activate or reserved a node on the wrong host
expectedNewNode.flatMap(node -> nodeRepository().getNode(node.hostname(), Node.State.reserved))
- .ifPresent(node -> nodeRepository().setDirty(node, Agent.system, "Expired by Rebalancer"));
+ .ifPresent(node -> nodeRepository().setDirty(node, Agent.Rebalancer, "Expired by Rebalancer"));
}
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java
index b0aa389fe7d..03d466dbf09 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java
@@ -28,6 +28,6 @@ public class ReservationExpirer extends Expirer {
}
@Override
- protected void expire(List<Node> expired) { nodeRepository.setDirty(expired, Agent.system, "Expired by ReservationExpirer"); }
+ protected void expire(List<Node> expired) { nodeRepository.setDirty(expired, Agent.ReservationExpirer, "Expired by ReservationExpirer"); }
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java
index f46e2f501bc..7522e411e42 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java
@@ -7,5 +7,15 @@ package com.yahoo.vespa.hosted.provision.node;
* @author bratseth
*/
public enum Agent {
- system, application, operator, NodeFailer
+ operator, // A hosted Vespa operator. Some logic recognizes these events.
+ application, // An application package change depoyment
+ system, // An unspecified system agent
+ // Specific system agents:
+ NodeFailer,
+ Rebalancer,
+ DirtyExpirer,
+ FailedExpirer,
+ InactiveExpirer,
+ ProvisionedExpirer,
+ ReservationExpirer
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java
index b06bbbb54b5..0622862f5ab 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java
@@ -33,6 +33,11 @@ public class OsVersion {
return wanted;
}
+ /** Returns whether this node is currently changing its version to the given version */
+ public boolean changingTo(Version version) {
+ return changing() && wanted.get().equals(version);
+ }
+
/** Returns whether this node is currently changing its version */
public boolean changing() {
return wanted.isPresent() && !current.equals(wanted);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java
index af32530a156..65cbc17aff5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java
@@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.provision.node;
import com.yahoo.slime.Cursor;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import java.time.Instant;
import java.util.Arrays;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java
index fd6094ae111..7885cec6b65 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java
@@ -71,6 +71,6 @@ public class Reports {
return this;
}
- public Reports build() { return new Reports(Collections.unmodifiableMap(reportMap)); }
+ public Reports build() { return new Reports(reportMap); }
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java
index 106595fbd47..e10ff3d24cd 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java
@@ -120,9 +120,10 @@ public class OsVersions {
/** Trigger upgrade of nodes of given type*/
private void upgrade(NodeType type, Version version) {
var nodes = nodeRepository.list().nodeType(type);
- var numberToUpgrade = Math.max(0, maxActiveUpgrades - nodes.changingOsVersion().size());
- var nodesToUpgrade = nodes.not().changingOsVersion()
+ var numberToUpgrade = Math.max(0, maxActiveUpgrades - nodes.changingOsVersionTo(version).size());
+ var nodesToUpgrade = nodes.not().changingOsVersionTo(version)
.not().onOsVersion(version)
+ .byIncreasingOsVersion()
.first(numberToUpgrade);
if (nodesToUpgrade.size() == 0) return;
log.info("Upgrading " + nodesToUpgrade.size() + " nodes of type " + type + " to OS version " + version);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index a28845109dc..f211ea9eac5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.persistence;
import com.google.common.util.concurrent.UncheckedTimeoutException;
@@ -38,6 +38,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Function;
+import java.util.function.Predicate;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
@@ -206,7 +207,7 @@ public class CuratorDatabaseClient {
toState,
toState.isAllocated() ? node.allocation() : Optional.empty(),
node.history().recordStateTransition(node.state(), toState, agent, clock.instant()),
- node.type(), node.reports(), node.modelName());
+ node.type(), node.reports(), node.modelName(), node.reservedTo());
writeNode(toState, curatorTransaction, node, newNode);
writtenNodes.add(newNode);
}
@@ -483,18 +484,16 @@ public class CuratorDatabaseClient {
// Load balancers
public List<LoadBalancerId> readLoadBalancerIds() {
- return curatorDatabase.getChildren(loadBalancersRoot).stream()
- .map(LoadBalancerId::fromSerializedForm)
- .collect(Collectors.toUnmodifiableList());
+ return readLoadBalancerIds((ignored) -> true);
}
- public Map<LoadBalancerId, LoadBalancer> readLoadBalancers() {
- return readLoadBalancerIds().stream()
- .map(this::readLoadBalancer)
- .filter(Optional::isPresent)
- .map(Optional::get)
- .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()),
- Collections::unmodifiableMap));
+ public Map<LoadBalancerId, LoadBalancer> readLoadBalancers(Predicate<LoadBalancerId> filter) {
+ return readLoadBalancerIds(filter).stream()
+ .map(this::readLoadBalancer)
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()),
+ Collections::unmodifiableMap));
}
public Optional<LoadBalancer> readLoadBalancer(LoadBalancerId id) {
@@ -522,14 +521,21 @@ public class CuratorDatabaseClient {
transaction.commit();
}
- public Lock lockLoadBalancers() {
- return lock(lockRoot.append("loadBalancersLock"), defaultLockTimeout);
+ public Lock lockLoadBalancers(ApplicationId application) {
+ return lock(lockRoot.append("loadBalancersLock2").append(application.serializedForm()), defaultLockTimeout);
}
private Path loadBalancerPath(LoadBalancerId id) {
return loadBalancersRoot.append(id.serializedForm());
}
+ private List<LoadBalancerId> readLoadBalancerIds(Predicate<LoadBalancerId> predicate) {
+ return curatorDatabase.getChildren(loadBalancersRoot).stream()
+ .map(LoadBalancerId::fromSerializedForm)
+ .filter(predicate)
+ .collect(Collectors.toUnmodifiableList());
+ }
+
private Transaction.Operation createOrSet(Path path, byte[] data) {
if (curatorDatabase.exists(path)) {
return CuratorOperations.setData(path.getAbsolute(), data);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
index ae4c93621e5..66172521d4c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
@@ -6,7 +6,7 @@ import com.yahoo.slime.ArrayTraverser;
import com.yahoo.slime.Cursor;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import com.yahoo.vespa.hosted.provision.lb.DnsZone;
import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index 2cbfbc349a6..9614c1aa5c7 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -19,7 +19,8 @@ import com.yahoo.slime.ArrayTraverser;
import com.yahoo.slime.Cursor;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.Type;
+import com.yahoo.slime.SlimeUtils;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.node.Allocation;
@@ -77,6 +78,7 @@ public class NodeSerializer {
private static final String firmwareCheckKey = "firmwareCheck";
private static final String reportsKey = "reports";
private static final String modelNameKey = "modelName";
+ private static final String reservedToKey = "reservedTo";
// Node resource fields
// ...for hosts and nodes allocated by legacy flavor specs
@@ -149,6 +151,7 @@ public class NodeSerializer {
node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong(firmwareCheckKey, instant.toEpochMilli()));
node.reports().toSlime(object, reportsKey);
node.modelName().ifPresent(modelName -> object.setString(modelNameKey, modelName));
+ node.reservedTo().ifPresent(tenant -> object.setString(reservedToKey, tenant.value()));
}
private void toSlime(Flavor flavor, Cursor object) {
@@ -222,7 +225,8 @@ public class NodeSerializer {
historyFromSlime(object.field(historyKey)),
nodeTypeFromString(object.field(nodeTypeKey).asString()),
Reports.fromSlime(object.field(reportsKey)),
- modelNameFromSlime(object));
+ modelNameFromSlime(object),
+ reservedToFromSlime(object.field(reservedToKey)));
}
private Status statusFromSlime(Inspector object) {
@@ -341,6 +345,13 @@ public class NodeSerializer {
return Optional.empty();
}
+ private Optional<TenantName> reservedToFromSlime(Inspector object) {
+ if (! object.valid()) return Optional.empty();
+ if (object.type() != Type.STRING)
+ throw new IllegalArgumentException("Expected 'reservedTo' to be a string but is " + object);
+ return Optional.of(TenantName.from(object.asString()));
+ }
+
// ----------------- Enum <-> string mappings ----------------------------------------
/** Returns the event type, or null if this event type should be ignored */
@@ -388,19 +399,31 @@ public class NodeSerializer {
private Agent eventAgentFromSlime(Inspector eventAgentField) {
switch (eventAgentField.asString()) {
+ case "operator" : return Agent.operator;
case "application" : return Agent.application;
case "system" : return Agent.system;
- case "operator" : return Agent.operator;
case "NodeFailer" : return Agent.NodeFailer;
+ case "Rebalancer" : return Agent.Rebalancer;
+ case "DirtyExpirer" : return Agent.DirtyExpirer;
+ case "FailedExpirer" : return Agent.FailedExpirer;
+ case "InactiveExpirer" : return Agent.InactiveExpirer;
+ case "ProvisionedExpirer" : return Agent.ProvisionedExpirer;
+ case "ReservationExpirer" : return Agent.ReservationExpirer;
}
throw new IllegalArgumentException("Unknown node event agent '" + eventAgentField.asString() + "'");
}
private String toString(Agent agent) {
switch (agent) {
+ case operator : return "operator";
case application : return "application";
case system : return "system";
- case operator : return "operator";
case NodeFailer : return "NodeFailer";
+ case Rebalancer : return "Rebalancer";
+ case DirtyExpirer : return "DirtyExpirer";
+ case FailedExpirer : return "FailedExpirer";
+ case InactiveExpirer : return "InactiveExpirer";
+ case ProvisionedExpirer : return "ProvisionedExpirer";
+ case ReservationExpirer : return "ReservationExpirer";
}
throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined");
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java
index 37dc8a8a1ad..6615dff24e5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java
@@ -7,7 +7,7 @@ import com.yahoo.slime.Cursor;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.ObjectTraverser;
import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import java.io.IOException;
import java.io.UncheckedIOException;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java
index dfa79a4fd9a..e27cdcd1842 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java
@@ -7,7 +7,7 @@ import com.yahoo.slime.Cursor;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.ObjectTraverser;
import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import java.io.IOException;
import java.io.UncheckedIOException;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java
index 14c8c7fa18e..915b1acedaa 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java
@@ -5,7 +5,7 @@ import com.yahoo.component.Version;
import com.yahoo.config.provision.NodeType;
import com.yahoo.slime.ObjectTraverser;
import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import com.yahoo.vespa.hosted.provision.node.OsVersion;
import java.io.IOException;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java
index 1839f7187fa..1149b15a2b0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java
@@ -5,7 +5,7 @@ import com.yahoo.slime.ArrayTraverser;
import com.yahoo.slime.Cursor;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import java.io.IOException;
import java.util.HashSet;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
index a7519de3776..36034b62cfb 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
@@ -6,7 +6,6 @@ import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.HostSpec;
import com.yahoo.config.provision.ParentHostUnavailableException;
-import com.yahoo.log.LogLevel;
import com.yahoo.transaction.Mutex;
import com.yahoo.transaction.NestedTransaction;
import com.yahoo.vespa.hosted.provision.Node;
@@ -90,6 +89,22 @@ class Activator {
nodeRepository.deactivate(activeToRemove, transaction);
nodeRepository.activate(updateFrom(hosts, continuedActive), transaction); // update active with any changes
nodeRepository.activate(updatePortsFrom(hosts, reservedToActivate), transaction);
+ unreserveParentsOf(reservedToActivate);
+ }
+
+ /** When a tenant node is activated on a host, we can open up that host for use by others */
+ private void unreserveParentsOf(List<Node> nodes) {
+ for (Node node : nodes) {
+ if ( node.parentHostname().isEmpty()) continue;
+ Optional<Node> parent = nodeRepository.getNode(node.parentHostname().get());
+ if (parent.isEmpty()) continue;
+ if (parent.get().reservedTo().isEmpty()) continue;
+ try (Mutex lock = nodeRepository.lock(parent.get())) {
+ Optional<Node> lockedParent = nodeRepository.getNode(parent.get().hostname());
+ if (lockedParent.isEmpty()) continue;
+ nodeRepository.write(lockedParent.get().withoutReservedTo(), lock);
+ }
+ }
}
/** Activate load balancers */
@@ -117,9 +132,9 @@ class Activator {
.filter(node -> node.state() != Node.State.active)
.map(Node::hostname)
.collect(Collectors.toSet());
- long numNonActive = nonActiveHosts.size();
- if (numNonActive > 0) {
- long numActive = parentHostnames.size() - numNonActive;
+
+ if (nonActiveHosts.size() > 0) {
+ long numActive = parentHostnames.size() - nonActiveHosts.size();
var messageBuilder = new StringBuilder()
.append(numActive).append("/").append(parentHostnames.size())
.append(" hosts for ")
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java
index a609103ac89..fb76dc54d1a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java
@@ -26,12 +26,6 @@ public class DockerHostCapacity {
this.hostResourcesCalculator = Objects.requireNonNull(hostResourcesCalculator, "hostResourcesCalculator must be non-null");
}
- /** Returns the allocation skew of this host */
- public double skew(Node host) {
- NodeResources free = freeCapacityOf(host, false);
- return Node.skew(host.flavor().resources(), free);
- }
-
int compareWithoutInactive(Node hostA, Node hostB) {
int result = compare(freeCapacityOf(hostB, true), freeCapacityOf(hostA, true));
if (result != 0) return result;
@@ -72,7 +66,7 @@ public class DockerHostCapacity {
NodeResources freeCapacityOf(Node host, boolean excludeInactive) {
// Only hosts have free capacity
if (!host.type().canRun(NodeType.tenant)) return new NodeResources(0, 0, 0, 0);
- NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(host.flavor().resources());
+ NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(host.flavor().name(), host.flavor().resources());
return allNodes.childrenOf(host).asList().stream()
.filter(node -> !(excludeInactive && isInactiveOrRetired(node)))
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java
index 9a06f2a980a..4416106f23e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java
@@ -3,9 +3,14 @@ package com.yahoo.vespa.hosted.provision.provisioning;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
+import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.DockerImage;
import com.yahoo.config.provision.NodeType;
import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.StringFlag;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
import java.time.Duration;
@@ -13,6 +18,7 @@ import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
import java.util.logging.Logger;
/**
@@ -28,6 +34,7 @@ public class DockerImages {
private final CuratorDatabaseClient db;
private final DockerImage defaultImage;
private final Duration cacheTtl;
+ private final StringFlag imageOverride;
/**
* Docker image is read on every request to /nodes/v2/node/[fqdn]. Cache current getDockerImages to avoid
@@ -36,20 +43,41 @@ public class DockerImages {
*/
private volatile Supplier<Map<NodeType, DockerImage>> dockerImages;
- public DockerImages(CuratorDatabaseClient db, DockerImage defaultImage) {
- this(db, defaultImage, defaultCacheTtl);
+ public DockerImages(CuratorDatabaseClient db, DockerImage defaultImage, StringFlag imageOverride) {
+ this(db, defaultImage, defaultCacheTtl, imageOverride);
}
- DockerImages(CuratorDatabaseClient db, DockerImage defaultImage, Duration cacheTtl) {
+ DockerImages(CuratorDatabaseClient db, DockerImage defaultImage, Duration cacheTtl, StringFlag imageOverride) {
this.db = db;
this.defaultImage = defaultImage;
this.cacheTtl = cacheTtl;
+ this.imageOverride = imageOverride;
createCache();
}
private void createCache() {
this.dockerImages = Suppliers.memoizeWithExpiration(() -> Collections.unmodifiableMap(db.readDockerImages()),
- cacheTtl.toMillis(), TimeUnit.MILLISECONDS);
+ cacheTtl.toMillis(), TimeUnit.MILLISECONDS);
+ }
+
+ /** Returns the image to use for given node and zone */
+ public DockerImage dockerImageFor(Node node) {
+ if (node.type().isDockerHost()) {
+ // Docker hosts do not run in containers, and thus has no image. Return the image of the child node type
+ // instead as this allows the host to pre-download the (likely) image its node will run.
+ //
+ // Note that if the Docker image has been overridden through feature flag, the preloaded image won't match.
+ return dockerImageFor(node.type().childNodeType());
+ }
+ return node.allocation()
+ .map(Allocation::owner)
+ .map(ApplicationId::serializedForm)
+ // Return overridden image for this application
+ .map(application -> imageOverride.with(FetchVector.Dimension.APPLICATION_ID, application).value())
+ .filter(Predicate.not(String::isEmpty))
+ .map(DockerImage::fromString)
+ // ... or default Docker image for this node type
+ .orElseGet(() -> dockerImageFor(node.type()));
}
/** Returns the current docker images for each node type */
@@ -58,7 +86,7 @@ public class DockerImages {
}
/** Returns the current docker image for given node type, or default */
- public DockerImage dockerImageFor(NodeType type) {
+ private DockerImage dockerImageFor(NodeType type) {
return getDockerImages().getOrDefault(type, defaultImage);
}
@@ -69,8 +97,8 @@ public class DockerImages {
}
try (Lock lock = db.lockDockerImages()) {
Map<NodeType, DockerImage> dockerImages = db.readDockerImages();
-
- dockerImage.ifPresentOrElse(image -> dockerImages.put(nodeType, image), () -> dockerImages.remove(nodeType));
+ dockerImage.ifPresentOrElse(image -> dockerImages.put(nodeType, image),
+ () -> dockerImages.remove(nodeType));
db.writeDockerImages(dockerImages);
createCache(); // Throw away current cache
log.info("Set docker image for " + nodeType + " nodes to " + dockerImage.map(DockerImage::asString).orElse(null));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java
index 05915b82bae..b5c4478cd5a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java
@@ -30,7 +30,7 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider {
public static class NoopHostResourcesCalculator implements HostResourcesCalculator {
@Override
- public NodeResources availableCapacityOf(NodeResources hostResources) {
+ public NodeResources availableCapacityOf(String flavorName, NodeResources hostResources) {
return hostResources;
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index eab8bb68863..5753bbb3c5a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -31,7 +31,6 @@ public class GroupPreparer {
private final Optional<HostProvisioner> hostProvisioner;
private final HostResourcesCalculator hostResourcesCalculator;
private final BooleanFlag dynamicProvisioningEnabledFlag;
- private final BooleanFlag enableInPlaceResize;
private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag;
public GroupPreparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner,
@@ -40,7 +39,6 @@ public class GroupPreparer {
this.hostProvisioner = hostProvisioner;
this.hostResourcesCalculator = hostResourcesCalculator;
this.dynamicProvisioningEnabledFlag = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource);
- this.enableInPlaceResize = Flags.ENABLE_IN_PLACE_RESIZE.bindTo(flagSource);
this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource);
}
@@ -65,25 +63,23 @@ public class GroupPreparer {
boolean dynamicProvisioningEnabled = hostProvisioner.isPresent() && dynamicProvisioningEnabledFlag
.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm())
.value();
- boolean inPlaceResizeEnabled = enableInPlaceResize
- .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm())
- .value();
+ boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty();
try (Mutex lock = nodeRepository.lock(application)) {
// Lock ready pool to ensure that the same nodes are not simultaneously allocated by others
- try (Mutex allocationLock = nodeRepository.lockAllocation()) {
+ try (Mutex allocationLock = nodeRepository.lockUnallocated()) {
// Create a prioritized set of nodes
LockedNodeList nodeList = nodeRepository.list(allocationLock);
NodePrioritizer prioritizer = new NodePrioritizer(nodeList, application, cluster, requestedNodes,
spareCount, wantedGroups, nodeRepository.nameResolver(),
- hostResourcesCalculator, inPlaceResizeEnabled);
+ hostResourcesCalculator, allocateFully);
prioritizer.addApplicationNodes();
prioritizer.addSurplusNodes(surplusActiveNodes);
prioritizer.addReadyNodes();
- prioritizer.addNewDockerNodes(dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty());
+ prioritizer.addNewDockerNodes();
// Allocate from the prioritized list
NodeAllocation allocation = new NodeAllocation(nodeList, application, cluster, requestedNodes,
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java
index c5808a53837..a5570dbf169 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java
@@ -9,6 +9,6 @@ import com.yahoo.config.provision.NodeResources;
public interface HostResourcesCalculator {
/** Calculates the resources that are reserved for host level processes and returns the remainder. */
- NodeResources availableCapacityOf(NodeResources hostResources);
+ NodeResources availableCapacityOf(String flavorName, NodeResources hostResources);
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
index f0caa358cb8..26f8cffa519 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.provisioning;
import com.yahoo.config.provision.ApplicationId;
@@ -51,8 +51,8 @@ public class LoadBalancerProvisioner {
this.db = nodeRepository.database();
this.service = service;
// Read and write all load balancers to make sure they are stored in the latest version of the serialization format
- try (var lock = db.lockLoadBalancers()) {
- for (var id : db.readLoadBalancerIds()) {
+ for (var id : db.readLoadBalancerIds()) {
+ try (var lock = db.lockLoadBalancers(id.application())) {
var loadBalancer = db.readLoadBalancer(id);
loadBalancer.ifPresent(db::writeLoadBalancer);
}
@@ -72,8 +72,9 @@ public class LoadBalancerProvisioner {
public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) {
if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type
if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type
- try (var loadBalancersLock = db.lockLoadBalancers()) {
- provision(application, cluster.id(), false, loadBalancersLock);
+ if (application.instance().isTester()) return; // Do not provision for tester instances
+ try (var lock = db.lockLoadBalancers(application)) {
+ provision(application, cluster.id(), false, lock);
}
}
@@ -89,11 +90,11 @@ public class LoadBalancerProvisioner {
*/
public void activate(ApplicationId application, Set<ClusterSpec> clusters,
@SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) {
- try (var loadBalancersLock = db.lockLoadBalancers()) {
+ try (var lock = db.lockLoadBalancers(application)) {
var containerClusters = containerClusterOf(clusters);
for (var clusterId : containerClusters) {
// Provision again to ensure that load balancer instance is re-configured with correct nodes
- provision(application, clusterId, true, loadBalancersLock);
+ provision(application, clusterId, true, lock);
}
// Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed
var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters);
@@ -107,16 +108,15 @@ public class LoadBalancerProvisioner {
*/
public void deactivate(ApplicationId application, NestedTransaction transaction) {
try (var applicationLock = nodeRepository.lock(application)) {
- try (Mutex loadBalancersLock = db.lockLoadBalancers()) {
- deactivate(nodeRepository.loadBalancers().owner(application).asList(), transaction);
+ try (var lock = db.lockLoadBalancers(application)) {
+ deactivate(nodeRepository.loadBalancers(application).asList(), transaction);
}
}
}
/** Returns load balancers of given application that are no longer referenced by given clusters */
private List<LoadBalancer> surplusLoadBalancersOf(ApplicationId application, Set<ClusterSpec.Id> activeClusters) {
- var activeLoadBalancersByCluster = nodeRepository.loadBalancers()
- .owner(application)
+ var activeLoadBalancersByCluster = nodeRepository.loadBalancers(application)
.in(LoadBalancer.State.active)
.asList()
.stream()
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
index 21e2c322594..c92f7889496 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
@@ -140,9 +140,9 @@ class NodeAllocation {
continue;
}
node.node = offered.allocate(application,
- ClusterMembership.from(cluster, highestIndex.add(1)),
- requestedNodes.resources().orElse(node.node.flavor().resources()),
- clock.instant());
+ ClusterMembership.from(cluster, highestIndex.add(1)),
+ requestedNodes.resources().orElse(node.node.flavor().resources()),
+ clock.instant());
accepted.add(acceptNode(node, false, false));
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
index a17d92117fb..1ef23209369 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
@@ -40,29 +40,30 @@ public class NodePrioritizer {
private final LockedNodeList allNodes;
private final DockerHostCapacity capacity;
private final NodeSpec requestedNodes;
- private final ApplicationId appId;
+ private final ApplicationId application;
private final ClusterSpec clusterSpec;
private final NameResolver nameResolver;
private final boolean isDocker;
private final boolean isAllocatingForReplacement;
private final boolean isTopologyChange;
- private final boolean inPlaceResizeEnabled;
+ /** If set, a host can only have nodes by single tenant and does not allow in-place resizing. */
+ private final boolean allocateFully;
private final int currentClusterSize;
private final Set<Node> spareHosts;
- NodePrioritizer(LockedNodeList allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec,
+ NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec,
int spares, int wantedGroups, NameResolver nameResolver, HostResourcesCalculator hostResourcesCalculator,
- boolean inPlaceResizeEnabled) {
+ boolean allocateFully) {
this.allNodes = allNodes;
this.capacity = new DockerHostCapacity(allNodes, hostResourcesCalculator);
this.requestedNodes = nodeSpec;
this.clusterSpec = clusterSpec;
- this.appId = appId;
+ this.application = application;
this.nameResolver = nameResolver;
this.spareHosts = findSpareHosts(allNodes, capacity, spares);
- this.inPlaceResizeEnabled = inPlaceResizeEnabled;
+ this.allocateFully = allocateFully;
- NodeList nodesInCluster = allNodes.owner(appId).type(clusterSpec.type()).cluster(clusterSpec.id());
+ NodeList nodesInCluster = allNodes.owner(application).type(clusterSpec.type()).cluster(clusterSpec.id());
NodeList nonRetiredNodesInCluster = nodesInCluster.not().retired();
long currentGroups = nonRetiredNodesInCluster.state(Node.State.active).stream()
.flatMap(node -> node.allocation()
@@ -117,23 +118,19 @@ public class NodePrioritizer {
}
}
- /**
- * Add a node on each docker host with enough capacity for the requested flavor
- *
- * @param exclusively whether the ready docker nodes should only be added on hosts that
- * already have nodes allocated to this tenant
- */
- void addNewDockerNodes(boolean exclusively) {
+ /** Add a node on each docker host with enough capacity for the requested flavor */
+ void addNewDockerNodes() {
if ( ! isDocker) return;
LockedNodeList candidates = allNodes
- .filter(node -> node.type() != NodeType.host || ALLOCATABLE_HOST_STATES.contains(node.state()));
+ .filter(node -> node.type() != NodeType.host || ALLOCATABLE_HOST_STATES.contains(node.state()))
+ .filter(node -> node.reservedTo().isEmpty() || node.reservedTo().get().equals(application.tenant()));
- if (exclusively) {
+ if (allocateFully) {
Set<String> candidateHostnames = candidates.asList().stream()
.filter(node -> node.type() == NodeType.tenant)
.filter(node -> node.allocation()
- .map(a -> a.owner().tenant().equals(appId.tenant()))
+ .map(a -> a.owner().tenant().equals(this.application.tenant()))
.orElse(false))
.flatMap(node -> node.parentHostname().stream())
.collect(Collectors.toSet());
@@ -152,7 +149,7 @@ public class NodePrioritizer {
if (host.status().wantToRetire()) continue;
boolean hostHasCapacityForWantedFlavor = capacity.hasCapacity(host, wantedResources);
- boolean conflictingCluster = allNodes.childrenOf(host).owner(appId).asList().stream()
+ boolean conflictingCluster = allNodes.childrenOf(host).owner(application).asList().stream()
.anyMatch(child -> child.allocation().get().membership().cluster().id().equals(clusterSpec.id()));
if (!hostHasCapacityForWantedFlavor || conflictingCluster) continue;
@@ -188,7 +185,7 @@ public class NodePrioritizer {
.filter(node -> node.type() == requestedNodes.type())
.filter(node -> legalStates.contains(node.state()))
.filter(node -> node.allocation().isPresent())
- .filter(node -> node.allocation().get().owner().equals(appId))
+ .filter(node -> node.allocation().get().owner().equals(application))
.map(node -> toPrioritizable(node, false, false))
.forEach(prioritizableNode -> nodes.put(prioritizableNode.node, prioritizableNode));
}
@@ -217,7 +214,7 @@ public class NodePrioritizer {
builder.parent(parent).freeParentCapacity(parentCapacity);
if (!isNewNode)
- builder.resizable(inPlaceResizeEnabled && requestedNodes.canResize(
+ builder.resizable(!allocateFully && requestedNodes.canResize(
node.flavor().resources(), parentCapacity, isTopologyChange, currentClusterSize));
if (spareHosts.contains(parent))
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
index 7cf55adc776..07cf297a314 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
@@ -131,7 +131,8 @@ public class NodeRepositoryProvisioner implements Provisioner {
private boolean hasQuota(ApplicationId application, int requestedNodes) {
if ( ! this.zone.system().isPublic()) return true; // no quota management
- if (application.tenant().value().hashCode() == 3857) return requestedNodes <= 60;
+ if (application.tenant().value().hashCode() == 3857) return requestedNodes <= 60;
+ if (application.tenant().value().hashCode() == -1271827001) return requestedNodes <= 75;
return requestedNodes <= 5;
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java
index c7732695069..6183bffe5ba 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java
@@ -87,6 +87,10 @@ class PrioritizableNode implements Comparable<PrioritizableNode> {
throw new IllegalStateException("Nodes " + this.node + " and " + other.node + " have different states");
if (this.parent.isPresent() && other.parent.isPresent()) {
+ // Prefer reserved hosts (that they are reserved to the right tenant is ensured elsewhere)
+ if ( this.parent.get().reservedTo().isPresent() && ! other.parent.get().reservedTo().isPresent()) return -1;
+ if ( ! this.parent.get().reservedTo().isPresent() && other.parent.get().reservedTo().isPresent()) return 1;
+
int diskCostDifference = NodeResources.DiskSpeed.compare(this.parent.get().flavor().resources().diskSpeed(),
other.parent.get().flavor().resources().diskSpeed());
if (diskCostDifference != 0)
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java
index 451ab089a8d..b979ccda740 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java
@@ -34,7 +34,7 @@ public class ProvisionedHost {
/** Generate {@link Node} instance representing the provisioned physical host */
public Node generateHost() {
- return Node.create(id, IP.Config.EMPTY, hostHostname, Optional.empty(), Optional.empty(), hostFlavor, NodeType.host);
+ return Node.create(id, IP.Config.EMPTY, hostHostname, Optional.empty(), Optional.empty(), hostFlavor, Optional.empty(), NodeType.host);
}
/** Generate {@link Node} instance representing the node running on this physical host */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java
index 9f8f4a804d1..3147d4caded 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.restapi.v2;
import com.yahoo.config.provision.ApplicationId;
@@ -37,10 +37,14 @@ public class LoadBalancersResponse extends HttpResponse {
}
private List<LoadBalancer> loadBalancers() {
- LoadBalancerList loadBalancers = nodeRepository.loadBalancers();
- return application().map(loadBalancers::owner)
- .map(LoadBalancerList::asList)
- .orElseGet(loadBalancers::asList);
+ LoadBalancerList loadBalancers;
+ var application = application();
+ if (application.isPresent()) {
+ loadBalancers = nodeRepository.loadBalancers(application.get());
+ } else {
+ loadBalancers = nodeRepository.loadBalancers();
+ }
+ return loadBalancers.asList();
}
@Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
index 81cf401d358..11be95604c8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
@@ -6,11 +6,13 @@ import com.yahoo.config.provision.DockerImage;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.NodeFlavors;
import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.TenantName;
import com.yahoo.io.IOUtils;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.ObjectTraverser;
import com.yahoo.slime.Type;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import com.yahoo.vespa.hosted.provision.LockedNodeList;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.node.Agent;
@@ -33,8 +35,8 @@ import java.util.stream.Collectors;
import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast;
import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow;
-import static com.yahoo.config.provision.NodeResources.StorageType.remote;
import static com.yahoo.config.provision.NodeResources.StorageType.local;
+import static com.yahoo.config.provision.NodeResources.StorageType.remote;
/**
* A class which can take a partial JSON node/v2 node JSON structure and apply it to a node object.
@@ -45,7 +47,6 @@ import static com.yahoo.config.provision.NodeResources.StorageType.local;
public class NodePatcher {
private static final String WANT_TO_RETIRE = "wantToRetire";
- private static final String WANT_TO_DEPROVISION = "wantToDeprovision";
private final NodeFlavors nodeFlavors;
private final Inspector inspector;
@@ -99,7 +100,6 @@ public class NodePatcher {
private List<Node> applyFieldRecursive(List<Node> childNodes, String name, Inspector value) {
switch (name) {
case WANT_TO_RETIRE:
- case WANT_TO_DEPROVISION:
return childNodes.stream()
.map(child -> applyField(child, name, value))
.collect(Collectors.toList());
@@ -138,7 +138,9 @@ public class NodePatcher {
return IP.Config.verify(node.with(node.ipConfig().with(IP.Pool.of(asStringSet(value)))), nodes);
case WANT_TO_RETIRE :
return node.withWantToRetire(asBoolean(value), Agent.operator, clock.instant());
- case WANT_TO_DEPROVISION :
+ case "wantToDeprovision" :
+ if (node.type() != NodeType.host && asBoolean(value))
+ throw new IllegalArgumentException("wantToDeprovision can only be set for hosts");
return node.with(node.status().withWantToDeprovision(asBoolean(value)));
case "reports" :
return nodeWithPatchedReports(node, value);
@@ -160,34 +162,53 @@ public class NodePatcher {
case "bandwidthGbps":
return node.with(node.flavor().with(node.flavor().resources().withBandwidthGbps(value.asDouble())));
case "modelName":
- if (value.type() == Type.NIX) {
- return node.withoutModelName();
- }
- return node.withModelName(asString(value));
+ return value.type() == Type.NIX ? node.withoutModelName() : node.withModelName(asString(value));
case "requiredDiskSpeed":
return patchRequiredDiskSpeed(asString(value));
+ case "reservedTo":
+ return value.type() == Type.NIX ? node.withoutReservedTo() : node.withReservedTo(TenantName.from(value.asString()));
default :
throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field");
}
}
private Node nodeWithPatchedReports(Node node, Inspector reportsInspector) {
+ Node patchedNode;
// "reports": null clears the reports
- if (reportsInspector.type() == Type.NIX) return node.with(new Reports());
+ if (reportsInspector.type() == Type.NIX) {
+ patchedNode = node.with(new Reports());
+ } else {
+ var reportsBuilder = new Reports.Builder(node.reports());
+ reportsInspector.traverse((ObjectTraverser) (reportId, reportInspector) -> {
+ if (reportInspector.type() == Type.NIX) {
+ // ... "reports": { "reportId": null } clears the report "reportId"
+ reportsBuilder.clearReport(reportId);
+ } else {
+ // ... "reports": { "reportId": {...} } overrides the whole report "reportId"
+ reportsBuilder.setReport(Report.fromSlime(reportId, reportInspector));
+ }
+ });
+ patchedNode = node.with(reportsBuilder.build());
+ }
- var reportsBuilder = new Reports.Builder(node.reports());
+ boolean hadHardFailReports = node.reports().getReports().stream()
+ .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL);
+ boolean hasHardFailReports = patchedNode.reports().getReports().stream()
+ .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL);
- reportsInspector.traverse((ObjectTraverser) (reportId, reportInspector) -> {
- if (reportInspector.type() == Type.NIX) {
- // ... "reports": { "reportId": null } clears the report "reportId"
- reportsBuilder.clearReport(reportId);
- } else {
- // ... "reports": { "reportId": {...} } overrides the whole report "reportId"
- reportsBuilder.setReport(Report.fromSlime(reportId, reportInspector));
- }
- });
+ // If this patch resulted in going from not having HARD_FAIL report to having one, or vice versa
+ if (hadHardFailReports != hasHardFailReports) {
+ // Do not automatically change wantToDeprovision when
+ // 1. Transitioning to having a HARD_FAIL report and being in state failed:
+ // To allow operators manually unset before the host is parked and deleted.
+ // 2. When in parked state: Deletion is imminent, possibly already underway
+ if ((hasHardFailReports && node.state() == Node.State.failed) || node.state() == Node.State.parked)
+ return patchedNode;
+
+ patchedNode = patchedNode.with(patchedNode.status().withWantToDeprovision(hasHardFailReports));
+ }
- return node.with(reportsBuilder.build());
+ return patchedNode;
}
private Set<String> asStringSet(Inspector field) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
index 692402ea914..f0d996ef595 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.HostFilter;
import com.yahoo.config.provision.NodeFlavors;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.TenantName;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.container.jdisc.LoggingRequestHandler;
@@ -19,7 +20,8 @@ import com.yahoo.restapi.ResourceResponse;
import com.yahoo.slime.ArrayTraverser;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.Type;
+import com.yahoo.slime.SlimeUtils;
import com.yahoo.vespa.hosted.provision.NoSuchNodeException;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -49,7 +51,7 @@ import java.util.function.Function;
import java.util.logging.Level;
import java.util.stream.Collectors;
-import static com.yahoo.vespa.config.SlimeUtils.optionalString;
+import static com.yahoo.slime.SlimeUtils.optionalString;
/**
* The implementation of the /nodes/v2 API.
@@ -225,14 +227,15 @@ public class NodesApiHandler extends LoggingRequestHandler {
Set<String> ipAddressPool = new HashSet<>();
inspector.field("additionalIpAddresses").traverse((ArrayTraverser) (i, item) -> ipAddressPool.add(item.asString()));
- return Node.create(
- inspector.field("openStackId").asString(),
- new IP.Config(ipAddresses, ipAddressPool),
- inspector.field("hostname").asString(),
- parentHostname,
- modelName,
- flavorFromSlime(inspector),
- nodeTypeFromSlime(inspector.field("type")));
+ return Node.create(inspector.field("openStackId").asString(),
+ new IP.Config(ipAddresses, ipAddressPool),
+ inspector.field("hostname").asString(),
+ parentHostname,
+ modelName,
+ flavorFromSlime(inspector),
+ reservedToFromSlime(inspector.field("reservedTo")),
+ nodeTypeFromSlime(inspector.field("type"))
+ );
}
private Flavor flavorFromSlime(Inspector inspector) {
@@ -277,6 +280,13 @@ public class NodesApiHandler extends LoggingRequestHandler {
return serializer.typeFrom(object.asString());
}
+ private Optional<TenantName> reservedToFromSlime(Inspector object) {
+ if (! object.valid()) return Optional.empty();
+ if ( object.type() != Type.STRING)
+ throw new IllegalArgumentException("Expected 'reservedTo' to be a string but is " + object);
+ return Optional.of(TenantName.from(object.asString()));
+ }
+
public static NodeFilter toNodeFilter(HttpRequest request) {
NodeFilter filter = NodeHostFilter.from(HostFilter.from(request.getProperty("hostname"),
request.getProperty("flavor"),
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
index 7ed05432e01..7f283452538 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
@@ -16,11 +16,10 @@ import com.yahoo.slime.Slime;
import com.yahoo.vespa.applicationmodel.HostName;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.node.History;
import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter;
import com.yahoo.vespa.orchestrator.Orchestrator;
-import com.yahoo.vespa.orchestrator.status.HostStatus;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
import java.io.IOException;
import java.io.OutputStream;
@@ -46,7 +45,7 @@ class NodesResponse extends HttpResponse {
private final NodeFilter filter;
private final boolean recursive;
- private final Function<HostName, Optional<HostStatus>> orchestrator;
+ private final Function<HostName, Optional<HostInfo>> orchestrator;
private final NodeRepository nodeRepository;
private final Slime slime;
private final NodeSerializer serializer = new NodeSerializer();
@@ -58,7 +57,7 @@ class NodesResponse extends HttpResponse {
this.nodeParentUrl = toNodeParentUrl(request);
filter = NodesApiHandler.toNodeFilter(request);
this.recursive = request.getBooleanProperty("recursive");
- this.orchestrator = orchestrator.getNodeStatuses();
+ this.orchestrator = orchestrator.getHostResolver();
this.nodeRepository = nodeRepository;
slime = new Slime();
@@ -146,6 +145,7 @@ class NodesResponse extends HttpResponse {
}
object.setString("openStackId", node.id());
object.setString("flavor", node.flavor().name());
+ node.reservedTo().ifPresent(reservedTo -> object.setString("reservedTo", reservedTo.value()));
if (node.flavor().isConfigured())
object.setDouble("cpuCores", node.flavor().getMinCpuCores());
toSlime(node.flavor().resources(), object.setObject("resources"));
@@ -157,13 +157,15 @@ class NodesResponse extends HttpResponse {
toSlime(allocation.membership(), object.setObject("membership"));
object.setLong("restartGeneration", allocation.restartGeneration().wanted());
object.setLong("currentRestartGeneration", allocation.restartGeneration().current());
- object.setString("wantedDockerImage", dockerImageFor(node.type()).withTag(allocation.membership().cluster().vespaVersion()).asString());
+ object.setString("wantedDockerImage", nodeRepository.dockerImage(node).withTag(allocation.membership().cluster().vespaVersion()).asString());
object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString());
toSlime(allocation.requestedResources(), object.setObject("requestedResources"));
allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts")));
orchestrator.apply(new HostName(node.hostname()))
- .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN)
- .ifPresent(allowedToBeDown -> object.setBool("allowedToBeDown", allowedToBeDown));
+ .ifPresent(info -> {
+ object.setBool("allowedToBeDown", info.status().isSuspended());
+ info.suspendedSince().ifPresent(since -> object.setLong("suspendedSinceMillis", since.toEpochMilli()));
+ });
});
object.setLong("rebootGeneration", node.status().reboot().wanted());
object.setLong("currentRebootGeneration", node.status().reboot().current());
@@ -220,16 +222,10 @@ class NodesResponse extends HttpResponse {
// TODO: Remove current + wanted docker image from response for non-docker types
private Optional<DockerImage> currentDockerImage(Node node) {
return node.status().dockerImage()
- .or(() -> Optional.of(node)
- .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER)
- .flatMap(n -> n.status().vespaVersion()
- .map(version -> dockerImageFor(n.type()).withTag(version))));
- }
-
- // Docker hosts are not running in an image, but return the image of the node type running on it anyway,
- // this allows the docker host to pre-download the (likely) image its node will run
- private DockerImage dockerImageFor(NodeType nodeType) {
- return nodeRepository.dockerImage(nodeType.isDockerHost() ? nodeType.childNodeType() : nodeType);
+ .or(() -> Optional.of(node)
+ .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER)
+ .flatMap(n -> n.status().vespaVersion()
+ .map(version -> nodeRepository.dockerImage(n).withTag(version))));
}
private void ipAddressesToSlime(Set<String> ipAddresses, Cursor array) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
index 4abf6d77268..d26accd7a84 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
@@ -13,7 +13,8 @@ public class ContainerConfig {
return "<container version='1.0'>\n" +
" <config name=\"container.handler.threadpool\">\n" +
" <maxthreads>20</maxthreads>\n" +
- " </config> \n" +
+ " </config>\n" +
+ " <accesslog type='disabled'/>\n" +
" <component id='com.yahoo.test.ManualClock'/>\n" +
" <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" +
" <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>\n" +
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 0d331ac0f3f..a2579bee0a1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -54,7 +54,7 @@ public class MockNodeRepository extends NodeRepository {
super(flavors, curator, Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(),
new MockNameResolver().mockAnyLookup(),
DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
- true);
+ true, new InMemoryFlagSource());
this.flavors = flavors;
curator.setZooKeeperEnsembleConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234");
@@ -70,34 +70,34 @@ public class MockNodeRepository extends NodeRepository {
// Regular nodes
nodes.add(createNode("node1", "host1.yahoo.com", ipConfig(1), Optional.empty(),
- new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant));
+ new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant));
nodes.add(createNode("node2", "host2.yahoo.com", ipConfig(2), Optional.empty(),
- new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant));
+ new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant));
nodes.add(createNode("node3", "host3.yahoo.com", ipConfig(3), Optional.empty(),
- new Flavor(new NodeResources(0.5, 48, 500, 1, fast, local)), NodeType.tenant));
+ new Flavor(new NodeResources(0.5, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant));
Node node4 = createNode("node4", "host4.yahoo.com", ipConfig(4), Optional.of("dockerhost1.yahoo.com"),
- new Flavor(new NodeResources(1, 4, 100, 1, fast, local)), NodeType.tenant);
+ new Flavor(new NodeResources(1, 4, 100, 1, fast, local)), Optional.empty(), NodeType.tenant);
node4 = node4.with(node4.status()
.withVespaVersion(new Version("6.41.0"))
.withDockerImage(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.41.0")));
nodes.add(node4);
Node node5 = createNode("node5", "host5.yahoo.com", ipConfig(5), Optional.of("dockerhost2.yahoo.com"),
- new Flavor(new NodeResources(1, 8, 100, 1, slow, remote)), NodeType.tenant);
+ new Flavor(new NodeResources(1, 8, 100, 1, slow, remote)), Optional.empty(), NodeType.tenant);
nodes.add(node5.with(node5.status()
.withVespaVersion(new Version("1.2.3"))
.withDockerImage(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:1.2.3"))));
nodes.add(createNode("node6", "host6.yahoo.com", ipConfig(6), Optional.empty(),
- new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant));
+ new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant));
Node node7 = createNode("node7", "host7.yahoo.com", ipConfig(7), Optional.empty(),
- new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant);
+ new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant);
nodes.add(node7);
// 8, 9, 11 and 12 are added by web service calls
Node node10 = createNode("node10", "host10.yahoo.com", ipConfig(10), Optional.of("parent1.yahoo.com"),
- new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant);
+ new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant);
Status node10newStatus = node10.status();
node10newStatus = node10newStatus
.withVespaVersion(Version.fromString("5.104.142"))
@@ -106,26 +106,26 @@ public class MockNodeRepository extends NodeRepository {
nodes.add(node10);
Node node55 = createNode("node55", "host55.yahoo.com", ipConfig(55), Optional.empty(),
- new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant);
+ new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant);
nodes.add(node55.with(node55.status().withWantToRetire(true).withWantToDeprovision(true)));
/* Setup docker hosts (two of these will be reserved for spares */
nodes.add(createNode("dockerhost1", "dockerhost1.yahoo.com", ipConfig(100, 1, 3), Optional.empty(),
- flavors.getFlavorOrThrow("large"), NodeType.host));
+ flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host));
nodes.add(createNode("dockerhost2", "dockerhost2.yahoo.com", ipConfig(101, 1, 3), Optional.empty(),
- flavors.getFlavorOrThrow("large"), NodeType.host));
+ flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host));
nodes.add(createNode("dockerhost3", "dockerhost3.yahoo.com", ipConfig(102, 1, 3), Optional.empty(),
- flavors.getFlavorOrThrow("large"), NodeType.host));
+ flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host));
nodes.add(createNode("dockerhost4", "dockerhost4.yahoo.com", ipConfig(103, 1, 3), Optional.empty(),
- flavors.getFlavorOrThrow("large"), NodeType.host));
+ flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host));
nodes.add(createNode("dockerhost5", "dockerhost5.yahoo.com", ipConfig(104, 1, 3), Optional.empty(),
- flavors.getFlavorOrThrow("large"), NodeType.host));
+ flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host));
// Config servers
nodes.add(createNode("cfg1", "cfg1.yahoo.com", ipConfig(201), Optional.empty(),
- flavors.getFlavorOrThrow("default"), NodeType.config));
+ flavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.config));
nodes.add(createNode("cfg2", "cfg2.yahoo.com", ipConfig(202), Optional.empty(),
- flavors.getFlavorOrThrow("default"), NodeType.config));
+ flavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.config));
// Ready all nodes, except 7 and 55
nodes = addNodes(nodes);
@@ -167,9 +167,9 @@ public class MockNodeRepository extends NodeRepository {
List<Node> largeNodes = new ArrayList<>();
largeNodes.add(createNode("node13", "host13.yahoo.com", ipConfig(13), Optional.empty(),
- new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), NodeType.tenant));
+ new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant));
largeNodes.add(createNode("node14", "host14.yahoo.com", ipConfig(14), Optional.empty(),
- new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), NodeType.tenant));
+ new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant));
addNodes(largeNodes);
setReady(largeNodes, Agent.system, getClass().getSimpleName());
ApplicationId app4 = ApplicationId.from(TenantName.from("tenant4"), ApplicationName.from("application4"), InstanceName.from("instance4"));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
index 96ec0349fb2..d183e62c96c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
@@ -1,16 +1,21 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.testutils;
+import com.yahoo.component.AbstractComponent;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.vespa.applicationmodel.HostName;
import com.yahoo.vespa.orchestrator.Host;
import com.yahoo.vespa.orchestrator.Orchestrator;
import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
import com.yahoo.vespa.orchestrator.status.HostStatus;
+import java.time.Instant;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
@@ -18,9 +23,9 @@ import java.util.function.Function;
/**
* @author bratseth
*/
-public class OrchestratorMock implements Orchestrator {
+public class OrchestratorMock extends AbstractComponent implements Orchestrator {
- private final Set<HostName> suspendedHosts = new HashSet<>();
+ private final Map<HostName, HostInfo> suspendedHosts = new HashMap<>();
private final Set<ApplicationId> suspendedApplications = new HashSet<>();
@Override
@@ -30,12 +35,13 @@ public class OrchestratorMock implements Orchestrator {
@Override
public HostStatus getNodeStatus(HostName hostName) {
- return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS;
+ HostInfo hostInfo = suspendedHosts.get(hostName);
+ return hostInfo == null ? HostStatus.NO_REMARKS : hostInfo.status();
}
@Override
- public Function<HostName, Optional<HostStatus>> getNodeStatuses() {
- return hostName -> Optional.of(getNodeStatus(hostName));
+ public Function<HostName, Optional<HostInfo>> getHostResolver() {
+ return hostName -> Optional.of(suspendedHosts.getOrDefault(hostName, HostInfo.createNoRemarks()));
}
@Override
@@ -48,7 +54,7 @@ public class OrchestratorMock implements Orchestrator {
@Override
public void suspend(HostName hostName) {
- suspendedHosts.add(hostName);
+ suspendedHosts.put(hostName, HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH));
}
@Override
@@ -73,7 +79,7 @@ public class OrchestratorMock implements Orchestrator {
}
@Override
- public void acquirePermissionToRemove(HostName hostName) {}
+ public void acquirePermissionToRemove(HostName hostName) { }
@Override
public void suspendAll(HostName parentHostname, List<HostName> hostNames) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
index f14b6d59ee0..95555185292 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.Zone;
import com.yahoo.config.provisioning.FlavorsConfig;
import com.yahoo.test.ManualClock;
import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
@@ -27,9 +28,12 @@ public class NodeRepositoryTester {
private final NodeRepository nodeRepository;
private final Clock clock;
private final MockCurator curator;
-
-
+
public NodeRepositoryTester() {
+ this(new InMemoryFlagSource());
+ }
+
+ public NodeRepositoryTester(InMemoryFlagSource flagSource) {
nodeFlavors = new NodeFlavors(createConfig());
clock = new ManualClock();
curator = new MockCurator();
@@ -37,7 +41,7 @@ public class NodeRepositoryTester {
nodeRepository = new NodeRepository(nodeFlavors, curator, clock, Zone.defaultZone(),
new MockNameResolver().mockAnyLookup(),
DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
- true);
+ true, flagSource);
}
public NodeRepository nodeRepository() { return nodeRepository; }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
index da174a8d38c..96236b5fb84 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
@@ -22,6 +22,7 @@ import com.yahoo.config.provision.Zone;
import com.yahoo.test.ManualClock;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.node.IP;
@@ -54,7 +55,8 @@ public class CapacityCheckerTester {
Curator curator = new MockCurator();
NodeFlavors f = new NodeFlavors(new FlavorConfigBuilder().build());
nodeRepository = new NodeRepository(f, curator, clock, zone, new MockNameResolver().mockAnyLookup(),
- DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true);
+ DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true,
+ new InMemoryFlagSource());
}
private void updateCapacityChecker() {
@@ -133,8 +135,8 @@ public class CapacityCheckerTester {
NodeResources nr = containingNodeResources(childResources,
excessCapacity);
Node node = nodeRepository.createNode(hostname, hostname,
- new IP.Config(Set.of("::"), availableIps), Optional.empty(),
- new Flavor(nr), NodeType.host);
+ new IP.Config(Set.of("::"), availableIps), Optional.empty(),
+ new Flavor(nr), Optional.empty(), NodeType.host);
hosts.add(node);
}
return hosts;
@@ -152,8 +154,8 @@ public class CapacityCheckerTester {
.mapToObj(n -> String.format("%04X::%04X", hostid, n))
.collect(Collectors.toSet());
Node node = nodeRepository.createNode(hostname, hostname,
- new IP.Config(Set.of("::"), availableIps), Optional.empty(),
- new Flavor(capacity), NodeType.host);
+ new IP.Config(Set.of("::"), availableIps), Optional.empty(),
+ new Flavor(capacity), Optional.empty(), NodeType.host);
hosts.add(node);
}
return hosts;
@@ -263,8 +265,8 @@ public class CapacityCheckerTester {
Flavor f = new Flavor(nr);
Node node = nodeRepository.createNode(nodeModel.id, nodeModel.hostname,
- new IP.Config(nodeModel.ipAddresses, nodeModel.additionalIpAddresses),
- nodeModel.parentHostname, f, nodeModel.type);
+ new IP.Config(nodeModel.ipAddresses, nodeModel.additionalIpAddresses),
+ nodeModel.parentHostname, f, Optional.empty(), nodeModel.type);
if (membership != null) {
return node.allocate(owner, membership, node.flavor().resources(), Instant.now());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
index 71788fb1a30..36f876fb6bb 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
@@ -26,9 +26,11 @@ import com.yahoo.vespa.hosted.provision.node.Status;
import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
import org.hamcrest.BaseMatcher;
import org.hamcrest.Description;
+import org.junit.Before;
import org.junit.Test;
import java.time.Duration;
@@ -49,6 +51,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -64,18 +67,23 @@ public class DynamicProvisioningMaintainerTest {
private final HostProvisionerTester tester = new HostProvisionerTester();
private final HostProvisioner hostProvisioner = mock(HostProvisioner.class);
+ private final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class);
private final InMemoryFlagSource flagSource = new InMemoryFlagSource()
.withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true)
.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(), PreprovisionCapacity.class);
private final DynamicProvisioningMaintainer maintainer = new DynamicProvisioningMaintainer(
- tester.nodeRepository, Duration.ofDays(1), hostProvisioner, flagSource);
+ tester.nodeRepository, Duration.ofDays(1), hostProvisioner, hostResourcesCalculator, flagSource);
@Test
public void delegates_to_host_provisioner_and_writes_back_result() {
addNodes();
+ Node host3 = tester.nodeRepository.getNode("host3").orElseThrow();
Node host4 = tester.nodeRepository.getNode("host4").orElseThrow();
Node host41 = tester.nodeRepository.getNode("host4-1").orElseThrow();
- assertTrue(Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty));
+ assertTrue(Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty));
+
+ Node host3new = host3.with(host3.ipConfig().with(Set.of("::5")));
+ when(hostProvisioner.provision(eq(host3), eq(Set.of()))).thenReturn(List.of(host3new));
Node host4new = host4.with(host4.ipConfig().with(Set.of("::2")));
Node host41new = host41.with(host4.ipConfig().with(Set.of("::4", "10.0.0.1")));
@@ -83,8 +91,10 @@ public class DynamicProvisioningMaintainerTest {
maintainer.updateProvisioningNodes(tester.nodeRepository.list(), () -> {});
verify(hostProvisioner).provision(eq(host4), eq(Set.of(host41)));
+ verify(hostProvisioner).provision(eq(host3), eq(Set.of()));
verifyNoMoreInteractions(hostProvisioner);
+ assertEquals(Optional.of(host3new), tester.nodeRepository.getNode("host3"));
assertEquals(Optional.of(host4new), tester.nodeRepository.getNode("host4"));
assertEquals(Optional.of(host41new), tester.nodeRepository.getNode("host4-1"));
}
@@ -127,7 +137,7 @@ public class DynamicProvisioningMaintainerTest {
@Test
public void provision_deficit_and_deprovision_excess() {
- flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(1, 3, 2, 1), new PreprovisionCapacity(2, 3, 2, 2)), PreprovisionCapacity.class);
+ flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(2, 4, 8, 1), new PreprovisionCapacity(2, 3, 2, 2)), PreprovisionCapacity.class);
addNodes();
maintainer.convergeToCapacity(tester.nodeRepository.list());
@@ -150,6 +160,15 @@ public class DynamicProvisioningMaintainerTest {
verifyNoMoreInteractions(hostProvisioner);
}
+ @Before
+ public void setup() {
+ doAnswer(invocation -> {
+ String flavorName = invocation.getArgument(0, String.class);
+ if ("default".equals(flavorName)) return new NodeResources(2, 4, 8, 1);
+ return invocation.getArguments()[1];
+ }).when(hostResourcesCalculator).availableCapacityOf(any(), any());
+ }
+
public void addNodes() {
List.of(createNode("host1", Optional.empty(), NodeType.host, Node.State.active, Optional.of(tenantHostApp)),
createNode("host1-1", Optional.of("host1"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)),
@@ -157,6 +176,7 @@ public class DynamicProvisioningMaintainerTest {
createNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)),
createNode("host2-1", Optional.of("host2"), NodeType.tenant, Node.State.failed, Optional.empty()),
+
createNode("host3", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()),
createNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()),
@@ -186,7 +206,8 @@ public class DynamicProvisioningMaintainerTest {
private final ManualClock clock = new ManualClock();
private final NodeRepository nodeRepository = new NodeRepository(
- nodeFlavors, new MockCurator(), clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), true);
+ nodeFlavors, new MockCurator(), clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(),
+ DockerImage.fromString("docker-image"), true, new InMemoryFlagSource());
Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) {
Node node = createNode(hostname, parentHostname, nodeType, state, application);
@@ -204,7 +225,7 @@ public class DynamicProvisioningMaintainerTest {
false));
var ipConfig = new IP.Config(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of());
return new Node("fake-id-" + hostname, ipConfig, hostname, parentHostname, flavor, Status.initial(),
- state, allocation, History.empty(), nodeType, new Reports(), Optional.empty());
+ state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty());
}
}
-} \ No newline at end of file
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
index 8509722b016..c293a3436b8 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
@@ -256,7 +256,7 @@ public class FailedExpirerTest {
this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone,
new MockNameResolver().mockAnyLookup(),
DockerImage.fromString("docker-image"),
- true);
+ true, new InMemoryFlagSource());
this.provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource());
this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30));
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
index 12b48fd7a35..7e8fcddb1ae 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.maintenance;
import com.yahoo.component.Vtag;
@@ -32,14 +32,14 @@ import static org.junit.Assert.assertTrue;
*/
public class LoadBalancerExpirerTest {
- private ProvisioningTester tester = new ProvisioningTester.Builder().build();
+ private final ProvisioningTester tester = new ProvisioningTester.Builder().build();
@Test
public void expire_inactive() {
LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(),
Duration.ofDays(1),
tester.loadBalancerService());
- Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers();
+ Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true);
// Deploy two applications with a total of three load balancers
ClusterSpec.Id cluster1 = ClusterSpec.Id.from("qrs");
@@ -67,14 +67,15 @@ public class LoadBalancerExpirerTest {
// Expirer prunes reals before expiration time of load balancer itself
expirer.maintain();
assertEquals(Set.of(), tester.loadBalancerService().instances().get(lb1).reals());
- assertEquals(Set.of(), tester.nodeRepository().loadBalancers().owner(lb1.application()).asList().get(0).instance().reals());
+ assertEquals(Set.of(), loadBalancers.get().get(lb1).instance().reals());
// Expirer defers removal of load balancer until expiration time passes
expirer.maintain();
+ assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb1).state());
assertTrue("Inactive load balancer not removed", tester.loadBalancerService().instances().containsKey(lb1));
// Expirer removes load balancers once expiration time passes
- tester.clock().advance(Duration.ofHours(1));
+ tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1)));
expirer.maintain();
assertFalse("Inactive load balancer removed", tester.loadBalancerService().instances().containsKey(lb1));
@@ -85,7 +86,7 @@ public class LoadBalancerExpirerTest {
// A single cluster is removed
deployApplication(app2, cluster1);
expirer.maintain();
- assertEquals(LoadBalancer.State.inactive, loadBalancers.get().get(lb3).state());
+ assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb3).state());
// Expirer defers removal while nodes are still allocated to cluster
expirer.maintain();
@@ -93,7 +94,7 @@ public class LoadBalancerExpirerTest {
dirtyNodesOf(app2, cluster2);
// Expirer removes load balancer for removed cluster
- tester.clock().advance(Duration.ofHours(1));
+ tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1)));
expirer.maintain();
assertFalse("Inactive load balancer removed", tester.loadBalancerService().instances().containsKey(lb3));
}
@@ -103,7 +104,7 @@ public class LoadBalancerExpirerTest {
LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(),
Duration.ofDays(1),
tester.loadBalancerService());
- Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers();
+ Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true);
// Prepare application
@@ -121,7 +122,7 @@ public class LoadBalancerExpirerTest {
// Application never activates and nodes are dirtied. Expirer moves load balancer to inactive after timeout
dirtyNodesOf(app, cluster);
- tester.clock().advance(Duration.ofHours(1));
+ tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1)));
expirer.maintain();
assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb).state());
@@ -130,7 +131,7 @@ public class LoadBalancerExpirerTest {
assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb).state());
// Expirer removes inactive load balancer
- tester.clock().advance(Duration.ofHours(1));
+ tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1)));
expirer.maintain();
assertFalse("Inactive load balancer removed", loadBalancers.get().containsKey(lb));
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
index a4b66d3cf9e..246f2509397 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.Zone;
import com.yahoo.test.ManualClock;
import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.node.Agent;
@@ -35,7 +36,7 @@ public class MaintenanceTester {
public final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone,
new MockNameResolver().mockAnyLookup(),
DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
- true);
+ true, new InMemoryFlagSource());
public MaintenanceTester() {
curator.setZooKeeperEnsembleConnectionSpec("zk1.host:1,zk2.host:2,zk3.host:3");
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
index 539d8c7cff2..321812497bd 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
@@ -10,8 +10,10 @@ import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.NodeType;
import com.yahoo.config.provision.Zone;
import com.yahoo.jdisc.Metric;
+import com.yahoo.test.ManualClock;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
import com.yahoo.vespa.hosted.provision.LockedNodeList;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -22,6 +24,7 @@ import com.yahoo.vespa.hosted.provision.node.IP;
import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
import com.yahoo.vespa.orchestrator.Orchestrator;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
import com.yahoo.vespa.orchestrator.status.HostStatus;
import com.yahoo.vespa.service.monitor.ServiceModel;
import com.yahoo.vespa.service.monitor.ServiceMonitor;
@@ -29,6 +32,7 @@ import org.junit.Test;
import java.time.Clock;
import java.time.Duration;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -53,7 +57,7 @@ public class MetricsReporterTest {
NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(),
new MockNameResolver().mockAnyLookup(),
DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
- true);
+ true, new InMemoryFlagSource());
Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant);
nodeRepository.addNodes(List.of(node));
Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy);
@@ -82,12 +86,17 @@ public class MetricsReporterTest {
expectedMetrics.put("wantToRetire", 0);
expectedMetrics.put("wantToDeprovision", 0);
expectedMetrics.put("failReport", 0);
- expectedMetrics.put("allowedToBeDown", 0);
+ expectedMetrics.put("allowedToBeDown", 1);
+ expectedMetrics.put("suspended", 1);
+ expectedMetrics.put("suspendedSeconds", 123L);
expectedMetrics.put("numberOfServices", 0L);
+ ManualClock clock = new ManualClock(Instant.ofEpochSecond(124));
Orchestrator orchestrator = mock(Orchestrator.class);
ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
- when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS));
+ when(orchestrator.getHostResolver()).thenReturn(hostName ->
+ Optional.of(HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.ofEpochSecond(1)))
+ );
ServiceModel serviceModel = mock(ServiceModel.class);
when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel);
when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of());
@@ -99,8 +108,8 @@ public class MetricsReporterTest {
orchestrator,
serviceMonitor,
() -> 42,
- Duration.ofMinutes(1)
- );
+ Duration.ofMinutes(1),
+ clock);
metricsReporter.maintain();
assertEquals(expectedMetrics, metric.values);
@@ -113,13 +122,13 @@ public class MetricsReporterTest {
NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(),
new MockNameResolver().mockAnyLookup(),
DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
- true);
+ true, new InMemoryFlagSource());
// Allow 4 containers
Set<String> ipAddressPool = Set.of("::2", "::3", "::4", "::5");
Node dockerHost = Node.create("openStackId1", new IP.Config(Set.of("::1"), ipAddressPool), "dockerHost",
- Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host);
+ Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host);
nodeRepository.addNodes(List.of(dockerHost));
nodeRepository.dirtyRecursively("dockerHost", Agent.system, getClass().getSimpleName());
nodeRepository.setReady("dockerHost", Agent.system, getClass().getSimpleName());
@@ -127,29 +136,30 @@ public class MetricsReporterTest {
Node container1 = Node.createDockerNode(Set.of("::2"), "container1",
"dockerHost", new NodeResources(1, 3, 2, 1), NodeType.tenant);
container1 = container1.with(allocation(Optional.of("app1"), container1).get());
- nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockAllocation()));
+ nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockUnallocated()));
Node container2 = Node.createDockerNode(Set.of("::3"), "container2",
"dockerHost", new NodeResources(2, 4, 4, 1), NodeType.tenant);
container2 = container2.with(allocation(Optional.of("app2"), container2).get());
- nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockAllocation()));
+ nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockUnallocated()));
Orchestrator orchestrator = mock(Orchestrator.class);
ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
- when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS));
+ when(orchestrator.getHostResolver()).thenReturn(hostName -> Optional.of(HostInfo.createNoRemarks()));
ServiceModel serviceModel = mock(ServiceModel.class);
when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel);
when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of());
TestMetric metric = new TestMetric();
+ ManualClock clock = new ManualClock();
MetricsReporter metricsReporter = new MetricsReporter(
nodeRepository,
metric,
orchestrator,
serviceMonitor,
() -> 42,
- Duration.ofMinutes(1)
- );
+ Duration.ofMinutes(1),
+ clock);
metricsReporter.maintain();
assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
index 5872a78e1e2..033ddcd827e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
@@ -75,7 +75,8 @@ public class NodeFailTester {
clock = new ManualClock();
curator = new MockCurator();
nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(),
- DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true);
+ DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true,
+ new InMemoryFlagSource());
provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
hostLivenessTracker = new TestHostLivenessTracker(clock);
orchestrator = new OrchestratorMock();
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
index 50c00c730bb..22d7f03c449 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
@@ -56,7 +56,7 @@ public class OperatorChangeApplicationMaintainerTest {
this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone,
new MockNameResolver().mockAnyLookup(),
DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
- true);
+ true, new InMemoryFlagSource());
this.fixture = new Fixture(zone, nodeRepository);
createReadyNodes(15, this.fixture.nodeResources, nodeRepository);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
index b1c3b23016c..913b8b53c46 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
@@ -62,7 +62,7 @@ public class PeriodicApplicationMaintainerTest {
this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone,
new MockNameResolver().mockAnyLookup(),
DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
- true);
+ true, new InMemoryFlagSource());
this.fixture = new Fixture(zone, nodeRepository);
createReadyNodes(15, fixture.nodeResources, nodeRepository);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java
index d1a330a3bd6..d0c678bdf45 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java
@@ -152,7 +152,7 @@ public class RebalancerTest {
private static class IdentityHostResourcesCalculator implements HostResourcesCalculator {
@Override
- public NodeResources availableCapacityOf(NodeResources hostResources) {
+ public NodeResources availableCapacityOf(String flavorName, NodeResources hostResources) {
return hostResources;
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
index 11ee6637720..96c3cc09b6b 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
@@ -47,7 +47,7 @@ public class ReservationExpirerTest {
NodeRepository nodeRepository = new NodeRepository(flavors, curator, clock, Zone.defaultZone(),
new MockNameResolver().mockAnyLookup(),
DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
- true);
+ true, new InMemoryFlagSource());
NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource());
List<Node> nodes = new ArrayList<>(2);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
index 67af2df36e7..bdbe046fbdf 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
@@ -64,8 +64,8 @@ public class RetiredExpirerTest {
private final Zone zone = new Zone(Environment.prod, RegionName.from("us-east"));
private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default");
private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone,
- new MockNameResolver().mockAnyLookup(),
- DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true);
+ new MockNameResolver().mockAnyLookup(),
+ DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, new InMemoryFlagSource());
private final NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
private final Orchestrator orchestrator = mock(Orchestrator.class);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java
index 5e44b2e903e..fcf2ab5a52d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java
@@ -179,7 +179,7 @@ public class IPTest {
private static Node createNode(Set<String> ipAddresses) {
return Node.create("id1", new IP.Config(Set.of("127.0.0.1"), ipAddresses),
"host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"),
- NodeType.host);
+ Optional.empty(), NodeType.host);
}
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
index ebb64d650f1..5e859cd3d25 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
@@ -5,13 +5,18 @@ import com.yahoo.component.Version;
import com.yahoo.config.provision.NodeType;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.node.OsVersion;
+import com.yahoo.vespa.hosted.provision.node.Status;
import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
import org.junit.Test;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
+import java.util.function.Function;
import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
+import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -75,8 +80,11 @@ public class OsVersionsTest {
tester.makeReadyNodes(totalNodes, "default", NodeType.host);
Supplier<NodeList> hostNodes = () -> tester.nodeRepository().list().nodeType(NodeType.host);
- // Some nodes have reported current version
- setCurrentVersion(hostNodes.get().asList().subList(0, 2), Version.fromString("7.0"));
+ // 5 nodes have no version. The other 15 are spread across different versions
+ var hostNodesList = hostNodes.get().asList();
+ for (int i = totalNodes - maxActiveUpgrades - 1; i >= 0; i--) {
+ setCurrentVersion(List.of(hostNodesList.get(i)), new Version(7, 0, i));
+ }
// Set target
var version1 = Version.fromString("7.1");
@@ -86,28 +94,68 @@ public class OsVersionsTest {
// Activate target
for (int i = 0; i < totalNodes; i += maxActiveUpgrades) {
versions.setActive(NodeType.host, true);
- var nodesUpgrading = hostNodes.get().changingOsVersion();
+ var nodes = hostNodes.get();
+ var nodesUpgrading = nodes.changingOsVersion();
assertEquals("Target is changed for a subset of nodes", maxActiveUpgrades, nodesUpgrading.size());
assertEquals("Wanted version is set for nodes upgrading", version1,
- nodesUpgrading.stream()
- .map(node -> node.status().osVersion().wanted().get())
- .min(Comparator.naturalOrder()).get());
+ minVersion(nodesUpgrading, OsVersion::wanted));
+ var nodesOnLowestVersion = nodes.asList().stream()
+ .sorted(Comparator.comparing(node -> node.status().osVersion().current().orElse(Version.emptyVersion)))
+ .collect(Collectors.toList())
+ .subList(0, maxActiveUpgrades);
+ assertEquals("Nodes on lowest version are told to upgrade",
+ nodesUpgrading.asList(), nodesOnLowestVersion);
completeUpgradeOf(nodesUpgrading.asList());
}
// Activating again after all nodes have upgraded does nothing
versions.setActive(NodeType.host, true);
- assertEquals(version1, hostNodes.get().stream()
- .map(n -> n.status().osVersion().current().get())
- .min(Comparator.naturalOrder()).get());
+ assertEquals("All nodes upgraded", version1, minVersion(hostNodes.get(), OsVersion::current));
+ }
+
+ @Test
+ public void test_newer_upgrade_aborts_upgrade_to_stale_version() {
+ var versions = new OsVersions(tester.nodeRepository(), Integer.MAX_VALUE);
+ tester.makeReadyNodes(10, "default", NodeType.host);
+ Supplier<NodeList> hostNodes = () -> tester.nodeRepository().list().nodeType(NodeType.host);
+
+ // Some nodes are targeting an older version
+ var version1 = Version.fromString("7.1");
+ setWantedVersion(hostNodes.get().asList().subList(0, 5), version1);
+
+ // Trigger upgrade to next version
+ var version2 = Version.fromString("7.2");
+ versions.setTarget(NodeType.host, version2, false);
+ versions.setActive(NodeType.host, true);
+
+ // Wanted version is changed to newest target for all nodes
+ assertEquals(version2, minVersion(hostNodes.get(), OsVersion::wanted));
+ }
+
+ private Version minVersion(NodeList nodes, Function<OsVersion, Optional<Version>> versionField) {
+ return nodes.asList().stream()
+ .map(Node::status)
+ .map(Status::osVersion)
+ .map(versionField)
+ .flatMap(Optional::stream)
+ .min(Comparator.naturalOrder())
+ .orElse(Version.emptyVersion);
+
+ }
+
+ private void setWantedVersion(List<Node> nodes, Version wantedVersion) {
+ writeNode(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withWanted(Optional.of(wantedVersion)))));
}
private void setCurrentVersion(List<Node> nodes, Version currentVersion) {
+ writeNode(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(Optional.of(currentVersion)))));
+ }
+
+ private void writeNode(List<Node> nodes, UnaryOperator<Node> updateFunc) {
for (var node : nodes) {
try (var lock = tester.nodeRepository().lock(node)) {
node = tester.nodeRepository().getNode(node.hostname()).get();
- node = node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(Optional.of(currentVersion))));
- tester.nodeRepository().write(node, lock);
+ tester.nodeRepository().write(updateFunc.apply(node), lock);
}
}
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
index 08e7772b5ba..8e048001b98 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
@@ -229,7 +229,7 @@ public class SerializationTest {
@Test
public void serialize_parentHostname() {
final String parentHostname = "parent.yahoo.com";
- Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", Optional.of(parentHostname), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant);
+ Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", Optional.of(parentHostname), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.tenant);
Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node));
assertEquals(parentHostname, deserializedNode.parentHostname().get());
@@ -441,7 +441,8 @@ public class SerializationTest {
Optional.empty(),
Optional.empty(),
nodeFlavors.getFlavorOrThrow("default"),
- NodeType.tenant);
+ Optional.empty(), NodeType.tenant
+ );
}
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
index 92d066e5f16..1d028f13340 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
@@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.provision.provisioning;
import com.yahoo.component.Version;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.Capacity;
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.HostSpec;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.NodeType;
import com.yahoo.vespa.hosted.provision.Node;
@@ -40,14 +38,14 @@ public class AclProvisioningTest {
tester.makeReadyNodes(10, new NodeResources(1, 4, 10, 1));
List<Node> dockerHost = tester.makeReadyNodes(1, new NodeResources(1, 4, 10, 1), NodeType.host);
ApplicationId zoneApplication = tester.makeApplicationId();
- deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.host));
+ tester.deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.host));
tester.makeReadyVirtualDockerNodes(1,new NodeResources(1, 4, 10, 1),
dockerHost.get(0).hostname());
List<Node> proxyNodes = tester.makeReadyNodes(3, new NodeResources(1, 4, 10, 1), NodeType.proxy);
// Allocate 2 nodes
ApplicationId application = tester.makeApplicationId();
- List<Node> activeNodes = deploy(application, Capacity.fromCount(2, new NodeResources(1, 4, 10, 1), false, true));
+ List<Node> activeNodes = tester.deploy(application, Capacity.fromCount(2, new NodeResources(1, 4, 10, 1), false, true));
assertEquals(2, activeNodes.size());
// Get trusted nodes for the first active node
@@ -112,7 +110,7 @@ public class AclProvisioningTest {
// Deploy zone application
ApplicationId zoneApplication = tester.makeApplicationId();
- deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.proxy));
+ tester.deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.proxy));
// Get trusted nodes for first proxy node
List<Node> proxyNodes = tester.nodeRepository().getNodes(zoneApplication);
@@ -154,7 +152,7 @@ public class AclProvisioningTest {
// Allocate
ApplicationId controllerApplication = tester.makeApplicationId();
- List<Node> controllers = deploy(controllerApplication, Capacity.fromRequiredNodeType(NodeType.controller));
+ List<Node> controllers = tester.deploy(controllerApplication, Capacity.fromRequiredNodeType(NodeType.controller));
// Controllers and hosts all trust each other
List<NodeAcl> controllerAcls = tester.nodeRepository().getNodeAcls(controllers.get(0), false);
@@ -164,12 +162,33 @@ public class AclProvisioningTest {
@Test
public void trusted_nodes_for_application_with_load_balancer() {
- // Populate repo
- tester.makeReadyNodes(10, nodeResources);
+ // Provision hosts and containers
+ var hosts = tester.makeReadyNodes(2, "default", NodeType.host);
+ tester.deployZoneApp();
+ for (var host : hosts) {
+ tester.makeReadyVirtualDockerNodes(2, new NodeResources(2, 8, 50, 1),
+ host.hostname());
+ }
- // Allocate 2 nodes
- List<Node> activeNodes = deploy(2);
+ // Deploy application
+ var application = tester.makeApplicationId();
+ List<Node> activeNodes = deploy(application, 2);
assertEquals(2, activeNodes.size());
+
+ // Load balancer is allocated to application
+ var loadBalancers = tester.nodeRepository().loadBalancers(application);
+ assertEquals(1, loadBalancers.asList().size());
+ var lbNetworks = loadBalancers.asList().get(0).instance().networks();
+ assertEquals(2, lbNetworks.size());
+
+ // ACL for nodes with allocation trust their respective load balancer networks, if any
+ for (var host : hosts) {
+ var acls = tester.nodeRepository().getNodeAcls(host, true);
+ assertEquals(2, acls.size());
+ assertEquals(Set.of(), acls.get(0).trustedNetworks());
+ assertEquals(application, acls.get(1).node().allocation().get().owner());
+ assertEquals(lbNetworks, acls.get(1).trustedNetworks());
+ }
}
@Test
@@ -191,15 +210,7 @@ public class AclProvisioningTest {
}
private List<Node> deploy(ApplicationId application, int nodeCount) {
- return deploy(application, Capacity.fromCount(nodeCount, nodeResources));
- }
-
- private List<Node> deploy(ApplicationId application, Capacity capacity) {
- ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"),
- Version.fromString("6.42"), false);
- List<HostSpec> prepared = tester.prepare(application, cluster, capacity, 1);
- tester.activate(application, Set.copyOf(prepared));
- return tester.getNodes(application, Node.State.active).asList();
+ return tester.deploy(application, Capacity.fromCount(nodeCount, nodeResources));
}
private static void assertAcls(List<List<Node>> expected, NodeAcl actual) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java
index 7e7338ab9ae..2c01cdde932 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java
@@ -81,7 +81,7 @@ public class AllocationSimulator {
var ipConfig = new IP.Config(Set.of("127.0.0.1"), parent.isPresent() ? Set.of() : getAdditionalIP());
return new Node("fake", ipConfig, hostname, parent, flavor, Status.initial(),
parent.isPresent() ? Node.State.ready : Node.State.active, allocation(tenant, flavor), History.empty(),
- parent.isPresent() ? NodeType.tenant : NodeType.host, new Reports(), Optional.empty());
+ parent.isPresent() ? NodeType.tenant : NodeType.host, new Reports(), Optional.empty(), Optional.empty());
}
private Set<String> getAdditionalIP() {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java
index e2015cfd30a..ba9a04573e1 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java
@@ -37,15 +37,15 @@ public class DockerHostCapacityTest {
@Before
public void setup() {
- doAnswer(invocation -> invocation.getArguments()[0]).when(hostResourcesCalculator).availableCapacityOf(any());
+ doAnswer(invocation -> invocation.getArguments()[1]).when(hostResourcesCalculator).availableCapacityOf(any(), any());
// Create flavors
NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2");
// Create three docker hosts
- host1 = Node.create("host1", new IP.Config(Set.of("::1"), generateIPs(2, 4)), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host);
- host2 = Node.create("host2", new IP.Config(Set.of("::11"), generateIPs(12, 3)), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host);
- host3 = Node.create("host3", new IP.Config(Set.of("::21"), generateIPs(22, 1)), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host);
+ host1 = Node.create("host1", new IP.Config(Set.of("::1"), generateIPs(2, 4)), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host);
+ host2 = Node.create("host2", new IP.Config(Set.of("::11"), generateIPs(12, 3)), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host);
+ host3 = Node.create("host3", new IP.Config(Set.of("::21"), generateIPs(22, 1)), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host);
// Add two containers to host1
var nodeA = Node.createDockerNode(Set.of("::2"), "nodeA", "host1", resources1, NodeType.tenant);
@@ -95,9 +95,9 @@ public class DockerHostCapacityTest {
capacity.freeCapacityOf(host3, false));
doAnswer(invocation -> {
- NodeResources totalHostResources = (NodeResources) invocation.getArguments()[0];
+ NodeResources totalHostResources = (NodeResources) invocation.getArguments()[1];
return totalHostResources.subtract(new NodeResources(1, 2, 3, 0.5, NodeResources.DiskSpeed.any));
- }).when(hostResourcesCalculator).availableCapacityOf(any());
+ }).when(hostResourcesCalculator).availableCapacityOf(any(), any());
assertEquals(new NodeResources(4, 2, 5, 1.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote),
capacity.freeCapacityOf(host1, false));
@@ -110,7 +110,7 @@ public class DockerHostCapacityTest {
// Dev host can assign both configserver and tenant containers.
var nodeFlavors = FlavorConfigBuilder.createDummies("devhost", "container");
- var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), generateIPs(2, 10)), "devhost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("devhost"), NodeType.devhost);
+ var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), generateIPs(2, 10)), "devhost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("devhost"), Optional.empty(), NodeType.devhost);
var cfg = Node.createDockerNode(Set.of("::2"), "cfg", "devhost", resources1, NodeType.config);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java
new file mode 100644
index 00000000000..70a57715d13
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java
@@ -0,0 +1,51 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mpolden
+ */
+public class DockerImagesTest {
+
+ @Test
+ public void image_selection() {
+ var flagSource = new InMemoryFlagSource();
+ var tester = new ProvisioningTester.Builder().flagSource(flagSource).build();
+
+ // Host uses tenant default image (for preload purposes)
+ var defaultImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa");
+ var hosts = tester.makeReadyNodes(2, "default", NodeType.host);
+ tester.deployZoneApp();
+ for (var host : hosts) {
+ assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(host));
+ }
+
+ // Tenant node uses tenant default image
+ var resources = new NodeResources(2, 8, 50, 1);
+ for (var host : hosts) {
+ var nodes = tester.makeReadyVirtualDockerNodes(2, resources, host.hostname());
+ for (var node : nodes) {
+ assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(node));
+ }
+ }
+
+ // Allocated containers uses overridden image when feature flag is set
+ var app = tester.makeApplicationId();
+ var nodes = tester.deploy(app, Capacity.fromCount(2, resources));
+ var customImage = DockerImage.fromString("docker.example.com/vespa/hosted");
+ flagSource.withStringFlag(Flags.DOCKER_IMAGE_OVERRIDE.id(), customImage.asString());
+ for (var node : nodes) {
+ assertEquals(customImage, tester.nodeRepository().dockerImages().dockerImageFor(node));
+ }
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
index b731ab309a6..1e59add8304 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
@@ -14,6 +14,7 @@ import com.yahoo.config.provision.NodeType;
import com.yahoo.config.provision.OutOfCapacityException;
import com.yahoo.config.provision.ParentHostUnavailableException;
import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.TenantName;
import com.yahoo.config.provision.Zone;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeList;
@@ -107,6 +108,45 @@ public class DockerProvisioningTest {
assertEquals(nodeCount, activeNodes.size());
}
+ @Test
+ public void reservations_are_respected() {
+ NodeResources resources = new NodeResources(10, 10, 10, 10);
+ ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
+ TenantName tenant1 = TenantName.from("tenant1");
+ TenantName tenant2 = TenantName.from("tenant2");
+ ApplicationId application1_1 = ApplicationId.from(tenant1, ApplicationName.from("application1"), InstanceName.defaultName());
+ ApplicationId application2_1 = ApplicationId.from(tenant2, ApplicationName.from("application1"), InstanceName.defaultName());
+ ApplicationId application2_2 = ApplicationId.from(tenant2, ApplicationName.from("application2"), InstanceName.defaultName());
+
+ tester.makeReadyNodes(10, resources, Optional.of(tenant1), NodeType.host, 1);
+ tester.makeReadyNodes(10, resources, Optional.empty(), NodeType.host, 1);
+ tester.deployZoneApp();
+
+ Version wantedVespaVersion = Version.fromString("6.39");
+ List<HostSpec> nodes = tester.prepare(application2_1,
+ ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false),
+ 6, 1, resources);
+ assertHostSpecParentReservation(nodes, Optional.empty(), tester); // We do not get nodes on hosts reserved to tenant1
+ tester.activate(application2_1, nodes);
+
+ try {
+ tester.prepare(application2_2,
+ ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false),
+ 5, 1, resources);
+ fail("Expected exception");
+ }
+ catch (OutOfCapacityException e) {
+ // Success: Not enough nonreserved hosts left
+ }
+
+ nodes = tester.prepare(application1_1,
+ ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false),
+ 10, 1, resources);
+ assertHostSpecParentReservation(nodes, Optional.of(tenant1), tester);
+ tester.activate(application1_1, nodes);
+ assertNodeParentReservation(tester.getNodes(application1_1).asList(), Optional.empty(), tester); // Reservation is cleared after activation
+ }
+
/** Exclusive app first, then non-exclusive: Should give the same result as below */
@Test
public void docker_application_deployment_with_exclusive_app_first() {
@@ -260,4 +300,16 @@ public class DockerProvisioningTest {
tester.activate(application, hosts);
}
+ private void assertNodeParentReservation(List<Node> nodes, Optional<TenantName> reservation, ProvisioningTester tester) {
+ for (Node node : nodes)
+ assertEquals(reservation, tester.nodeRepository().getNode(node.parentHostname().get()).get().reservedTo());
+ }
+
+ private void assertHostSpecParentReservation(List<HostSpec> hostSpecs, Optional<TenantName> reservation, ProvisioningTester tester) {
+ for (HostSpec hostSpec : hostSpecs) {
+ Node node = tester.nodeRepository().getNode(hostSpec.hostname()).get();
+ assertEquals(reservation, tester.nodeRepository().getNode(node.parentHostname().get()).get().reservedTo());
+ }
+ }
+
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java
index d1498507a7c..53fecbf6095 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java
@@ -423,7 +423,9 @@ public class DynamicDockerAllocationTest {
}
private void addAndAssignNode(ApplicationId id, String hostname, String parentHostname, ClusterSpec clusterSpec, NodeResources flavor, int index, ProvisioningTester tester) {
- Node node1a = Node.create("open1", new IP.Config(Set.of("127.0.233." + index), Set.of()), hostname, Optional.of(parentHostname), Optional.empty(), new Flavor(flavor), NodeType.tenant);
+ Node node1a = Node.create("open1", new IP.Config(Set.of("127.0.233." + index), Set.of()), hostname,
+ Optional.of(parentHostname), Optional.empty(), new Flavor(flavor), Optional.empty(), NodeType.tenant
+ );
ClusterMembership clusterMembership1 = ClusterMembership.from(
clusterSpec.with(Optional.of(ClusterSpec.Group.from(0))), index); // Need to add group here so that group is serialized in node allocation
Node node1aAllocation = node1a.allocate(id, clusterMembership1, node1a.flavor().resources(), Instant.now());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java
index c0e06ef3dff..0ad7d37d13b 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java
@@ -11,7 +11,6 @@ import com.yahoo.config.provision.NodeType;
import com.yahoo.config.provision.OutOfCapacityException;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.Zone;
-import com.yahoo.vespa.flags.Flags;
import com.yahoo.vespa.flags.InMemoryFlagSource;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeList;
@@ -47,6 +46,7 @@ import static org.junit.Assert.fail;
* @author freva
*/
public class InPlaceResizeProvisionTest {
+
private static final NodeResources smallResources = new NodeResources(2, 4, 8, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any);
private static final NodeResources mediumResources = new NodeResources(4, 8, 16, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any);
private static final NodeResources largeResources = new NodeResources(8, 16, 32, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any);
@@ -55,8 +55,7 @@ public class InPlaceResizeProvisionTest {
private static final ClusterSpec container2 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container2"), Version.fromString("7.157.9"), false);
private static final ClusterSpec content1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("content1"), Version.fromString("7.157.9"), false);
- private final InMemoryFlagSource flagSource = new InMemoryFlagSource()
- .withBooleanFlag(Flags.ENABLE_IN_PLACE_RESIZE.id(), true);
+ private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
private final ProvisioningTester tester = new ProvisioningTester.Builder()
.flagSource(flagSource)
.zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
@@ -158,17 +157,6 @@ public class InPlaceResizeProvisionTest {
assertTrue("All initial nodes should still be allocated to the application", initialHostnames.isEmpty());
}
- @Test(expected = OutOfCapacityException.class)
- public void no_in_place_resize_if_flag_not_set() {
- flagSource.withBooleanFlag(Flags.ENABLE_IN_PLACE_RESIZE.id(), false);
- addParentHosts(4, mediumResources.with(fast).with(local));
-
- new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate();
- assertClusterSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
-
- new PrepareHelper(tester, app).prepare(container1, 4, 1, smallResources);
- }
-
@Test(expected = OutOfCapacityException.class)
public void cannot_inplace_decrease_resources_while_increasing_cluster_size() {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
index 6dfca4d2c04..ee9a582c4db 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.provisioning;
import com.google.common.collect.Iterators;
@@ -40,15 +40,14 @@ public class LoadBalancerProvisionerTest {
private final ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default");
private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default");
-
private final ApplicationId infraApp1 = ApplicationId.from("vespa", "tenant-host", "default");
- private ProvisioningTester tester = new ProvisioningTester.Builder().build();
+ private final ProvisioningTester tester = new ProvisioningTester.Builder().build();
@Test
public void provision_load_balancer() {
- Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers().owner(app1).asList();
- Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers().owner(app2).asList();
+ Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers(app1).asList();
+ Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers(app2).asList();
ClusterSpec.Id containerCluster1 = ClusterSpec.Id.from("qrs1");
ClusterSpec.Id contentCluster = ClusterSpec.Id.from("content");
@@ -80,7 +79,7 @@ public class LoadBalancerProvisionerTest {
tester.activate(app1, prepare(app1,
clusterRequest(ClusterSpec.Type.container, containerCluster1),
clusterRequest(ClusterSpec.Type.content, contentCluster)));
- LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers().owner(app1).asList().get(0);
+ LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers(app1).asList().get(0);
assertEquals(2, loadBalancer.instance().reals().size());
assertTrue("Failed node is removed", loadBalancer.instance().reals().stream()
.map(Real::hostname)
@@ -158,7 +157,7 @@ public class LoadBalancerProvisionerTest {
var nodes = prepare(app1, Capacity.fromCount(2, new NodeResources(1, 4, 10, 0.3), false, true),
true,
clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs")));
- Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers().owner(app1).asList().get(0);
+ Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers(app1).asList().get(0);
assertTrue("Load balancer provisioned with empty reals", tester.loadBalancerService().instances().get(lb.get().id()).reals().isEmpty());
assignIps(tester.nodeRepository().getNodes(app1));
tester.activate(app1, nodes);
@@ -189,7 +188,7 @@ public class LoadBalancerProvisionerTest {
clusterRequest(ClusterSpec.Type.container,
ClusterSpec.Id.from("tenant-host"))));
assertTrue("No load balancer provisioned", tester.loadBalancerService().instances().isEmpty());
- assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(infraApp1).asList());
+ assertEquals(List.of(), tester.nodeRepository().loadBalancers(infraApp1).asList());
}
@Test
@@ -197,12 +196,12 @@ public class LoadBalancerProvisionerTest {
tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.content,
ClusterSpec.Id.from("tenant-host"))));
assertTrue("No load balancer provisioned", tester.loadBalancerService().instances().isEmpty());
- assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(app1).asList());
+ assertEquals(List.of(), tester.nodeRepository().loadBalancers(app1).asList());
}
@Test
public void provision_load_balancer_combined_cluster() {
- Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().owner(app1).asList();
+ Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(app1).asList();
ClusterSpec.Id cluster = ClusterSpec.Id.from("foo");
var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, cluster));
@@ -247,7 +246,7 @@ public class LoadBalancerProvisionerTest {
}
private void assignIps(List<Node> nodes) {
- try (var lock = tester.nodeRepository().lockAllocation()) {
+ try (var lock = tester.nodeRepository().lockUnallocated()) {
for (int i = 0; i < nodes.size(); i++) {
tester.nodeRepository().write(nodes.get(i).with(IP.Config.EMPTY.with(Set.of("127.0.0." + i))), lock);
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java
index 718abf7d73d..1d9c135b4b5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java
@@ -124,18 +124,26 @@ public class PrioritizableNodeTest {
}
private static Node node(String hostname, Node.State state) {
- return new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), new Flavor(new NodeResources(2, 2, 2, 2)),
- Status.initial(), state, Optional.empty(), History.empty(), NodeType.tenant, new Reports(), Optional.empty());
+ return new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(),
+ new Flavor(new NodeResources(2, 2, 2, 2)),
+ Status.initial(), state, Optional.empty(), History.empty(), NodeType.tenant, new Reports(),
+ Optional.empty(), Optional.empty());
}
private static PrioritizableNode node(String hostname,
NodeResources nodeResources,
NodeResources allocatedHostResources, // allocated before adding nodeResources
NodeResources totalHostResources) {
- Node node = new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.of(hostname + "parent"), new Flavor(nodeResources),
- Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.tenant, new Reports(), Optional.empty());
- Node parent = new Node(hostname + "parent", new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), new Flavor(totalHostResources),
- Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.host, new Reports(), Optional.empty());
- return new PrioritizableNode(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), false, false, true, false);
+ Node node = new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.of(hostname + "parent"),
+ new Flavor(nodeResources),
+ Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.tenant,
+ new Reports(), Optional.empty(), Optional.empty());
+ Node parent = new Node(hostname + "parent", new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(),
+ new Flavor(totalHostResources),
+ Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.host,
+ new Reports(), Optional.empty(), Optional.empty());
+ return new PrioritizableNode(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent),
+ false, false, true, false);
}
+
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index 679c22e4df2..85a6ed31073 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -87,7 +87,7 @@ public class ProvisioningTester {
this.nodeFlavors = nodeFlavors;
this.clock = new ManualClock();
this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, nameResolver,
- DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true);
+ DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, flagSource);
this.orchestrator = orchestrator;
ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner);
this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource);
@@ -241,13 +241,16 @@ public class ProvisioningTester {
}
public List<Node> makeReadyNodes(int n, String flavor, NodeType type) {
- return makeReadyNodes(n, asFlavor(flavor, type), type, 0);
+ return makeReadyNodes(n, asFlavor(flavor, type), Optional.empty(), type, 0);
}
public List<Node> makeReadyNodes(int n, NodeResources resources, NodeType type) {
- return makeReadyNodes(n, new Flavor(resources), type, 0);
+ return makeReadyNodes(n, new Flavor(resources), Optional.empty(), type, 0);
}
public List<Node> makeReadyNodes(int n, NodeResources resources, NodeType type, int ipAddressPoolSize) {
- return makeReadyNodes(n, new Flavor(resources), type, ipAddressPoolSize);
+ return makeReadyNodes(n, resources, Optional.empty(), type, ipAddressPoolSize);
+ }
+ public List<Node> makeReadyNodes(int n, NodeResources resources, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize) {
+ return makeReadyNodes(n, new Flavor(resources), reservedTo, type, ipAddressPoolSize);
}
public List<Node> makeProvisionedNodes(int count, String flavor, NodeType type, int ipAddressPoolSize) {
@@ -255,9 +258,9 @@ public class ProvisioningTester {
}
public List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
- return makeProvisionedNodes(n, asFlavor(flavor, type), type, ipAddressPoolSize, dualStack);
+ return makeProvisionedNodes(n, asFlavor(flavor, type), Optional.empty(), type, ipAddressPoolSize, dualStack);
}
- public List<Node> makeProvisionedNodes(int n, Flavor flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
+ public List<Node> makeProvisionedNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize, boolean dualStack) {
List<Node> nodes = new ArrayList<>(n);
for (int i = 0; i < n; i++) {
@@ -299,6 +302,7 @@ public class ProvisioningTester {
new IP.Config(hostIps, ipAddressPool),
Optional.empty(),
flavor,
+ reservedTo,
type));
}
nodes = nodeRepository.addNodes(nodes);
@@ -315,11 +319,12 @@ public class ProvisioningTester {
nameResolver.addRecord(hostname, ipv4);
Node node = nodeRepository.createNode(hostname,
- hostname,
- new IP.Config(Set.of(ipv4), Set.of()),
- Optional.empty(),
- nodeFlavors.getFlavorOrThrow(flavor),
- NodeType.config);
+ hostname,
+ new IP.Config(Set.of(ipv4), Set.of()),
+ Optional.empty(),
+ nodeFlavors.getFlavorOrThrow(flavor),
+ Optional.empty(),
+ NodeType.config);
nodes.add(node);
}
@@ -338,17 +343,20 @@ public class ProvisioningTester {
}
public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize) {
- return makeReadyNodes(n, asFlavor(flavor, type), type, ipAddressPoolSize);
+ return makeReadyNodes(n, asFlavor(flavor, type), Optional.empty(), type, ipAddressPoolSize);
}
- public List<Node> makeReadyNodes(int n, Flavor flavor, NodeType type, int ipAddressPoolSize) {
- return makeReadyNodes(n, flavor, type, ipAddressPoolSize, false);
+ public List<Node> makeReadyNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize) {
+ return makeReadyNodes(n, flavor, reservedTo, type, ipAddressPoolSize, false);
}
public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
return makeReadyNodes(n, asFlavor(flavor, type), type, ipAddressPoolSize, dualStack);
}
public List<Node> makeReadyNodes(int n, Flavor flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
- List<Node> nodes = makeProvisionedNodes(n, flavor, type, ipAddressPoolSize, dualStack);
+ return makeReadyNodes(n, flavor, Optional.empty(), type, ipAddressPoolSize, dualStack);
+ }
+ public List<Node> makeReadyNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize, boolean dualStack) {
+ List<Node> nodes = makeProvisionedNodes(n, flavor, reservedTo, type, ipAddressPoolSize, dualStack);
nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
}
@@ -410,21 +418,20 @@ public class ProvisioningTester {
activate(applicationId, Set.copyOf(list));
}
+ public List<Node> deploy(ApplicationId application, Capacity capacity) {
+ ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"),
+ Version.fromString("6.42"), false);
+ List<HostSpec> prepared = prepare(application, cluster, capacity, 1);
+ activate(application, Set.copyOf(prepared));
+ return getNodes(application, Node.State.active).asList();
+ }
+
+
/** Returns the hosts from the input list which are not retired */
public List<HostSpec> nonRetired(Collection<HostSpec> hosts) {
return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList());
}
- public void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) {
- long actualNodesWithFlavor = hostSpecs.stream()
- .map(HostSpec::hostname)
- .map(this::getNodeFlavor)
- .map(Flavor::name)
- .filter(name -> name.equals(flavor))
- .count();
- assertEquals(expectedCount, actualNodesWithFlavor);
- }
-
public void assertAllocatedOn(String explanation, String hostFlavor, ApplicationId app) {
for (Node node : nodeRepository.getNodes(app)) {
Node parent = nodeRepository.getNode(node.parentHostname().get()).get();
@@ -432,17 +439,6 @@ public class ProvisioningTester {
}
}
- public void printFreeResources() {
- for (Node host : nodeRepository().getNodes(NodeType.host)) {
- NodeResources free = host.flavor().resources();
- for (Node child : nodeRepository().getNodes(NodeType.tenant)) {
- if (child.parentHostname().get().equals(host.hostname()))
- free = free.subtract(child.flavor().resources());
- }
- System.out.println(host.flavor().name() + " node. Free resources: " + free);
- }
- }
-
public int hostFlavorCount(String hostFlavor, ApplicationId app) {
return (int)nodeRepository().getNodes(app).stream()
.map(n -> nodeRepository().getNode(n.parentHostname().get()).get())
@@ -450,10 +446,6 @@ public class ProvisioningTester {
.count();
}
- private Flavor getNodeFlavor(String hostname) {
- return nodeRepository.getNode(hostname).map(Node::flavor).orElseThrow(() -> new RuntimeException("No flavor for host " + hostname));
- }
-
public static final class Builder {
private Curator curator;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index 18f0d989900..c26614c630c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -6,12 +6,15 @@ import com.yahoo.application.container.JDisc;
import com.yahoo.application.container.handler.Request;
import com.yahoo.application.container.handler.Response;
import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.TenantName;
import com.yahoo.io.IOUtils;
import com.yahoo.text.Utf8;
+import com.yahoo.vespa.applicationmodel.HostName;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.maintenance.OsUpgradeActivator;
import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig;
import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository;
+import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock;
import org.junit.After;
import org.junit.Before;
import org.junit.ComparisonFailure;
@@ -23,6 +26,7 @@ import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Arrays;
import java.util.List;
+import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -106,7 +110,7 @@ public class RestApiTest {
assertResponse(new Request("http://localhost:8080/nodes/v2/node",
("[" + asNodeJson("host8.yahoo.com", "default", "127.0.8.1") + "," + // test with only 1 ip address
asNodeJson("host9.yahoo.com", "large-variant", "127.0.9.1", "::9:1") + "," +
- asHostJson("parent2.yahoo.com", "large-variant", "127.0.127.1", "::127:1") + "," +
+ asHostJson("parent2.yahoo.com", "large-variant", Optional.of(TenantName.from("myTenant")), "127.0.127.1", "::127:1") + "," +
asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com", "::11") + "]").
getBytes(StandardCharsets.UTF_8),
Request.Method.POST),
@@ -214,9 +218,6 @@ public class RestApiTest {
Utf8.toBytes("{\"wantToRetire\": true}"), Request.Method.PATCH),
"{\"message\":\"Updated host4.yahoo.com\"}");
assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
- Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH),
- "{\"message\":\"Updated host4.yahoo.com\"}");
- assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
Utf8.toBytes("{\"currentVespaVersion\": \"6.43.0\",\"currentDockerImage\": \"docker-registry.domain.tld:8080/dist/vespa:6.45.0\"}"), Request.Method.PATCH),
"{\"message\":\"Updated host4.yahoo.com\"}");
assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
@@ -225,6 +226,9 @@ public class RestApiTest {
assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com",
Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH),
"{\"message\":\"Updated dockerhost1.yahoo.com\"}");
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com",
+ Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH),
+ "{\"message\":\"Updated dockerhost1.yahoo.com\"}");
assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\"");
assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com",
Utf8.toBytes("{\"modelName\": null}"), Request.Method.PATCH),
@@ -232,6 +236,9 @@ public class RestApiTest {
assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "modelName", false);
container.handleRequest((new Request("http://localhost:8080/nodes/v2/upgrade/tenant", Utf8.toBytes("{\"dockerImage\": \"docker.domain.tld/my/image\"}"), Request.Method.PATCH)));
+ ((OrchestratorMock) container.components().getComponent(OrchestratorMock.class.getName()))
+ .suspend(new HostName("host4.yahoo.com"));
+
assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json");
}
@@ -292,7 +299,7 @@ public class RestApiTest {
// Attempt to POST host node with already assigned IP
assertResponse(new Request("http://localhost:8080/nodes/v2/node",
- "[" + asHostJson("host200.yahoo.com", "default", "127.0.2.1") + "]",
+ "[" + asHostJson("host200.yahoo.com", "default", Optional.empty(), "127.0.2.1") + "]",
Request.Method.POST), 400,
"{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.2.1] to host200.yahoo.com: [127.0.2.1] already assigned to host2.yahoo.com\"}");
@@ -304,7 +311,7 @@ public class RestApiTest {
// Node types running a single container can share their IP address with child node
assertResponse(new Request("http://localhost:8080/nodes/v2/node",
- "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", "127.0.42.1") + "]",
+ "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.42.1") + "]",
Request.Method.POST), 200,
"{\"message\":\"Added 1 nodes to the provisioned state\"}");
assertResponse(new Request("http://localhost:8080/nodes/v2/node",
@@ -320,7 +327,7 @@ public class RestApiTest {
// ... nor with child node on different host
assertResponse(new Request("http://localhost:8080/nodes/v2/node",
- "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", "127.0.43.1") + "]",
+ "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.43.1") + "]",
Request.Method.POST), 200,
"{\"message\":\"Added 1 nodes to the provisioned state\"}");
assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com",
@@ -534,11 +541,13 @@ public class RestApiTest {
" \"actualCpuCores\": {" +
" \"createdMillis\": 1, " +
" \"description\": \"Actual number of CPU cores (2) differs from spec (4)\"," +
+ " \"type\": \"HARD_FAIL\"," +
" \"value\":2" +
" }," +
" \"diskSpace\": {" +
" \"createdMillis\": 2, " +
" \"description\": \"Actual disk space (2TB) differs from spec (3TB)\"," +
+ " \"type\": \"HARD_FAIL\"," +
" \"details\": {" +
" \"inGib\": 3," +
" \"disks\": [\"/dev/sda1\", \"/dev/sdb3\"]" +
@@ -921,14 +930,15 @@ public class RestApiTest {
"\"flavor\":\"" + flavor + "\"}";
}
- private static String asHostJson(String hostname, String flavor, String... ipAddress) {
- return asNodeJson(hostname, NodeType.host, flavor, ipAddress);
+ private static String asHostJson(String hostname, String flavor, Optional<TenantName> reservedTo, String... ipAddress) {
+ return asNodeJson(hostname, NodeType.host, flavor, reservedTo, ipAddress);
}
- private static String asNodeJson(String hostname, NodeType nodeType, String flavor, String... ipAddress) {
+ private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional<TenantName> reservedTo, String... ipAddress) {
return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\"," +
createIpAddresses(ipAddress) +
"\"flavor\":\"" + flavor + "\"" +
+ (reservedTo.isPresent() ? ", \"reservedTo\":\"" + reservedTo.get().value() + "\"" : "") +
", \"type\":\"" + nodeType + "\"}";
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
index f663f4adb8d..af3552945d9 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
@@ -27,14 +27,15 @@
"wantedDockerImage": "docker.domain.tld/my/image:6.42.0",
"wantedVespaVersion": "6.42.0",
"requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" },
- "allowedToBeDown": false,
+ "allowedToBeDown": true,
+ "suspendedSinceMillis": 0,
"rebootGeneration": 3,
"currentRebootGeneration": 1,
"vespaVersion": "6.43.0",
"currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.45.0",
"failCount": 1,
"wantToRetire": true,
- "wantToDeprovision": true,
+ "wantToDeprovision": false,
"history": [
{
"event": "provisioned",
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json
index d33c1c9e743..a3d53798d7c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json
@@ -30,7 +30,7 @@
"currentRebootGeneration": 0,
"failCount": 0,
"wantToRetire": false,
- "wantToDeprovision": false,
+ "wantToDeprovision": true,
"history": [
{
"event": "provisioned",
@@ -65,6 +65,7 @@
"diskSpace": {
"createdMillis": 2,
"description": "Actual disk space (2TB) differs from spec (3TB)",
+ "type":"HARD_FAIL",
"details": {
"inGib": 3,
"disks": [
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json
index 4119e46e225..67b8d67c7f1 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json
@@ -30,7 +30,7 @@
"currentRebootGeneration": 0,
"failCount": 0,
"wantToRetire": false,
- "wantToDeprovision": false,
+ "wantToDeprovision": true,
"history": [
{
"event": "provisioned",
@@ -62,11 +62,13 @@
"actualCpuCores": {
"createdMillis": 1,
"description": "Actual number of CPU cores (2) differs from spec (4)",
+ "type":"HARD_FAIL",
"value": 2
},
"diskSpace": {
"createdMillis": 2,
"description": "Actual disk space (2TB) differs from spec (3TB)",
+ "type":"HARD_FAIL",
"details": {
"inGib": 3,
"disks": [
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json
index 8be034cb036..ecc497172c7 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json
@@ -6,6 +6,7 @@
"hostname": "parent2.yahoo.com",
"openStackId": "parent2.yahoo.com",
"flavor": "large-variant",
+ "reservedTo": "myTenant",
"cpuCores": 64.0,
"resources": {"vcpu":64.0,"memoryGb":128.0,"diskGb":2000.0,"bandwidthGbps":15.0,"diskSpeed":"fast","storageType":"remote"},
"environment": "BARE_METAL",