summaryrefslogtreecommitdiffstats
path: root/node-repository
diff options
context:
space:
mode:
authorArnstein Ressem <aressem@gmail.com>2017-08-14 13:01:26 +0200
committerGitHub <noreply@github.com>2017-08-14 13:01:26 +0200
commit2b29283afac9713c23a8160c1f44374fdafede0c (patch)
tree2e98587efb4983eed6e98f6b2058a6fb856e774a /node-repository
parent3c745e5796d1d8b875ecd2ff68d677c228254b06 (diff)
Revert "Allocation rewrite"
Diffstat (limited to 'node-repository')
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java153
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java102
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java36
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java7
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareChecker.java1
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCount.java1
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java151
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java111
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java301
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java8
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java101
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java1
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java34
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java9
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java17
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java277
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java6
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java42
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json56
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json44
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json51
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json53
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json67
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json42
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json44
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json59
26 files changed, 624 insertions, 1150 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java
new file mode 100644
index 00000000000..fca15283e5d
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java
@@ -0,0 +1,153 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeFlavors;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.provision.Node;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+
+/**
+ * Set of methods to allocate new docker nodes
+ * <p>
+ * The nodes are not added to the repository here - this is done by caller.
+ *
+ * @author smorgrav
+ */
+public class DockerAllocator {
+
+ /**
+ * The docker container allocation algorithm
+ */
+ static List<Node> allocateNewDockerNodes(NodeAllocation allocation,
+ NodeSpec requestedNodes,
+ List<Node> allNodes,
+ List<Node> nodesBefore,
+ NodeFlavors flavors,
+ Flavor flavor,
+ int nofSpares,
+ BiConsumer<List<Node>, String> recorder) {
+ // Try allocate new nodes with all constraints in place
+ List<Node> nodesWithHeadroomAndSpares = DockerCapacityConstraints.addHeadroomAndSpareNodes(allNodes, flavors, nofSpares);
+ recorder.accept(nodesWithHeadroomAndSpares, "Headroom and spares");
+ List<Node> accepted = DockerAllocator.allocate(allocation, flavor, nodesWithHeadroomAndSpares);
+
+ List<Node> allNodesIncludingAccepted = new ArrayList<>(allNodes);
+ allNodesIncludingAccepted.addAll(accepted);
+ recorder.accept(allNodesIncludingAccepted, "1st dynamic docker allocation - fullfilled: " + allocation.fullfilled());
+
+ // If still not fully allocated - try to allocate the remaining nodes with only hard constraints
+ if (!allocation.fullfilled()) {
+ List<Node> nodesWithSpares = DockerCapacityConstraints.addSpareNodes(allNodesIncludingAccepted, nofSpares);
+ recorder.accept(nodesWithSpares, "Spares only");
+
+ List<Node> acceptedWithHard = DockerAllocator.allocate(allocation, flavor, nodesWithSpares);
+ accepted.addAll(acceptedWithHard);
+ allNodesIncludingAccepted.addAll(acceptedWithHard);
+ recorder.accept(allNodesIncludingAccepted, "2nd dynamic docker allocation - fullfilled: " + allocation.fullfilled());
+
+ // If still not fully allocated and this is a replacement - drop all constraints
+ boolean isReplacement = DockerAllocator.isReplacement(requestedNodes, nodesBefore, allNodes);
+ if (!allocation.fullfilled() && isReplacement) {
+ List<Node> finalTry = DockerAllocator.allocate(allocation, flavor, allNodesIncludingAccepted);
+ accepted.addAll(finalTry);
+ allNodesIncludingAccepted.addAll(finalTry);
+ recorder.accept(allNodesIncludingAccepted, "Final dynamic docker alloction - fullfilled: " + allocation.fullfilled());
+ }
+ }
+
+ return accepted;
+ }
+
+ /**
+ * Offer the node allocation a prioritized set of new nodes according to capacity constraints
+ *
+ * @param allocation The allocation we want to fulfill
+ * @param flavor Since we create nodes here we need to know the exact flavor
+ * @param nodes The nodes relevant for the allocation (all nodes from node repo give or take)
+ * @return Nodes accepted by the node allocation - these nodes does not exist in the noderepo yet.
+ * @see DockerHostCapacity
+ */
+ public static List<Node> allocate(NodeAllocation allocation, Flavor flavor, List<Node> nodes) {
+
+ DockerHostCapacity dockerCapacity = new DockerHostCapacity(nodes);
+
+ // Get all active docker hosts with enough capacity and ip slots - sorted on free capacity
+ List<Node> dockerHosts = nodes.stream()
+ .filter(node -> node.type().equals(NodeType.host))
+ .filter(dockerHost -> dockerHost.state().equals(Node.State.active))
+ .filter(dockerHost -> dockerCapacity.hasCapacity(dockerHost, flavor))
+ .sorted(dockerCapacity::compare)
+ .collect(Collectors.toList());
+
+ // Create one node pr. docker host that we can offer to the allocation
+ List<Node> offers = new LinkedList<>();
+ for (Node parentHost : dockerHosts) {
+ Set<String> ipAddresses = DockerHostCapacity.findFreeIps(parentHost, nodes);
+ if (ipAddresses.isEmpty()) continue;
+ String ipAddress = ipAddresses.stream().findFirst().get();
+ String hostname = lookupHostname(ipAddress);
+ if (hostname == null) continue;
+ Node node = Node.createDockerNode("fake-" + hostname, Collections.singleton(ipAddress),
+ Collections.emptySet(), hostname, Optional.of(parentHost.hostname()), flavor, NodeType.tenant);
+ offers.add(node);
+ }
+
+ return allocation.offer(offers, false);
+ }
+
+ /**
+ * From ipAddress - get hostname
+ *
+ * @return hostname or null if not able to do the loopup
+ */
+ private static String lookupHostname(String ipAddress) {
+ try {
+ return InetAddress.getByName(ipAddress).getHostName();
+ } catch (UnknownHostException e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ /**
+ * This is an heuristic way to find if new nodes are to replace failing nodes
+ * or are to expand the cluster.
+ *
+ * The current implementation does not account for failed nodes that are not in the application
+ * anymore. The consequence is that we will ignore the spare capacity constraints too often - in
+ * particular when the number of failed nodes (not in the application anymore)
+ * for the cluster equal to the upscaling of the cluster.
+ *
+ * The deployment algorithm will still try to allocate the the capacity outside the spare capacity if possible.
+ *
+ * TODO propagate this information either through the node object or from the configserver deployer
+ */
+ private static boolean isReplacement(NodeSpec nodeSpec, List<Node> nodesBefore, List<Node> nodesReserved) {
+ int wantedCount = 0;
+ if (nodeSpec instanceof NodeSpec.CountNodeSpec) {
+ NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) nodeSpec;
+ wantedCount = countSpec.getCount();
+ }
+
+ List<Node> failedNodes = new ArrayList<>();
+ for (Node node : nodesBefore) {
+ if (node.state() == Node.State.failed) {
+ failedNodes.add(node);
+ }
+ }
+
+ if (failedNodes.size() == 0) return false;
+ return (wantedCount <= nodesReserved.size() + failedNodes.size());
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java
new file mode 100644
index 00000000000..3be6b1d6df3
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java
@@ -0,0 +1,102 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.cloud.config.ApplicationIdConfig;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeFlavors;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.lang.MutableInteger;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Enforce allocation constraints for docker by manipulating the NodeList we operate on.
+ *
+ * The constraints comes in two flavors: headroom and spare.
+ *
+ * <b>Headroom</b> is the number of docker nodes (of various flavors) we want to reserve for new applications.
+ * This is e.g. to make sure we don't smear out small docker flavors on all hosts
+ * starving allocations for bigger flavors.
+ *
+ * <b>Spares</b> is to make sure we have replacement for applications if one or more hosts go down.
+ * It is more important to safeguard already onboarded applications than accept new applications.
+ *
+ * For now - we will use spare also as a means to reserve capacity for future applications that
+ * have had a separate AI process.
+ *
+ * When using spares - we will relay on maintenance jobs to reclaim the spare capacity whenever the
+ * capacity has been recovered (e.g. when the dead docker host is replaced)
+ *
+ * @author smorgrav
+ */
+public class DockerCapacityConstraints {
+
+ /** This is a static utility class */
+ private DockerCapacityConstraints() {}
+
+ /**
+ * Spare nodes in first iteration is a node that fills up the two
+ * largest hosts (in terms of free capacity)
+ */
+ public static List<Node> addSpareNodes(List<Node> nodes, int spares) {
+ DockerHostCapacity capacity = new DockerHostCapacity(nodes);
+ List<Flavor> spareFlavors = nodes.stream()
+ .filter(node -> node.type().equals(NodeType.host))
+ .filter(dockerHost -> dockerHost.state().equals(Node.State.active))
+ .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0)
+ .sorted(capacity::compare)
+ .limit(spares)
+ .map(dockerHost -> freeCapacityAsFlavor(dockerHost, nodes))
+ .collect(Collectors.toList());
+
+ return addNodes(nodes, spareFlavors, "spare");
+ }
+
+ public static List<Node> addHeadroomAndSpareNodes(List<Node> nodes, NodeFlavors flavors, int nofSpares) {
+ List<Node> sparesAndHeadroom = addSpareNodes(nodes, nofSpares);
+ return addNodes(sparesAndHeadroom, flavors.getFlavors(), "headroom");
+ }
+
+ private static List<Node> addNodes(List<Node> nodes, List<Flavor> flavors, String id) {
+ List<Node> headroom = new ArrayList<>(nodes);
+ for (Flavor flavor : flavors) {
+ int headroomCount = flavor.getIdealHeadroom();
+ if (headroomCount > 0) {
+ NodeAllocation allocation = createHeadRoomAllocation(flavor, headroomCount, id);
+ List<Node> acceptedNodes = DockerAllocator.allocate(allocation, flavor, headroom);
+ headroom.addAll(acceptedNodes);
+ }
+ }
+ return headroom;
+ }
+
+ private static Flavor freeCapacityAsFlavor(Node host, List<Node> nodes) {
+ ResourceCapacity hostCapacity = new ResourceCapacity(host);
+ for (Node container : new NodeList(nodes).childNodes(host).asList()) {
+ hostCapacity.subtract(container);
+ }
+ return hostCapacity.asFlavor();
+ }
+
+ private static NodeAllocation createHeadRoomAllocation(Flavor flavor, int count, String id) {
+ ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container,
+ new ClusterSpec.Id(id), new Version());
+ ApplicationId appId = new ApplicationId(
+ new ApplicationIdConfig(
+ new ApplicationIdConfig.Builder()
+ .tenant(id)
+ .application(id + "-" + flavor.name())
+ .instance("temporarynode")));
+
+ return new NodeAllocation(appId, cluster, new NodeSpec.CountNodeSpec(count, flavor),
+ new MutableInteger(0), Clock.systemUTC());
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java
index 77d91c7bea7..0d8a50df568 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java
@@ -38,7 +38,7 @@ public class DockerHostCapacity {
* Used in prioritizing hosts for allocation in <b>descending</b> order.
*/
int compare(Node hostA, Node hostB) {
- int comp = freeCapacityOf(hostB, false).compare(freeCapacityOf(hostA, false));
+ int comp = freeCapacityOf(hostB, true).compare(freeCapacityOf(hostA, true));
if (comp == 0) {
comp = freeCapacityOf(hostB, false).compare(freeCapacityOf(hostA, false));
if (comp == 0) {
@@ -49,27 +49,11 @@ public class DockerHostCapacity {
return comp;
}
- int compareWithoutInactive(Node hostA, Node hostB) {
- int comp = freeCapacityOf(hostB, true).compare(freeCapacityOf(hostA, true));
- if (comp == 0) {
- comp = freeCapacityOf(hostB, true).compare(freeCapacityOf(hostA, true));
- if (comp == 0) {
- // If resources are equal - we want to assign to the one with the most IPaddresses free
- comp = freeIPs(hostB) - freeIPs(hostA);
- }
- }
- return comp;
- }
-
/**
* Checks the node capacity and free ip addresses to see
* if we could allocate a flavor on the docker host.
*/
boolean hasCapacity(Node dockerHost, Flavor flavor) {
- return freeCapacityOf(dockerHost, false).hasCapacityFor(flavor) && freeIPs(dockerHost) > 0;
- }
-
- boolean hasCapacityWhenRetiredAndInactiveNodesAreGone(Node dockerHost, Flavor flavor) {
return freeCapacityOf(dockerHost, true).hasCapacityFor(flavor) && freeIPs(dockerHost) > 0;
}
@@ -117,34 +101,20 @@ public class DockerHostCapacity {
/**
* Calculate the remaining capacity for the dockerHost.
- * @param dockerHost The host to find free capacity of.
- *
- * @return A default (empty) capacity if not a docker host, otherwise the free/unallocated/rest capacity
*/
- public ResourceCapacity freeCapacityOf(Node dockerHost, boolean treatInactiveOrRetiredAsUnusedCapacity) {
+ private ResourceCapacity freeCapacityOf(Node dockerHost, boolean includeHeadroom) {
// Only hosts have free capacity
if (!dockerHost.type().equals(NodeType.host)) return new ResourceCapacity();
ResourceCapacity hostCapacity = new ResourceCapacity(dockerHost);
for (Node container : allNodes.childNodes(dockerHost).asList()) {
- boolean isUsedCapacity = !(treatInactiveOrRetiredAsUnusedCapacity && isInactiveOrRetired(container));
- if (isUsedCapacity) {
+ if (includeHeadroom || !(container.allocation().isPresent() && container.allocation().get().owner().tenant().value().equals(HEADROOM_TENANT))) {
hostCapacity.subtract(container);
}
}
return hostCapacity;
}
- private boolean isInactiveOrRetired(Node node) {
- boolean isInactive = node.state().equals(Node.State.inactive);
- boolean isRetired = false;
- if (node.allocation().isPresent()) {
- isRetired = node.allocation().get().membership().retired();
- }
-
- return isInactive || isRetired;
- }
-
/**
* Compare the additional ip addresses against the set of used addresses on
* child nodes.
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java
index b52506c268c..cff62508ec6 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java
@@ -19,7 +19,7 @@ public class FlavorConfigBuilder {
return new FlavorsConfig(builder);
}
- public FlavorsConfig.Flavor.Builder addFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type, int headRoom) {
+ public FlavorsConfig.Flavor.Builder addFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type) {
FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder();
flavor.name(flavorName);
flavor.description("Flavor-name-is-" + flavorName);
@@ -27,15 +27,10 @@ public class FlavorConfigBuilder {
flavor.minCpuCores(cpu);
flavor.minMainMemoryAvailableGb(mem);
flavor.environment(type.name());
- flavor.idealHeadroom(headRoom);
builder.flavor(flavor);
return flavor;
}
- public FlavorsConfig.Flavor.Builder addFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type) {
- return addFlavor(flavorName, cpu, mem, disk, type, 0);
- }
-
public FlavorsConfig.Flavor.Builder addNonStockFlavor(String flavorName, double cpu, double mem, double disk, Flavor.Type type) {
FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder();
flavor.name(flavorName);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareChecker.java
index 5f81fed2a04..a7a0eadf515 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareChecker.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareChecker.java
@@ -29,7 +29,6 @@ import java.util.Set;
* @author freva
*/
public class FlavorSpareChecker {
-
private final SpareNodesPolicy spareNodesPolicy;
private final Map<Flavor, FlavorSpareCount> spareCountByFlavor;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCount.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCount.java
index 217f4999bfb..2a3f44df42f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCount.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorSpareCount.java
@@ -17,7 +17,6 @@ import java.util.stream.Collectors;
* @author freva
*/
public class FlavorSpareCount {
-
private final Flavor flavor;
private Set<FlavorSpareCount> possibleWantedFlavors;
private Set<FlavorSpareCount> immediateReplacees;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index c5d60c30356..bf7942a1c56 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -1,8 +1,11 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.provisioning;
+import com.google.common.collect.ComparisonChain;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeFlavors;
import com.yahoo.config.provision.OutOfCapacityException;
import com.yahoo.lang.MutableInteger;
import com.yahoo.transaction.Mutex;
@@ -10,7 +13,9 @@ import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import java.time.Clock;
+import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.function.BiConsumer;
/**
@@ -23,6 +28,8 @@ class GroupPreparer {
private final NodeRepository nodeRepository;
private final Clock clock;
+ private static final boolean canChangeGroup = true;
+
public GroupPreparer(NodeRepository nodeRepository, Clock clock) {
this.nodeRepository = nodeRepository;
this.clock = clock;
@@ -31,63 +38,123 @@ class GroupPreparer {
/**
* Ensure sufficient nodes are reserved or active for the given application, group and cluster
*
- * @param application the application we are allocating to
- * @param cluster the cluster and group we are allocating to
- * @param requestedNodes a specification of the requested nodes
+ * @param application the application we are allocating to
+ * @param cluster the cluster and group we are allocating to
+ * @param requestedNodes a specification of the requested nodes
* @param surplusActiveNodes currently active nodes which are available to be assigned to this group.
- * This method will remove from this list if it finds it needs additional nodes
- * @param highestIndex the current highest node index among all active nodes in this cluster.
- * This method will increase this number when it allocates new nodes to the cluster.
- * @param nofSpares The number of spare docker hosts we want when dynamically allocate docker containers
- * @param debugRecorder Debug facility to step through the allocation process after the fact
+ * This method will remove from this list if it finds it needs additional nodes
+ * @param highestIndex the current highest node index among all active nodes in this cluster.
+ * This method will increase this number when it allocates new nodes to the cluster.
+ * @param nofSpares The number of spare docker hosts we want when dynamically allocate docker containers
+ * @param debugRecorder Debug facility to step through the allocation process after the fact
* @return the list of nodes this cluster group will have allocated if activated
*/
- // Note: This operation may make persisted changes to the set of reserved and inactive nodes,
- // but it may not change the set of active nodes, as the active nodes must stay in sync with the
- // active config model which is changed on activate
+ // Note: This operation may make persisted changes to the set of reserved and inactive nodes,
+ // but it may not change the set of active nodes, as the active nodes must stay in sync with the
+ // active config model which is changed on activate
public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes,
List<Node> surplusActiveNodes, MutableInteger highestIndex, int nofSpares, BiConsumer<List<Node>, String> debugRecorder) {
try (Mutex lock = nodeRepository.lock(application)) {
+ // A snapshot of nodes before we start the process - used to determine if this is a replacement
+ List<Node> nodesBefore = nodeRepository.getNodes(application, Node.State.values());
+ NodeAllocation allocation = new NodeAllocation(application, cluster, requestedNodes, highestIndex, clock);
+
+ // Use active nodes
+ allocation.offer(nodeRepository.getNodes(application, Node.State.active), !canChangeGroup);
+ if (allocation.saturated()) return allocation.finalNodes(surplusActiveNodes);
+
+ // Use active nodes from other groups that will otherwise be retired
+ List<Node> accepted = allocation.offer(prioritizeNodes(surplusActiveNodes, requestedNodes), canChangeGroup);
+ surplusActiveNodes.removeAll(accepted);
+ if (allocation.saturated()) return allocation.finalNodes(surplusActiveNodes);
+
+ // Use previously reserved nodes
+ allocation.offer(nodeRepository.getNodes(application, Node.State.reserved), !canChangeGroup);
+ if (allocation.saturated()) return allocation.finalNodes(surplusActiveNodes);
+
+ // Use inactive nodes
+ accepted = allocation.offer(prioritizeNodes(nodeRepository.getNodes(application, Node.State.inactive), requestedNodes), !canChangeGroup);
+ allocation.update(nodeRepository.reserve(accepted));
+ if (allocation.saturated()) return allocation.finalNodes(surplusActiveNodes);
+
// Use new, ready nodes. Lock ready pool to ensure that nodes are not grabbed by others.
try (Mutex readyLock = nodeRepository.lockUnallocated()) {
- // Create a prioritized set of nodes
- NodePrioritizer prioritizer = new NodePrioritizer(nodeRepository.getNodes(),
- application,
- cluster,
- requestedNodes,
- nodeRepository.getAvailableFlavors(),
- nofSpares);
-
- prioritizer.addApplicationNodes();
- prioritizer.addSurplusNodes(surplusActiveNodes);
- prioritizer.addReadyNodes();
- if (nodeRepository.dynamicAllocationEnabled())
- prioritizer.addNewDockerNodes();
-
- // Allocate from the prioritized list
- NodeAllocation allocation = new NodeAllocation(application, cluster, requestedNodes, highestIndex, clock);
- allocation.offer(prioritizer.prioritize());
- if (! allocation.fullfilled())
- throw new OutOfCapacityException("Could not satisfy " + requestedNodes + " for " + cluster +
- outOfCapacityDetails(allocation));
-
- // Carry out and return allocation
- nodeRepository.reserve(allocation.acceptedInactiveAndReadyNodes());
- nodeRepository.addDockerNodes(allocation.acceptedNewNodes());
- surplusActiveNodes.removeAll(allocation.acceptedSurplusNodes());
- return allocation.finalNodes(surplusActiveNodes);
+ // Check if we have ready nodes that we can allocate
+ List<Node> readyNodes = nodeRepository.getNodes(requestedNodes.type(), Node.State.ready);
+ accepted = allocation.offer(prioritizeNodes(readyNodes, requestedNodes), !canChangeGroup);
+ allocation.update(nodeRepository.reserve(accepted));
+
+ if (nodeRepository.dynamicAllocationEnabled()) {
+ // Check if we have available capacity on docker hosts that we can allocate
+ if (!allocation.fullfilled()) {
+ // The new dynamic allocation method
+ Optional<Flavor> flavor = getFlavor(requestedNodes);
+ if (flavor.isPresent() && flavor.get().getType().equals(Flavor.Type.DOCKER_CONTAINER)) {
+ List<Node> allNodes = nodeRepository.getNodes(Node.State.values());
+ NodeFlavors flavors = nodeRepository.getAvailableFlavors();
+ accepted = DockerAllocator.allocateNewDockerNodes(allocation, requestedNodes, allNodes,
+ nodesBefore, flavors, flavor.get(), nofSpares, debugRecorder);
+
+ // Add nodes to the node repository
+ if (allocation.fullfilled()) {
+ List<Node> nodesAddedToNodeRepo = nodeRepository.addDockerNodes(accepted);
+ allocation.update(nodesAddedToNodeRepo);
+ }
+ }
+ }
+ }
}
+
+ if (allocation.fullfilled())
+ return allocation.finalNodes(surplusActiveNodes);
+ else
+ throw new OutOfCapacityException("Could not satisfy " + requestedNodes + " for " + cluster +
+ outOfCapacityDetails(allocation));
}
}
+ private Optional<Flavor> getFlavor(NodeSpec nodeSpec) {
+ if (nodeSpec instanceof NodeSpec.CountNodeSpec) {
+ NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) nodeSpec;
+ return Optional.of(countSpec.getFlavor());
+ }
+ return Optional.empty();
+ }
+
private String outOfCapacityDetails(NodeAllocation allocation) {
- if (allocation.wouldBeFulfilledWithClashingParentHost())
+ if (allocation.wouldBeFulfilledWithClashingParentHost()) {
return ": Not enough nodes available on separate physical hosts.";
- if (allocation.wouldBeFulfilledWithRetiredNodes())
+ }
+ if (allocation.wouldBeFulfilledWithRetiredNodes()) {
return ": Not enough nodes available due to retirement.";
- else
- return ".";
+ }
+ return ".";
+ }
+
+ /**
+ * Returns the node list in prioritized order, where the nodes we would most prefer the application
+ * to use comes first
+ */
+ private List<Node> prioritizeNodes(List<Node> nodeList, NodeSpec nodeSpec) {
+ if ( nodeSpec.specifiesNonStockFlavor()) { // prefer exact flavor, docker hosts, lower cost, tie break by hostname
+ Collections.sort(nodeList, (n1, n2) -> ComparisonChain.start()
+ .compareTrueFirst(nodeSpec.matchesExactly(n1.flavor()), nodeSpec.matchesExactly(n2.flavor()))
+ .compareTrueFirst(n1.parentHostname().isPresent(), n2.parentHostname().isPresent())
+ .compare(n1.flavor().cost(), n2.flavor().cost())
+ .compare(n1.hostname(), n2.hostname())
+ .result()
+ );
+ }
+ else { // prefer docker hosts, lower cost, tie break by hostname
+ Collections.sort(nodeList, (n1, n2) -> ComparisonChain.start()
+ .compareTrueFirst(n1.parentHostname().isPresent(), n2.parentHostname().isPresent())
+ .compare(n1.flavor().cost(), n2.flavor().cost())
+ .compare(n1.hostname(), n2.hostname())
+ .result()
+ );
+ }
+ return nodeList;
}
-} \ No newline at end of file
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
index b9f682bc79f..b8d0495095b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
@@ -21,10 +21,8 @@ import java.util.stream.Collectors;
/**
* Used to manage a list of nodes during the node reservation process
* in order to fulfill the nodespec.
- *
- * @author bratseth
*/
-class NodeAllocation {
+public class NodeAllocation {
/** The application this list is for */
private final ApplicationId application;
@@ -36,7 +34,7 @@ class NodeAllocation {
private final NodeSpec requestedNodes;
/** The nodes this has accepted so far */
- private final Set<PrioritizableNode> nodes = new LinkedHashSet<>();
+ private final Set<Node> nodes = new LinkedHashSet<>();
/** The number of nodes in the accepted nodes which are of the requested flavor */
private int acceptedOfRequestedFlavor = 0;
@@ -56,7 +54,7 @@ class NodeAllocation {
/** Used to record event timestamps **/
private final Clock clock;
- NodeAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, MutableInteger highestIndex, Clock clock) {
+ public NodeAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, MutableInteger highestIndex, Clock clock) {
this.application = application;
this.cluster = cluster;
this.requestedNodes = requestedNodes;
@@ -67,24 +65,23 @@ class NodeAllocation {
/**
* Offer some nodes to this. The nodes may have an allocation to a different application or cluster,
* an allocation to this cluster, or no current allocation (in which case one is assigned).
- *
+ * <p>
* Note that if unallocated nodes are offered before allocated nodes, this will unnecessarily
* reject allocated nodes due to index duplicates.
*
- * @param nodesPrioritized the nodes which are potentially on offer. These may belong to a different application etc.
+ * @param offeredNodes the nodes which are potentially on offer. These may belong to a different application etc.
+ * @param canChangeGroup whether it is ok to change the group the offered node is to belong to if necessary
* @return the subset of offeredNodes which was accepted, with the correct allocation assigned
*/
- List<Node> offer(List<PrioritizableNode> nodesPrioritized) {
+ public List<Node> offer(List<Node> offeredNodes, boolean canChangeGroup) {
List<Node> accepted = new ArrayList<>();
- for (PrioritizableNode offeredPriority : nodesPrioritized) {
- Node offered = offeredPriority.node;
-
+ for (Node offered : offeredNodes) {
if (offered.allocation().isPresent()) {
boolean wantToRetireNode = false;
ClusterMembership membership = offered.allocation().get().membership();
if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application
if ( ! membership.cluster().equalsIgnoringGroupAndVespaVersion(cluster)) continue; // wrong cluster id/type
- if ((! offeredPriority.isSurplusNode || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it
+ if ((! canChangeGroup || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it
if ( offered.allocation().get().isRemovable()) continue; // don't accept; causes removal
if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure)
@@ -94,9 +91,8 @@ class NodeAllocation {
if ( offered.flavor().isRetired()) wantToRetireNode = true;
if ( offered.status().wantToRetire()) wantToRetireNode = true;
- if ((!saturated() && hasCompatibleFlavor(offered)) || acceptToRetire(offered) ) {
- accepted.add(acceptNode(offeredPriority, wantToRetireNode));
- }
+ if ((!saturated() && hasCompatibleFlavor(offered)) || acceptToRetire(offered) )
+ accepted.add(acceptNode(offered, wantToRetireNode));
}
else if (! saturated() && hasCompatibleFlavor(offered)) {
if ( offeredNodeHasParentHostnameAlreadyAccepted(this.nodes, offered)) {
@@ -109,18 +105,18 @@ class NodeAllocation {
if (offered.status().wantToRetire()) {
continue;
}
- offeredPriority.node = offered.allocate(application, ClusterMembership.from(cluster, highestIndex.add(1)), clock.instant());
- accepted.add(acceptNode(offeredPriority, false));
+ Node alloc = offered.allocate(application, ClusterMembership.from(cluster, highestIndex.add(1)), clock.instant());
+ accepted.add(acceptNode(alloc, false));
}
}
return accepted;
}
- private boolean offeredNodeHasParentHostnameAlreadyAccepted(Collection<PrioritizableNode> accepted, Node offered) {
- for (PrioritizableNode acceptedNode : accepted) {
- if (acceptedNode.node.parentHostname().isPresent() && offered.parentHostname().isPresent() &&
- acceptedNode.node.parentHostname().get().equals(offered.parentHostname().get())) {
+ private boolean offeredNodeHasParentHostnameAlreadyAccepted(Collection<Node> accepted, Node offered) {
+ for (Node acceptedNode : accepted) {
+ if (acceptedNode.parentHostname().isPresent() && offered.parentHostname().isPresent() &&
+ acceptedNode.parentHostname().get().equals(offered.parentHostname().get())) {
return true;
}
}
@@ -154,29 +150,31 @@ class NodeAllocation {
return requestedNodes.isCompatible(node.flavor());
}
- private Node acceptNode(PrioritizableNode prioritizableNode, boolean wantToRetire) {
- Node node = prioritizableNode.node;
+ /** Updates the state of some existing nodes in this list by replacing them by id with the given instances. */
+ public void update(List<Node> updatedNodes) {
+ nodes.removeAll(updatedNodes);
+ nodes.addAll(updatedNodes);
+ }
+
+ private Node acceptNode(Node node, boolean wantToRetire) {
if (! wantToRetire) {
if ( ! node.state().equals(Node.State.active)) {
// reactivated node - make sure its not retired
node = node.unretire();
- prioritizableNode.node= node;
}
acceptedOfRequestedFlavor++;
} else {
++wasRetiredJustNow;
// Retire nodes which are of an unwanted flavor, retired flavor or have an overlapping parent host
node = node.retire(clock.instant());
- prioritizableNode.node= node;
}
if ( ! node.allocation().get().membership().cluster().equals(cluster)) {
// group may be different
node = setCluster(cluster, node);
- prioritizableNode.node= node;
}
indexes.add(node.allocation().get().membership().index());
highestIndex.set(Math.max(highestIndex.get(), node.allocation().get().membership().index()));
- nodes.add(prioritizableNode);
+ nodes.add(node);
return node;
}
@@ -186,20 +184,20 @@ class NodeAllocation {
}
/** Returns true if no more nodes are needed in this list */
- private boolean saturated() {
+ public boolean saturated() {
return requestedNodes.saturatedBy(acceptedOfRequestedFlavor);
}
/** Returns true if the content of this list is sufficient to meet the request */
- boolean fullfilled() {
+ public boolean fullfilled() {
return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor);
}
- boolean wouldBeFulfilledWithRetiredNodes() {
+ public boolean wouldBeFulfilledWithRetiredNodes() {
return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + wasRetiredJustNow);
}
- boolean wouldBeFulfilledWithClashingParentHost() {
+ public boolean wouldBeFulfilledWithClashingParentHost() {
return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + rejectedWithClashingParentHost);
}
@@ -212,61 +210,42 @@ class NodeAllocation {
* @param surplusNodes this will add nodes not any longer needed by this group to this list
* @return the final list of nodes
*/
- List<Node> finalNodes(List<Node> surplusNodes) {
- long currentRetired = nodes.stream().filter(node -> node.node.allocation().get().membership().retired()).count();
+ public List<Node> finalNodes(List<Node> surplusNodes) {
+ long currentRetired = nodes.stream().filter(node -> node.allocation().get().membership().retired()).count();
long surplus = requestedNodes.surplusGiven(nodes.size()) - currentRetired;
+ List<Node> changedNodes = new ArrayList<>();
if (surplus > 0) { // retire until surplus is 0, prefer to retire higher indexes to minimize redistribution
- for (PrioritizableNode node : byDecreasingIndex(nodes)) {
- if ( ! node.node.allocation().get().membership().retired() && node.node.state().equals(Node.State.active)) {
- node.node = node.node.retire(Agent.application, clock.instant());
- surplusNodes.add(node.node); // offer this node to other groups
+ for (Node node : byDecreasingIndex(nodes)) {
+ if ( ! node.allocation().get().membership().retired() && node.state().equals(Node.State.active)) {
+ changedNodes.add(node.retire(Agent.application, clock.instant()));
+ surplusNodes.add(node); // offer this node to other groups
if (--surplus == 0) break;
}
}
}
else if (surplus < 0) { // unretire until surplus is 0
- for (PrioritizableNode node : byIncreasingIndex(nodes)) {
- if ( node.node.allocation().get().membership().retired() && hasCompatibleFlavor(node.node)) {
- node.node = node.node.unretire();
+ for (Node node : byIncreasingIndex(nodes)) {
+ if ( node.allocation().get().membership().retired() && hasCompatibleFlavor(node)) {
+ changedNodes.add(node.unretire());
if (++surplus == 0) break;
}
}
}
-
- return nodes.stream().map(n -> n.node).collect(Collectors.toList());
- }
-
- List<Node> acceptedInactiveAndReadyNodes() {
- return nodes.stream().map(n -> n.node)
- .filter(n -> n.state().equals(Node.State.inactive) || n.state().equals(Node.State.ready))
- .collect(Collectors.toList());
- }
-
- List<Node> acceptedSurplusNodes() {
- return nodes.stream()
- .filter(n -> n.isSurplusNode)
- .map(n -> n.node)
- .collect(Collectors.toList());
- }
-
- List<Node> acceptedNewNodes() {
- return nodes.stream()
- .filter(n -> n.isNewNode)
- .map(n -> n.node)
- .collect(Collectors.toList());
+ update(changedNodes);
+ return new ArrayList<>(nodes);
}
- private List<PrioritizableNode> byDecreasingIndex(Set<PrioritizableNode> nodes) {
+ private List<Node> byDecreasingIndex(Set<Node> nodes) {
return nodes.stream().sorted(nodeIndexComparator().reversed()).collect(Collectors.toList());
}
- private List<PrioritizableNode> byIncreasingIndex(Set<PrioritizableNode> nodes) {
+ private List<Node> byIncreasingIndex(Set<Node> nodes) {
return nodes.stream().sorted(nodeIndexComparator()).collect(Collectors.toList());
}
- private Comparator<PrioritizableNode> nodeIndexComparator() {
- return Comparator.comparing((PrioritizableNode n) -> n.node.allocation().get().membership().index());
+ private Comparator<Node> nodeIndexComparator() {
+ return Comparator.comparing((Node n) -> n.allocation().get().membership().index());
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
deleted file mode 100644
index 24e33d31a7a..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
+++ /dev/null
@@ -1,301 +0,0 @@
-package com.yahoo.vespa.hosted.provision.provisioning;
-
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.Flavor;
-import com.yahoo.config.provision.NodeFlavors;
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.vespa.hosted.provision.Node;
-import com.yahoo.vespa.hosted.provision.NodeList;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * Builds up data structures necessary for node prioritization. It wraps each node
- * up in a PrioritizableNode object with attributes used in sorting.
- *
- * The actual sorting/prioritization is implemented in the PrioritizableNode class as a compare method.
- *
- * @author smorgrav
- */
-public class NodePrioritizer {
-
- private final Map<Node, PrioritizableNode> nodes = new HashMap<>();
- private final List<Node> allNodes;
- private final DockerHostCapacity capacity;
- private final NodeSpec requestedNodes;
- private final ApplicationId appId;
- private final ClusterSpec clusterSpec;
-
- private final boolean isAllocatingForReplacement;
- private final Set<Node> spareHosts;
- private final Map<Node, Boolean> headroomHosts;
- private final boolean isDocker;
-
- NodePrioritizer(List<Node> allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec, NodeFlavors nodeFlavors, int spares) {
- this.allNodes = Collections.unmodifiableList(allNodes);
- this.requestedNodes = nodeSpec;
- this.clusterSpec = clusterSpec;
- this.appId = appId;
-
- spareHosts = findSpareHosts(allNodes, spares);
- headroomHosts = findHeadroomHosts(allNodes, spareHosts, nodeFlavors);
-
- this.capacity = new DockerHostCapacity(allNodes);
-
- long nofFailedNodes = allNodes.stream()
- .filter(node -> node.state().equals(Node.State.failed))
- .filter(node -> node.allocation().isPresent())
- .filter(node -> node.allocation().get().owner().equals(appId))
- .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterSpec.id()))
- .count();
-
- long nofNodesInCluster = allNodes.stream()
- .filter(node -> node.allocation().isPresent())
- .filter(node -> node.allocation().get().owner().equals(appId))
- .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterSpec.id()))
- .count();
-
- isAllocatingForReplacement = isReplacement(nofNodesInCluster, nofFailedNodes);
- isDocker = isDocker();
- }
-
- /**
- * From ipAddress - get hostname
- *
- * @return hostname or null if not able to do the loopup
- */
- private static String lookupHostname(String ipAddress) {
- try {
- return InetAddress.getByName(ipAddress).getHostName();
- } catch (UnknownHostException e) {
- e.printStackTrace();
- }
- return null;
- }
-
- /**
- * Spare hosts are the two hosts in the system with the most free capacity.
- *
- * We do not count retired or inactive nodes as used capacity (as they could have been
- * moved to create space for the spare node in the first place).
- */
- private static Set<Node> findSpareHosts(List<Node> nodes, int spares) {
- DockerHostCapacity capacity = new DockerHostCapacity(new ArrayList<>(nodes));
- return nodes.stream()
- .filter(node -> node.type().equals(NodeType.host))
- .filter(dockerHost -> dockerHost.state().equals(Node.State.active))
- .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0)
- .sorted(capacity::compareWithoutInactive)
- .limit(spares)
- .collect(Collectors.toSet());
- }
-
- /**
- * Headroom are the nodes with the least but sufficient space for the requested headroom.
- *
- * If not enough headroom - the headroom violating hosts are the once that are closest to fulfull
- * a headroom request.
- */
- private static Map<Node, Boolean> findHeadroomHosts(List<Node> nodes, Set<Node> spareNodes, NodeFlavors flavors) {
- DockerHostCapacity capacity = new DockerHostCapacity(nodes);
- Map<Node, Boolean> headroomNodesToViolation = new HashMap<>();
-
- List<Node> hostsSortedOnLeastCapacity = nodes.stream()
- .filter(n -> !spareNodes.contains(n))
- .filter(node -> node.type().equals(NodeType.host))
- .filter(dockerHost -> dockerHost.state().equals(Node.State.active))
- .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0)
- .sorted((a, b) -> capacity.compareWithoutInactive(b, a))
- .collect(Collectors.toList());
-
- for (Flavor flavor : flavors.getFlavors().stream().filter(f -> f.getIdealHeadroom() > 0).collect(Collectors.toList())) {
- Set<Node> tempHeadroom = new HashSet<>();
- Set<Node> notEnoughCapacity = new HashSet<>();
- for (Node host : hostsSortedOnLeastCapacity) {
- if (headroomNodesToViolation.containsKey(host)) continue;
- if (capacity.hasCapacityWhenRetiredAndInactiveNodesAreGone(host, flavor)) {
- headroomNodesToViolation.put(host, false);
- tempHeadroom.add(host);
- } else {
- notEnoughCapacity.add(host);
- }
-
- if (tempHeadroom.size() == flavor.getIdealHeadroom()) {
- continue;
- }
- }
-
- // Now check if we have enough headroom - if not choose the nodes that almost has it
- if (tempHeadroom.size() < flavor.getIdealHeadroom()) {
- List<Node> violations = notEnoughCapacity.stream()
- .sorted((a, b) -> capacity.compare(b, a))
- .limit(flavor.getIdealHeadroom() - tempHeadroom.size())
- .collect(Collectors.toList());
-
- for (Node nodeViolatingHeadrom : violations) {
- headroomNodesToViolation.put(nodeViolatingHeadrom, true);
- }
-
- }
- }
-
- return headroomNodesToViolation;
- }
-
- /**
- * @return The list of nodes sorted by PrioritizableNode::compare
- */
- List<PrioritizableNode> prioritize() {
- List<PrioritizableNode> priorityList = new ArrayList<>(nodes.values());
- Collections.sort(priorityList);
- return priorityList;
- }
-
- /**
- * Add nodes that have been previously reserved to the same application from
- * an earlier downsizing of a cluster
- */
- void addSurplusNodes(List<Node> surplusNodes) {
- for (Node node : surplusNodes) {
- PrioritizableNode nodePri = toNodePriority(node, true, false);
- if (!nodePri.violatesSpares || isAllocatingForReplacement) {
- nodes.put(node, nodePri);
- }
- }
- }
-
- /**
- * Add a node on each docker host with enough capacity for the requested flavor
- */
- void addNewDockerNodes() {
- if (!isDocker) return;
- DockerHostCapacity capacity = new DockerHostCapacity(allNodes);
-
- for (Node node : allNodes) {
- if (node.type() == NodeType.host) {
- boolean conflictingCluster = false;
- NodeList list = new NodeList(allNodes);
- NodeList childrenWithSameApp = list.childNodes(node).owner(appId);
- for (Node child : childrenWithSameApp.asList()) {
- // Look for nodes from the same cluster
- if (child.allocation().get().membership().cluster().id().equals(clusterSpec.id())) {
- conflictingCluster = true;
- break;
- }
- }
-
- if (!conflictingCluster && capacity.hasCapacity(node, getFlavor())) {
- Set<String> ipAddresses = DockerHostCapacity.findFreeIps(node, allNodes);
- if (ipAddresses.isEmpty()) continue;
- String ipAddress = ipAddresses.stream().findFirst().get();
- String hostname = lookupHostname(ipAddress);
- if (hostname == null) continue;
- Node newNode = Node.createDockerNode("fake-" + hostname, Collections.singleton(ipAddress),
- Collections.emptySet(), hostname, Optional.of(node.hostname()), getFlavor(), NodeType.tenant);
- PrioritizableNode nodePri = toNodePriority(newNode, false, true);
- if (!nodePri.violatesSpares || isAllocatingForReplacement) {
- nodes.put(newNode, nodePri);
- }
- }
- }
- }
- }
-
- /**
- * Add existing nodes allocated to the application
- */
- void addApplicationNodes() {
- List<Node.State> legalStates = Arrays.asList(Node.State.active, Node.State.inactive, Node.State.reserved);
- allNodes.stream()
- .filter(node -> node.type().equals(requestedNodes.type()))
- .filter(node -> legalStates.contains(node.state()))
- .filter(node -> node.allocation().isPresent())
- .filter(node -> node.allocation().get().owner().equals(appId))
- .map(node -> toNodePriority(node, false, false))
- .forEach(prioritizableNode -> nodes.put(prioritizableNode.node, prioritizableNode));
- }
-
- /**
- * Add nodes already provisioned, but not allocatied to any application
- */
- void addReadyNodes() {
- allNodes.stream()
- .filter(node -> node.type().equals(requestedNodes.type()))
- .filter(node -> node.state().equals(Node.State.ready))
- .map(node -> toNodePriority(node, false, false))
- .filter(n -> !n.violatesSpares || isAllocatingForReplacement)
- .forEach(prioritizableNode -> nodes.put(prioritizableNode.node, prioritizableNode));
- }
-
- /**
- * Convert a list of nodes to a list of node priorities. This includes finding, calculating
- * parameters to the priority sorting procedure.
- */
- private PrioritizableNode toNodePriority(Node node, boolean isSurplusNode, boolean isNewNode) {
- PrioritizableNode pri = new PrioritizableNode();
- pri.node = node;
- pri.isSurplusNode = isSurplusNode;
- pri.isNewNode = isNewNode;
- pri.preferredOnFlavor = requestedNodes.specifiesNonStockFlavor() && node.flavor().equals(getFlavor());
- pri.parent = findParentNode(node);
-
- if (pri.parent.isPresent()) {
- Node parent = pri.parent.get();
- pri.freeParentCapacity = capacity.freeCapacityOf(parent, false);
-
- if (spareHosts.contains(parent)) {
- pri.violatesSpares = true;
- }
-
- if (headroomHosts.containsKey(parent)) {
- pri.violatesHeadroom = headroomHosts.get(parent);
- }
- }
-
- return pri;
- }
-
- private boolean isReplacement(long nofNodesInCluster, long nodeFailedNodes) {
- if (nodeFailedNodes == 0) return false;
-
- int wantedCount = 0;
- if (requestedNodes instanceof NodeSpec.CountNodeSpec) {
- NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) requestedNodes;
- wantedCount = countSpec.getCount();
- }
-
- return (wantedCount > nofNodesInCluster - nodeFailedNodes);
- }
-
- private Flavor getFlavor() {
- if (requestedNodes instanceof NodeSpec.CountNodeSpec) {
- NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) requestedNodes;
- return countSpec.getFlavor();
- }
- return null;
- }
-
- private boolean isDocker() {
- Flavor flavor = getFlavor();
- return (flavor != null) && flavor.getType().equals(Flavor.Type.DOCKER_CONTAINER);
- }
-
- private Optional<Node> findParentNode(Node node) {
- if (!node.parentHostname().isPresent()) return Optional.empty();
- return allNodes.stream()
- .filter(n -> n.hostname().equals(node.parentHostname().orElse(" NOT A NODE")))
- .findAny();
- }
-} \ No newline at end of file
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
index a0f9452ef9f..639d6ea17f4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
@@ -88,12 +88,7 @@ class Preparer {
}
}
}
-
- /**
- * Nodes are immutable so when changing attributes to the node we create a new instance.
- *
- * This method is used to both add new nodes and replaces old node references with the new references.
- */
+
private List<Node> replace(List<Node> list, List<Node> changed) {
list.removeAll(changed);
list.addAll(changed);
@@ -126,4 +121,5 @@ class Preparer {
}
return retired;
}
+
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java
deleted file mode 100644
index 219778ab7a0..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java
+++ /dev/null
@@ -1,101 +0,0 @@
-package com.yahoo.vespa.hosted.provision.provisioning;
-
-import com.yahoo.vespa.hosted.provision.Node;
-
-import java.util.Optional;
-
-/**
- * A node with additional information required to prioritize it for allocation.
- *
- * @author smorgrav
- */
-class PrioritizableNode implements Comparable<PrioritizableNode> {
-
- Node node;
-
- /** The free capacity excluding headroom, including retired allocations */
- ResourceCapacity freeParentCapacity = new ResourceCapacity();
-
- /** The parent host (docker or hypervisor) */
- Optional<Node> parent = Optional.empty();
-
- /** True if the node is allocated to a host that should be dedicated as a spare */
- boolean violatesSpares;
-
- /** True if the node is allocated on slots that should be dedicated to headroom */
- boolean violatesHeadroom;
-
- /** True if this is a node that has been retired earlier in the allocation process */
- boolean isSurplusNode;
-
- /** This node does not exist in the node repository yet */
- boolean isNewNode;
-
- /** True if exact flavor is specified by the allocation request and this node has this flavor */
- boolean preferredOnFlavor;
-
- /**
- * Compares two prioritizable nodes
- *
- * @return negative if first priority is higher than second node
- */
- @Override
- public int compareTo(PrioritizableNode other) {
- // First always pick nodes without violation above nodes with violations
- if (!this.violatesSpares && other.violatesSpares) return -1;
- if (!other.violatesSpares && this.violatesSpares) return 1;
- if (!this.violatesHeadroom && other.violatesHeadroom) return -1;
- if (!other.violatesHeadroom && this.violatesHeadroom) return 1;
-
- // Choose active nodes
- if (this.node.state().equals(Node.State.active) && !other.node.state().equals(Node.State.active)) return -1;
- if (other.node.state().equals(Node.State.active) && !this.node.state().equals(Node.State.active)) return 1;
-
- // Choose active node that is not retired first (surplus is active but retired)
- if (!this.isSurplusNode && other.isSurplusNode) return -1;
- if (!other.isSurplusNode && this.isSurplusNode) return 1;
-
- // Choose inactive nodes
- if (this.node.state().equals(Node.State.inactive) && !other.node.state().equals(Node.State.inactive)) return -1;
- if (other.node.state().equals(Node.State.inactive) && !this.node.state().equals(Node.State.inactive)) return 1;
-
- // Choose reserved nodes from a previous allocation attempt (the exist in node repo)
- if (isInNodeRepoAndReserved(this) && !isInNodeRepoAndReserved(other)) return -1;
- if (isInNodeRepoAndReserved(other) && !isInNodeRepoAndReserved(this)) return 1;
-
- // Choose ready nodes
- if (this.node.state().equals(Node.State.ready) && !other.node.state().equals(Node.State.ready)) return -1;
- if (other.node.state().equals(Node.State.ready) && !this.node.state().equals(Node.State.ready)) return 1;
-
- // The node state should be equal here
- if (!this.node.state().equals(other.node.state())) {
- throw new RuntimeException(
- String.format("Error during node priority comparison. Node states are not equal as expected. Got %s and %s.",
- this.node.state(), other.node.state()));
- }
-
- // Choose exact flavor
- if (this.preferredOnFlavor && !other.preferredOnFlavor) return -1;
- if (other.preferredOnFlavor && !this.preferredOnFlavor) return 1;
-
- // Choose docker node over non-docker node (is this to differentiate between docker replaces non-docker flavors?)
- if (this.parent.isPresent() && !other.parent.isPresent()) return -1;
- if (other.parent.isPresent() && !this.parent.isPresent()) return 1;
-
- // Choose the node with parent node with the least capacity (TODO parameterize this as this is pretty much the core of the algorithm)
- int freeCapacity = this.freeParentCapacity.compare(other.freeParentCapacity);
- if (freeCapacity != 0) return freeCapacity;
-
- // Choose cheapest node
- if (this.node.flavor().cost() < other.node.flavor().cost()) return -1;
- if (other.node.flavor().cost() < this.node.flavor().cost()) return 1;
-
- // All else equal choose hostname alphabetically
- return this.node.hostname().compareTo(other.node.hostname());
- }
-
- private static boolean isInNodeRepoAndReserved(PrioritizableNode nodePri) {
- if (nodePri.isNewNode) return false;
- return nodePri.node.state().equals(Node.State.reserved);
- }
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java
index fdec29d5b97..e56409d47b9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java
@@ -91,5 +91,4 @@ public class ResourceCapacity {
b.addFlavor("spareflavor", cpu, memory, disk, Flavor.Type.DOCKER_CONTAINER).idealHeadroom(1);
return new Flavor(b.build().flavor(0));
}
-
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 0885b941401..ad66f86de69 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -51,11 +51,6 @@ public class MockNodeRepository extends NodeRepository {
populate();
}
- @Override
- public boolean dynamicAllocationEnabled() {
- return true;
- }
-
private void populate() {
NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, flavors, Zone.defaultZone());
@@ -65,22 +60,18 @@ public class MockNodeRepository extends NodeRepository {
Collections.sort(ipAddressesForAllHost);
final HashSet<String> ipAddresses = new HashSet<>(ipAddressesForAllHost);
- final List<String> additionalIpAddressesForAllHost = Arrays.asList("::2", "::3", "::4");
- Collections.sort(additionalIpAddressesForAllHost);
- final HashSet<String> additionalIpAddresses = new HashSet<>(additionalIpAddressesForAllHost);
-
nodes.add(createNode("node1", "host1.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant));
nodes.add(createNode("node2", "host2.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant));
nodes.add(createNode("node3", "host3.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("expensive"), NodeType.tenant));
- Node node4 = createNode("node4", "host4.yahoo.com", ipAddresses, Optional.of("dockerhost1.yahoo.com"), flavors.getFlavorOrThrow("docker"), NodeType.tenant);
+ // TODO: Use docker flavor
+ Node node4 = createNode("node4", "host4.yahoo.com", ipAddresses, Optional.of("dockerhost4"), flavors.getFlavorOrThrow("default"), NodeType.tenant);
node4 = node4.with(node4.status().withVespaVersion(new Version("6.41.0")));
nodes.add(node4);
- Node node5 = createNode("node5", "host5.yahoo.com", ipAddresses, Optional.of("dockerhost2.yahoo.com"), flavors.getFlavorOrThrow("docker"), NodeType.tenant);
+ Node node5 = createNode("node5", "host5.yahoo.com", ipAddresses, Optional.of("parent1.yahoo.com"), flavors.getFlavorOrThrow("default"), NodeType.tenant);
nodes.add(node5.with(node5.status().withVespaVersion(new Version("1.2.3"))));
-
nodes.add(createNode("node6", "host6.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant));
nodes.add(createNode("node7", "host7.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant));
// 8 and 9 are added by web service calls
@@ -92,13 +83,7 @@ public class MockNodeRepository extends NodeRepository {
nodes.add(node10);
nodes.add(createNode("node55", "host55.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant));
-
- /** Setup docker hosts (two of these will be reserved for spares */
- nodes.add(createNode("dockerhost1", "dockerhost1.yahoo.com", ipAddresses, additionalIpAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host));
- nodes.add(createNode("dockerhost2", "dockerhost2.yahoo.com", ipAddresses, additionalIpAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host));
- nodes.add(createNode("dockerhost3", "dockerhost3.yahoo.com", ipAddresses, additionalIpAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host));
- nodes.add(createNode("dockerhost4", "dockerhost4.yahoo.com", ipAddresses, additionalIpAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host));
- nodes.add(createNode("dockerhost5", "dockerhost5.yahoo.com", ipAddresses, additionalIpAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.host));
+ nodes.add(createNode("parent1", "parent1.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.host));
nodes = addNodes(nodes);
nodes.remove(6);
@@ -108,12 +93,6 @@ public class MockNodeRepository extends NodeRepository {
fail("host5.yahoo.com", Agent.system, "Failing to unit test");
setDirty("host55.yahoo.com");
- ApplicationId zoneApp = ApplicationId.from(TenantName.from("zoneapp"), ApplicationName.from("zoneapp"), InstanceName.from("zoneapp"));
- ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container,
- ClusterSpec.Id.from("node-admin"),
- Version.fromString("6.42"));
- activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), 1, null), zoneApp, provisioner);
-
ApplicationId app1 = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1"));
ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Version.fromString("6.42"));
provisioner.prepare(app1, cluster1, Capacity.fromNodeCount(2), 1, null);
@@ -124,7 +103,7 @@ public class MockNodeRepository extends NodeRepository {
ApplicationId app3 = ApplicationId.from(TenantName.from("tenant3"), ApplicationName.from("application3"), InstanceName.from("instance3"));
ClusterSpec cluster3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id3"), Version.fromString("6.42"));
- activate(provisioner.prepare(app3, cluster3, Capacity.fromNodeCount(2, "docker"), 1, null), app3, provisioner);
+ activate(provisioner.prepare(app3, cluster3, Capacity.fromNodeCount(2), 1, null), app3, provisioner);
}
private void activate(List<HostSpec> hosts, ApplicationId application, NodeRepositoryProvisioner provisioner) {
@@ -132,4 +111,5 @@ public class MockNodeRepository extends NodeRepository {
provisioner.activate(transaction, application, hosts);
transaction.commit();
}
-} \ No newline at end of file
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java
index dc30f0ed1a8..985277d17ea 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java
@@ -117,7 +117,14 @@ public class AllocationSimulator {
NodeSpec.CountNodeSpec nodeSpec = new NodeSpec.CountNodeSpec(count, flavor);
NodeAllocation allocation = new NodeAllocation(app(id), cluster(), nodeSpec, new MutableInteger(0), Clock.systemUTC());
- List<Node> accepted = new ArrayList<>(); //TODO adpot the new allocation algoritm
+ List<Node> accepted = DockerAllocator.allocateNewDockerNodes(allocation,
+ nodeSpec,
+ new ArrayList<>(nodes.asList()),
+ new ArrayList<>(nodes.asList()),
+ flavors,
+ flavor,
+ 2,
+ (nodes, message)-> visualizer.addStep(nodes, id, message));
accepted.addAll(nodes.asList());
nodes = new NodeList(accepted);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java
index 55e1ff8de9f..eae846abb2a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java
@@ -1,11 +1,15 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.provisioning;
+import com.yahoo.component.Version;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.NodeFlavors;
import com.yahoo.config.provision.NodeType;
import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.Generation;
import org.junit.Before;
import org.junit.Test;
@@ -63,11 +67,22 @@ public class DockerHostCapacityTest {
@Test
public void compare_used_to_sort_in_decending_order() {
assertEquals(host1, nodes.get(0)); //Make sure it is unsorted here
-
Collections.sort(nodes, capacity::compare);
assertEquals(host3, nodes.get(0));
assertEquals(host1, nodes.get(1));
assertEquals(host2, nodes.get(2));
+
+ // Replace a node for host 2 with a headroom node - host2 should then be prioritized
+ Allocation allocation = new Allocation(app(DockerHostCapacity.HEADROOM_TENANT), ClusterMembership.from("container/id1/3", new Version()), Generation.inital(), false);
+ Node nodeF = Node.create("nodeF", Collections.singleton("::6"), Collections.emptySet(), "nodeF", Optional.of("host2"), flavorDocker, NodeType.tenant);
+ Node nodeFWithAllocation = nodeF.with(allocation);
+ nodes.add(nodeFWithAllocation);
+ nodes.remove(nodeC);
+ capacity = new DockerHostCapacity(nodes);
+ Collections.sort(nodes, capacity::compare);
+ assertEquals(host3, nodes.get(0));
+ assertEquals(host2, nodes.get(1));
+ assertEquals(host1, nodes.get(2));
}
@Test
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java
index 8254cd23030..a5d72947fcc 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java
@@ -5,7 +5,6 @@ import com.google.common.collect.ImmutableSet;
import com.yahoo.component.Version;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.Capacity;
-import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.Flavor;
@@ -16,27 +15,15 @@ import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.Zone;
import com.yahoo.config.provisioning.FlavorsConfig;
import com.yahoo.path.Path;
-import com.yahoo.transaction.NestedTransaction;
-import com.yahoo.vespa.curator.transaction.CuratorTransaction;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeList;
-import com.yahoo.vespa.hosted.provision.node.Agent;
-import org.junit.Assert;
import org.junit.Test;
-import java.time.Instant;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
import java.util.stream.Collectors;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.Matchers.greaterThan;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
@@ -45,218 +32,6 @@ import static org.junit.Assert.fail;
*/
public class DynamicDockerProvisioningTest {
- /**
- * Test reloaction of nodes that violates headroom.
- *
- * Setup 4 docker hosts and allocate one container on each (from two different applications)
- * No spares - only headroom (4xd-2)
- *
- * One application is now violating headroom and need relocation
- *
- * Initial allocation of app 1 and 2 --> final allocation (headroom marked as H):
- *
- * | H | H | H | H | | | | | |
- * | H | H | H1a | H1b | --> | | | | |
- * | | | 2a | 2b | | 1a | 1b | 2a | 2b |
- *
- */
- @Test
- public void relocate_nodes_from_headroom_hosts() {
- ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true));
- enableDynamicAllocation(tester);
- tester.makeReadyNodes(4, "host", "host-small", NodeType.host, 32);
- deployZoneApp(tester);
- List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active);
- Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1");
-
- // Application 1
- ApplicationId application1 = makeApplicationId("t1", "a1");
- ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"));
- addAndAssignNode(application1, "1a", dockerHosts.get(2).hostname(), flavor, 0, tester);
- addAndAssignNode(application1, "1b", dockerHosts.get(3).hostname(), flavor, 1, tester);
-
- // Application 2
- ApplicationId application2 = makeApplicationId("t2", "a2");
- ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"));
- addAndAssignNode(application2, "2a", dockerHosts.get(2).hostname(), flavor, 0, tester);
- addAndAssignNode(application2, "2b", dockerHosts.get(3).hostname(), flavor, 1, tester);
-
- // Redeploy one of the applications
- deployapp(application1, clusterSpec1, flavor, tester, 2);
-
- // Assert that the nodes are spread across all hosts (to allow headroom)
- Set<String> hostsWithChildren = new HashSet<>();
- for (Node node : tester.nodeRepository().getNodes(NodeType.tenant, Node.State.active)) {
- if (!isInactiveOrRetired(node)) {
- hostsWithChildren.add(node.parentHostname().get());
- }
- }
- Assert.assertEquals(4, hostsWithChildren.size());
- }
-
- /**
- * Test relocation of nodes from spare hosts.
- *
- * Setup 4 docker hosts and allocate one container on each (from two different applications)
- * No headroom defined - only 2 spares.
- *
- * Check that it relocates containers away from the 2 spares
- *
- * Initial allocation of app 1 and 2 --> final allocation:
- *
- * | | | | | | | | | |
- * | | | | | --> | 2a | 2b | | |
- * | 1a | 1b | 2a | 2b | | 1a | 1b | | |
- *
- */
- @Test
- public void relocate_nodes_from_spare_hosts() {
- ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig());
- enableDynamicAllocation(tester);
- tester.makeReadyNodes(4, "host", "host-small", NodeType.host, 32);
- deployZoneApp(tester);
- List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active);
- Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1");
-
- // Application 1
- ApplicationId application1 = makeApplicationId("t1", "a1");
- ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"));
- addAndAssignNode(application1, "1a", dockerHosts.get(0).hostname(), flavor, 0, tester);
- addAndAssignNode(application1, "1b", dockerHosts.get(1).hostname(), flavor, 1, tester);
-
- // Application 2
- ApplicationId application2 = makeApplicationId("t2", "a2");
- ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"));
- addAndAssignNode(application2, "2a", dockerHosts.get(2).hostname(), flavor, 0, tester);
- addAndAssignNode(application2, "2b", dockerHosts.get(3).hostname(), flavor, 1, tester);
-
- // Redeploy both applications (to be agnostic on which hosts are picked as spares)
- deployapp(application1, clusterSpec1, flavor, tester, 2);
- deployapp(application2, clusterSpec2, flavor, tester, 2);
-
- // Assert that we have two spare nodes (two hosts that are don't have allocations)
- Set<String> hostsWithChildren = new HashSet<>();
- for (Node node : tester.nodeRepository().getNodes(NodeType.tenant, Node.State.active)) {
- if (!isInactiveOrRetired(node)) {
- hostsWithChildren.add(node.parentHostname().get());
- }
- }
- Assert.assertEquals(2, hostsWithChildren.size());
- }
-
- /**
- * Test an allocation workflow:
- *
- * 5 Hosts of capacity 3 (2 spares)
- * - Allocate app with 3 nodes
- * - Allocate app with 2 nodes
- * - Fail host and check redistribution
- */
- @Test
- public void reloacte_failed_nodes() {
- ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig());
- enableDynamicAllocation(tester);
- tester.makeReadyNodes(5, "host", "host-small", NodeType.host, 32);
- deployZoneApp(tester);
- List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active);
- Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1");
-
- // Application 1
- ApplicationId application1 = makeApplicationId("t1", "a1");
- ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"));
- deployapp(application1, clusterSpec1, flavor, tester, 3);
-
- // Application 2
- ApplicationId application2 = makeApplicationId("t2", "a2");
- ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"));
- deployapp(application2, clusterSpec2, flavor, tester, 2);
-
- // Application 3
- ApplicationId application3 = makeApplicationId("t3", "a3");
- ClusterSpec clusterSpec3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"));
- deployapp(application3, clusterSpec3, flavor, tester, 2);
-
- // App 2 and 3 should have been allocated to the same nodes - fail on of the parent hosts from there
- String parent = tester.nodeRepository().getNodes(application2).stream().findAny().get().parentHostname().get();
- tester.nodeRepository().failRecursively(parent, Agent.system, "Testing");
-
- // Redeploy all applications
- deployapp(application1, clusterSpec1, flavor, tester, 3);
- deployapp(application2, clusterSpec2, flavor, tester, 2);
- deployapp(application3, clusterSpec3, flavor, tester, 2);
-
- Map<Integer, Integer> numberOfChildrenStat = new HashMap<>();
- for (Node node : dockerHosts) {
- int nofChildren = tester.nodeRepository().getChildNodes(node.hostname()).size();
- if (!numberOfChildrenStat.containsKey(nofChildren)) {
- numberOfChildrenStat.put(nofChildren, 0);
- }
- numberOfChildrenStat.put(nofChildren, numberOfChildrenStat.get(nofChildren) + 1);
- }
-
- assertEquals(3l, (long)numberOfChildrenStat.get(3));
- assertEquals(1l, (long)numberOfChildrenStat.get(0));
- assertEquals(1l, (long)numberOfChildrenStat.get(1));
- }
-
- /**
- * Test redeployment of nodes that violates spare headroom - but without alternatives
- *
- * Setup 2 docker hosts and allocate one app with a container on each
- * No headroom defined - only 2 spares.
- *
- * Initial allocation of app 1 --> final allocation:
- *
- * | | | | | |
- * | | | --> | | |
- * | 1a | 1b | | 1a | 1b |
- *
- */
- @Test
- public void do_not_relocate_nodes_from_spare_if_no_where_to_reloacte_them() {
- ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig());
- enableDynamicAllocation(tester);
- tester.makeReadyNodes(2, "host", "host-small", NodeType.host, 32);
- deployZoneApp(tester);
- List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active);
- Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1");
-
- // Application 1
- ApplicationId application1 = makeApplicationId("t1", "a1");
- ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"));
- addAndAssignNode(application1, "1a", dockerHosts.get(0).hostname(), flavor, 0, tester);
- addAndAssignNode(application1, "1b", dockerHosts.get(1).hostname(), flavor, 1, tester);
-
- // Redeploy both applications (to be agnostic on which hosts are picked as spares)
- deployapp(application1, clusterSpec1, flavor, tester, 2);
-
- // Assert that we have two spare nodes (two hosts that are don't have allocations)
- Set<String> hostsWithChildren = new HashSet<>();
- for (Node node : tester.nodeRepository().getNodes(NodeType.tenant, Node.State.active)) {
- if (!isInactiveOrRetired(node)) {
- hostsWithChildren.add(node.parentHostname().get());
- }
- }
- Assert.assertEquals(2, hostsWithChildren.size());
- }
-
- @Test(expected = OutOfCapacityException.class)
- public void multiple_groups_are_on_separate_parent_hosts() {
- ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig());
- enableDynamicAllocation(tester);
- tester.makeReadyNodes(5, "host-small", NodeType.host, 32);
- deployZoneApp(tester);
- Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1");
-
- //Deploy an application of 6 nodes of 3 nodes in each cluster. We only have 3 docker hosts available
- ApplicationId application1 = tester.makeApplicationId();
- tester.prepare(application1,
- ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")),
- 6, 2, flavor.canonicalName());
-
- fail("Two groups have been allocated to the same parent host");
- }
-
@Test
public void spare_capacity_used_only_when_replacement() {
// Use spare capacity only when replacement (i.e one node is failed)
@@ -301,6 +76,9 @@ public class DynamicDockerProvisioningTest {
List<Node> finalSpareCapacity = findSpareCapacity(tester);
assertThat(finalSpareCapacity.size(), is(1));
+
+ // Uncomment the statement below to walk through the allocation events visually
+ //AllocationVisualizer.visualize(tester.getAllocationSnapshots());
}
@Test
@@ -321,29 +99,6 @@ public class DynamicDockerProvisioningTest {
assertThat(initialSpareCapacity.size(), is(0));
}
- private ApplicationId makeApplicationId(String tenant, String appName) {
- return ApplicationId.from(tenant, appName, "default");
- }
-
- private void deployapp(ApplicationId id, ClusterSpec spec, Flavor flavor, ProvisioningTester tester, int nodecount) {
- List<HostSpec> hostSpec = tester.prepare(id, spec, nodecount,1, flavor.canonicalName());
- tester.activate(id, new HashSet<>(hostSpec));
- }
-
- private Node addAndAssignNode(ApplicationId id, String hostname, String parentHostname, Flavor flavor, int index, ProvisioningTester tester) {
- Node node1a = Node.create("open1", Collections.singleton("127.0.0.100"), new HashSet<>(), hostname, Optional.of(parentHostname), flavor, NodeType.tenant);
- ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")).changeGroup(Optional.of(ClusterSpec.Group.from(0)));
- ClusterMembership clusterMembership1 = ClusterMembership.from(clusterSpec,index);
- Node node1aAllocation = node1a.allocate(id,clusterMembership1, Instant.now());
-
- tester.nodeRepository().addNodes(Collections.singletonList(node1aAllocation));
- NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(tester.getCurator()));
- tester.nodeRepository().activate(Collections.singletonList(node1aAllocation), transaction);
- transaction.commit();
-
- return node1aAllocation;
- }
-
private List<Node> findSpareCapacity(ProvisioningTester tester) {
List<Node> nodes = tester.nodeRepository().getNodes(Node.State.values());
NodeList nl = new NodeList(nodes);
@@ -353,22 +108,6 @@ public class DynamicDockerProvisioningTest {
.collect(Collectors.toList());
}
- private FlavorsConfig flavorsConfig(boolean includeHeadroom) {
- FlavorConfigBuilder b = new FlavorConfigBuilder();
- b.addFlavor("host-large", 6., 6., 6, Flavor.Type.BARE_METAL);
- b.addFlavor("host-small", 3., 3., 3, Flavor.Type.BARE_METAL);
- b.addFlavor("d-1", 1, 1., 1, Flavor.Type.DOCKER_CONTAINER);
- b.addFlavor("d-2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER);
- if (includeHeadroom) {
- b.addFlavor("d-2-4", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER, 4);
- }
- b.addFlavor("d-3", 3, 3., 3, Flavor.Type.DOCKER_CONTAINER);
- b.addFlavor("d-3-disk", 3, 3., 5, Flavor.Type.DOCKER_CONTAINER);
- b.addFlavor("d-3-mem", 3, 5., 3, Flavor.Type.DOCKER_CONTAINER);
- b.addFlavor("d-3-cpu", 5, 3., 3, Flavor.Type.DOCKER_CONTAINER);
- return b.build();
- }
-
private FlavorsConfig flavorsConfig() {
FlavorConfigBuilder b = new FlavorConfigBuilder();
b.addFlavor("host-large", 6., 6., 6, Flavor.Type.BARE_METAL);
@@ -397,14 +136,4 @@ public class DynamicDockerProvisioningTest {
private void enableDynamicAllocation(ProvisioningTester tester) {
tester.getCurator().set(Path.fromString("/provision/v1/dynamicDockerAllocation"), new byte[0]);
}
-
- private boolean isInactiveOrRetired(Node node) {
- boolean isInactive = node.state().equals(Node.State.inactive);
- boolean isRetired = false;
- if (node.allocation().isPresent()) {
- isRetired = node.allocation().get().membership().retired();
- }
-
- return isInactive || isRetired;
- }
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index bd50a9fc581..f576f11cd64 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -226,10 +226,6 @@ public class ProvisioningTester implements AutoCloseable {
}
public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int additionalIps) {
- return makeReadyNodes(n, UUID.randomUUID().toString(), flavor, type, additionalIps);
- }
-
- public List<Node> makeReadyNodes(int n, String prefix, String flavor, NodeType type, int additionalIps) {
List<Node> nodes = new ArrayList<>(n);
for (int i = 0; i < n; i++) {
Set<String> ips = IntStream.range(additionalIps * i, additionalIps * (i+1))
@@ -237,7 +233,7 @@ public class ProvisioningTester implements AutoCloseable {
.collect(Collectors.toSet());
nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(),
- prefix + i,
+ UUID.randomUUID().toString(),
Collections.emptySet(),
ips,
Optional.empty(),
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index 8a74f3aff24..dd2b64a7854 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -13,7 +13,6 @@ import com.yahoo.vespa.config.SlimeUtils;
import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig;
import org.junit.After;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
@@ -42,7 +41,6 @@ public class RestApiTest {
/** This test gives examples of all the requests that can be made to nodes/v2 */
@Test
- @Ignore /** TODO re-enable this and verify correctness */
public void test_requests() throws Exception {
// GET
assertFile(new Request("http://localhost:8080/nodes/v2/"), "root.json");
@@ -312,33 +310,33 @@ public class RestApiTest {
assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/" + hostname,
new byte[0], Request.Method.PUT),
"{\"message\":\"Moved foo.yahoo.com to ready\"}");
- Pattern responsePattern = Pattern.compile("\\{\"trustedNodes\":\\[.*" +
+ Pattern responsePattern = Pattern.compile("\\{\"trustedNodes\":\\[" +
"\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"foo.yahoo.com\"}," +
"\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"foo.yahoo.com\"}," +
"\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"foo.yahoo.com\"}" +
- ".*],\"trustedNetworks\":\\[\\]}");
+ "],\"trustedNetworks\":\\[\\]}");
assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/" + hostname), responsePattern);
}
@Test
public void acl_request_by_config_server() throws Exception {
- Pattern responsePattern = Pattern.compile("\\{\"trustedNodes\":\\[.*" +
+ Pattern responsePattern = Pattern.compile("\\{\"trustedNodes\":\\[" +
"\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"cfg1\"}," +
"\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"cfg1\"}," +
"\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"cfg1\"}" +
- ".*],\"trustedNetworks\":\\[\\]}");
+ "],\"trustedNetworks\":\\[\\]}");
assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/cfg1"), responsePattern);
}
@Test
public void acl_request_by_docker_host() throws Exception {
Pattern responsePattern = Pattern.compile("\\{\"trustedNodes\":\\[" +
- "\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"dockerhost1.yahoo.com\"}," +
- "\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"dockerhost1.yahoo.com\"}," +
- "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"dockerhost1.yahoo.com\"}]," +
+ "\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"parent1.yahoo.com\"}," +
+ "\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"parent1.yahoo.com\"}," +
+ "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"parent1.yahoo.com\"}]," +
"\"trustedNetworks\":\\[" +
- "\\{\"network\":\"172.17.0.0/16\",\"trustedBy\":\"dockerhost1.yahoo.com\"}]}");
- assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/dockerhost1.yahoo.com"), responsePattern);
+ "\\{\"network\":\"172.17.0.0/16\",\"trustedBy\":\"parent1.yahoo.com\"}]}");
+ assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/parent1.yahoo.com"), responsePattern);
}
@Test
@@ -349,8 +347,8 @@ public class RestApiTest {
"\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"host1.yahoo.com\"}," +
"\\{\"hostname\":\"host1.yahoo.com\",\"ipAddress\":\"::1\",\"trustedBy\":\"host1.yahoo.com\"}," +
"\\{\"hostname\":\"host1.yahoo.com\",\"ipAddress\":\"127.0.0.1\",\"trustedBy\":\"host1.yahoo.com\"}," +
- "\\{\"hostname\":\"host10.yahoo.com\",\"ipAddress\":\"::1\",\"trustedBy\":\"host1.yahoo.com\"}," +
- "\\{\"hostname\":\"host10.yahoo.com\",\"ipAddress\":\"127.0.0.1\",\"trustedBy\":\"host1.yahoo.com\"}" +
+ "\\{\"hostname\":\"host2.yahoo.com\",\"ipAddress\":\"::1\",\"trustedBy\":\"host1.yahoo.com\"}," +
+ "\\{\"hostname\":\"host2.yahoo.com\",\"ipAddress\":\"127.0.0.1\",\"trustedBy\":\"host1.yahoo.com\"}" +
"],\"trustedNetworks\":\\[\\]}");
assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/host1.yahoo.com"), responsePattern);
}
@@ -372,8 +370,7 @@ public class RestApiTest {
"{\"message\":\"Moved host1.yahoo.com to failed\"}");
assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com",
new byte[0], Request.Method.PUT),
- 400,
- "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can not set failed node host1.yahoo.com allocated to tenant1.application1.instance1 as 'container/id1/0/0' ready. It is not dirty.\"}");
+ 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can not set failed node host1.yahoo.com allocated to tenant2.application2.instance2 as 'content/id2/0/0' ready. It is not dirty.\"}");
// (... while dirty then ready works (the ready move will be initiated by node maintenance))
assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host1.yahoo.com",
@@ -389,7 +386,7 @@ public class RestApiTest {
"{\"message\":\"Moved host2.yahoo.com to parked\"}");
assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com",
new byte[0], Request.Method.PUT),
- 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can not set parked node host2.yahoo.com allocated to tenant2.application2.instance2 as 'content/id2/0/0' ready. It is not dirty.\"}");
+ 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can not set parked node host2.yahoo.com allocated to tenant2.application2.instance2 as 'content/id2/0/1' ready. It is not dirty.\"}");
// (... while dirty then ready works (the ready move will be initiated by node maintenance))
assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host2.yahoo.com",
new byte[0], Request.Method.PUT),
@@ -446,18 +443,18 @@ public class RestApiTest {
@Test
public void test_hardware_patching_of_docker_host() throws Exception {
assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com"), Optional.of(false));
- assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), Optional.of(false));
+ assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/parent1.yahoo.com"), Optional.of(false));
- assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com",
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/parent1.yahoo.com",
Utf8.toBytes("{" +
"\"hardwareFailureType\": \"memory_mcelog\"" +
"}"
),
Request.Method.PATCH),
- "{\"message\":\"Updated dockerhost2.yahoo.com\"}");
+ "{\"message\":\"Updated parent1.yahoo.com\"}");
assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com"), Optional.of(true));
- assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), Optional.of(true));
+ assertHardwareFailure(new Request("http://localhost:8080/nodes/v2/node/parent1.yahoo.com"), Optional.of(true));
}
@Test
@@ -509,8 +506,7 @@ public class RestApiTest {
private JDisc container;
@Before
- public void startContainer() {
- container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); }
+ public void startContainer() { container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); }
@After
public void stopContainer() { container.close(); }
@@ -562,10 +558,10 @@ public class RestApiTest {
private void assertHardwareFailure(Request request, Optional<Boolean> expectedHardwareFailure) throws CharacterCodingException {
Response response = container.handleRequest(request);
+ assertEquals(response.getStatus(), 200);
String json = response.getBodyAsString();
Optional<Boolean> actualHardwareFailure = getHardwareFailure(json);
assertEquals(expectedHardwareFailure, actualHardwareFailure);
- assertEquals(200, response.getStatus());
}
/** Asserts a particular response and 200 as response status */
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json
index 075ce1693cb..9c299cb6fe8 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json
@@ -1,55 +1,41 @@
{
"url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com",
"id": "host1.yahoo.com",
- "state": "reserved",
+ "state": "active",
"type": "tenant",
"hostname": "host1.yahoo.com",
"openStackId": "node1",
"flavor": "default",
"canonicalFlavor": "default",
- "minDiskAvailableGb": 400.0,
- "minMainMemoryAvailableGb": 16.0,
- "description": "Flavor-name-is-default",
- "minCpuCores": 2.0,
- "fastDisk": true,
- "environment": "BARE_METAL",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "fastDisk":true,
+ "environment":"BARE_METAL",
"owner": {
- "tenant": "tenant1",
- "application": "application1",
- "instance": "instance1"
+ "tenant": "tenant2",
+ "application": "application2",
+ "instance": "instance2"
},
"membership": {
- "clustertype": "container",
- "clusterid": "id1",
+ "clustertype": "content",
+ "clusterid": "id2",
"group": "0",
"index": 0,
"retired": false
},
"restartGeneration": 0,
"currentRestartGeneration": 0,
- "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0",
- "wantedVespaVersion": "6.42.0",
+ "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0",
+ "wantedVespaVersion":"6.42.0",
"rebootGeneration": 1,
"currentRebootGeneration": 0,
"failCount": 0,
- "hardwareFailure": false,
- "wantToRetire": false,
- "wantToDeprovision": false,
- "history": [
- {
- "event": "readied",
- "at": 123,
- "agent": "system"
- },
- {
- "event": "reserved",
- "at": 123,
- "agent": "application"
- }
- ],
- "ipAddresses": [
- "::1",
- "127.0.0.1"
- ],
- "additionalIpAddresses": []
-} \ No newline at end of file
+ "hardwareFailure" : false,
+ "wantToRetire" : false,
+ "wantToDeprovision" : false,
+ "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}],
+ "ipAddresses":["::1", "127.0.0.1"],
+ "additionalIpAddresses":[]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json
index 120d6286634..f788b5c6e59 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json
@@ -8,12 +8,12 @@
"openStackId": "node10",
"flavor": "default",
"canonicalFlavor": "default",
- "minDiskAvailableGb": 400.0,
- "minMainMemoryAvailableGb": 16.0,
- "description": "Flavor-name-is-default",
- "minCpuCores": 2.0,
- "fastDisk": true,
- "environment": "BARE_METAL",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "fastDisk":true,
+ "environment":"BARE_METAL",
"owner": {
"tenant": "tenant1",
"application": "application1",
@@ -23,12 +23,12 @@
"clustertype": "container",
"clusterid": "id1",
"group": "0",
- "index": 1,
+ "index": 0,
"retired": false
},
"restartGeneration": 0,
"currentRestartGeneration": 0,
- "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0",
+ "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0",
"wantedVespaVersion": "6.42.0",
"rebootGeneration": 1,
"currentRebootGeneration": 0,
@@ -37,24 +37,10 @@
"hostedVersion": "5.104.142",
"convergedStateVersion": "5.104.142",
"failCount": 0,
- "hardwareFailure": false,
- "wantToRetire": false,
- "wantToDeprovision": false,
- "history": [
- {
- "event": "readied",
- "at": 123,
- "agent": "system"
- },
- {
- "event": "reserved",
- "at": 123,
- "agent": "application"
- }
- ],
- "ipAddresses": [
- "::1",
- "127.0.0.1"
- ],
- "additionalIpAddresses": []
-} \ No newline at end of file
+ "hardwareFailure" : false,
+ "wantToRetire" : false,
+ "wantToDeprovision" : false,
+ "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"}],
+ "ipAddresses":["::1", "127.0.0.1"],
+ "additionalIpAddresses":[]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json
index 52864fc165c..6a4522bf0a4 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json
@@ -7,12 +7,12 @@
"openStackId": "node2",
"flavor": "default",
"canonicalFlavor": "default",
- "minDiskAvailableGb": 400.0,
- "minMainMemoryAvailableGb": 16.0,
- "description": "Flavor-name-is-default",
- "minCpuCores": 2.0,
- "fastDisk": true,
- "environment": "BARE_METAL",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "fastDisk":true,
+ "environment":"BARE_METAL",
"owner": {
"tenant": "tenant2",
"application": "application2",
@@ -22,39 +22,20 @@
"clustertype": "content",
"clusterid": "id2",
"group": "0",
- "index": 0,
+ "index": 1,
"retired": false
},
"restartGeneration": 0,
"currentRestartGeneration": 0,
- "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0",
- "wantedVespaVersion": "6.42.0",
+ "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0",
+ "wantedVespaVersion":"6.42.0",
"rebootGeneration": 1,
"currentRebootGeneration": 0,
"failCount": 0,
- "hardwareFailure": false,
- "wantToRetire": false,
- "wantToDeprovision": false,
- "history": [
- {
- "event": "readied",
- "at": 123,
- "agent": "system"
- },
- {
- "event": "reserved",
- "at": 123,
- "agent": "application"
- },
- {
- "event": "activated",
- "at": 123,
- "agent": "application"
- }
- ],
- "ipAddresses": [
- "::1",
- "127.0.0.1"
- ],
- "additionalIpAddresses": []
-} \ No newline at end of file
+ "hardwareFailure" : false,
+ "wantToRetire" : false,
+ "wantToDeprovision" : false,
+ "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}],
+ "ipAddresses":["::1", "127.0.0.1"],
+ "additionalIpAddresses":[]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json
index 7782cf15e50..f05e2cd578a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json
@@ -1,32 +1,39 @@
{
"url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com",
"id": "host3.yahoo.com",
- "state": "ready",
+ "state": "active",
"type": "tenant",
"hostname": "host3.yahoo.com",
"openStackId": "node3",
- "flavor": "expensive",
- "canonicalFlavor": "default",
- "description": "Flavor-name-is-expensive",
- "cost": 200,
- "fastDisk": true,
- "environment": "BARE_METAL",
+ "flavor":"expensive",
+ "canonicalFlavor":"default",
+ "description":"Flavor-name-is-expensive",
+ "cost":200,
+ "fastDisk":true,
+ "environment":"BARE_METAL",
+ "owner": {
+ "tenant": "tenant3",
+ "application": "application3",
+ "instance": "instance3"
+ },
+ "membership": {
+ "clustertype": "content",
+ "clusterid": "id3",
+ "group": "0",
+ "index": 1,
+ "retired": false
+ },
+ "restartGeneration": 0,
+ "currentRestartGeneration": 0,
+ "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0",
+ "wantedVespaVersion":"6.42.0",
"rebootGeneration": 1,
"currentRebootGeneration": 0,
"failCount": 0,
- "hardwareFailure": false,
- "wantToRetire": false,
- "wantToDeprovision": false,
- "history": [
- {
- "event": "readied",
- "at": 123,
- "agent": "system"
- }
- ],
- "ipAddresses": [
- "::1",
- "127.0.0.1"
- ],
- "additionalIpAddresses": []
-} \ No newline at end of file
+ "hardwareFailure" : false,
+ "wantToRetire" : false,
+ "wantToDeprovision" : false,
+ "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}],
+ "ipAddresses":["::1", "127.0.0.1"],
+ "additionalIpAddresses":[]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json
index 10b5689f8ce..efdde53ffb8 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json
@@ -1,34 +1,34 @@
{
"url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com",
"id": "host4.yahoo.com",
- "state": "active",
+ "state": "reserved",
"type": "tenant",
"hostname": "host4.yahoo.com",
- "parentHostname": "dockerhost1.yahoo.com",
+ "parentHostname":"dockerhost4",
"openStackId": "node4",
- "flavor": "docker",
- "canonicalFlavor": "docker",
- "minDiskAvailableGb": 100.0,
- "minMainMemoryAvailableGb": 0.5,
- "description": "Flavor-name-is-docker",
- "minCpuCores": 0.2,
- "fastDisk": true,
- "environment": "DOCKER_CONTAINER",
+ "flavor": "default",
+ "canonicalFlavor": "default",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "fastDisk":true,
+ "environment":"BARE_METAL",
"owner": {
- "tenant": "tenant3",
- "application": "application3",
- "instance": "instance3"
+ "tenant": "tenant1",
+ "application": "application1",
+ "instance": "instance1"
},
"membership": {
- "clustertype": "content",
- "clusterid": "id3",
+ "clustertype": "container",
+ "clusterid": "id1",
"group": "0",
- "index": 0,
+ "index": 1,
"retired": false
},
"restartGeneration": 0,
"currentRestartGeneration": 0,
- "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0",
+ "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0",
"wantedVespaVersion": "6.42.0",
"rebootGeneration": 1,
"currentRebootGeneration": 0,
@@ -37,29 +37,10 @@
"hostedVersion": "6.41.0",
"convergedStateVersion": "6.41.0",
"failCount": 0,
- "hardwareFailure": false,
- "wantToRetire": false,
- "wantToDeprovision": false,
- "history": [
- {
- "event": "readied",
- "at": 123,
- "agent": "system"
- },
- {
- "event": "reserved",
- "at": 123,
- "agent": "application"
- },
- {
- "event": "activated",
- "at": 123,
- "agent": "application"
- }
- ],
- "ipAddresses": [
- "::1",
- "127.0.0.1"
- ],
- "additionalIpAddresses": []
-} \ No newline at end of file
+ "hardwareFailure" : false,
+ "wantToRetire" : false,
+ "wantToDeprovision" : false,
+ "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"}],
+ "ipAddresses":["::1", "127.0.0.1"],
+ "additionalIpAddresses":[]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json
index bf81509b79a..0d0fda0b594 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json
@@ -4,37 +4,23 @@
"state": "failed",
"type": "tenant",
"hostname": "host5.yahoo.com",
- "parentHostname": "dockerhost2.yahoo.com",
+ "parentHostname":"parent1.yahoo.com",
"openStackId": "node5",
- "flavor": "docker",
- "canonicalFlavor": "docker",
- "minDiskAvailableGb": 100.0,
- "minMainMemoryAvailableGb": 0.5,
- "description": "Flavor-name-is-docker",
- "minCpuCores": 0.2,
- "fastDisk": true,
- "environment": "DOCKER_CONTAINER",
+ "flavor": "default",
+ "canonicalFlavor": "default",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "fastDisk":true,
+ "environment":"BARE_METAL",
"rebootGeneration": 1,
"currentRebootGeneration": 0,
"failCount": 1,
"hardwareFailure": false,
"wantToRetire": false,
- "wantToDeprovision": false,
- "history": [
- {
- "event": "readied",
- "at": 123,
- "agent": "system"
- },
- {
- "event": "failed",
- "at": 123,
- "agent": "system"
- }
- ],
- "ipAddresses": [
- "::1",
- "127.0.0.1"
- ],
- "additionalIpAddresses": []
-} \ No newline at end of file
+ "wantToDeprovision" : false,
+ "history":[{"event":"readied","at":123,"agent":"system"},{"event":"failed","at":123,"agent":"system"}],
+ "ipAddresses":["::1", "127.0.0.1"],
+ "additionalIpAddresses":[]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json
index 1fc001fa224..35805e3b20f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json
@@ -4,16 +4,16 @@
"state": "failed",
"type": "tenant",
"hostname": "host5.yahoo.com",
- "parentHostname": "dockerhost2.yahoo.com",
+ "parentHostname":"parent1.yahoo.com",
"openStackId": "node5",
- "flavor": "docker",
- "canonicalFlavor": "docker",
- "minDiskAvailableGb": 100.0,
- "minMainMemoryAvailableGb": 0.5,
- "description": "Flavor-name-is-docker",
- "minCpuCores": 0.2,
- "fastDisk": true,
- "environment": "DOCKER_CONTAINER",
+ "flavor": "default",
+ "canonicalFlavor": "default",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "fastDisk":true,
+ "environment":"BARE_METAL",
"rebootGeneration": 1,
"currentRebootGeneration": 0,
"vespaVersion": "1.2.3",
@@ -21,24 +21,10 @@
"hostedVersion": "1.2.3",
"convergedStateVersion": "1.2.3",
"failCount": 1,
- "hardwareFailure": false,
- "wantToRetire": false,
- "wantToDeprovision": false,
- "history": [
- {
- "event": "readied",
- "at": 123,
- "agent": "system"
- },
- {
- "event": "failed",
- "at": 123,
- "agent": "system"
- }
- ],
- "ipAddresses": [
- "::1",
- "127.0.0.1"
- ],
- "additionalIpAddresses": []
+ "hardwareFailure" : false,
+ "wantToRetire" : false,
+ "wantToDeprovision" : false,
+ "history":[{"event":"readied","at":123,"agent":"system"},{"event":"failed","at":123,"agent":"system"}],
+ "ipAddresses":["::1", "127.0.0.1"],
+ "additionalIpAddresses":[]
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json
index 750ebbd695e..25a9b8554d8 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json
@@ -7,54 +7,35 @@
"openStackId": "node6",
"flavor": "default",
"canonicalFlavor": "default",
- "minDiskAvailableGb": 400.0,
- "minMainMemoryAvailableGb": 16.0,
- "description": "Flavor-name-is-default",
- "minCpuCores": 2.0,
- "fastDisk": true,
- "environment": "BARE_METAL",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "fastDisk":true,
+ "environment":"BARE_METAL",
"owner": {
- "tenant": "tenant2",
- "application": "application2",
- "instance": "instance2"
+ "tenant": "tenant3",
+ "application": "application3",
+ "instance": "instance3"
},
"membership": {
"clustertype": "content",
- "clusterid": "id2",
+ "clusterid": "id3",
"group": "0",
- "index": 1,
+ "index": 0,
"retired": false
},
"restartGeneration": 0,
"currentRestartGeneration": 0,
- "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0",
- "wantedVespaVersion": "6.42.0",
+ "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0",
+ "wantedVespaVersion":"6.42.0",
"rebootGeneration": 1,
"currentRebootGeneration": 0,
"failCount": 0,
- "hardwareFailure": false,
- "wantToRetire": false,
- "wantToDeprovision": false,
- "history": [
- {
- "event": "readied",
- "at": 123,
- "agent": "system"
- },
- {
- "event": "reserved",
- "at": 123,
- "agent": "application"
- },
- {
- "event": "activated",
- "at": 123,
- "agent": "application"
- }
- ],
- "ipAddresses": [
- "::1",
- "127.0.0.1"
- ],
- "additionalIpAddresses": []
-} \ No newline at end of file
+ "hardwareFailure" : false,
+ "wantToRetire" : false,
+ "wantToDeprovision" : false,
+ "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}],
+ "ipAddresses":["::1", "127.0.0.1"],
+ "additionalIpAddresses":[]
+}