aboutsummaryrefslogtreecommitdiffstats
path: root/node-repository
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
committerJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
commit72231250ed81e10d66bfe70701e64fa5fe50f712 (patch)
tree2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /node-repository
Publish
Diffstat (limited to 'node-repository')
-rw-r--r--node-repository/OWNERS2
-rw-r--r--node-repository/README1
-rw-r--r--node-repository/pom.xml137
-rw-r--r--node-repository/src/main/config/node-repository.xml21
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java242
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java318
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClient.java157
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/XmlUtils.java68
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java57
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java39
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java66
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java62
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java39
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java65
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java160
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java104
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java33
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java70
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetrics.java59
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java71
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Configuration.java31
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Flavor.java124
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Generation.java48
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java123
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeFlavors.java70
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java93
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java61
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java31
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeHostFilter.java56
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java38
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java38
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java52
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java7
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java51
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java197
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java256
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorMutex.java51
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java273
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java112
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java66
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java345
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java103
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java116
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java7
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializer.java49
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ContainersForHost.java41
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/HostInfo.java27
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionEndpoint.java25
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResource.java151
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionStatus.java17
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/TenantStatus.java31
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/package-info.java10
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v1/NodesApiHandler.java117
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ErrorResponse.java58
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/MessageResponse.java39
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java109
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java232
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java214
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java15
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ResourcesResponse.java48
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java24
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/FlavorConfigBuilder.java52
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeFlavors.java32
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java98
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md3
-rw-r--r--node-repository/src/main/resources/configdefinitions/node-repository.def34
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeList.java53
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClientTest.java103
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java148
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java119
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java102
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MockDeployer.java104
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java350
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java68
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java128
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java89
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/NodeFlavorsTest.java57
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java68
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java119
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java240
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java59
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java196
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionTest.java525
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java249
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java302
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializerTest.java44
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResourceTest.java148
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java113
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java268
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json8
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json6
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json33
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json38
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json20
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json33
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json30
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json48
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json36
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json21
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json33
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json19
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json19
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json19
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json12
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json28
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json5
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json13
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json47
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json25
-rw-r--r--node-repository/src/test/resources/hosts.xml27
-rw-r--r--node-repository/src/test/resources/services.xml22
111 files changed, 9740 insertions, 0 deletions
diff --git a/node-repository/OWNERS b/node-repository/OWNERS
new file mode 100644
index 00000000000..cc4d4971a75
--- /dev/null
+++ b/node-repository/OWNERS
@@ -0,0 +1,2 @@
+bratseth
+musum
diff --git a/node-repository/README b/node-repository/README
new file mode 100644
index 00000000000..c6050e560ed
--- /dev/null
+++ b/node-repository/README
@@ -0,0 +1 @@
+Node repository component that manages node (de)allocation in hosted vespa.
diff --git a/node-repository/pom.xml b/node-repository/pom.xml
new file mode 100644
index 00000000000..da48ff9eb4b
--- /dev/null
+++ b/node-repository/pom.xml
@@ -0,0 +1,137 @@
+<?xml version="1.0"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>parent</artifactId>
+ <version>6-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
+ </parent>
+ <artifactId>node-repository</artifactId>
+ <version>6-SNAPSHOT</version>
+ <packaging>container-plugin</packaging>
+ <description>Keeps track of node assignment in a multi-application setup.</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-dev</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>application</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespa_jersey2</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ <type>pom</type>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-jersey2</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-provisioning</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>application-model</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>service-monitor</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>zkfacade</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>testutil</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-joda</artifactId>
+ <version>${jackson2.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>orchestrator</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <compilerArgs>
+ <arg>-Xlint:all</arg>
+ <arg>-Xlint:-try</arg>
+ <arg>-Xlint:-serial</arg>
+ <arg>-Werror</arg>
+ </compilerArgs>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml
new file mode 100644
index 00000000000..de9d5da1eef
--- /dev/null
+++ b/node-repository/src/main/config/node-repository.xml
@@ -0,0 +1,21 @@
+<!-- services.xml snippet for the node repository. Included in config server services.xml if the package is installed-->
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<component id="com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner" bundle="node-repository" />
+<component id="NodeRepository" class="com.yahoo.vespa.hosted.provision.NodeRepository" bundle="node-repository"/>
+<component id="com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance" bundle="node-repository"/>
+<component id="com.yahoo.vespa.hosted.provision.monitoring.ProvisionMetrics" bundle="node-repository" />
+<component id="com.yahoo.vespa.hosted.provision.node.NodeFlavors" bundle="node-repository" />
+
+<rest-api path="hack" jersey2="true">
+ <components bundle="node-repository" />
+</rest-api>
+
+<handler id="com.yahoo.vespa.hosted.provision.restapi.v1.NodesApiHandler" bundle="node-repository">
+ <binding>http://*/nodes/v1/</binding>
+</handler>
+
+<handler id="com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler" bundle="node-repository">
+ <binding>http://*/nodes/v2/*</binding>
+</handler>
+
+<preprocess:include file="node-flavors.xml" required="false" />
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
new file mode 100644
index 00000000000..2ab98f0c582
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
@@ -0,0 +1,242 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.Flavor;
+import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.Status;
+import com.yahoo.vespa.hosted.provision.node.Generation;
+
+import java.time.Instant;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * A node in the node repository. The identity of a node is given by its id.
+ * The classes making up the node model are found in the node package.
+ * This (and hence all classes referenced from it) is immutable.
+ *
+ * @author bratseth
+ */
+public final class Node {
+
+ private final String id;
+ private final String hostname;
+ private final String openStackId;
+ private final Optional<String> parentHostname;
+ private final Configuration configuration;
+ private final Status status;
+ private final State state;
+
+ /** Record of the last event of each type happening to this node */
+ private final History history;
+
+ /** The current allocation of this node, if any */
+ private Optional<Allocation> allocation;
+
+ /** Creates a node in the initial state (provisioned) */
+ public static Node create(String openStackId, String hostname, Optional<String> parentHostname, Configuration configuration) {
+ return new Node(openStackId, hostname, parentHostname, configuration, Status.initial(), State.provisioned,
+ Optional.empty(), History.empty());
+ }
+
+ /** Do not use. Construct nodes by calling {@link NodeRepository#createNode} */
+ public Node(String openStackId, String hostname, Optional<String> parentHostname,
+ Configuration configuration, Status status, State state, Allocation allocation, History history) {
+ this(openStackId, hostname, parentHostname, configuration, status, state, Optional.of(allocation), history);
+ }
+
+ public Node(String openStackId, String hostname, Optional<String> parentHostname,
+ Configuration configuration, Status status, State state, Optional<Allocation> allocation, History history) {
+ Objects.requireNonNull(openStackId, "A node must have an openstack id");
+ Objects.requireNonNull(hostname, "A node must have a hostname");
+ Objects.requireNonNull(parentHostname, "A null parentHostname is not permitted.");
+ Objects.requireNonNull(configuration, "A node must have a configuration");
+ Objects.requireNonNull(status, "A node must have a status");
+ Objects.requireNonNull(state, "A null node state is not permitted");
+ Objects.requireNonNull(allocation, "A null node allocation is not permitted");
+ Objects.requireNonNull(history, "A null node history is not permitted");
+
+ this.id = hostname;
+ this.hostname = hostname;
+ this.parentHostname = parentHostname;
+ this.openStackId = openStackId;
+ this.configuration = configuration;
+ this.status = status;
+ this.state = state;
+ this.allocation = allocation;
+ this.history = history;
+ }
+
+ /**
+ * Returns the unique id of this host.
+ * This may be the host name or some other opaque id which is unique across hosts
+ */
+ public String id() { return id; }
+
+ /** Returns the host name of this node */
+ public String hostname() { return hostname; }
+
+ // TODO: Different meaning for vms and docker hosts?
+ /** Returns the OpenStack id of this node, or of its docker host if this is a virtual node */
+ public String openStackId() { return openStackId; }
+
+ /** Returns the parent hostname for this node if this node is a docker container or a VM (i.e. it has a parent host). Otherwise, empty **/
+ public Optional<String> parentHostname() { return parentHostname; }
+
+ /** Returns the hardware configuration of this node */
+ public Configuration configuration() { return configuration; }
+
+ /** Returns the known information about the nodes ephemeral status */
+ public Status status() { return status; }
+
+ /** Returns the current state of this node (in the node state machine) */
+ public State state() { return state; }
+
+ /** Returns the current allocation of this, if any */
+ public Optional<Allocation> allocation() { return allocation; }
+
+ /** Returns a history of the last events happening to this node */
+ public History history() { return history; }
+
+ /**
+ * Returns a copy of this node which is retired by the application owning it.
+ * If the node was already retired it is returned as-is.
+ */
+ public Node retireByApplication(Instant retiredAt) {
+ if (allocation().get().membership().retired()) return this;
+ return setAllocation(allocation.get().retire())
+ .setHistory(history.record(new History.RetiredEvent(retiredAt, History.RetiredEvent.Agent.application)));
+ }
+
+ /** Returns a copy of this node which is retired by the system */
+ // We will use this when we support operators retiring a flavor completely from hosted Vespa
+ public Node retireBySystem(Instant retiredAt) {
+ return setAllocation(allocation.get().retire())
+ .setHistory(history.record(new History.RetiredEvent(retiredAt, History.RetiredEvent.Agent.system)));
+ }
+
+ /** Returns a copy of this node which is not retired */
+ public Node unretire() {
+ return setAllocation(allocation.get().unretire());
+ }
+
+ /** Returns a copy of this with the current generation set to generation */
+ public Node setRestart(Generation generation) {
+ final Optional<Allocation> allocation = this.allocation;
+ if ( ! allocation.isPresent())
+ throw new IllegalArgumentException("Cannot set restart generation for " + hostname() + ": The node is unallocated");
+
+ return setAllocation(allocation.get().setRestart(generation));
+ }
+
+ /** Returns a node with the status assigned to the given value */
+ public Node setStatus(Status status) {
+ return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history);
+ }
+
+ /** Returns a node with the hardware configuration assigned to the given value */
+ public Node setConfiguration(Configuration configuration) {
+ return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history);
+ }
+
+ /** Returns a copy of this with the current generation set to generation */
+ public Node setReboot(Generation generation) {
+ return new Node(openStackId, hostname, parentHostname, configuration, status.setReboot(generation), state,
+ allocation, history);
+ }
+
+ /** Returns a copy of this with the flavor set to flavor */
+ public Node setFlavor(Flavor flavor) {
+ return new Node(openStackId, hostname, parentHostname, new Configuration(flavor), status, state,
+ allocation, history);
+ }
+
+ /** Returns a copy of this with a history record saying it was detected to be down at this instant */
+ public Node setDown(Instant instant) {
+ return setHistory(history.record(new History.Event(History.Event.Type.down, instant)));
+ }
+
+ /** Returns a copy of this with any history record saying it has been detected down removed */
+ public Node setUp() {
+ return setHistory(history.clear(History.Event.Type.down));
+ }
+
+ /** Returns a copy of this with allocation set as specified. <code>node.state</code> is *not* changed. */
+ public Node allocate(ApplicationId owner, ClusterMembership membership, Instant at) {
+ return setAllocation(new Allocation(owner, membership, new Generation(0, 0), false))
+ .setHistory(history.record(new History.Event(History.Event.Type.reserved, at)));
+ }
+
+ /**
+ * Returns a copy of this node with the allocation assigned to the given allocation.
+ * Do not use this to allocate a node.
+ */
+ public Node setAllocation(Allocation allocation) {
+ return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history);
+ }
+
+ /** Returns a copy of this node with the parent hostname assigned to the given value. */
+ public Node setParentHostname(String parentHostname) {
+ return new Node(openStackId, hostname, Optional.of(parentHostname), configuration, status, state, allocation, history);
+ }
+
+ /** Returns a copy of this node with the given history. */
+ private Node setHistory(History history) {
+ return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history);
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) return true;
+ if ( ! other.getClass().equals(this.getClass())) return false;
+ return ((Node)other).id.equals(this.id);
+ }
+
+ @Override
+ public String toString() {
+ return state + " node " +
+ (hostname !=null ? hostname : id) +
+ (allocation.isPresent() ? " " + allocation.get() : "") +
+ (parentHostname.isPresent() ? " [on: " + parentHostname.get() + "]" : "");
+ }
+
+ public enum State {
+
+ /** This node has been requested (from OpenStack) but is not yet read for use */
+ provisioned,
+
+ /** This node is free and ready for use */
+ ready,
+
+ /** This node has been reserved by an application but is not yet used by it */
+ reserved,
+
+ /** This node is in active use by an application */
+ active,
+
+ /** This node has been used by an application, is still allocated to it and retains the data needed for its allocated role */
+ inactive,
+
+ /** This node is not allocated to an application but may contain data which must be cleaned before it is ready */
+ dirty,
+
+ /** This node has failed and must be repaired or removed. The node retains any allocation data for diagnosis. */
+ failed;
+
+ /** Returns whether this is a state where the node is assigned to an application */
+ public boolean isAllocated() {
+ return this == reserved || this == active || this == inactive || this == failed;
+ }
+
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
new file mode 100644
index 00000000000..1980b1f5318
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -0,0 +1,318 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision;
+
+import com.google.inject.Inject;
+import com.yahoo.collections.ListMap;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.transaction.Mutex;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter;
+import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter;
+import com.yahoo.vespa.hosted.provision.node.filter.StateFilter;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.UnaryOperator;
+import java.util.stream.Collectors;
+
+/**
+ * The hosted Vespa production node repository, which stores its state in Zookeeper.
+ * The node repository knows about all nodes in a zone, their states and manages all transitions between
+ * node states.
+ * <p>
+ * Node repo locking: Locks must be acquired before making changes to the set of nodes, or to the content
+ * of the nodes.
+ * Unallocated states use a single lock, while application level locks are used for all allocated states
+ * such that applications can mostly change in parallel.
+ * If both locks are needed acquire the application lock first, then the unallocated lock.
+ * <p>
+ * Changes to the set of active nodes must be accompanied by changes to the config model of the application.
+ * Such changes are not handled by the node repository but by the classes calling it - see
+ * {@link com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner} for such changes initiated
+ * by the application package and {@link com.yahoo.vespa.hosted.provision.maintenance.ApplicationMaintainer}
+ * for changes initiated by the node repository.
+ * Refer to {@link com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance} for timing details
+ * of the node state transitions.
+ *
+ * @author bratseth
+ */
+// Node state transitions:
+// 1) (new) - > provisioned -> ready -> reserved -> active -> inactive -> dirty -> ready
+// 2) inactive -> reserved
+// 3) reserved -> dirty
+// 3) * -> failed -> dirty | active | (removed)
+// Nodes have an application assigned when in states reserved, active and inactive.
+// Nodes might have an application assigned in dirty.
+public class NodeRepository extends AbstractComponent {
+
+ private final CuratorDatabaseClient zkClient;
+
+ /**
+ * Creates a node repository form a zookeeper provider.
+ * This will use the system time to make time-sensitive decisions
+ */
+ @Inject
+ public NodeRepository(NodeFlavors flavors, Curator curator) {
+ this(flavors, curator, Clock.systemUTC());
+ }
+
+ /**
+ * Creates a node repository form a zookeeper provider and a clock instance
+ * which will be used for time-sensitive decisions.
+ */
+ public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock) {
+ this.zkClient = new CuratorDatabaseClient(flavors, curator, clock);
+
+ // read and write all nodes to make sure they are stored in the latest version of the serialized format
+ for (Node.State state : Node.State.values())
+ zkClient.writeTo(state, zkClient.getNodes(state));
+ }
+
+ // ---------------- Query API ----------------------------------------------------------------
+
+ /** Finds and returns the node with the given hostname */
+ public Optional<Node> getNode(String hostname) {
+ for (Node.State state : Node.State.values()) {
+ Optional<Node> node = getNode(state, hostname);
+ if (node.isPresent())
+ return node;
+ }
+ return Optional.empty();
+ }
+
+ /** Finds and returns the node with the given state and hostname, or empty if not found */
+ public Optional<Node> getNode(Node.State state, String hostname) {
+ return zkClient.getNode(state, hostname);
+ }
+
+ public List<Node> getNodes(Node.State ... inState) { return zkClient.getNodes(inState); }
+ public List<Node> getNodes(ApplicationId id, Node.State ... inState) { return zkClient.getNodes(id, inState); }
+ public List<Node> getInactive() { return zkClient.getNodes(Node.State.inactive); }
+ public List<Node> getFailed() { return zkClient.getNodes(Node.State.failed); }
+
+ public int getNodeCount(String tenantId, Node.State ... inState) {
+ return zkClient.getNodes(inState).stream()
+ .filter(node -> node.allocation().get().owner().tenant().value().equals(tenantId))
+ .collect(Collectors.counting()).intValue();
+ }
+
+ // ----------------- Node lifecycle -----------------------------------------------------------
+
+ /** Creates a new node object, without adding it to the node repo */
+ public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Configuration configuration) {
+ return Node.create(openStackId, hostname, parentHostname, configuration);
+ }
+
+ /** Adds a list of (newly created) nodes to the node repository as <i>provisioned</i> nodes */
+ public List<Node> addNodes(List<Node> nodes) {
+ for (Node node : nodes) {
+ Optional<Node> existing = getNode(node.hostname());
+ if (existing.isPresent())
+ throw new IllegalArgumentException("Cannot add " + node.hostname() + ": A node with this name already exists");
+ }
+ try (Mutex lock = lockUnallocated()) {
+ return zkClient.addNodes(nodes);
+ }
+ }
+
+ /** Sets a list of nodes ready and returns the nodes in the ready state */
+ public List<Node> setReady(List<Node> nodes) {
+ for (Node node : nodes)
+ if (node.state() != Node.State.provisioned && node.state() != Node.State.dirty)
+ throw new IllegalArgumentException("Can not set " + node + " ready. It is not provisioned or dirty.");
+ try (Mutex lock = lockUnallocated()) {
+ return zkClient.writeTo(Node.State.ready, nodes);
+ }
+ }
+
+ /** Reserve nodes. This method does <b>not</b> lock the node repository */
+ public List<Node> reserve(List<Node> nodes) { return zkClient.writeTo(Node.State.reserved, nodes); }
+
+ /** Activate nodes. This method does <b>not</b> lock the node repository */
+ public List<Node> activate(List<Node> nodes, NestedTransaction transaction) {
+ return zkClient.writeTo(Node.State.active, nodes, transaction);
+ }
+
+ /**
+ * Sets a list of nodes to have their allocation removable (active to inactive) in the node repository.
+ *
+ * @param application the application the nodes belong to
+ * @param nodes the nodes to make removable. These nodes MUST be in the active state.
+ */
+ public void setRemovable(ApplicationId application, List<Node> nodes) {
+ try (Mutex lock = lock(application)) {
+ List<Node> removableNodes =
+ nodes.stream().map(node -> node.setAllocation(node.allocation().get().makeRemovable()))
+ .collect(Collectors.toList());
+ write(removableNodes);
+ }
+ }
+
+ public void deactivate(ApplicationId application) {
+ try (Mutex lock = lock(application)) {
+ zkClient.writeTo(Node.State.inactive, zkClient.getNodes(application, Node.State.reserved, Node.State.active));
+ }
+ }
+
+ /**
+ * Deactivates these nodes in a transaction and returns
+ * the nodes in the new state which will hold if the transaction commits.
+ * This method does <b>not</b> lock
+ */
+ public List<Node> deactivate(List<Node> nodes, NestedTransaction transaction) {
+ return zkClient.writeTo(Node.State.inactive, nodes, transaction);
+ }
+
+ /** Deallocates these nodes, causing them to move to the dirty state */
+ public List<Node> deallocate(List<Node> nodes) {
+ return performOn(NodeListFilter.from(nodes), node -> zkClient.writeTo(Node.State.dirty, node));
+ }
+
+ /** Deallocate a node which is in the failed state. Use this to recycle failed nodes which have been repaired. */
+ public Node deallocate(String hostname) {
+ Optional<Node> nodeToDeallocate = getNode(Node.State.failed, hostname);
+ if ( ! nodeToDeallocate.isPresent())
+ throw new IllegalArgumentException("Could not deallocate " + hostname + ": Node not found in the failed state");
+ return deallocate(Collections.singletonList(nodeToDeallocate.get())).get(0);
+ }
+
+ /**
+ * Fails this node and returns it in its new state.
+ *
+ * @return the node in its new state
+ * @throws IllegalArgumentException if the node is not found
+ */
+ public Node fail(String hostname) {
+ return move(hostname, Node.State.failed);
+ }
+
+ /**
+ * Moves a previously failed node back to the active state.
+ *
+ * @return the node in its new state
+ * @throws IllegalArgumentException if the node is not found
+ */
+ public Node unfail(String hostname) {
+ return move(hostname, Node.State.active);
+ }
+
+ public Node move(String hostname, Node.State toState) {
+ Optional<Node> node = getNode(hostname);
+ if ( ! node.isPresent())
+ throw new IllegalArgumentException("Could not move " + hostname + " to " + toState + ": Node not found");
+ try (Mutex lock = lock(node.get())) {
+ return zkClient.writeTo(toState, node.get());
+ }
+ }
+
+ /**
+ * Removes a node. A node must be in the failed state before it can be removed.
+ *
+ * @return true if the node was removed, false if it was not found
+ */
+ public boolean remove(String hostname) {
+ Optional<Node> nodeToRemove = getNode(Node.State.failed, hostname);
+ if ( ! nodeToRemove.isPresent()) return false;
+ try (Mutex lock = lock(nodeToRemove.get())) {
+ return zkClient.removeNode(Node.State.failed, hostname);
+ }
+ }
+
+ /**
+ * Increases the restart generation of the active nodes matching the filter.
+ * Returns the nodes in their new state.
+ */
+ public List<Node> restart(NodeFilter filter) {
+ return performOn(StateFilter.from(Node.State.active, filter), node -> write(node.setRestart(node.allocation().get().restartGeneration().increaseWanted())));
+ }
+
+ /**
+ * Increases the reboot generation of the nodes matching the filter.
+ * Returns the nodes in their new state.
+ */
+ public List<Node> reboot(NodeFilter filter) {
+ return performOn(filter, node -> write(node.setReboot(node.status().reboot().increaseWanted())));
+ }
+
+ /**
+ * Writes this node after it has changed some internal state but NOT changed its state field.
+ * This does NOT lock the node repository.
+ *
+ * @return the written node for convenience
+ */
+ public Node write(Node node) { return zkClient.writeTo(node.state(), node); }
+
+ /**
+ * Writes these nodes after they have changed some internal state but NOT changed their state field.
+ * This does NOT lock the node repository.
+ *
+ * @return the written nodes for convenience
+ */
+ public List<Node> write(List<Node> nodes) {
+ if (nodes.isEmpty()) return Collections.emptyList();
+
+ // decide current state and make sure all nodes have it (alternatively we could create a transaction here)
+ Node.State state = nodes.get(0).state();
+ for (Node node : nodes) {
+ if ( node.state() != state)
+ throw new IllegalArgumentException("Multiple states: " + node.state() + " and " + state);
+ }
+ return zkClient.writeTo(state, nodes);
+ }
+
+ /**
+ * Performs an operation requiring locking on all nodes matching some filter.
+ *
+ * @param filter the filter determining the set of nodes where the operation will be performed
+ * @param action the action to perform
+ * @return the set of nodes on which the action was performed, as they became as a result of the operation
+ */
+ private List<Node> performOn(NodeFilter filter, UnaryOperator<Node> action) {
+ List<Node> unallocatedNodes = new ArrayList<>();
+ ListMap<ApplicationId, Node> allocatedNodes = new ListMap<>();
+
+ // Group matching nodes by the lock needed
+ for (Node node : zkClient.getNodes()) {
+ if ( ! filter.matches(node)) continue;
+ if (node.allocation().isPresent())
+ allocatedNodes.put(node.allocation().get().owner(), node);
+ else
+ unallocatedNodes.add(node);
+ }
+
+ // perform operation while holding locks
+ List<Node> resultingNodes = new ArrayList<>();
+ try (Mutex lock = lockUnallocated()) {
+ for (Node node : unallocatedNodes)
+ resultingNodes.add(action.apply(node));
+ }
+ for (Map.Entry<ApplicationId, List<Node>> applicationNodes : allocatedNodes.entrySet()) {
+ try (Mutex lock = lock(applicationNodes.getKey())) {
+ for (Node node : applicationNodes.getValue())
+ resultingNodes.add(action.apply(node));
+ }
+ }
+ return resultingNodes;
+ }
+
+ /** Create a lock which provides exclusive rights to making changes to the given application */
+ public Mutex lock(ApplicationId application) { return zkClient.lock(application); }
+
+ /** Create a lock which provides exclusive rights to changing the set of ready nodes */
+ public Mutex lockUnallocated() { return zkClient.lockInactive(); }
+
+ /** Acquires the appropriate lock for this node */
+ private Mutex lock(Node node) {
+ return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockUnallocated();
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClient.java
new file mode 100644
index 00000000000..1ec20f6df0c
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClient.java
@@ -0,0 +1,157 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.assimilate;
+
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.config.provision.*;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.Flavor;
+import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.node.Status;
+import com.yahoo.vespa.hosted.provision.persistence.NodeSerializer;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import javax.xml.xpath.XPathConstants;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+* @author vegard
+*/
+// TODO Moved here from hosted repo as is, more work to be done before it is usable
+public class PopulateClient {
+ static final Map<String, String> CLUSTER_TYPE_ELEMENT = ImmutableMap.of("container", "jdisc", "content", "content");
+ static final String CONTAINER_CLUSTER_TYPE = "container";
+ static final String CONTENT_CLUSTER_TYPE = "content";
+
+ private final String tenantId;
+ private final String applicationId;
+ private final String instanceId;
+ private final Document servicesXml;
+ private final Map<String, String> flavorSpec;
+ private final Map<String, String> hostMapping;
+ private final boolean dryRun;
+
+ private final NodeFlavors nodeFlavors;
+
+ private Clock creationClock = Clock.systemUTC();
+ private CuratorDatabaseClient zkClient;
+
+ // TODO NodeFlavors is now based on configuration, so callers need to do some work to get a proper NodeFlavors injected
+ public PopulateClient(Curator curator, NodeFlavors nodeFlavors, String tenantId, String applicationId, String instanceId,
+ String servicesXmlFilename, String hostsXmlFilename, Map<String, String> flavorSpec, boolean dryRun) {
+ this.nodeFlavors = nodeFlavors;
+ this.tenantId = tenantId;
+ this.applicationId = applicationId;
+ this.instanceId = instanceId;
+ this.servicesXml = XmlUtils.parseXml(servicesXmlFilename);
+ this.hostMapping = XmlUtils.getHostMapping(XmlUtils.parseXml(hostsXmlFilename));
+ this.flavorSpec = flavorSpec;
+ this.dryRun = dryRun;
+ this.zkClient = new CuratorDatabaseClient(nodeFlavors, curator, creationClock);
+
+ ensureFlavorIsDefinedForEveryCluster();
+ }
+
+ public void populate(String clusterType) {
+ final List<Node> nodes = getNodesForCluster(clusterType);
+
+ if (dryRun) {
+ System.out.println("Will populate zookeeper with the following:");
+ nodes.stream().forEach(node -> System.out.println(byteArrayToUTF8(new NodeSerializer(nodeFlavors).toJson(node))));
+ return;
+ }
+
+ zkClient.addNodesInState(nodes, Node.State.active);
+ }
+
+ private Optional<Flavor> getFlavor(String clusterType, String clusterId) {
+ return nodeFlavors.getFlavor(flavorSpec.get(clusterType + "." + clusterId));
+ }
+
+ private boolean hasDefinedFlavor(String clusterType, String clusterId) {
+ return flavorSpec.containsKey(clusterType + "." + clusterId);
+ }
+
+ private Node buildNode(String hostname, String clusterType, String clusterId, int nodeIndex) {
+ return new Node(
+ hostname, // Id
+ hostname, // Hostname
+ Optional.empty(), // parent hostname
+ new Configuration(getFlavor(clusterType, clusterId).get()), // Flavor
+ Status.initial(),
+ Node.State.active, // State = active/allocated
+ Optional.empty(), // Allocation
+ History.empty()) // History
+
+ .allocate(
+ ApplicationId.from(
+ TenantName.from(tenantId),
+ ApplicationName.from(applicationId),
+ InstanceName.from(instanceId)),
+ ClusterMembership.from(
+ ClusterSpec.from(ClusterSpec.Type.from(clusterType),
+ ClusterSpec.Id.from(clusterId)),
+ nodeIndex),
+ creationClock.instant());
+ }
+
+ private List<Node> getNodesForCluster(String clusterType) {
+ List<Node> nodes = new ArrayList<>();
+
+ final String elementName = CLUSTER_TYPE_ELEMENT.get(clusterType);
+ final NodeList clusterList = (NodeList) XmlUtils.evalXPath(servicesXml, String.format("services/%s", elementName), XPathConstants.NODESET);
+ for (int i = 0; i < clusterList.getLength(); i++) {
+ Element cluster = (Element) clusterList.item(i);
+ String clusterId = XmlUtils.attributeOrDefault(cluster, "id", "default");
+
+ // An empty cluster id is interpreted as 'default'
+ final String partialPath = clusterId.equals("default") ?
+ String.format("services/%s[@id='default' or string-length(@id)=0]", elementName) :
+ String.format("services/%s[@id='%s']", elementName, clusterId);
+
+ // Get all 'node' elements under 'group' or 'nodes'
+ final NodeList nodeList = (NodeList) XmlUtils.evalXPath(servicesXml, String.format("%s/group/node | %s/nodes/node", partialPath, partialPath), XPathConstants.NODESET);
+
+ for (int nodeIndex = 0; nodeIndex < nodeList.getLength(); nodeIndex++) {
+ Element node = (Element) nodeList.item(nodeIndex);
+ String hostname = hostMapping.get(node.getAttribute("hostalias"));
+ String indexString = XmlUtils.attributeOrDefault(node, "distribution-key", "");
+
+ final int index = !indexString.isEmpty() ? Integer.valueOf(indexString) : nodeIndex;
+ nodes.add(buildNode(hostname, clusterType, clusterId, index));
+ }
+ }
+
+ return nodes;
+ }
+
+ private void ensureFlavorIsDefinedForEveryCluster() {
+ CLUSTER_TYPE_ELEMENT.forEach((clusterType, element) -> {
+ final NodeList clusters = (NodeList) XmlUtils.evalXPath(servicesXml, "/services/" + element, XPathConstants.NODESET);
+
+ for (int i = 0; i < clusters.getLength(); i++) {
+ String clusterId = XmlUtils.attributeOrDefault((Element) clusters.item(i), "id", "default");
+
+ if (!hasDefinedFlavor(clusterType, clusterId)) {
+ throw new RuntimeException(String.format("Flavor is not defined for %s.%s\n", clusterType, clusterId));
+ }
+ }
+ });
+ }
+
+ private static String byteArrayToUTF8(byte[] array) {
+ return Charset.forName("UTF-8").decode(ByteBuffer.wrap(array)).toString();
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/XmlUtils.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/XmlUtils.java
new file mode 100644
index 00000000000..931d7dd9878
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/XmlUtils.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.assimilate;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import javax.xml.namespace.QName;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathExpression;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author vegard
+*/
+class XmlUtils {
+ static String attributeOrDefault(Element element, String attributeName, String defaultValue) {
+ String value = element.getAttribute(attributeName);
+ return !value.isEmpty() ? value : defaultValue;
+ }
+
+ static Document parseXml(String filename) {
+ try {
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ return builder.parse(new File(filename));
+ }
+ catch (ParserConfigurationException | SAXException | IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ static Object evalXPath(Document doc, String xPathExpr, QName returnType) {
+ try {
+ XPathFactory xPathfactory = XPathFactory.newInstance();
+ XPath xpath = xPathfactory.newXPath();
+ XPathExpression expr = xpath.compile(xPathExpr);
+ return expr.evaluate(doc, returnType);
+ }
+ catch (XPathExpressionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Reads hosts.xml into a map (alias to hostname)
+ */
+ static Map<String, String> getHostMapping(Document hostsXml) {
+ Map<String, String> hostMapping = new HashMap<>();
+
+ NodeList hostList = hostsXml.getElementsByTagName("host");
+ for (int i = 0; i < hostList.getLength(); i++) {
+ Element host = (Element) hostList.item(i);
+ hostMapping.put(host.getElementsByTagName("alias").item(0).getTextContent(), host.getAttribute("name"));
+ }
+
+ return hostMapping;
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
new file mode 100644
index 00000000000..fd5e229fa1b
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Deployer;
+import com.yahoo.config.provision.Deployment;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.yolean.Exceptions;
+
+import java.time.Duration;
+import java.util.Optional;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.stream.Collectors;
+
+/**
+ * The application maintainer regularly redeploys all applications.
+ * This is necessary because applications may gain and lose active nodes due to nodes being moved to and from the
+ * failed state. This is corrected by redeploying the applications periodically.
+ * It can not (at this point) be done reliably synchronously as part of the fail/unfail call due to the need for this
+ * to happen at a node having the deployer.
+ *
+ * @author bratseth
+ */
+public class ApplicationMaintainer extends Maintainer {
+
+ private final Deployer deployer;
+
+ public ApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration rate) {
+ super(nodeRepository, rate);
+ this.deployer = deployer;
+ }
+
+ @Override
+ protected void maintain() {
+ Set<ApplicationId> applications =
+ nodeRepository().getNodes(Node.State.active).stream().map(node -> node.allocation().get().owner()).collect(Collectors.toSet());
+
+ for (ApplicationId application : applications) {
+ try {
+ Optional<Deployment> deployment = deployer.deployFromLocalActive(application, Duration.ofMinutes(30));
+ if ( ! deployment.isPresent()) continue; // this will be done at another config server
+
+ deployment.get().prepare();
+ deployment.get().activate();
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, "Exception on maintenance redeploy of " + application, e);
+ }
+ }
+ }
+
+ @Override
+ public String toString() { return "Periodic application redeployer"; }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java
new file mode 100644
index 00000000000..4d106098ab5
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.History;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * This moves nodes from dirty to failed if they have been in dirty too long
+ * with the assumption that a node is stuck in dirty because it has failed.
+ * <p>
+ * As the nodes are moved back to dirty their failure count is increased,
+ * and if the count is sufficiently low they will be attempted recycled to dirty again.
+ * The upshot is nodes may get multiple attempts at clearing through dirty, but they will
+ * eventually stay in failed.
+ *
+ * @author bratseth
+ */
+public class DirtyExpirer extends Expirer {
+
+ private final NodeRepository nodeRepository;
+
+ public DirtyExpirer(NodeRepository nodeRepository, Clock clock, Duration dirtyTimeout) {
+ super(Node.State.dirty, History.Event.Type.deallocated, nodeRepository, clock, dirtyTimeout);
+ this.nodeRepository = nodeRepository;
+ }
+
+ @Override
+ protected void expire(List<Node> expired) {
+ for (Node expiredNode : expired.stream().collect(Collectors.toList()))
+ nodeRepository.fail(expiredNode.hostname());
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
new file mode 100644
index 00000000000..8333996c23e
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.History;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+/**
+ * Superclass of expiry tasks which moves nodes from some state to the dirty state.
+ * These jobs runs at least every 25 minutes.
+ *
+ * @author bratseth
+ */
+public abstract class Expirer extends Maintainer {
+
+ protected static final Logger log = Logger.getLogger(Expirer.class.getName());
+
+ /** The state to expire from */
+ private final Node.State fromState;
+
+ /** The event record type which contains the timestamp to use for expiry */
+ private final History.Event.Type eventType;
+
+ private final Clock clock;
+
+ private final Duration expiryTime;
+
+ public Expirer(Node.State fromState, History.Event.Type eventType, NodeRepository nodeRepository, Clock clock, Duration expiryTime) {
+ super(nodeRepository, min(Duration.ofMinutes(25), expiryTime));
+ this.fromState = fromState;
+ this.eventType = eventType;
+ this.clock = clock;
+ this.expiryTime = expiryTime;
+ }
+
+ private static Duration min(Duration a, Duration b) {
+ return a.toMillis() < b.toMillis() ? a : b;
+ }
+
+ @Override
+ protected void maintain() {
+ List<Node> expired = new ArrayList<>();
+ for (Node node : nodeRepository().getNodes(fromState)) {
+ Optional<History.Event> event = node.history().event(eventType);
+ if (event.isPresent() && event.get().at().plus(expiryTime).isBefore(clock.instant()))
+ expired.add(node);
+ }
+ if ( ! expired.isEmpty())
+ log.info(fromState + " expirer found " + expired.size() + " expired nodes");
+ expire(expired);
+ }
+
+ @Override
+ public String toString() { return "Expiry from " + fromState; }
+
+ /** Implement this callback to take action to expire these nodes */
+ protected abstract void expire(List<Node> node);
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
new file mode 100644
index 00000000000..ef9c65c8a01
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.History;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * This moves nodes from failed back to dirty if
+ * <ul>
+ * <li>No hardware failure is known to be detected on the node
+ * <li>The node has failed less than 5 times OR the environment is dev, test or perf,
+ * as those environments have no protection against users running bogus applications, so
+ * we cannot use the node failure count to conclude the node has a failure.
+ * </ul>
+ * Failed nodes are typically given a long expiry time to enable us to manually moved them back to
+ * active to recover data in cases where the node was failed accidentally.
+ * <p>
+ * The purpose of the automatic recycling to dirty + fail count is that nodes which were moved
+ * to failed due to some undetected hardware failure will end up being failed again.
+ * When that has happened enough they will not be recycled.
+ * <p>
+ * The Chef recipe running locally on the node may set the hardwareFailure flag to avoid the node
+ * being automatically recycled in cases where an error has been positively detected.
+ *
+ * @author bratseth
+ */
+public class FailedExpirer extends Expirer {
+
+ private final NodeRepository nodeRepository;
+ private final Zone zone;
+
+ public FailedExpirer(NodeRepository nodeRepository, Zone zone, Clock clock, Duration failTimeout) {
+ super(Node.State.failed, History.Event.Type.failed, nodeRepository, clock, failTimeout);
+ this.nodeRepository = nodeRepository;
+ this.zone = zone;
+ }
+
+ @Override
+ protected void expire(List<Node> expired) {
+ List<Node> nodesToRecycle = new ArrayList<>();
+ for (Node recycleCandidate : expired) {
+ if (recycleCandidate.status().hardwareFailure()) continue;
+ if (failCountIndicatesHwFail(zone) && recycleCandidate.status().failCount() >= 5) continue;
+ nodesToRecycle.add(recycleCandidate);
+ }
+ nodeRepository.deallocate(nodesToRecycle);
+ }
+
+ private boolean failCountIndicatesHwFail(Zone zone) {
+ return zone.environment() == Environment.prod || zone.environment() == Environment.staging;
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
new file mode 100644
index 00000000000..652a3783d4b
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.History;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.util.List;
+
+/**
+ * Maintenance job which moves inactive nodes to dirty after timeout.
+ * The timeout is in place for two reasons:
+ * <ul>
+ * <li>To ensure that the new application configuration has time to
+ * propagate before the node is used for something else
+ * <li>To provide a grace period in which nodes can be brought back to active
+ * if they were deactivated in error. As inactive nodes retain their state
+ * they can be brought back to active and correct state faster than a new node.
+ * </ul>
+ *
+ * @author bratseth
+ */
+public class InactiveExpirer extends Expirer {
+
+ private final NodeRepository nodeRepository;
+
+ public InactiveExpirer(NodeRepository nodeRepository, Clock clock, Duration inactiveTimeout) {
+ super(Node.State.inactive, History.Event.Type.deactivated, nodeRepository, clock, inactiveTimeout);
+ this.nodeRepository = nodeRepository;
+ }
+
+ @Override
+ protected void expire(List<Node> expired) {
+ nodeRepository.deallocate(expired);
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
new file mode 100644
index 00000000000..59cd8f5f85c
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.yolean.Exceptions;
+
+import java.time.Duration;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A maintainer is some job which runs at a fixed rate to perform some maintenance task on the node repo.
+ *
+ * @author bratseth
+ */
+public abstract class Maintainer extends AbstractComponent implements Runnable {
+
+ protected static final Logger log = Logger.getLogger(Maintainer.class.getName());
+
+ private final NodeRepository nodeRepository;
+ private final Duration rate;
+
+ private final ScheduledExecutorService service;
+
+ public Maintainer(NodeRepository nodeRepository, Duration rate) {
+ this.nodeRepository = nodeRepository;
+ this.rate = rate;
+
+ this.service = new ScheduledThreadPoolExecutor(1);
+ this.service.scheduleAtFixedRate(this, rate.toMillis(), rate.toMillis(), TimeUnit.MILLISECONDS);
+ }
+
+ /** Returns the node repository */
+ protected NodeRepository nodeRepository() { return nodeRepository; }
+
+ /** Returns the rate at which this job is set to run */
+ protected Duration rate() { return rate; }
+
+ @Override
+ public void run() {
+ try {
+ maintain();
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, this + " failed. Will retry in " + rate.toMinutes() + " minutes", e);
+ }
+ }
+
+ @Override
+ public void deconstruct() {
+ this.service.shutdown();
+ }
+
+ /** Returns a textual description of this job */
+ @Override
+ public abstract String toString();
+
+ /** Called once each time this maintenance job should run */
+ protected abstract void maintain();
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
new file mode 100644
index 00000000000..c18aacea284
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -0,0 +1,160 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.Deployer;
+import com.yahoo.config.provision.Deployment;
+import com.yahoo.log.LogLevel;
+import com.yahoo.transaction.Mutex;
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ServiceCluster;
+import com.yahoo.vespa.applicationmodel.ServiceInstance;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException;
+import com.yahoo.vespa.orchestrator.Orchestrator;
+import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
+import com.yahoo.vespa.service.monitor.ServiceMonitor;
+import com.yahoo.vespa.service.monitor.ServiceMonitorStatus;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Maintains information in the node repo about when this node last responded to ping
+ * and fails nodes which have not responded within the given time limit.
+ *
+ * @author bratseth
+ */
+public class NodeFailer extends Maintainer {
+
+ private static final Logger log = Logger.getLogger(NodeFailer.class.getName());
+
+ private final Deployer deployer;
+ private final ServiceMonitor serviceMonitor;
+ private final Duration downTimeLimit;
+ private final Clock clock;
+ private final Orchestrator orchestrator;
+
+ public NodeFailer(Deployer deployer, ServiceMonitor serviceMonitor, NodeRepository nodeRepository,
+ Duration downTimeLimit, Clock clock, Orchestrator orchestrator) {
+ // check ping status every five minutes, but at least twice as often as the down time limit
+ super(nodeRepository, min(downTimeLimit.dividedBy(2), Duration.ofMinutes(5)));
+ this.deployer = deployer;
+ this.serviceMonitor = serviceMonitor;
+ this.downTimeLimit = downTimeLimit;
+ this.clock = clock;
+ this.orchestrator = orchestrator;
+ }
+
+ private static Duration min(Duration d1, Duration d2) {
+ return d1.toMillis() < d2.toMillis() ? d1 : d2;
+ }
+
+ @Override
+ protected void maintain() {
+ List<Node> downNodes = maintainDownStatus();
+
+ for (Node node : downNodes) {
+ // Grace time before failing the node
+ Instant graceTimeEnd = node.history().event(History.Event.Type.down).get().at().plus(downTimeLimit);
+
+ if (graceTimeEnd.isBefore(clock.instant()) && ! applicationSuspended(node))
+ fail(node);
+ }
+ }
+
+ private boolean applicationSuspended(Node node) {
+ try {
+ return orchestrator.getApplicationInstanceStatus(node.allocation().get().owner())
+ == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN;
+ } catch (ApplicationIdNotFoundException e) {
+ //Treat it as not suspended and allow to fail the node anyway
+ return false;
+ }
+ }
+
+ /**
+ * If the node is positively DOWN, and there is no "down" history record, we add it.
+ * If the node is positively UP we remove any "down" history record.
+ *
+ * @return a list of all nodes which are positively currently in the down state
+ */
+ private List<Node> maintainDownStatus() {
+ List<Node> downNodes = new ArrayList<>();
+ for (ApplicationInstance<ServiceMonitorStatus> application : serviceMonitor.queryStatusOfAllApplicationInstances().values()) {
+ for (ServiceCluster<ServiceMonitorStatus> cluster : application.serviceClusters()) {
+ for (ServiceInstance<ServiceMonitorStatus> service : cluster.serviceInstances()) {
+ Optional<Node> node = nodeRepository().getNode(Node.State.active, service.hostName().s());
+ if ( ! node.isPresent()) continue; // we also get status from infrastructure nodes, which are not in the repo
+
+ if (service.serviceStatus().equals(ServiceMonitorStatus.DOWN))
+ downNodes.add(recordAsDown(node.get()));
+ else if (service.serviceStatus().equals(ServiceMonitorStatus.UP))
+ clearDownRecord(node.get());
+ // else: we don't know current status; don't take any action until we have positive information
+ }
+ }
+ }
+ return downNodes;
+ }
+
+ /**
+ * Record a node as down if not already recorded and returns the node in the new state.
+ * This assumes the node is found in the node
+ * repo and that the node is allocated. If we get here otherwise something is truly odd.
+ */
+ private Node recordAsDown(Node node) {
+ if (node.history().event(History.Event.Type.down).isPresent()) return node; // already down: Don't change down timestamp
+
+ try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) {
+ node = nodeRepository().getNode(Node.State.active, node.hostname()).get(); // re-get inside lock
+ return nodeRepository().write(node.setDown(clock.instant()));
+ }
+ }
+
+ private void clearDownRecord(Node node) {
+ if ( ! node.history().event(History.Event.Type.down).isPresent()) return;
+
+ try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) {
+ node = nodeRepository().getNode(Node.State.active, node.hostname()).get(); // re-get inside lock
+ nodeRepository().write(node.setUp());
+ }
+ }
+
+ /**
+ * Called when a node should be moved to the failed state: Do that if it seems safe,
+ * which is when the node repo has available capacity to replace the node.
+ * Otherwise not replacing the node ensures (by Orchestrator check) that no further action will be taken.
+ */
+ private void fail(Node node) {
+ Optional<Deployment> deployment =
+ deployer.deployFromLocalActive(node.allocation().get().owner(), Duration.ofMinutes(30));
+ if ( ! deployment.isPresent()) return; // this will be done at another config server
+
+ try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) {
+ node = nodeRepository().fail(node.hostname());
+ try {
+ deployment.get().prepare();
+ deployment.get().activate();
+ }
+ catch (RuntimeException e) {
+ // The expected reason for deployment to fail here is that there is no capacity available to redeploy.
+ // In that case we should leave the node in the active state to avoid failing additional nodes.
+ nodeRepository().unfail(node.hostname());
+ log.log(Level.WARNING, "Attempted to fail " + node + " for " + node.allocation().get().owner() +
+ ", but redeploying without the node failed", e);
+ }
+ }
+ }
+
+ @Override
+ public String toString() { return "Node failer"; }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
new file mode 100644
index 00000000000..c23a0fa5f56
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -0,0 +1,104 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.provision.Deployer;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.orchestrator.Orchestrator;
+import com.yahoo.vespa.service.monitor.ServiceMonitor;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.util.Optional;
+
+/**
+ * A component which sets up all the node repo maintenance jobs.
+ *
+ * @author bratseth
+ */
+public class NodeRepositoryMaintenance extends AbstractComponent {
+
+ private final NodeFailer nodeFailer;
+ private final ApplicationMaintainer applicationMaintainer;
+ private final ReservationExpirer reservationExpirer;
+ private final InactiveExpirer inactiveExpirer;
+ private final RetiredExpirer retiredExpirer;
+ private final FailedExpirer failedExpirer;
+ private final DirtyExpirer dirtyExpirer;
+
+ @Inject
+ public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, ServiceMonitor serviceMonitor, Zone zone, Orchestrator orchestrator) {
+ this(nodeRepository, deployer, serviceMonitor, zone, Clock.systemUTC(), orchestrator);
+ }
+
+ public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, ServiceMonitor serviceMonitor, Zone zone, Clock clock, Orchestrator orchestrator) {
+ DefaultTimes defaults = new DefaultTimes(zone.environment());
+ nodeFailer = new NodeFailer(deployer, serviceMonitor, nodeRepository, fromEnv("fail_grace").orElse(defaults.failGrace), clock, orchestrator);
+ applicationMaintainer = new ApplicationMaintainer(deployer, nodeRepository, fromEnv("redeploy_frequency").orElse(defaults.redeployFrequency));
+ reservationExpirer = new ReservationExpirer(nodeRepository, clock, fromEnv("reservation_expiry").orElse(defaults.reservationExpiry));
+ retiredExpirer = new RetiredExpirer(nodeRepository, deployer, clock, fromEnv("retired_expiry").orElse(defaults.retiredExpiry));
+ inactiveExpirer = new InactiveExpirer(nodeRepository, clock, fromEnv("inactive_expiry").orElse(defaults.inactiveExpiry));
+ failedExpirer = new FailedExpirer(nodeRepository, zone, clock, fromEnv("failed_expiry").orElse(defaults.failedExpiry));
+ dirtyExpirer = new DirtyExpirer(nodeRepository, clock, fromEnv("dirty_expiry").orElse(defaults.dirtyExpiry));
+ }
+
+ private Optional<Duration> fromEnv(String envVariable) {
+ String prefix = "vespa_node_repository__";
+ return Optional.ofNullable(System.getenv(prefix + envVariable)).map(Long::parseLong).map(Duration::ofSeconds);
+ }
+
+ @Override
+ public void deconstruct() {
+ nodeFailer.deconstruct();
+ applicationMaintainer.deconstruct();
+ reservationExpirer.deconstruct();
+ inactiveExpirer.deconstruct();
+ retiredExpirer.deconstruct();
+ failedExpirer.deconstruct();
+ dirtyExpirer.deconstruct();
+ }
+
+ private static class DefaultTimes {
+
+ /** All applications are redeployed with this frequency */
+ private final Duration redeployFrequency;
+
+ /** The time a node must be continuously nonresponsive before it is failed */
+ private final Duration failGrace;
+
+ private final Duration reservationExpiry;
+ private final Duration inactiveExpiry;
+ private final Duration retiredExpiry;
+ private final Duration failedExpiry;
+ private final Duration dirtyExpiry;
+
+ DefaultTimes(Environment environment) {
+ if (environment.equals(Environment.prod)) {
+ // These values are to avoid losing data (retired), and to be able to return an application
+ // back to a previous state fast (inactive)
+ redeployFrequency = Duration.ofMinutes(30);
+ failGrace = Duration.ofMinutes(60);
+ reservationExpiry = Duration.ofMinutes(15);
+ inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy
+ retiredExpiry = Duration.ofDays(4); // enough time to migrate data
+ failedExpiry = Duration.ofDays(4); // enough time to recover data even if it happens friday night
+ dirtyExpiry = Duration.ofHours(2); // enough time to clean the node
+ } else {
+ redeployFrequency = Duration.ofMinutes(30);
+ failGrace = Duration.ofMinutes(60);
+ // These values ensure tests and development is not delayed due to nodes staying around
+ // Use non-null values as these also determine the maintenance interval
+ reservationExpiry = Duration.ofMinutes(10); // Need to be long enough for deployment to be finished for all config model versions
+ inactiveExpiry = Duration.ofMinutes(1);
+ retiredExpiry = Duration.ofMinutes(1);
+ failedExpiry = Duration.ofMinutes(10);
+ dirtyExpiry = Duration.ofMinutes(30);
+ }
+ }
+
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java
new file mode 100644
index 00000000000..f45f8ebd086
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.History;
+
+import java.time.Duration;
+import java.time.Clock;
+import java.util.List;
+
+/**
+ * Maintenance job which moves reserved nodes to dirty after timeout.
+ * Nodes need to time out in case someone reserves nodes (by calling prepare) but never commits.
+ * reserved nodes may in some cases come from the inactive state, in which case they are dirty.
+ * For this reason, all reserved nodes go through the dirty state before going back to ready.
+ *
+ * @author bratseth
+ * @version $Id$
+ */
+public class ReservationExpirer extends Expirer {
+
+ private final NodeRepository nodeRepository;
+
+ public ReservationExpirer(NodeRepository nodeRepository, Clock clock, Duration reservationPeriod) {
+ super(Node.State.reserved, History.Event.Type.reserved, nodeRepository, clock, reservationPeriod);
+ this.nodeRepository = nodeRepository;
+ }
+
+ @Override
+ protected void expire(List<Node> expired) { nodeRepository.deallocate(expired); }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
new file mode 100644
index 00000000000..392d72f9f8e
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.collections.ListMap;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Deployer;
+import com.yahoo.config.provision.Deployment;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.History;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.logging.Level;
+
+/**
+ * Maintenance job which deactivates nodes which has been retired.
+ * This should take place after the system has been given sufficient time to migrate data to other nodes.
+ * <p>
+ * As these nodes are active, and therefore part of the configuration the impacted applications must be
+ * reconfigured after inactivation.
+ *
+ * @author bratseth
+ * @version $Id$
+ */
+public class RetiredExpirer extends Expirer {
+
+ private final NodeRepository nodeRepository;
+ private final Deployer deployer;
+
+ public RetiredExpirer(NodeRepository nodeRepository, Deployer deployer, Clock clock, Duration retiredDuration) {
+ super(Node.State.active, History.Event.Type.retired, nodeRepository, clock, retiredDuration);
+ this.nodeRepository = nodeRepository;
+ this.deployer = deployer;
+ }
+
+ @Override
+ protected void expire(List<Node> expired) {
+ // Only expire nodes which are retired. Do one application at the time.
+ ListMap<ApplicationId, Node> applicationNodes = new ListMap<>();
+ for (Node node : expired) {
+ if (node.allocation().isPresent() && node.allocation().get().membership().retired())
+ applicationNodes.put(node.allocation().get().owner(), node);
+ }
+
+ for (Map.Entry<ApplicationId, List<Node>> entry : applicationNodes.entrySet()) {
+ ApplicationId application = entry.getKey();
+ List<Node> nodesToRemove = entry.getValue();
+ try {
+ Optional<Deployment> deployment = deployer.deployFromLocalActive(application, Duration.ofMinutes(30));
+ if ( ! deployment.isPresent()) continue; // this will be done at another config server
+
+ nodeRepository.setRemovable(application, nodesToRemove);
+
+ deployment.get().prepare();
+ deployment.get().activate();
+
+ log.info("Redeployed " + application + " to deactivate " + nodesToRemove.size() + " retired nodes");
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, "Exception trying to remove previously retired nodes " + nodesToRemove +
+ "from " + application, e);
+ }
+ }
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetrics.java
new file mode 100644
index 00000000000..2753f435bde
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetrics.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.monitoring;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+/**
+ * @author oyving
+ */
+public class ProvisionMetrics extends AbstractComponent {
+
+ private static final Logger log = Logger.getLogger(ProvisionMetrics.class.getName());
+ private final ScheduledExecutorService executorService;
+
+ // TODO: make report interval configurable
+ public ProvisionMetrics(Metric metric, NodeRepository nodeRepository) {
+ this.executorService = new ScheduledThreadPoolExecutor(1);
+ this.executorService.scheduleAtFixedRate(
+ new ProvisionMetricsTask(metric, nodeRepository),
+ 0, // start immediately
+ 1, // report every minute
+ TimeUnit.MINUTES
+ );
+ }
+
+ @Override
+ public void deconstruct() {
+ this.executorService.shutdown();
+ }
+
+ private static class ProvisionMetricsTask implements Runnable {
+ private final Metric metric;
+ private final NodeRepository nodeRepository;
+
+ private ProvisionMetricsTask(Metric metric, NodeRepository nodeRepository) {
+ this.metric = metric;
+ this.nodeRepository = nodeRepository;
+ }
+
+ @Override
+ public void run() {
+ log.log(LogLevel.DEBUG, "Running provision metrics task");
+ try {
+ for (Node.State state : Node.State.values())
+ metric.set("hostedVespa." + state.name() + "Hosts", nodeRepository.getNodes(state).size(), null);
+ } catch (RuntimeException e) {
+ log.log(LogLevel.INFO, "Failed gathering metrics data: " + e.getMessage());
+ }
+ }
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java
new file mode 100644
index 00000000000..8a15c04a7df
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.vespa.hosted.provision.Node;
+
+/**
+ * The allocation of a node
+ *
+ * @author bratseth
+ */
+public class Allocation {
+
+ private final ApplicationId owner;
+ private final ClusterMembership clusterMembership;
+
+ /**
+ * Restart generation, see {@link com.yahoo.vespa.hosted.provision.node.Generation},
+ * wanted is increased when a restart of the services on the node is needed, current is updated
+ * when a restart has been done on the node.
+ */
+ private final Generation restartGeneration;
+
+ /** This node can (and should) be removed from the cluster on the next deployment */
+ private final boolean removable;
+
+ public Allocation(ApplicationId owner, ClusterMembership clusterMembership,
+ Generation restartGeneration, boolean removable) {
+ this.owner = owner;
+ this.clusterMembership = clusterMembership;
+ this.restartGeneration = restartGeneration;
+ this.removable = removable;
+ }
+
+ /** Returns the id of the application this is allocated to */
+ public ApplicationId owner() { return owner; }
+
+ /** Returns the role this node is allocated to */
+ public ClusterMembership membership() { return clusterMembership; }
+
+ /** Returns the restart generation (wanted and current) of this */
+ public Generation restartGeneration() { return restartGeneration; }
+
+ /** Returns a copy of this which is retired */
+ public Allocation retire() { return new Allocation(owner, clusterMembership.retire(), restartGeneration, removable); }
+
+ /** Returns a copy of this which is not retired */
+ public Allocation unretire() { return new Allocation(owner, clusterMembership.unretire(), restartGeneration, removable); }
+
+ /** Return whether this node is ready to be removed from the application */
+ public boolean removable() { return removable; }
+
+ /** Returns a copy of this with the current restart generation set to generation */
+ public Allocation setRestart(Generation generation) {
+ return new Allocation(owner, clusterMembership, generation, removable);
+ }
+
+ /** Returns a copy of this allocation where removable is set to true */
+ public Allocation makeRemovable() {
+ return new Allocation(owner, clusterMembership, restartGeneration, true);
+ }
+
+ public Allocation changeMembership(ClusterMembership newMembership) {
+ return new Allocation(owner, newMembership, restartGeneration, removable);
+ }
+
+ @Override
+ public String toString() { return "allocated to " + owner + " as '" + clusterMembership + "'"; }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Configuration.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Configuration.java
new file mode 100644
index 00000000000..9647c3f938b
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Configuration.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node;
+
+import java.util.Objects;
+
+/**
+ * The hardware configuration of a node
+ *
+ * @author bratseth
+ */
+public class Configuration {
+
+ private final Flavor flavor;
+
+ public Configuration(Flavor flavor) {
+ Objects.requireNonNull(flavor, "A node configuration must have a flavor");
+ this.flavor = flavor;
+ }
+
+ /** Returns the name of this hardware configuration */
+ public Flavor flavor() { return flavor; }
+
+ /** Returns a configuration with the flavor set to the given value */
+ public Configuration setFlavor(Flavor flavor) { return new Configuration(flavor); }
+
+ @Override
+ public String toString() {
+ return flavor.toString();
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Flavor.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Flavor.java
new file mode 100644
index 00000000000..bbb9a95a155
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Flavor.java
@@ -0,0 +1,124 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.vespa.config.nodes.NodeRepositoryConfig;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * A host flavor (type). This is a value object where the identity is the name.
+ * Use {@link NodeFlavors} to create a flavor.
+ *
+ * @author bratseth
+ */
+public class Flavor {
+
+ private final String name;
+ private final int cost;
+ private final String environment;
+ private final double minCpuCores;
+ private final double minMainMemoryAvailableGb;
+ private final double minDiskAvailableGb;
+ private final String description;
+ private List<Flavor> replacesFlavors;
+
+ /**
+ * Creates a Flavor, but does not set the replacesFlavors.
+ * @param flavorConfig config to be used for Flavor.
+ */
+ public Flavor(NodeRepositoryConfig.Flavor flavorConfig) {
+ this.name = flavorConfig.name();
+ this.replacesFlavors = new ArrayList<>();
+ this.cost = flavorConfig.cost();
+ this.environment = flavorConfig.environment();
+ this.minCpuCores = flavorConfig.minCpuCores();
+ this.minMainMemoryAvailableGb = flavorConfig.minMainMemoryAvailableGb();
+ this.minDiskAvailableGb = flavorConfig.minDiskAvailableGb();
+ this.description = flavorConfig.description();
+ }
+
+ public String name() { return name; }
+
+ /**
+ * Get the monthly cost (total cost of ownership) in USD for this flavor, typically total cost
+ * divided by 36 months.
+ * @return Monthly cost in USD
+ */
+ public int cost() { return cost; }
+
+ public double getMinMainMemoryAvailableGb() {
+ return minMainMemoryAvailableGb;
+ }
+
+ public double getMinDiskAvailableGb() {
+ return minDiskAvailableGb;
+ }
+
+ public double getMinCpuCores() {
+ return minCpuCores;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public String getEnvironment() {
+ return environment;
+ }
+
+ /**
+ * Returns the canonical name of this flavor - which is the name which should be used as an interface to users.
+ * The canonical name of this flavor is
+ * <ul>
+ * <li>If it replaces one flavor, the canonical name of the flavor it replaces
+ * <li>If it replaces multiple or no flavors - itself
+ * </ul>
+ *
+ * The logic is that we can use this to capture the gritty details of configurations in exact flavor names
+ * but also encourage users to refer to them by a common name by letting such flavor variants declare that they
+ * replace the caninical name we want. However, if a node replaces multiple names, it means that a former
+ * flavor distinction has become obsolete so this name becomes one of the canonical names users should refer to.
+ */
+ public String canonicalName() {
+ return replacesFlavors.size() == 1 ? replacesFlavors.get(0).canonicalName() : name;
+ }
+
+ /**
+ * The flavors this (directly) replaces.
+ * This is immutable if this is frozen, and a mutable list otherwise.
+ */
+ public List<Flavor> replaces() { return replacesFlavors; }
+
+ /**
+ * Returns whether this flavor satisfies the requested flavor, either directly
+ * (by being the same), or by directly or indirectly replacing it
+ */
+ public boolean satisfies(Flavor flavor) {
+ if (this.equals(flavor)) return true;
+ for (Flavor replaces : replacesFlavors)
+ if (replaces.satisfies(flavor))
+ return true;
+ return false;
+ }
+
+ /** Irreversibly freezes the content of this */
+ public void freeze() {
+ replacesFlavors = ImmutableList.copyOf(replacesFlavors);
+ }
+
+ @Override
+ public int hashCode() { return name.hashCode(); }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) return true;
+ if ( ! (other instanceof Flavor)) return false;
+ return ((Flavor)other).name.equals(this.name);
+ }
+
+ @Override
+ public String toString() { return "flavor '" + name + "'"; }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Generation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Generation.java
new file mode 100644
index 00000000000..b1be9da62fe
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Generation.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * An immutable generation, with wanted and current generation fields. Wanted generation
+ * is increased when an action (restart services or reboot are the available
+ * actions) is wanted, current is updated when the action has been done on the node.
+ *
+ * @author musum
+ */
+@Immutable
+public class Generation {
+
+ private final long wanted;
+ private final long current;
+
+ public Generation(long wanted, long current) {
+ this.wanted = wanted;
+ this.current = current;
+ }
+
+ public long wanted() {
+ return wanted;
+ }
+
+ public long current() {
+ return current;
+ }
+
+ public Generation increaseWanted() {
+ return new Generation(wanted + 1, current);
+ }
+
+ public Generation setCurrent(long newValue) {
+ return new Generation(wanted, newValue);
+ }
+
+ @Override
+ public String toString() {
+ return "current generation: " + current + ", wanted: " + wanted;
+ }
+
+ /** Returns the initial generation (0, 0) */
+ public static Generation inital() { return new Generation(0, 0); }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
new file mode 100644
index 00000000000..42134d082b7
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node;
+
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.vespa.hosted.provision.Node;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * An immutable record of the last event of each type happening to this node.
+ * Note that the history cannot be used to find the nodes current state - it will have a record of some
+ * event happening in the past even if that event is later undone.
+ *
+ * @author bratseth
+ */
+public class History {
+
+ private final ImmutableMap<Event.Type, Event> events;
+
+ public History(Collection<Event> events) {
+ this(toImmutableMap(events));
+ }
+
+ private History(ImmutableMap<Event.Type, Event> events) {
+ this.events = events;
+ }
+
+ private static ImmutableMap<Event.Type, Event> toImmutableMap(Collection<Event> events) {
+ ImmutableMap.Builder<Event.Type, Event> builder = new ImmutableMap.Builder<>();
+ for (Event event : events)
+ builder.put(event.type(), event);
+ return builder.build();
+ }
+
+ /** Returns this event if it is present in this history */
+ public Optional<Event> event(Event.Type type) { return Optional.ofNullable(events.get(type)); }
+
+ public Collection<Event> events() { return events.values(); }
+
+ /** Returns a copy of this history with the given event added */
+ public History record(Event event) {
+ ImmutableMap.Builder<Event.Type, Event> builder = builderWithout(event.type());
+ builder.put(event.type(), event);
+ return new History(builder.build());
+ }
+
+ /** Returns a copy of this history with the given event type removed (or an identical if it was not present) */
+ public History clear(Event.Type type) {
+ return new History(builderWithout(type).build());
+ }
+
+ private ImmutableMap.Builder<Event.Type, Event> builderWithout(Event.Type type) {
+ ImmutableMap.Builder<Event.Type, Event> builder = new ImmutableMap.Builder<>();
+ for (Event event : events.values())
+ if (event.type() != type)
+ builder.put(event.type(), event);
+ return builder;
+ }
+
+ /** Returns a copy of this history with a record of this state transition added, if applicable */
+ public History recordStateTransition(Node.State from, Node.State to, Instant at) {
+ if (from == to) return this;
+ switch (to) {
+ case ready: return record(new Event(Event.Type.readied, at));
+ case active: return record(new Event(Event.Type.activated, at));
+ case inactive: return record(new Event(Event.Type.deactivated, at));
+ case reserved: return record(new Event(Event.Type.reserved, at));
+ case failed: return record(new Event(Event.Type.failed, at));
+ case dirty: return record(new Event(Event.Type.deallocated, at));
+ default: return this;
+ }
+ }
+
+ /** Returns the empty history */
+ public static History empty() { return new History(Collections.emptyList()); }
+
+ /** An event which may happen to a node */
+ public static class Event {
+
+ private final Instant at;
+ private final Event.Type type;
+
+ public Event(Event.Type type, Instant at) {
+ this.type = type;
+ this.at = at;
+ }
+
+ /** Returns the type of event */
+ public Event.Type type() { return type; }
+
+ /** Returns the instant this even took place */
+ public Instant at() { return at; }
+
+ public enum Type { readied, reserved, activated, retired, deactivated, failed, deallocated, down }
+
+ @Override
+ public String toString() { return type + " event at " + at; }
+
+ }
+
+ /** A retired event includes additional information about the causing agent. */
+ public static class RetiredEvent extends Event {
+
+ private final RetiredEvent.Agent agent;
+
+ public RetiredEvent(Instant at, RetiredEvent.Agent agent) {
+ super(Type.retired, at);
+ this.agent = agent;
+ }
+
+ /** Returns the agent which caused retirement */
+ public RetiredEvent.Agent agent() { return agent; }
+
+ public enum Agent { system, application }
+
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeFlavors.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeFlavors.java
new file mode 100644
index 00000000000..9a07be3f86c
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeFlavors.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+import com.yahoo.vespa.config.nodes.NodeRepositoryConfig;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * All the available node flavors.
+ *
+ * @author bratseth
+ */
+public class NodeFlavors {
+
+ /** Flavors <b>which are configured</b> in this zone */
+ private final ImmutableMap<String, Flavor> flavors;
+
+ @Inject
+ public NodeFlavors(NodeRepositoryConfig config) {
+ ImmutableMap.Builder<String, Flavor> b = new ImmutableMap.Builder<>();
+ for (Flavor flavor : toFlavors(config))
+ b.put(flavor.name(), flavor);
+ this.flavors = b.build();
+ }
+
+ /** Returns a flavor by name, or empty if there is no flavor with this name. */
+ public Optional<Flavor> getFlavor(String name) {
+ return Optional.ofNullable(flavors.get(name));
+ }
+
+ /** Returns the flavor with the given name or throws an IllegalArgumentException if it does not exist */
+ public Flavor getFlavorOrThrow(String flavorName) {
+ Optional<Flavor> flavor = getFlavor(flavorName);
+ if ( flavor.isPresent()) return flavor.get();
+ throw new IllegalArgumentException("Unknown flavor '" + flavorName + "' Flavors are " + canonicalFlavorNames());
+ }
+
+ private List<String> canonicalFlavorNames() {
+ return flavors.values().stream().map(Flavor::canonicalName).distinct().sorted().collect(Collectors.toList());
+ }
+
+ private static Collection<Flavor> toFlavors(NodeRepositoryConfig config) {
+ Map<String, Flavor> flavors = new HashMap<>();
+ // First pass, create all flavors, but do not include flavorReplacesConfig.
+ for (NodeRepositoryConfig.Flavor flavorConfig : config.flavor()) {
+ flavors.put(flavorConfig.name(), new Flavor(flavorConfig));
+ }
+ // Second pass, set flavorReplacesConfig to point to correct flavor.
+ for (NodeRepositoryConfig.Flavor flavorConfig : config.flavor()) {
+ Flavor flavor = flavors.get(flavorConfig.name());
+ for (NodeRepositoryConfig.Flavor.Replaces flavorReplacesConfig : flavorConfig.replaces()) {
+ if (! flavors.containsKey(flavorReplacesConfig.name())) {
+ throw new IllegalStateException("Replaces for "
+ + flavor.name() + " pointing to a non existing flavor: " + flavorReplacesConfig.name());
+ }
+ flavor.replaces().add(flavors.get(flavorReplacesConfig.name()));
+ }
+ flavor.freeze();
+ }
+ return flavors.values();
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
new file mode 100644
index 00000000000..a26f02a53dc
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
@@ -0,0 +1,93 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node;
+
+import com.yahoo.component.Version;
+
+import javax.annotation.concurrent.Immutable;
+import java.util.Optional;
+
+/**
+ * Information about current status of a node
+ *
+ * @author bratseth
+ */
+@Immutable
+public class Status {
+
+ private final Generation reboot;
+ private final Optional<Version> vespaVersion;
+ private final Optional<Version> hostedVersion;
+ private final Optional<String> stateVersion;
+ private final Optional<String> dockerImage;
+ private final int failCount;
+ private final boolean hardwareFailure;
+
+ public Status(Generation generation,
+ Optional<Version> vespaVersion,
+ Optional<Version> hostedVersion,
+ Optional<String> stateVersion,
+ Optional<String> dockerImage,
+ int failCount,
+ boolean hardwareFailure) {
+ this.reboot = generation;
+ this.vespaVersion = vespaVersion;
+ this.hostedVersion = hostedVersion;
+ this.stateVersion = stateVersion;
+ this.dockerImage = dockerImage;
+ this.failCount = failCount;
+ this.hardwareFailure = hardwareFailure;
+ }
+
+ /** Returns a copy of this with the reboot generation changed */
+ public Status setReboot(Generation reboot) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, dockerImage, failCount, hardwareFailure); }
+
+ /** Returns the reboot generation of this node */
+ public Generation reboot() { return reboot; }
+
+ /** Returns a copy of this with the vespa version changed */
+ public Status setVespaVersion(Version version) { return new Status(reboot, Optional.of(version), hostedVersion, stateVersion, dockerImage, failCount, hardwareFailure); }
+
+ /** Returns the Vespa version installed on the node, if known */
+ public Optional<Version> vespaVersion() { return vespaVersion; }
+
+ /** Returns a copy of this with the hosted version changed */
+ public Status setHostedVersion(Version version) { return new Status(reboot, vespaVersion, Optional.of(version), stateVersion, dockerImage, failCount, hardwareFailure); }
+
+ /** Returns the hosted version installed on the node, if known */
+ public Optional<Version> hostedVersion() { return hostedVersion; }
+
+ /** Returns a copy of this with the state version changed */
+ public Status setStateVersion(String version) { return new Status(reboot, vespaVersion, hostedVersion, Optional.of(version), dockerImage, failCount, hardwareFailure); }
+
+ /**
+ * Returns the state version the node last successfully converged with.
+ * The state version contains the version-specific parts in identifying state
+ * files on dist, and is of the form VESPAVERSION-HOSTEDVERSION in CI, or otherwise HOSTEDVERSION.
+ * It's also used to uniquely identify a hosted Vespa release.
+ */
+ public Optional<String> stateVersion() { return stateVersion; }
+
+ /** Returns a copy of this with the docker image changed */
+ public Status setDockerImage(String dockerImage) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, Optional.of(dockerImage), failCount, hardwareFailure); }
+
+ /** Returns the current docker image the node is running, if known. */
+ public Optional<String> dockerImage() { return dockerImage; }
+
+ public Status increaseFailCount() { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, dockerImage, failCount+1, hardwareFailure); }
+
+ public Status decreaseFailCount() { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, dockerImage, failCount-1, hardwareFailure); }
+
+ public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, dockerImage, value, hardwareFailure); }
+
+ /** Returns how many times this node has been moved to the failed state. */
+ public int failCount() { return failCount; }
+
+ /** Returns whether a hardware failure has been detected on this node */
+ public boolean hardwareFailure() { return hardwareFailure; }
+
+ public Status setHardwareFailure(boolean hardwareFailure) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, dockerImage, failCount, hardwareFailure); }
+
+ /** Returns the initial status of a newly provisioned node */
+ public static Status initial() { return new Status(Generation.inital(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), 0, false); }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java
new file mode 100644
index 00000000000..b728b862686
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node.filter;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.hosted.provision.Node;
+
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * A node filter which matches a set of applications.
+ *
+ * @author bratseth
+ */
+public class ApplicationFilter extends NodeFilter {
+
+ private final Set<ApplicationId> applications;
+
+ /** Creates a node filter which filters using the given host filter */
+ private ApplicationFilter(Set<ApplicationId> applications, NodeFilter next) {
+ super(next);
+ Objects.requireNonNull(applications, "Applications set cannot be null, use an empty set");
+ this.applications = applications;
+ }
+
+ @Override
+ public boolean matches(Node node) {
+ if ( ! applications.isEmpty() && ! (node.allocation().isPresent() && applications.contains(node.allocation().get().owner()))) return false;
+ return nextMatches(node);
+ }
+
+ public static ApplicationFilter from(ApplicationId applicationId, NodeFilter next) {
+ return new ApplicationFilter(ImmutableSet.of(applicationId), next);
+ }
+
+ public static ApplicationFilter from(Set<ApplicationId> applicationIds, NodeFilter next) {
+ return new ApplicationFilter(ImmutableSet.copyOf(applicationIds), next);
+ }
+
+ public static ApplicationFilter from(String applicationIds, NodeFilter next) {
+ return new ApplicationFilter(HostFilter.split(applicationIds).stream().map(ApplicationFilter::toApplicationId).collect(Collectors.toSet()), next);
+ }
+
+ public static ApplicationId toApplicationId(String applicationIdString) {
+ String[] parts = applicationIdString.split("\\.");
+ if (parts.length != 3)
+ throw new IllegalArgumentException("Application id must be on the form tenant.application.instance, got '" +
+ applicationIdString + "'");
+ return ApplicationId.from(TenantName.from(parts[0]), ApplicationName.from(parts[1]), InstanceName.from(parts[2]));
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java
new file mode 100644
index 00000000000..6a53d935311
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node.filter;
+
+import com.yahoo.vespa.hosted.provision.Node;
+
+import java.util.Objects;
+
+/**
+ * A chainable node filter
+ *
+ * @author bratseth
+ */
+public abstract class NodeFilter {
+
+ private final NodeFilter next;
+
+ /** Creates a node filter with a nchained filter, or null if this is the last filter */
+ protected NodeFilter(NodeFilter next) {
+ this.next = next;
+ }
+
+ /** Returns whether this node matches this filter */
+ public abstract boolean matches(Node node);
+
+ /** Returns whether this is a match according to the chained filter */
+ protected final boolean nextMatches(Node node) {
+ if (next == null) return true;
+ return next.matches(node);
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeHostFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeHostFilter.java
new file mode 100644
index 00000000000..1753461afea
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeHostFilter.java
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node.filter;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.hosted.provision.Node;
+
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * A node filter adaption of a host filter
+ *
+ * @author bratseth
+ */
+public class NodeHostFilter extends NodeFilter {
+
+ private final HostFilter filter;
+
+ /** Creates a node filter which filters using the given host filter */
+ private NodeHostFilter(HostFilter filter, NodeFilter next) {
+ super(next);
+ Objects.requireNonNull(filter, "filter cannot be null, use HostFilter.all()");
+ this.filter = filter;
+ }
+
+ @Override
+ public boolean matches(Node node) {
+ if ( ! filter.matches(node.hostname(), node.configuration().flavor().name(), membership(node))) return false;
+ return nextMatches(node);
+ }
+
+ private Optional<ClusterMembership> membership(Node node) {
+ if (node.allocation().isPresent())
+ return Optional.of(node.allocation().get().membership());
+ else
+ return Optional.empty();
+ }
+
+ public static NodeHostFilter from(HostFilter hostFilter) {
+ return new NodeHostFilter(hostFilter, null);
+ }
+
+ public static NodeHostFilter from(HostFilter hostFilter, NodeFilter next) {
+ return new NodeHostFilter(hostFilter, next);
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java
new file mode 100644
index 00000000000..63e0493d53d
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node.filter;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.vespa.hosted.provision.Node;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * A node filter which matches a particular list of nodes
+ *
+ * @author bratseth
+ */
+public class NodeListFilter extends NodeFilter {
+
+ private final Set<Node> nodes;
+
+ private NodeListFilter(List<Node> nodes, NodeFilter next) {
+ super(next);
+ this.nodes = ImmutableSet.copyOf(nodes);
+ }
+
+ @Override
+ public boolean matches(Node node) {
+ return nodes.contains(node);
+ }
+
+ public static NodeListFilter from(List<Node> nodes) {
+ return new NodeListFilter(nodes, null);
+ }
+
+ public static NodeListFilter from(List<Node> nodes, NodeFilter next) {
+ return new NodeListFilter(nodes, next);
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java
new file mode 100644
index 00000000000..3e79acffce3
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node.filter;
+
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.vespa.hosted.provision.Node;
+
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Filter based on the parent host value (for virtualized nodes).
+ * @author dybdahl
+ */
+public class ParentHostFilter extends NodeFilter {
+
+ private final Set<String> parentHostNames;
+
+ /** Creates a node filter which filters using the given parent host name */
+ private ParentHostFilter(Set<String> parentHostNames, NodeFilter next) {
+ super(next);
+ Objects.requireNonNull(parentHostNames, "parentHostNames cannot be null.");
+ this.parentHostNames = parentHostNames;
+ }
+
+ @Override
+ public boolean matches(Node node) {
+ if (! parentHostNames.isEmpty() && (
+ ! node.parentHostname().isPresent() || ! parentHostNames.contains(node.parentHostname().get())))
+ return false;
+ return nextMatches(node);
+ }
+
+ /** Returns a copy of the given filter which only matches for the given parent */
+ public static ParentHostFilter from(String parentNames, NodeFilter filter) {
+ return new ParentHostFilter(HostFilter.split(parentNames).stream().collect(Collectors.toSet()), filter);
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java
new file mode 100644
index 00000000000..a605835c11c
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node.filter;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.hosted.provision.Node;
+
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * A node filter which filters on node states.
+ *
+ * @author bratseth
+ */
+public class StateFilter extends NodeFilter {
+
+ private final Set<Node.State> states;
+
+ /** Creates a node filter which filters using the given host filter */
+ private StateFilter(Set<Node.State> states, NodeFilter next) {
+ super(next);
+ Objects.requireNonNull(states, "state cannot be null, use an empty optional");
+ this.states = states;
+ }
+
+ @Override
+ public boolean matches(Node node) {
+ if ( ! states.isEmpty() && ! states.contains(node.state())) return false;
+ return nextMatches(node);
+ }
+
+ /** Returns a copy of the given filter which only matches for the given state */
+ public static StateFilter from(Node.State state, NodeFilter filter) {
+ return new StateFilter(Collections.singleton(state), filter);
+ }
+
+ /** Returns a node filter which matches a comma or space-separated list of states */
+ public static StateFilter from(String states, NodeFilter next) {
+ return new StateFilter(HostFilter.split(states).stream().map(Node.State::valueOf).collect(Collectors.toSet()), next);
+ }
+
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java
new file mode 100644
index 00000000000..be45c0df8e0
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.provision;
+
+import com.yahoo.osgi.annotation.ExportPackage;
+
+/** The node repository controls and allocates the nodes available in a hosted Vespa zone */ \ No newline at end of file
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java
new file mode 100644
index 00000000000..de7513de242
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.yahoo.transaction.Transaction;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.recipes.CuratorCounter;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A curator transaction which increases a change counter on commit.
+ * As this only ever does a single thing it needs no operations.
+ */
+class CountingCuratorTransaction implements Transaction {
+
+ private final CuratorCounter counter;
+
+ public CountingCuratorTransaction(CuratorCounter counter) {
+ this.counter = counter;
+ }
+
+ @Override
+ public Transaction add(Operation operation) { return this; }
+
+ @Override
+ public Transaction add(List<Operation> operation) { return this; }
+
+ @Override
+ public List<Operation> operations() { return Collections.emptyList(); }
+
+ @Override
+ public void prepare() {
+ // Increase the counter also if there are prepare errors to throw away the cached state
+ // in case that state leads to the rollback
+ counter.next();
+ }
+
+ @Override
+ public void rollbackOrLog() { }
+
+ @Override
+ public void close() { }
+
+ @Override
+ public void commit() { }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
new file mode 100644
index 00000000000..c7134de5ae6
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
@@ -0,0 +1,197 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.path.Path;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.recipes.CuratorCounter;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Thius encapsulated the curator database of the node repo.
+ * It serves reads from an in-memory cache of the content which is invalidated when changed on another node
+ * using a global, shared counter. The counter is updated on all write operations, ensured by wrapping write
+ * operations in a 2pc transaction containing the counter update.
+ *
+ * @author bratseth
+ */
+public class CuratorDatabase {
+
+ private final Curator curator;
+
+ /** A shared atomic counter which is incremented every time we write to the curator database */
+ private final CuratorCounter changeGenerationCounter;
+
+ /** A partial cache of the Curator database, which is only valid if generations match */
+ private final AtomicReference<CuratorDatabaseCache> cache = new AtomicReference<>();
+
+ /** Whether we should return data from the cache or always read fro ZooKeeper */
+ private final boolean useCache;
+
+ /**
+ * All keys, to allow reentrancy.
+ * This will grow forever with the number of applications seen, but this should be too slow to be a problem.
+ */
+ private final ConcurrentHashMap<Path, CuratorMutex> locks = new ConcurrentHashMap<>();
+
+ /**
+ * Creates a curator database
+ *
+ * @param curator the curator instance
+ * @param root the file system root of the db
+ */
+ public CuratorDatabase(Curator curator, Path root, boolean useCache) {
+ this.useCache = useCache;
+ this.curator = curator;
+ changeGenerationCounter = new CuratorCounter(curator, root.append("changeCounter").getAbsolute());
+ cache.set(newCache(changeGenerationCounter.get()));
+ }
+
+ /** Create a reentrant lock */
+ // Locks are not cached in the in-memory state
+ public CuratorMutex lock(Path path) {
+ CuratorMutex lock = locks.computeIfAbsent(path, (pathArg) -> new CuratorMutex(pathArg.getAbsolute(), curator.framework()));
+ lock.acquire();
+ return lock;
+
+ }
+
+ // --------- Write operations ------------------------------------------------------------------------------
+ // These must either create a nested transaction ending in a counter increment or not depend on prior state
+
+ /**
+ * Creates a new curator transaction against this database and adds it to the given nested transaction.
+ * Important: It is the nested transaction which must be committed - never the curator transaction directly.
+ */
+ public CuratorTransaction newCuratorTransactionIn(NestedTransaction transaction) {
+ // Add a counting transaction first, to make sure we always invalidate the current state on any transaction commit
+ transaction.add(new CountingCuratorTransaction(changeGenerationCounter), CuratorTransaction.class);
+ CuratorTransaction curatorTransaction = new CuratorTransaction(curator);
+ transaction.add(curatorTransaction);
+ return curatorTransaction;
+ }
+
+ /** Creates a path in curator and all its parents as necessary. If the path already exists this does nothing. */
+ // As this operation does not depend on the prior state we do not need to increment the write counter
+ public void create(Path path) {
+ curator.create(path);
+ }
+
+ // --------- Read operations -------------------------------------------------------------------------------
+ // These can read from the memory file system, which accurately mirrors the ZooKeeper content IF
+
+ /** Returns the immediate, local names of the children under this node in any order */
+ public List<String> getChildren(Path path) {
+ CuratorDatabaseCache cache = getCache();
+ List<String> children = cache.children(path);
+ if (children == null) { // children are not in this cache - get and add
+ children = curator.getChildren(path);
+ cache.addChildren(path, children);
+ }
+ return children;
+ }
+
+ public Optional<byte[]> getData(Path path) {
+ CuratorDatabaseCache cache = getCache();
+ Optional<byte[]> data = cache.data(path);
+ if (data == null) { // data is not in this cache - get and add
+ data = curator.getData(path);
+ cache.addData(path, data);
+ }
+ return data;
+ }
+
+ private CuratorDatabaseCache getCache() {
+ CuratorDatabaseCache cache = this.cache.get();
+ long currentCuratorGeneration = changeGenerationCounter.get();
+ if (currentCuratorGeneration != cache.generation()) { // current cache is invalid - start new
+ cache = newCache(currentCuratorGeneration);
+ this.cache.set(cache);
+ }
+ return cache;
+ }
+
+ /** Caches must only be instantiated using this method */
+ private CuratorDatabaseCache newCache(long generation) {
+ return useCache ? new CuratorDatabaseCache(generation) : new DeactivatedCache(generation);
+ }
+
+ /**
+ * A thread safe partial snapshot of the curator database content with a given generation.
+ * Note that a snapshot is not necessarily consistent - consistency is handled by pessimistic and optimistic locking
+ * in other layers.
+ * This is merely what Curator returned at various points in time it had the counter at this generation.
+ */
+ private static class CuratorDatabaseCache {
+
+ private final long generation;
+
+ // The data of this partial state mirror. The amount of curator state mirrored in this may grow
+ // over time by multiple threads. Growing is the only operation permitted by this.
+ // The content of the map is immutable.
+ private final Map<Path, List<String>> children = new ConcurrentHashMap<>();
+ private final Map<Path, Optional<byte[]>> data = new ConcurrentHashMap<>();
+
+ /** Create an empty snapshot at a given generation (as empty snapshot is a valid "partial snapshot" */
+ public CuratorDatabaseCache(long generation) {
+ this.generation = generation;
+ }
+
+ public long generation() { return generation; }
+
+ /**
+ * Returns the children of this path, which may be empty.
+ * Returns null only if it is not present in this state mirror
+ */
+ public List<String> children(Path path) { return children.get(path); }
+
+ public void addChildren(Path path, List<String> childrenAtPath) {
+ if (children.containsKey(path)) throw new RuntimeException("Programming error");
+ children.put(path, ImmutableList.copyOf(childrenAtPath));
+ }
+
+ /**
+ * Returns the content of this child - which may be empty.
+ * Returns null only if it is not present in this state mirror
+ */
+ public Optional<byte[]> data(Path path) {
+ Optional<byte[]> dataAtPath = data.get(path);
+ if (dataAtPath == null) return null;
+ return dataAtPath.map(d -> Arrays.copyOf(d, d.length));
+ }
+
+ public void addData(Path path, Optional<byte[]> dataAtPath) {
+ if (data.containsKey(path)) throw new RuntimeException("Programming error");
+ data.put(path, dataAtPath);
+ }
+
+ }
+
+ /** An implementation of the curator database cache which does no caching */
+ private static class DeactivatedCache extends CuratorDatabaseCache {
+
+ public DeactivatedCache(long generation) { super(generation); }
+
+ @Override
+ public List<String> children(Path path) { return null; }
+
+ @Override
+ public void addChildren(Path path, List<String> childrenAtPath) {}
+
+ @Override
+ public Optional<byte[]> data(Path path) { return null; }
+
+ @Override
+ public void addData(Path path, Optional<byte[]> dataAtPath) {}
+
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
new file mode 100644
index 00000000000..5ff1f41272a
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -0,0 +1,256 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.joda.JodaModule;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.log.LogLevel;
+import com.yahoo.path.Path;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.transaction.Transaction;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+
+import com.yahoo.vespa.curator.transaction.CuratorOperations;
+import com.yahoo.vespa.hosted.provision.node.Status;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+/**
+ * Client which reads and writes nodes to a curator database.
+ * Nodes are stored in files named <code>/provision/v1/[nodestate]/[hostname]</code>.
+ * The responsibility of this class is to turn operations on the level of node states, applications and nodes
+ * into operations on the level of file paths and bytes.
+ *
+ * @author bratseth
+ */
+public class CuratorDatabaseClient {
+
+ private static final Logger log = Logger.getLogger(CuratorDatabaseClient.class.getName());
+
+ private static final Path root = Path.fromString("/provision/v1");
+
+ private final NodeSerializer nodeSerializer;
+
+ private final CuratorDatabase curatorDatabase;
+
+ /** Used to serialize and de-serialize JSON data stored in ZK */
+ private final ObjectMapper jsonMapper = new ObjectMapper();
+
+ private final Clock clock;
+
+ public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock) {
+ this.nodeSerializer = new NodeSerializer(flavors);
+ jsonMapper.registerModule(new JodaModule());
+ this.curatorDatabase = new CuratorDatabase(curator, root, /* useCache: */ false);
+ this.clock = clock;
+ initZK();
+ }
+
+ private void initZK() {
+ curatorDatabase.create(root);
+ for (Node.State state : Node.State.values())
+ curatorDatabase.create(toPath(state));
+ }
+
+ /**
+ * Adds a set of nodes. Rollbacks/fails transaction if any node is not in the expected state.
+ */
+ public List<Node> addNodesInState(List<Node> nodes, Node.State expectedState) {
+ NestedTransaction transaction = new NestedTransaction();
+ CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction);
+ for (Node node : nodes) {
+ if (node.state() != expectedState)
+ throw new IllegalArgumentException(node + " is not in the " + node.state() + " state");
+ curatorTransaction.add(CuratorOperations.create(toPath(node).getAbsolute(), nodeSerializer.toJson(node)));
+ }
+ transaction.commit();
+
+ for (Node node : nodes)
+ log.log(LogLevel.INFO, "Added " + node);
+
+ return nodes;
+ }
+
+ /**
+ * Adds a set of nodes in the initial, provisioned state.
+ *
+ * @return the given nodes for convenience.
+ */
+ public List<Node> addNodes(List<Node> nodes) {
+ return addNodesInState(nodes, Node.State.provisioned);
+ }
+
+ /**
+ * Removes a node.
+ *
+ * @param state the current state of the node
+ * @param hostName the host name of the node to remove
+ * @return true if the node was removed, false if it was not found
+ */
+ public boolean removeNode(Node.State state, String hostName) {
+ Path path = toPath(state, hostName);
+ NestedTransaction transaction = new NestedTransaction();
+ CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction);
+ curatorTransaction.add(CuratorOperations.delete(path.getAbsolute()));
+ transaction.commit();
+ log.log(LogLevel.INFO, "Removed: " + state + " node " + hostName);
+ return true;
+ }
+
+ /**
+ * Writes the given nodes to the given state (whether or not they are already in this state or another),
+ * and returns a copy of the incoming nodes in their persisted state.
+ *
+ * @param toState the state to write the nodes to
+ * @param nodes the list of nodes to write
+ * @return the nodes in their persisted state
+ */
+ public List<Node> writeTo(Node.State toState, List<Node> nodes) {
+ try (NestedTransaction nestedTransaction = new NestedTransaction()) {
+ List<Node> writtenNodes = writeTo(toState, nodes, nestedTransaction);
+ nestedTransaction.commit();
+ return writtenNodes;
+ }
+ }
+ public Node writeTo(Node.State toState, Node node) {
+ return writeTo(toState, Collections.singletonList(node)).get(0);
+ }
+
+ /**
+ * Adds to the given transaction operations to write the given nodes to the given state,
+ * and returns a copy of the nodes in the state they will have if the transaction is committed.
+ *
+ * @param toState the state to write the nodes to
+ * @param nodes the list of nodes to write
+ * @param transaction the transaction to which write operations are added by this
+ * @return the nodes in their state as it will be written if committed
+ */
+ public List<Node> writeTo(Node.State toState, List<Node> nodes, NestedTransaction transaction) {
+ if (nodes.isEmpty()) return nodes;
+
+ List<Node> writtenNodes = new ArrayList<>(nodes.size());
+
+ CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction);
+ for (Node node : nodes) {
+ Node newNode = new Node(node.openStackId(), node.hostname(), node.parentHostname(), node.configuration(),
+ newNodeStatus(node, toState),
+ toState,
+ toState.isAllocated() ? node.allocation() : Optional.empty(),
+ newNodeHistory(node, toState));
+ curatorTransaction.add(CuratorOperations.delete(toPath(node).getAbsolute()))
+ .add(CuratorOperations.create(toPath(toState, newNode.hostname()).getAbsolute(), nodeSerializer.toJson(newNode)));
+ writtenNodes.add(newNode);
+ }
+
+ transaction.onCommitted(() -> { // schedule logging on commit of nodes which changed state
+ for (Node node : nodes) {
+ if (toState != node.state())
+ log.log(LogLevel.INFO, "Moved to " + toState + ": " + node);
+ }
+ });
+ return writtenNodes;
+ }
+
+ private Status newNodeStatus(Node node, Node.State toState) {
+ if (node.state() != Node.State.failed && toState == Node.State.failed) return node.status().increaseFailCount();
+ if (node.state() == Node.State.failed && toState == Node.State.active) return node.status().decreaseFailCount(); // fail undo
+ return node.status();
+ }
+
+ private History newNodeHistory(Node node, Node.State toState) {
+ History history = node.history();
+
+ // wipe history to avoid expiring based on events under the previous allocation
+ if (toState == Node.State.ready)
+ history = History.empty();
+
+ return history.recordStateTransition(node.state(), toState, clock.instant());
+ }
+
+ /**
+ * Returns all nodes which are in one of the given states.
+ * If no states are given this returns all nodes.
+ */
+ public List<Node> getNodes(Node.State ... states) {
+ List<Node> nodes = new ArrayList<>();
+ if (states.length == 0)
+ states = Node.State.values();
+ for (Node.State state : states) {
+ for (String hostname : curatorDatabase.getChildren(toPath(state))) {
+ final Optional<Node> node = getNode(state, hostname);
+ if (node.isPresent()) nodes.add(node.get()); // node might disappear between getChildren and getNode
+ }
+ }
+ return nodes;
+ }
+
+ /** Returns all nodes allocated to the given application which are in one of the given states */
+ public List<Node> getNodes(ApplicationId applicationId, Node.State ... states) {
+ List<Node> nodes = getNodes(states);
+ nodes.removeIf(node -> ! node.allocation().isPresent() || ! node.allocation().get().owner().equals(applicationId));
+ return nodes;
+ }
+
+ /** Returns a particular node, or empty if there is no such node in this state */
+ public Optional<Node> getNode(Node.State state, String hostname) {
+ return curatorDatabase.getData(toPath(state, hostname)).map((data) -> nodeSerializer.fromJson(state, data));
+ }
+
+ private Path toPath(Node.State nodeState) { return root.append(toDir(nodeState)); }
+
+ private Path toPath(Node node) {
+ return root.append(toDir(node.state())).append(node.hostname());
+ }
+
+ private Path toPath(Node.State nodeState, String nodeName) {
+ return root.append(toDir(nodeState)).append(nodeName);
+ }
+
+ /** Creates an returns the path to the lock for this application */
+ private Path lockPath(ApplicationId application) {
+ Path lockPath = root.append("locks")
+ .append(application.tenant().value())
+ .append(application.application().value())
+ .append(application.instance().value());
+ curatorDatabase.create(lockPath);
+ return lockPath;
+ }
+
+ private String toDir(Node.State state) {
+ switch (state) {
+ case provisioned: return "provisioned";
+ case ready: return "ready";
+ case reserved: return "reserved";
+ case active: return "allocated"; // legacy name
+ case inactive: return "deallocated"; // legacy name
+ case dirty: return "dirty";
+ case failed: return "failed";
+ default: throw new RuntimeException("Node state " + state + " does not map to a directory name");
+ }
+ }
+
+ /** Acquires the single cluster-global, reentrant lock for all non-active nodes */
+ public CuratorMutex lockInactive() {
+ return lock(root.append("locks").append("unallocatedLock"));
+ }
+
+ /** Acquires the single cluster-global, reentrant lock for active nodes of this application */
+ public CuratorMutex lock(ApplicationId application) {
+ return lock(lockPath(application));
+ }
+
+ /** Acquires the single cluster-global, reentrant lock for all non-active nodes */
+ public CuratorMutex lock(Path path) {
+ return curatorDatabase.lock(path);
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorMutex.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorMutex.java
new file mode 100644
index 00000000000..6f8a7aae3d5
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorMutex.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.yahoo.transaction.Mutex;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.locks.InterProcessMutex;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * A cluster-wide reentrant mutex which is released on (the last symmetric) close
+ *
+ * @author bratseth
+ */
+public class CuratorMutex implements Mutex {
+
+ private final InterProcessMutex mutex;
+ private final String lockPath;
+
+ public CuratorMutex(String lockPath, CuratorFramework curator) {
+ this.lockPath = lockPath;
+ mutex = new InterProcessMutex(curator, lockPath);
+ }
+
+ /** Take the lock. This may be called multiple times from the same thread - each matched by a close */
+ public void acquire() {
+ try {
+ boolean acquired = mutex.acquire(60, TimeUnit.SECONDS);
+ if ( ! acquired) {
+ throw new TimeoutException("Timed out after waiting 60 seconds");
+ }
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Exception acquiring lock '" + lockPath + "'", e);
+ }
+ }
+
+ @Override
+ public void close() {
+ try {
+ mutex.release();
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Exception releasing lock '" + lockPath + "'");
+ }
+ }
+
+}
+
+
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
new file mode 100644
index 00000000000..9e0a26be308
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -0,0 +1,273 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.Generation;
+import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.node.Status;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import static com.yahoo.vespa.config.SlimeUtils.optionalString;
+
+/**
+ * Serializes a node to/from JSON.
+ * Instances of this are multithread safe and can be reused
+ *
+ * @author bratseth
+ */
+public class NodeSerializer {
+
+ /** The configured node flavors */
+ private final NodeFlavors flavors;
+
+ // Node fields
+ private static final String hostnameKey = "hostname";
+ private static final String openStackIdKey = "openStackId";
+ // TODO Legacy name. Remove when 5.120 is released everywhere
+ private static final String dockerHostHostNameKey = "dockerHostHostName";
+ private static final String parentHostnameKey = "parentHostname";
+ private static final String configurationKey ="configuration";
+ private static final String historyKey = "history";
+ private static final String instanceKey = "instance"; // legacy name, TODO: change to allocation with backwards compat
+ private static final String rebootGenerationKey = "rebootGeneration";
+ private static final String currentRebootGenerationKey = "currentRebootGeneration";
+ private static final String vespaVersionKey = "vespaVersion";
+ private static final String hostedVersionKey = "hostedVersion";
+ private static final String stateVersionKey = "stateVersion";
+ private static final String failCountKey = "failCount";
+ private static final String hardwareFailureKey = "hardwareFailure";
+
+ // Configuration fields
+ private static final String flavorKey = "flavor";
+
+ // Allocation fields
+ private static final String tenantIdKey = "tenantId";
+ private static final String applicationIdKey = "applicationId";
+ private static final String instanceIdKey = "instanceId";
+ private static final String serviceIdKey = "serviceId"; // legacy name, TODO: change to membership with backwards compat
+ private static final String restartGenerationKey = "restartGeneration";
+ private static final String currentRestartGenerationKey = "currentRestartGeneration";
+ private static final String removableKey = "removable";
+ //Saved as part of allocation instead of serviceId, since serviceId serialized form is not easily extendable.
+ private static final String dockerImageKey = "dockerImage";
+
+ // History event fields
+ private static final String typeKey = "type";
+ private static final String atKey = "at";
+ private static final String agentKey = "agent"; // retired events only
+
+ // ---------------- Serialization ----------------------------------------------------
+
+ public NodeSerializer(NodeFlavors flavors) {
+ this.flavors = flavors;
+ }
+
+ public byte[] toJson(Node node) {
+ try {
+ Slime slime = new Slime();
+ toSlime(node, slime.setObject());
+ return SlimeUtils.toJsonBytes(slime);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Serialization of " + node + " to json failed", e);
+ }
+ }
+
+ private void toSlime(Node node, Cursor object) {
+ object.setString(hostnameKey, node.hostname());
+ object.setString(openStackIdKey, node.openStackId());
+ node.parentHostname().ifPresent(hostname -> object.setString(parentHostnameKey, hostname));
+ toSlime(node.configuration(), object.setObject(configurationKey));
+ object.setLong(rebootGenerationKey, node.status().reboot().wanted());
+ object.setLong(currentRebootGenerationKey, node.status().reboot().current());
+ node.status().vespaVersion().ifPresent(version -> object.setString(vespaVersionKey, version.toString()));
+ node.status().hostedVersion().ifPresent(version -> object.setString(hostedVersionKey, version.toString()));
+ node.status().stateVersion().ifPresent(version -> object.setString(stateVersionKey, version.toString()));
+ node.status().dockerImage().ifPresent(image -> object.setString(dockerImageKey, image));
+ object.setLong(failCountKey, node.status().failCount());
+ object.setBool(hardwareFailureKey, node.status().hardwareFailure());
+ node.allocation().ifPresent(allocation -> toSlime(allocation, object.setObject(instanceKey)));
+ toSlime(node.history(), object.setArray(historyKey));
+ }
+
+ private void toSlime(Configuration configuration, Cursor object) {
+ object.setString(flavorKey, configuration.flavor().name());
+ }
+
+ private void toSlime(Allocation allocation, Cursor object) {
+ object.setString(tenantIdKey, allocation.owner().tenant().value());
+ object.setString(applicationIdKey, allocation.owner().application().value());
+ object.setString(instanceIdKey, allocation.owner().instance().value());
+ object.setString(serviceIdKey, allocation.membership().stringValue());
+ object.setLong(restartGenerationKey, allocation.restartGeneration().wanted());
+ object.setLong(currentRestartGenerationKey, allocation.restartGeneration().current());
+ object.setBool(removableKey, allocation.removable());
+ allocation.membership().cluster().dockerImage().ifPresent( dockerImage ->
+ object.setString(dockerImageKey, dockerImage));
+ }
+
+ private void toSlime(History history, Cursor array) {
+ for (History.Event event : history.events())
+ toSlime(event, array.addObject());
+ }
+
+ private void toSlime(History.Event event, Cursor object) {
+ object.setString(typeKey, toString(event.type()));
+ object.setLong(atKey, event.at().toEpochMilli());
+ if (event instanceof History.RetiredEvent)
+ object.setString(agentKey, toString(((History.RetiredEvent)event).agent()));
+ }
+
+ // ---------------- Deserialization --------------------------------------------------
+
+ public Node fromJson(Node.State state, byte[] data) {
+ return nodeFromSlime(state, SlimeUtils.jsonToSlime(data).get());
+ }
+
+ private Node nodeFromSlime(Node.State state, Inspector object) {
+ return new Node(object.field(openStackIdKey).asString(),
+ object.field(hostnameKey).asString(),
+ parentHostnameFromSlime(object),
+ configurationFromSlime(object.field(configurationKey)),
+ statusFromSlime(object),
+ state,
+ allocationFromSlime(object.field(instanceKey)),
+ historyFromSlime(object.field(historyKey)));
+ }
+
+ private Status statusFromSlime(Inspector object) {
+ return new Status(
+ generationFromSlime(object, rebootGenerationKey, currentRebootGenerationKey),
+ softwareVersionFromSlime(object.field(vespaVersionKey)),
+ softwareVersionFromSlime(object.field(hostedVersionKey)),
+ optionalString(object.field(stateVersionKey)),
+ optionalString(object.field(dockerImageKey)),
+ (int)object.field(failCountKey).asLong(),
+ object.field(hardwareFailureKey).asBool());
+ }
+
+ private Configuration configurationFromSlime(Inspector object) {
+ return new Configuration(flavors.getFlavorOrThrow(object.field(flavorKey).asString()));
+ }
+
+ private Optional<Allocation> allocationFromSlime(Inspector object) {
+ if ( ! object.valid()) return Optional.empty();
+ return Optional.of(new Allocation(
+ applicationIdFromSlime(object),
+ ClusterMembership.from(object.field(serviceIdKey).asString(), optionalString(object.field(dockerImageKey))),
+ generationFromSlime(object, restartGenerationKey, currentRestartGenerationKey),
+ object.field(removableKey).asBool()));
+ }
+
+ private ApplicationId applicationIdFromSlime(Inspector object) {
+ return ApplicationId.from(TenantName.from(object.field(tenantIdKey).asString()),
+ ApplicationName.from(object.field(applicationIdKey).asString()),
+ InstanceName.from(object.field(instanceIdKey).asString()));
+ }
+
+ private History historyFromSlime(Inspector array) {
+ List<History.Event> events = new ArrayList<>();
+ array.traverse((ArrayTraverser) (int i, Inspector item) -> {
+ History.Event event = eventFromSlime(item);
+ if (event != null)
+ events.add(event);
+ });
+ return new History(events);
+ }
+
+ private History.Event eventFromSlime(Inspector object) {
+ History.Event.Type type = eventTypeFromString(object.field(typeKey).asString());
+ if (type == null) return null;
+ Instant at = Instant.ofEpochMilli(object.field(atKey).asLong());
+ if (type.equals(History.Event.Type.retired))
+ return new History.RetiredEvent(at, eventAgentFromString(object.field(agentKey).asString()));
+ else
+ return new History.Event(type, at);
+
+ }
+
+ private Generation generationFromSlime(Inspector object, String wantedField, String currentField) {
+ Inspector current = object.field(currentField);
+ return new Generation(object.field(wantedField).asLong(), current.asLong());
+ }
+
+ private Optional<Version> softwareVersionFromSlime(Inspector object) {
+ if ( ! object.valid()) return Optional.empty();
+ return Optional.of(Version.fromString(object.asString()));
+ }
+
+ private Optional<String> parentHostnameFromSlime(Inspector object) {
+ if (object.field(parentHostnameKey).valid())
+ return Optional.of(object.field(parentHostnameKey).asString());
+ // TODO Remove when 5.120 is released everywhere
+ else if (object.field(dockerHostHostNameKey).valid())
+ return Optional.of(object.field(dockerHostHostNameKey).asString());
+ else
+ return Optional.empty();
+ }
+
+ /** Returns the event type, or null if this event type should be ignored */
+ private History.Event.Type eventTypeFromString(String eventTypeString) {
+ switch (eventTypeString) {
+ case "readied" : return History.Event.Type.readied;
+ case "reserved" : return History.Event.Type.reserved;
+ case "activated" : return History.Event.Type.activated;
+ case "retired" : return History.Event.Type.retired;
+ case "deactivated" : return History.Event.Type.deactivated;
+ case "failed" : return History.Event.Type.failed;
+ case "deallocated" : return History.Event.Type.deallocated;
+ case "down" : return History.Event.Type.down;
+ }
+ throw new IllegalArgumentException("Unknown node event type '" + eventTypeString + "'");
+ }
+
+ private String toString(History.Event.Type nodeEventType) {
+ switch (nodeEventType) {
+ case readied : return "readied";
+ case reserved : return "reserved";
+ case activated : return "activated";
+ case retired : return "retired";
+ case deactivated : return "deactivated";
+ case failed : return "failed";
+ case deallocated : return "deallocated";
+ case down : return "down";
+ }
+ throw new IllegalArgumentException("Serialized form of '" + nodeEventType + "' not defined");
+ }
+
+ private History.RetiredEvent.Agent eventAgentFromString(String eventAgentString) {
+ switch (eventAgentString) {
+ case "application" : return History.RetiredEvent.Agent.application;
+ case "system" : return History.RetiredEvent.Agent.system;
+ }
+ throw new IllegalArgumentException("Unknown node event agent '" + eventAgentString + "'");
+ }
+
+ private String toString(History.RetiredEvent.Agent agent) {
+ switch (agent) {
+ case application : return "application";
+ case system : return "system";
+ }
+ throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined");
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
new file mode 100644
index 00000000000..3e288bd7785
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
@@ -0,0 +1,112 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.transaction.Mutex;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Performs activation of nodes for an application
+ *
+ * @author bratseth
+ */
+class Activator {
+
+ private final NodeRepository nodeRepository;
+ private final Clock clock;
+
+ public Activator(NodeRepository nodeRepository, Clock clock) {
+ this.nodeRepository = nodeRepository;
+ this.clock = clock;
+ }
+
+ /**
+ * Add operations to activates nodes for an application to the given transaction.
+ * The operations are not effective until the transaction is committed.
+ * <p>
+ * Pre condition: The application has a possibly empty set of nodes in each of reserved and active.
+ * <p>
+ * Post condition: Nodes in reserved which are present in <code>hosts</code> are moved to active.
+ * Nodes in active which are not present in <code>hosts</code> are moved to inactive.
+ *
+ * @param transaction Transaction with operations to commit together with any operations done within the repository.
+ * @param application the application to allocate nodes for
+ * @param hosts the hosts to make the set of active nodes of this
+ */
+ public void activate(ApplicationId application, Collection<HostSpec> hosts, NestedTransaction transaction) {
+ try (Mutex lock = nodeRepository.lock(application)) {
+ Set<String> hostnames = hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet());
+
+ List<Node> reserved = nodeRepository.getNodes(application, Node.State.reserved);
+ List<Node> reservedToActivate = retainHostsInList(hostnames, reserved);
+ List<Node> active = nodeRepository.getNodes(application, Node.State.active);
+ List<Node> continuedActive = retainHostsInList(hostnames, active);
+ List<Node> allActive = new ArrayList<>(continuedActive);
+ allActive.addAll(reservedToActivate);
+ if ( ! containsAll(hostnames, allActive))
+ throw new IllegalArgumentException("Activation of " + application + " failed. " +
+ "Could not find all requested hosts." +
+ "\nRequested: " + hosts +
+ "\nReserved: " + toHostNames(reserved) +
+ "\nActive: " + toHostNames(active));
+
+ List<Node> activeToRemove = removeHostsFromList(hostnames, active);
+ activeToRemove = activeToRemove.stream().map(Node::unretire).collect(Collectors.toList()); // only active nodes can be retired
+ nodeRepository.deactivate(activeToRemove, transaction);
+ nodeRepository.activate(updateFrom(hosts, continuedActive), transaction); // update active with any changes
+ nodeRepository.activate(reservedToActivate, transaction);
+ }
+ }
+
+ private List<Node> retainHostsInList(Set<String> hosts, List<Node> nodes) {
+ return nodes.stream().filter(node -> hosts.contains(node.hostname())).collect(Collectors.toList());
+ }
+
+ private List<Node> removeHostsFromList(Set<String> hosts, List<Node> nodes) {
+ return nodes.stream().filter(node -> ! hosts.contains(node.hostname())).collect(Collectors.toList());
+ }
+
+ private Set<String> toHostNames(List<Node> nodes) {
+ return nodes.stream().map(Node::hostname).collect(Collectors.toSet());
+ }
+
+ private boolean containsAll(Set<String> hosts, List<Node> nodes) {
+ Set<String> notFoundHosts = new HashSet<>(hosts);
+ for (Node node : nodes)
+ notFoundHosts.remove(node.hostname());
+ return notFoundHosts.isEmpty();
+ }
+
+ /**
+ * Returns the input nodes with the changes resulting from applying the settings in hosts to the given list of nodes.
+ */
+ private List<Node> updateFrom(Collection<HostSpec> hosts, List<Node> nodes) {
+ List<Node> updated = new ArrayList<>();
+ for (Node node : nodes) {
+ HostSpec hostSpec = getHost(node.hostname(), hosts);
+ node = hostSpec.membership().get().retired() ? node.retireByApplication(clock.instant()) : node.unretire();
+ node = node.setAllocation(node.allocation().get().changeMembership(hostSpec.membership().get()));
+ updated.add(node);
+ }
+ return updated;
+ }
+
+ private HostSpec getHost(String hostname, Collection<HostSpec> fromHosts) {
+ for (HostSpec host : fromHosts)
+ if (host.hostname().equals(hostname))
+ return host;
+ return null;
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
new file mode 100644
index 00000000000..c96a7c6dab4
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.provision.node.Flavor;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+
+import java.util.Optional;
+
+/**
+ * Defines the policies for assigning cluster capacity in various environments
+ *
+ * @author bratseth
+ */
+public class CapacityPolicies {
+
+ private final Zone zone;
+ private final NodeFlavors flavors;
+
+ public CapacityPolicies(Zone zone, NodeFlavors flavors) {
+ this.zone = zone;
+ this.flavors = flavors;
+ }
+
+ /** provides capacity defaults for various environments */
+ public int decideSize(Capacity requestedCapacity) {
+ int requestedNodes = requestedCapacity.nodeCount();
+ if (requestedCapacity.isRequired()) return requestedNodes;
+
+ switch(zone.environment()) {
+ case dev : case test : return 1;
+ case perf : return Math.min(requestedCapacity.nodeCount(), 3);
+ case staging: return requestedNodes <= 1 ? requestedNodes : Math.max(2, requestedNodes / 10);
+ case prod : return ensureRedundancy(requestedCapacity.nodeCount());
+ default : throw new IllegalArgumentException("Unsupported environment " + zone.environment());
+ }
+ }
+
+ public Flavor decideFlavor(Capacity requestedCapacity, ClusterSpec cluster) {
+ // for now, always use requested docker flavor when requested
+ final Optional<String> requestedFlavor = requestedCapacity.flavor();
+ if (requestedFlavor.isPresent() && flavors.getFlavorOrThrow(requestedFlavor.get()).getEnvironment().equals("DOCKER_CONTAINER"))
+ return flavors.getFlavorOrThrow(requestedFlavor.get());
+
+ switch(zone.environment()) {
+ case dev : case test : case staging : return flavors.getFlavorOrThrow(zone.defaultFlavor(cluster.type()));
+ default : return flavors.getFlavorOrThrow(requestedFlavor.orElse(zone.defaultFlavor(cluster.type())));
+ }
+ }
+
+ /**
+ * Throw if the node count is 1
+
+ * @return the argument node count
+ * @throws IllegalArgumentException if only one node is requested
+ */
+ private int ensureRedundancy(int nodeCount) {
+ // TODO: Reactivate this check when we have sufficient capacity in ap-northeast
+ // if (nodeCount == 1)
+ // throw new IllegalArgumentException("Deployments to prod require at least 2 nodes per cluster for redundancy");
+ return nodeCount;
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
new file mode 100644
index 00000000000..6b5d37d812a
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -0,0 +1,345 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.OutOfCapacityException;
+import com.yahoo.lang.MutableInteger;
+import com.yahoo.transaction.Mutex;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Flavor;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Performs preparation of node activation changes for a single host group in an application.
+ *
+ * @author bratseth
+ */
+class GroupPreparer {
+
+ private final NodeRepository nodeRepository;
+ private final Clock clock;
+
+ private static final boolean canChangeGroup = true;
+
+ public GroupPreparer(NodeRepository nodeRepository, Clock clock) {
+ this.nodeRepository = nodeRepository;
+ this.clock = clock;
+ }
+
+ /**
+ * Ensure sufficient nodes are reserved or active for the given application, group and cluster
+ *
+ * @param application the application we are allocating to
+ * @param cluster the cluster and group we are allocating to
+ * @param nodeCount the desired number of nodes to return
+ * @param flavor the desired flavor of those nodes
+ * @param surplusActiveNodes currently active nodes which are available to be assigned to this group.
+ * This method will remove from this list if it finds it needs additional nodes
+ * @param highestIndex the current highest node index among all active nodes in this cluster.
+ * This method will increase this number when it allocates new nodes to the cluster.
+ * @return the list of nodes this cluster group will have allocated if activated
+ */
+ // Note: This operation may make persisted changes to the set of reserved and inactive nodes,
+ // but it may not change the set of active nodes, as the active nodes must stay in sync with the
+ // active config model which is changed on activate
+ public List<Node> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, Flavor flavor, List<Node> surplusActiveNodes, MutableInteger highestIndex) {
+ try (Mutex lock = nodeRepository.lock(application)) {
+ NodeList nodeList = new NodeList(application, cluster, nodeCount, flavor, highestIndex);
+
+ // Use active nodes
+ nodeList.offer(nodeRepository.getNodes(application, Node.State.active), !canChangeGroup);
+ if (nodeList.satisfied()) return nodeList.finalNodes(surplusActiveNodes);
+
+ // Use active nodes that will otherwise be retired
+ List<Node> accepted = nodeList.offer(surplusActiveNodes, canChangeGroup);
+ surplusActiveNodes.removeAll(accepted);
+ if (nodeList.satisfied()) return nodeList.finalNodes(surplusActiveNodes);
+
+ // Use previously reserved nodes
+ nodeList.offer(nodeRepository.getNodes(application, Node.State.reserved), !canChangeGroup);
+ if (nodeList.satisfied()) return nodeList.finalNodes(surplusActiveNodes);
+
+ // Use inactive nodes
+ accepted = nodeList.offer(nodeRepository.getNodes(application, Node.State.inactive), !canChangeGroup);
+ nodeList.update(nodeRepository.reserve(accepted));
+ if (nodeList.satisfied()) return nodeList.finalNodes(surplusActiveNodes);
+
+ // Use new, ready nodes. Need to lock ready pool to ensure that nodes are not grabbed by others.
+ try (Mutex readyLock = nodeRepository.lockUnallocated()) {
+ List<Node> readyNodes = nodeRepository.getNodes(Node.State.ready);
+ accepted = nodeList.offer(optimize(readyNodes), !canChangeGroup);
+ nodeList.update(nodeRepository.reserve(accepted));
+ }
+ if (nodeList.satisfied()) return nodeList.finalNodes(surplusActiveNodes);
+
+ if (nodeList.whatAboutUsingRetiredNodes()) {
+ throw new OutOfCapacityException("Could not satisfy request for " + nodeCount +
+ " nodes of " + flavor + " for " + cluster +
+ " because we want to retire existing nodes.");
+ }
+ if (nodeList.whatAboutUsingVMs()) {
+ throw new OutOfCapacityException("Could not satisfy request for " + nodeCount +
+ " nodes of " + flavor + " for " + cluster +
+ " because too many have same parentHost.");
+ }
+ throw new OutOfCapacityException("Could not satisfy request for " + nodeCount +
+ " nodes of " + flavor + " for " + cluster + ".");
+ }
+ }
+
+ // optimize based on parent hosts
+ static List<Node> optimize(List<Node> input) {
+ int cnt = input.size();
+ List<Node> output = new ArrayList<Node>(cnt);
+
+ // first deal with VMs.
+ long vms = input.stream()
+ .filter(n -> n.parentHostname().isPresent())
+ .collect(Collectors.counting());
+ if (vms > 0) {
+ // Make a map where each parenthost maps to a list of VMs:
+ Map<String, List<Node>> byParentHosts = input.stream()
+ .filter(n -> n.parentHostname().isPresent())
+ .collect(Collectors.groupingBy(n -> n.parentHostname().get()));
+
+ // sort keys, those parent hosts with the most (remaining) ready VMs first
+ List<String> sortedParentHosts = byParentHosts
+ .keySet().stream()
+ .sorted((k1, k2) -> byParentHosts.get(k2).size() - byParentHosts.get(k1).size())
+ .collect(Collectors.toList());
+ while (vms > 0) {
+ // take one VM from each parent host, round-robin.
+ for (String k : sortedParentHosts) {
+ List<Node> leftFromHost = byParentHosts.get(k);
+ if (! leftFromHost.isEmpty()) {
+ output.add(leftFromHost.remove(0));
+ --vms;
+ }
+ }
+ }
+ }
+
+ // now add non-VMs (nodes without a parent):
+ input.stream()
+ .filter(n -> ! n.parentHostname().isPresent())
+ .forEach(n -> output.add(n));
+ return output;
+ }
+
+ /** Used to manage a list of nodes during the node reservation process */
+ private class NodeList {
+
+ /** The application this list is for */
+ private final ApplicationId application;
+
+ /** The cluster this list is for */
+ private final ClusterSpec cluster;
+
+ /** The requested capacity of the list */
+ private final int requestedNodes;
+
+ /** The requested node flavor */
+ private final Flavor requestedFlavor;
+
+ /** The nodes this has accepted so far */
+ private final Set<Node> nodes = new LinkedHashSet<>();
+
+ /** The number of nodes in the accepted nodes which are of the requested flavor */
+ private int acceptedOfRequestedFlavor = 0;
+
+ /** The number of nodes rejected because of clashing parentHostname */
+ private int rejectedWithClashingParentHost = 0;
+
+ /** The number of nodes that just now was changed to retired */
+ private int wasRetiredJustNow = 0;
+
+ /** The node indexes to verify uniqueness of each members index */
+ private Set<Integer> indexes = new HashSet<>();
+
+ /** The next membership index to assign to a new node */
+ private MutableInteger highestIndex;
+
+ public NodeList(ApplicationId application, ClusterSpec cluster, int requestedNodes, Flavor requestedFlavor, MutableInteger highestIndex) {
+ this.application = application;
+ this.cluster = cluster;
+ this.requestedNodes = requestedNodes;
+ this.requestedFlavor = requestedFlavor;
+ this.highestIndex = highestIndex;
+ }
+
+ /**
+ * Offer some nodes to this. The nodes may have an allocation to a different application or cluster,
+ * an allocation to this cluster, or no current allocation (in which case one is assigned).
+ * <p>
+ * Note that if unallocated nodes are offered before allocated nodes, this will unnecessarily
+ * reject allocated nodes due to index duplicates.
+ *
+ * @param offeredNodes the nodes which are potentially on offer. These may belong to a different application etc.
+ * @param canChangeGroup whether it is ok to change the group the offered node is to belong to if necessary
+ * @return the subset of offeredNodes which was accepted, with the correct allocation assigned
+ */
+ public List<Node> offer(List<Node> offeredNodes, boolean canChangeGroup) {
+ List<Node> accepted = new ArrayList<>();
+ for (Node offered : offeredNodes) {
+ boolean wantToRetireNode = false;
+ if (offered.allocation().isPresent()) {
+ ClusterMembership membership = offered.allocation().get().membership();
+ if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application
+ if ( ! membership.cluster().equalsIgnoringGroup(cluster)) continue; // wrong cluster id/type
+ if ( (! canChangeGroup || satisfied()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it
+ if ( offered.allocation().get().removable()) continue; // don't accept; causes removal
+ if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure)
+
+ // conditions on which we want to retire nodes that were allocated previously
+ if ( offeredNodeHasParentHostnameAlreadyAccepted(this.nodes, offered)) wantToRetireNode = true;
+ if ( !hasCompatibleFlavor(offered)) wantToRetireNode = true;
+
+ if ( ( !satisfied() && hasCompatibleFlavor(offered)) || acceptToRetire(offered) )
+ accepted.add(acceptNode(offered, wantToRetireNode));
+ }
+ else if (! satisfied() && hasCompatibleFlavor(offered)) {
+ if ( offeredNodeHasParentHostnameAlreadyAccepted(this.nodes, offered)) {
+ ++rejectedWithClashingParentHost;
+ continue;
+ }
+ Node alloc = offered.allocate(application, ClusterMembership.from(cluster, highestIndex.add(1)), clock.instant());
+ accepted.add(acceptNode(alloc, wantToRetireNode));
+ }
+ }
+
+ return accepted;
+ }
+
+ private boolean offeredNodeHasParentHostnameAlreadyAccepted(Collection<Node> accepted, Node offered) {
+ for (Node acceptedNode : accepted) {
+ if (acceptedNode.parentHostname().isPresent() && offered.parentHostname().isPresent() &&
+ acceptedNode.parentHostname().get().equals(offered.parentHostname().get())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns whether this node should be accepted into the cluster even if it is not currently desired
+ * (already enough nodes, or wrong flavor).
+ * Such nodes will be marked retired during finalization of the list of accepted nodes.
+ * The conditions for this are
+ * <ul>
+ * <li>This is a content node. These must always be retired before being removed to allow the cluster to
+ * migrate away data.
+ * <li>This is a container node and it is not desired due to having the wrong flavor. In this case this
+ * will (normally) obtain for all the current nodes in the cluster and so retiring before removing must
+ * be used to avoid removing all the current nodes at once, before the newly allocated replacements are
+ * initialized. (In the other case, where a container node is not desired because we have enough nodes we
+ * do want to remove it immediately to get immediate feedback on how the size reduction works out.)
+ * </ul>
+ */
+ private boolean acceptToRetire(Node node) {
+ if (node.state() != Node.State.active) return false;
+ if (! node.allocation().get().membership().cluster().group().equals(cluster.group())) return false;
+
+ return (cluster.type() == ClusterSpec.Type.content) ||
+ (cluster.type() == ClusterSpec.Type.container && ! hasCompatibleFlavor(node));
+ }
+
+ private boolean hasCompatibleFlavor(Node node) {
+ return node.configuration().flavor().satisfies(requestedFlavor);
+ }
+
+ /** Updates the state of some existing nodes in this list by replacing them by id with the given instances. */
+ public void update(List<Node> updatedNodes) {
+ nodes.removeAll(updatedNodes);
+ nodes.addAll(updatedNodes);
+ }
+
+ private Node acceptNode(Node node, boolean wantToRetire) {
+ if (! wantToRetire) {
+ if ( ! node.state().equals(Node.State.active)) {
+ // reactivated node - make sure its not retired
+ node = node.unretire();
+ }
+ acceptedOfRequestedFlavor++;
+ } else {
+ ++wasRetiredJustNow;
+ // retire nodes which are of an unwanted flavor
+ // or have an overlapping parent host
+ node = node.retireByApplication(clock.instant());
+ }
+ if ( ! node.allocation().get().membership().cluster().equals(cluster)) {
+ // group may be different
+ node = setCluster(cluster, node);
+ }
+ indexes.add(node.allocation().get().membership().index());
+ highestIndex.set(Math.max(highestIndex.get(), node.allocation().get().membership().index()));
+ nodes.add(node);
+ return node;
+ }
+
+ private Node setCluster(ClusterSpec cluster, Node node) {
+ ClusterMembership membership = node.allocation().get().membership().changeCluster(cluster);
+ return node.setAllocation(node.allocation().get().changeMembership(membership));
+ }
+
+ /** Returns true if we have accepted at least the requested number of nodes of the requested flavor */
+ public boolean satisfied() {
+ return acceptedOfRequestedFlavor >= requestedNodes;
+ }
+
+ public boolean whatAboutUsingRetiredNodes() {
+ return acceptedOfRequestedFlavor + wasRetiredJustNow >= requestedNodes;
+ }
+
+ public boolean whatAboutUsingVMs() {
+ return acceptedOfRequestedFlavor + rejectedWithClashingParentHost >= requestedNodes;
+ }
+
+ /**
+ * Make the number of <i>non-retired</i> nodes in the list equal to the requested number
+ * of nodes, and retire the rest of the list. Only retire currently active nodes.
+ * Prefer to retire nodes of the wrong flavor.
+ * Make as few changes to the retired set as possible.
+ *
+ * @return the final list of nodes
+ */
+ public List<Node> finalNodes(List<Node> surplusNodes) {
+ long currentRetired = nodes.stream().filter(node -> node.allocation().get().membership().retired()).count();
+ long surplus = nodes.size() - requestedNodes - currentRetired;
+
+ List<Node> changedNodes = new ArrayList<>();
+ if (surplus > 0) { // retire until surplus is 0
+ for (Node node : nodes) {
+ if ( ! node.allocation().get().membership().retired() && node.state().equals(Node.State.active)) {
+ changedNodes.add(node.retireByApplication(clock.instant()));
+ surplusNodes.add(node); // will be used in another group or retired
+ if (--surplus == 0) break;
+ }
+ }
+ }
+ else if (surplus < 0) { // unretire until surplus is 0
+ for (Node node : nodes) {
+ if ( node.allocation().get().membership().retired() && hasCompatibleFlavor(node)) {
+ changedNodes.add(node.unretire());
+ if (++surplus == 0) break;
+ }
+ }
+ }
+ update(changedNodes);
+ return new ArrayList<>(nodes);
+ }
+
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
new file mode 100644
index 00000000000..e2ff947de80
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.google.inject.Inject;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.ProvisionLogger;
+import com.yahoo.config.provision.Provisioner;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.log.LogLevel;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Flavor;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter;
+import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Implementation of the host provisioner API for hosted Vespa, using the node repository to allocate nodes.
+ * Does not allocate hosts for the routing application, see VespaModelFactory.createHostProvisioner
+ *
+ * @author bratseth
+ */
+public class NodeRepositoryProvisioner implements Provisioner {
+
+ private static Logger log = Logger.getLogger(NodeRepositoryProvisioner.class.getName());
+
+ private final NodeRepository nodeRepository;
+ private final CapacityPolicies capacityPolicies;
+ private final Zone zone;
+ private final Preparer preparer;
+ private final Activator activator;
+
+ @Inject
+ public NodeRepositoryProvisioner(NodeRepository nodeRepository, NodeFlavors flavors, Zone zone) {
+ this(nodeRepository, flavors, zone, Clock.systemUTC());
+ }
+
+ public NodeRepositoryProvisioner(NodeRepository nodeRepository, NodeFlavors flavors, Zone zone, Clock clock) {
+ this.nodeRepository = nodeRepository;
+ this.capacityPolicies = new CapacityPolicies(zone, flavors);
+ this.zone = zone;
+ this.preparer = new Preparer(nodeRepository, clock);
+ this.activator = new Activator(nodeRepository, clock);
+ }
+
+ /**
+ * Returns a list of nodes in the prepared or active state, matching the given constraints.
+ * The nodes are ordered by increasing index number.
+ */
+ @Override
+ public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity requestedCapacity, int groups, ProvisionLogger logger) {
+ log.log(LogLevel.DEBUG, () -> "Received deploy prepare request for " + requestedCapacity + " in " +
+ groups + " groups for application " + application + ", cluster " + cluster);
+
+ Flavor flavor = capacityPolicies.decideFlavor(requestedCapacity, cluster);
+ int nodeCount = capacityPolicies.decideSize(requestedCapacity);
+ int effectiveGroups = groups > nodeCount ? nodeCount : groups; // cannot have more groups than nodes
+
+ if (zone.environment().isManuallyDeployed() && nodeCount < requestedCapacity.nodeCount())
+ logger.log(Level.WARNING, "Requested " + requestedCapacity.nodeCount() + " nodes for " + cluster +
+ ", downscaling to " + nodeCount + " nodes in " + zone.environment());
+
+ return asSortedHosts(preparer.prepare(application, cluster, nodeCount, flavor, effectiveGroups));
+ }
+
+ @Override
+ public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) {
+ activator.activate(application, hosts, transaction);
+ }
+
+ @Override
+ public void restart(ApplicationId application, HostFilter filter) {
+ nodeRepository.restart(ApplicationFilter.from(application, NodeHostFilter.from(filter)));
+ }
+
+ @Override
+ public void removed(ApplicationId application) {
+ nodeRepository.deactivate(application);
+ }
+
+ private List<HostSpec> asSortedHosts(List<Node> nodes) {
+ nodes.sort(Comparator.comparingInt((Node node) -> node.allocation().get().membership().index()));
+ List<HostSpec> hosts = new ArrayList<>(nodes.size());
+ for (Node node : nodes)
+ hosts.add(new HostSpec(node.hostname(),
+ node.allocation().orElseThrow(IllegalStateException::new).membership()));
+ return hosts;
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
new file mode 100644
index 00000000000..52eeb7e536f
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
@@ -0,0 +1,116 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.lang.MutableInteger;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Flavor;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Performs preparation of node activation changes for an application.
+ *
+ * @author bratseth
+ */
+class Preparer {
+
+ private final NodeRepository nodeRepository;
+ private final Clock clock;
+ private final GroupPreparer groupPreparer;
+
+ public Preparer(NodeRepository nodeRepository, Clock clock) {
+ this.nodeRepository = nodeRepository;
+ this.clock = clock;
+ groupPreparer = new GroupPreparer(nodeRepository, clock);
+ }
+
+ /**
+ * Ensure sufficient nodes are reserved or active for the given application and cluster
+ *
+ * @return the list of nodes this cluster will have allocated if activated
+ */
+ // Note: This operation may make persisted changes to the set of reserved and inactive nodes,
+ // but it may not change the set of active nodes, as the active nodes must stay in sync with the
+ // active config model which is changed on activate
+ public List<Node> prepare(ApplicationId application, ClusterSpec cluster, int nodes, Flavor flavor, int groups) {
+ if (cluster.group().isPresent() && groups > 1)
+ throw new IllegalArgumentException("Cannot specify both a particular group and request multiple groups");
+ if (nodes > 0 && nodes % groups != 0)
+ throw new IllegalArgumentException("Requested " + nodes + " nodes in " + groups + " groups, " +
+ "which doesn't allow the nodes to be divided evenly into groups");
+
+ // no group -> this asks for the entire cluster -> we are free to remove groups we won't need
+ List<Node> surplusActiveNodes =
+ cluster.group().isPresent() ? new ArrayList<>() : findNodesInRemovableGroups(application, cluster, groups);
+
+ MutableInteger highestIndex = new MutableInteger(findHighestIndex(application, cluster));
+ List<Node> acceptedNodes = new ArrayList<>();
+ for (int groupIndex = 0; groupIndex < groups; groupIndex++) {
+ // Generated groups always have contiguous indexes starting from 0
+ ClusterSpec clusterGroup =
+ cluster.group().isPresent() ? cluster : cluster.changeGroup(Optional.of(ClusterSpec.Group.from(String.valueOf(groupIndex))));
+
+ List<Node> accepted = groupPreparer.prepare(application, clusterGroup, nodes/groups, flavor, surplusActiveNodes, highestIndex);
+ replace(acceptedNodes, accepted);
+ }
+ replace(acceptedNodes, retire(surplusActiveNodes));
+ return acceptedNodes;
+ }
+
+ /**
+ * Returns a list of the nodes which are
+ * in groups with index number above or equal the group count
+ */
+ private List<Node> findNodesInRemovableGroups(ApplicationId application, ClusterSpec requestedCluster, int groups) {
+ List<Node> surplusActiveNodes = new ArrayList<>(0);
+ for (Node node : nodeRepository.getNodes(application, Node.State.active)) {
+ ClusterSpec nodeCluster = node.allocation().get().membership().cluster();
+ if ( ! nodeCluster.id().equals(requestedCluster.id())) continue;
+ if ( ! nodeCluster.type().equals(requestedCluster.type())) continue;
+ if (Integer.parseInt(nodeCluster.group().get().value()) >= groups)
+ surplusActiveNodes.add(node);
+ }
+ return surplusActiveNodes;
+ }
+
+ private List<Node> replace(List<Node> list, List<Node> changed) {
+ list.removeAll(changed);
+ list.addAll(changed);
+ return list;
+ }
+
+ /**
+ * Returns the highest index number of all active and failed nodes in this cluster, or -1 if there are no nodes.
+ * We include failed nodes to avoid reusing the index of the failed node in the case where the failed node is the
+ * node with the highest index.
+ */
+ private int findHighestIndex(ApplicationId application, ClusterSpec cluster) {
+ int highestIndex = -1;
+ for (Node node : nodeRepository.getNodes(application, Node.State.active, Node.State.failed)) {
+ ClusterSpec nodeCluster = node.allocation().get().membership().cluster();
+ if ( ! nodeCluster.id().equals(cluster.id())) continue;
+ if ( ! nodeCluster.type().equals(cluster.type())) continue;
+
+ highestIndex = Math.max(node.allocation().get().membership().index(), highestIndex);
+ }
+ return highestIndex;
+ }
+
+ /** Returns retired copies of the given nodes, unless they are removable */
+ private List<Node> retire(List<Node> nodes) {
+ List<Node> retired = new ArrayList<>(nodes.size());
+ for (Node node : nodes) {
+ if ( ! node.allocation().get().removable())
+ retired.add(node.retireByApplication(clock.instant()));
+ }
+ return retired;
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java
new file mode 100644
index 00000000000..f85d8ae2924
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.osgi.annotation.ExportPackage;
+
+/** Implements the provisioning API to perform node provisioning form a node repository */ \ No newline at end of file
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializer.java
new file mode 100644
index 00000000000..321a75421a2
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializer.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi;
+
+import com.yahoo.vespa.hosted.provision.Node;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Converts {@link Node.State} to/from serialized form in REST APIs.
+ *
+ * @author bakksjo
+ */
+public class NodeStateSerializer {
+
+ private static final Map<Node.State, String> serializationMap = new HashMap<>();
+ private static final Map<String, Node.State> deserializationMap = new HashMap<>();
+
+ private static void addMapping(final Node.State nodeState, final String wireName) {
+ serializationMap.put(nodeState, wireName);
+ deserializationMap.put(wireName, nodeState);
+ }
+
+ static {
+ // Alphabetical order. No cheating, please - don't use .name(), .toString(), reflection etc. to get wire name.
+ addMapping(Node.State.active, "active");
+ addMapping(Node.State.dirty, "dirty");
+ addMapping(Node.State.failed, "failed");
+ addMapping(Node.State.inactive, "inactive");
+ addMapping(Node.State.provisioned, "provisioned");
+ addMapping(Node.State.ready, "ready");
+ addMapping(Node.State.reserved, "reserved");
+ }
+
+ private NodeStateSerializer() {} // Utility class, no instances.
+
+ public static Optional<Node.State> fromWireName(final String wireName) {
+ return Optional.ofNullable(deserializationMap.get(wireName));
+ }
+
+ public static String wireNameOf(final Node.State nodeState) {
+ final String wireName = serializationMap.get(nodeState);
+ if (wireName == null) {
+ throw new RuntimeException("Bug: Unknown serialization form of node state " + nodeState.name());
+ }
+ return wireName;
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ContainersForHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ContainersForHost.java
new file mode 100644
index 00000000000..a501e7f5a0a
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ContainersForHost.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.legacy;
+
+import java.util.List;
+
+/**
+ * Represents the JSON reply for getContainersForHost.
+ * Serialized by jackson, and therefore uses public fields to avoid writing cruft.
+ *
+ * @author tonytv
+ */
+public class ContainersForHost {
+
+ public List<DockerContainer> dockerContainers;
+
+ public static class DockerContainer {
+ public String containerHostname;
+ public String dockerImage;
+ public String nodeState;
+ public long wantedRestartGeneration;
+ public long currentRestartGeneration;
+
+ public DockerContainer(
+ String containerHostname,
+ String dockerImage,
+ String nodeState,
+ long wantedRestartGeneration,
+ long currentRestartGeneration) {
+ this.containerHostname = containerHostname;
+ this.dockerImage = dockerImage;
+ this.nodeState = nodeState;
+ this.wantedRestartGeneration = wantedRestartGeneration;
+ this.currentRestartGeneration = currentRestartGeneration;
+ }
+ }
+
+ public ContainersForHost(List<DockerContainer> dockerContainers) {
+ this.dockerContainers = dockerContainers;
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/HostInfo.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/HostInfo.java
new file mode 100644
index 00000000000..81211f978f7
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/HostInfo.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.legacy;
+
+/**
+ * Value class used to automatically convert to/from JSON.
+ *
+ * @author Oyvind Gronnesby
+ */
+class HostInfo {
+
+ public String hostname;
+ public String openStackId;
+ public String flavor;
+
+ public static HostInfo createHostInfo(String hostname, String openStackId, String flavor) {
+ HostInfo hostInfo = new HostInfo();
+ hostInfo.hostname = hostname;
+ hostInfo.openStackId = openStackId;
+ hostInfo.flavor = flavor;
+ return hostInfo;
+ }
+
+ public String toString(){
+ return String.format("%s/%s", openStackId, hostname);
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionEndpoint.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionEndpoint.java
new file mode 100644
index 00000000000..caef4630544
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionEndpoint.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.legacy;
+
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+/**
+ * To avoid duplication of URI construction.
+ * This class should be deleted when there's a provision client configured in services xml.
+ * @author tonytv
+ */
+public class ProvisionEndpoint {
+
+ public static final int configServerPort = 19071;
+
+ public static URI provisionUri(String configServerHostName, int port) {
+ try {
+ return new URL("http", configServerHostName, port, "/hack/provision").toURI();
+ } catch (URISyntaxException | MalformedURLException e) {
+ throw new IllegalArgumentException("Failed creating provisionUri from " + configServerHostName, e);
+ }
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResource.java
new file mode 100644
index 00000000000..f4c52010415
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResource.java
@@ -0,0 +1,151 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.legacy;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.container.jaxrs.annotation.Component;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.Node.State;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.restapi.NodeStateSerializer;
+import com.yahoo.vespa.hosted.provision.restapi.legacy.ContainersForHost.DockerContainer;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.MediaType;
+import java.util.*;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * The provisioning web service used by the provisioning controller to provide nodes to a node repository.
+ *
+ * @author mortent
+ */
+@Path("/provision")
+@Produces(MediaType.APPLICATION_JSON)
+public class ProvisionResource {
+ private static final Logger log = Logger.getLogger(ProvisionResource.class.getName());
+
+ private final NodeRepository nodeRepository;
+
+ private final NodeFlavors nodeFlavors;
+
+ public ProvisionResource(@Component NodeRepository nodeRepository, @Component NodeFlavors nodeFlavors) {
+ super();
+ this.nodeRepository = nodeRepository;
+ this.nodeFlavors = nodeFlavors;
+ }
+
+
+ @POST
+ @Path("/node")
+ @Consumes(MediaType.APPLICATION_JSON)
+ public void addNodes(List<HostInfo> hostInfoList) {
+ List<Node> nodes = new ArrayList<>();
+ for (HostInfo hostInfo : hostInfoList)
+ nodes.add(nodeRepository.createNode(hostInfo.openStackId, hostInfo.hostname, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow(hostInfo.flavor))));
+ nodeRepository.addNodes(nodes);
+ }
+
+ @GET
+ @Path("/node/required")
+ public ProvisionStatus getStatus() {
+ ProvisionStatus provisionStatus = new ProvisionStatus();
+ provisionStatus.requiredNodes = 0; // This concept has no meaning any more ...
+ provisionStatus.decomissionNodes = toHostInfo(nodeRepository.getInactive());
+ provisionStatus.failedNodes = toHostInfo(nodeRepository.getFailed());
+
+ return provisionStatus;
+ }
+
+ private List<HostInfo> toHostInfo(List<Node> nodes) {
+ List<HostInfo> hostInfoList = new ArrayList<>(nodes.size());
+ for (Node node : nodes)
+ hostInfoList.add(HostInfo.createHostInfo(node.hostname(), node.openStackId(), "medium"));
+ return hostInfoList;
+ }
+
+
+ @PUT
+ @Path("/node/ready")
+ public void setReady(String hostName) {
+ if ( nodeRepository.getNode(Node.State.ready, hostName).isPresent()) return; // node already 'ready'
+
+ Optional<Node> node = nodeRepository.getNode(Node.State.provisioned, hostName);
+ if ( ! node.isPresent())
+ node = nodeRepository.getNode(Node.State.dirty, hostName);
+ if ( ! node.isPresent())
+ throw new IllegalArgumentException("Could not set " + hostName + " ready: Not registered as provisioned or dirty");
+
+ nodeRepository.setReady(Collections.singletonList(node.get()));
+ }
+
+ @GET
+ @Path("/node/usage/{tenantId}")
+ public TenantStatus getTenantUsage(@PathParam("tenantId") String tenantId) {
+ TenantStatus ts = new TenantStatus();
+ ts.tenantId = tenantId;
+ ts.allocated = nodeRepository.getNodeCount(tenantId, Node.State.active);
+ ts.reserved = nodeRepository.getNodeCount(tenantId, Node.State.reserved);
+
+ Map<String, TenantStatus.ApplicationUsage> appinstanceUsageMap = new HashMap<>();
+
+ nodeRepository.getNodes(Node.State.active).stream()
+ .filter(node -> {
+ return node.allocation().get().owner().tenant().value().equals(tenantId);
+ })
+ .forEach(node -> {
+ ApplicationId owner = node.allocation().get().owner();
+ appinstanceUsageMap.merge(
+ String.format("%s:%s", owner.application().value(), owner.instance().value()),
+ TenantStatus.ApplicationUsage.create(owner.application().value(), owner.instance().value(), 1),
+ (a, b) -> {
+ a.usage += b.usage;
+ return a;
+ }
+ );
+ });
+
+ ts.applications = new ArrayList<>(appinstanceUsageMap.values());
+ return ts;
+ }
+
+ //TODO: move this to nodes/v2/ when the spec for this has been nailed.
+ @GET
+ @Path("/dockerhost/{hostname}")
+ public ContainersForHost getContainersForHost(@PathParam("hostname") String hostname) {
+ List<DockerContainer> dockerContainersForHost =
+ nodeRepository.getNodes(State.active, State.inactive).stream()
+ .filter(runsOnDockerHost(hostname))
+ .flatMap(ProvisionResource::toDockerContainer)
+ .collect(Collectors.toList());
+
+ return new ContainersForHost(dockerContainersForHost);
+ }
+
+ //returns stream since there is no conversion from optional to stream in java.
+ private static Stream<DockerContainer> toDockerContainer(Node node) {
+ try {
+ String dockerImage = node.allocation().get().membership().cluster().dockerImage().orElseThrow(() ->
+ new Exception("Docker image not set for node " + node));
+
+ return Stream.of(new DockerContainer(
+ node.hostname(),
+ dockerImage,
+ NodeStateSerializer.wireNameOf(node.state()),
+ node.allocation().get().restartGeneration().wanted(),
+ node.allocation().get().restartGeneration().current()));
+ } catch (Exception e) {
+ log.log(LogLevel.ERROR, "Ignoring docker container.", e);
+ return Stream.empty();
+ }
+ }
+
+ private static Predicate<Node> runsOnDockerHost(String hostname) {
+ return node -> node.parentHostname().map(hostname::equals).orElse(false);
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionStatus.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionStatus.java
new file mode 100644
index 00000000000..7e0eb41627f
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionStatus.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.legacy;
+
+import java.util.List;
+
+/**
+ * Value class used to convert to/from JSON.
+ *
+ * @author mortent
+ */
+class ProvisionStatus {
+
+ public int requiredNodes;
+ public List<HostInfo> decomissionNodes;
+ public List<HostInfo> failedNodes;
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/TenantStatus.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/TenantStatus.java
new file mode 100644
index 00000000000..4f20670fa12
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/TenantStatus.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.legacy;
+
+import java.util.List;
+
+/**
+ * Value class used to convert to/from JSON.
+ *
+ * @author Oyvind Gronnesby
+ */
+class TenantStatus {
+
+ public String tenantId;
+ public int allocated;
+ public int reserved;
+ public List<ApplicationUsage> applications;
+
+ public static class ApplicationUsage {
+ public String application;
+ public String instance;
+ public int usage;
+
+ public static ApplicationUsage create(String applicationId, String instanceId, int usage) {
+ ApplicationUsage appUsage = new ApplicationUsage();
+ appUsage.application = applicationId;
+ appUsage.instance = instanceId;
+ appUsage.usage = usage;
+ return appUsage;
+ }
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/package-info.java
new file mode 100644
index 00000000000..75ffa3e240e
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/package-info.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.provision.restapi.legacy;
+
+import com.yahoo.osgi.annotation.ExportPackage;
+
+/**
+ * Rest API which allows nodes to be added and removed from this node repository
+ * This API, aptly named "hack" will be removed once the dependencies are off it - Jon, March 2015
+ */ \ No newline at end of file
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v1/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v1/NodesApiHandler.java
new file mode 100644
index 00000000000..00e232dcfd3
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v1/NodesApiHandler.java
@@ -0,0 +1,117 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v1;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.jdisc.Response;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Executor;
+
+/**
+ * The implementation of the /state/v1 API.
+ * This dumps the content of the node repository on request, possibly with a host filter to return just the single
+ * matching node.
+ *
+ * @author bratseth
+ */
+public class
+ NodesApiHandler extends LoggingRequestHandler {
+
+ private final NodeRepository nodeRepository;
+
+ public NodesApiHandler(Executor executor, AccessLog accessLog, NodeRepository nodeRepository) {
+ super(executor, accessLog);
+ this.nodeRepository = nodeRepository;
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ return new NodesResponse(Response.Status.OK,
+ Optional.ofNullable(request.getProperty("hostname")), nodeRepository);
+ }
+
+ private static class NodesResponse extends HttpResponse {
+
+ /** If present only the node with this hostname will be present in the response */
+ private final Optional<String> hostnameFilter;
+ private final NodeRepository nodeRepository;
+
+ public NodesResponse(int status, Optional<String> hostnameFilter, NodeRepository nodeRepository) {
+ super(status);
+ this.hostnameFilter = hostnameFilter;
+ this.nodeRepository = nodeRepository;
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ stream.write(toJson());
+ }
+
+ @Override
+ public String getContentType() {
+ return "application/json";
+ }
+
+ private byte[] toJson() throws IOException {
+ Slime slime = new Slime();
+ toSlime(slime.setObject());
+ return SlimeUtils.toJsonBytes(slime);
+ }
+
+ private void toSlime(Cursor root) {
+ for (Node.State state : Node.State.values())
+ toSlime(state, root);
+ }
+
+ private void toSlime(Node.State state, Cursor object) {
+ List<Node> nodes = nodeRepository.getNodes(state);
+ Cursor nodeArray = null; // create if there are nodes
+ for (Node node : nodes) {
+ if (hostnameFilter.isPresent() && ! node.hostname().equals(hostnameFilter.get())) continue;
+ if (nodeArray == null)
+ nodeArray = object.setArray(state.name());
+ toSlime(node, nodeArray.addObject());
+ }
+ }
+
+ private void toSlime(Node node, Cursor object) {
+ object.setString("id", node.openStackId());
+ object.setString("hostname", node.hostname());
+ object.setString("flavor", node.configuration().flavor().name());
+ Optional<Allocation> allocation = node.allocation();
+ if (! allocation.isPresent()) return;
+ toSlime(allocation.get().owner(), object.setObject("owner"));
+ toSlime(allocation.get().membership(), object.setObject("membership"));
+ object.setLong("restartGeneration", allocation.get().restartGeneration().wanted());
+ }
+
+ private void toSlime(ApplicationId id, Cursor object) {
+ object.setString("tenant", id.tenant().value());
+ object.setString("application", id.application().value());
+ object.setString("instance", id.instance().value());
+ }
+
+ private void toSlime(ClusterMembership membership, Cursor object) {
+ object.setString("clustertype", membership.cluster().type().name());
+ object.setString("clusterid", membership.cluster().id().value());
+ object.setLong("index", membership.index());
+ object.setBool("retired", membership.retired());
+ }
+
+ }
+
+} \ No newline at end of file
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ErrorResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ErrorResponse.java
new file mode 100644
index 00000000000..7c5a1fffbc0
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ErrorResponse.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v2;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.JsonFormat;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import static com.yahoo.jdisc.Response.Status.*;
+
+public class ErrorResponse extends HttpResponse {
+
+ private final Slime slime = new Slime();
+
+ public ErrorResponse(int code, String errorType, String message) {
+ super(code);
+ Cursor root = slime.setObject();
+ root.setString("error-code", errorType);
+ root.setString("message", message);
+ }
+
+ public enum errorCodes {
+ NOT_FOUND,
+ BAD_REQUEST,
+ METHOD_NOT_ALLOWED,
+ INTERNAL_SERVER_ERROR,
+ INVALID_APPLICATION_PACKAGE,
+ UNKNOWN_VESPA_VERSION
+ }
+
+ public static ErrorResponse notFoundError(String message) {
+ return new ErrorResponse(NOT_FOUND, errorCodes.NOT_FOUND.name(), message);
+ }
+
+ public static ErrorResponse internalServerError(String message) {
+ return new ErrorResponse(INTERNAL_SERVER_ERROR, errorCodes.INTERNAL_SERVER_ERROR.name(), message);
+ }
+
+ public static ErrorResponse badRequest(String message) {
+ return new ErrorResponse(BAD_REQUEST, errorCodes.BAD_REQUEST.name(), message);
+ }
+
+ public static ErrorResponse methodNotAllowed(String message) {
+ return new ErrorResponse(METHOD_NOT_ALLOWED, errorCodes.METHOD_NOT_ALLOWED.name(), message);
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ new JsonFormat(true).encode(stream, slime);
+ }
+
+ @Override
+ public String getContentType() { return "application/json"; }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/MessageResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/MessageResponse.java
new file mode 100644
index 00000000000..0c91efa823b
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/MessageResponse.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v2;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import static com.yahoo.jdisc.Response.Status.BAD_REQUEST;
+import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR;
+import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED;
+import static com.yahoo.jdisc.Response.Status.NOT_FOUND;
+
+/**
+ * A 200 ok response with a message in JSON
+ *
+ * @author bratseth
+ */
+public class MessageResponse extends HttpResponse {
+
+ private final Slime slime = new Slime();
+
+ public MessageResponse(String message) {
+ super(200);
+ slime.setObject().setString("message", message);
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ new JsonFormat(true).encode(stream, slime);
+ }
+
+ @Override
+ public String getContentType() { return "application/json"; }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
new file mode 100644
index 00000000000..5d1b8a65b3c
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
@@ -0,0 +1,109 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v2;
+
+import com.yahoo.component.Version;
+import com.yahoo.io.IOUtils;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Type;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Optional;
+
+/**
+ * A class which can take a partial JSON node/v2 node JSON structure and apply it to a node object.
+ * This is a one-time use object.
+ *
+ * @author bratseth
+ */
+public class NodePatcher {
+
+ private final NodeFlavors nodeFlavors;
+
+ private final Inspector inspector;
+ private Node node;
+
+ public NodePatcher(NodeFlavors nodeFlavors, InputStream json, Node node) {
+ try {
+ inspector = SlimeUtils.jsonToSlime(IOUtils.readBytes(json, 1000 * 1000)).get();
+ this.node = node;
+ this.nodeFlavors = nodeFlavors;
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Error reading request body", e);
+ }
+ }
+
+ /**
+ * Apply the json to the node and return the resulting node
+ */
+ public Node apply() {
+ inspector.traverse((String name, Inspector value) -> {
+ try {
+ node = applyField(name, value);
+ }
+ catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Could not set field '" + name + "'", e);
+ }
+ } );
+ return node;
+ }
+
+ private Node applyField(String name, Inspector value) {
+ switch (name) {
+ case "convergedStateVersion" :
+ return node.setStatus(node.status().setStateVersion(asString(value)));
+ case "currentRebootGeneration" :
+ return node.setStatus(node.status().setReboot(node.status().reboot().setCurrent(asLong(value))));
+ case "currentRestartGeneration" :
+ return patchCurrentRestartGeneration(asLong(value));
+ case "currentDockerImage" :
+ return node.setStatus(node.status().setDockerImage(asString(value)));
+ case "currentVespaVersion" :
+ return node.setStatus(node.status().setVespaVersion(Version.fromString(asString(value))));
+ case "currentHostedVersion" :
+ return node.setStatus(node.status().setHostedVersion(Version.fromString(asString(value))));
+ case "failCount" :
+ return node.setStatus(node.status().setFailCount(asLong(value).intValue()));
+ case "flavor" :
+ return node.setConfiguration(node.configuration().setFlavor(nodeFlavors.getFlavorOrThrow(asString(value))));
+ case "hardwareFailure" :
+ return node.setStatus(node.status().setHardwareFailure(asBoolean(value)));
+ case "parentHostname" :
+ return node.setParentHostname(asString(value));
+ default :
+ throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field");
+ }
+ }
+
+ private Node patchCurrentRestartGeneration(Long value) {
+ Optional<Allocation> allocation = node.allocation();
+ if (allocation.isPresent())
+ return node.setAllocation(allocation.get().setRestart(allocation.get().restartGeneration().setCurrent(value)));
+ else
+ throw new IllegalArgumentException("Node is not allocated");
+ }
+
+ private Long asLong(Inspector field) {
+ if ( ! field.type().equals(Type.LONG))
+ throw new IllegalArgumentException("Expected a LONG value, got a " + field.type());
+ return field.asLong();
+ }
+
+ private String asString(Inspector field) {
+ if ( ! field.type().equals(Type.STRING))
+ throw new IllegalArgumentException("Expected a STRING value, got a " + field.type());
+ return field.asString();
+ }
+
+ private boolean asBoolean(Inspector field) {
+ if ( ! field.type().equals(Type.BOOL))
+ throw new IllegalArgumentException("Expected a BOOL value, got a " + field.type());
+ return field.asBool();
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
new file mode 100644
index 00000000000..9981602e4d0
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
@@ -0,0 +1,232 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v2;
+
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.io.IOUtils;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter;
+import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter;
+import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter;
+import com.yahoo.vespa.hosted.provision.node.filter.StateFilter;
+import com.yahoo.vespa.hosted.provision.restapi.v2.NodesResponse.ResponseType;
+import com.yahoo.yolean.Exceptions;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Executor;
+import java.util.logging.Level;
+
+import static com.yahoo.vespa.config.SlimeUtils.optionalString;
+
+/**
+ * The implementation of the /state/v2 API.
+ * See RestApiTest for documentation.
+ *
+ * @author bratseth
+ */
+public class NodesApiHandler extends LoggingRequestHandler {
+
+ private final NodeRepository nodeRepository;
+ private final NodeFlavors nodeFlavors;
+
+ public NodesApiHandler(Executor executor, AccessLog accessLog, NodeRepository nodeRepository, NodeFlavors flavors) {
+ super(executor, accessLog);
+ this.nodeRepository = nodeRepository;
+ this.nodeFlavors = flavors;
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ try {
+ switch (request.getMethod()) {
+ case GET: return handleGET(request);
+ case PUT: return handlePUT(request);
+ case POST: return handlePOST(request);
+ case DELETE: return handleDELETE(request);
+ case PATCH: return handlePATCH(request);
+ default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
+ }
+ }
+ catch (NotFoundException e) {
+ return ErrorResponse.notFoundError(Exceptions.toMessageString(e));
+ }
+ catch (IllegalArgumentException e) {
+ return ErrorResponse.badRequest(Exceptions.toMessageString(e));
+ }
+ catch (RuntimeException e) {
+ e.printStackTrace();
+ log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e);
+ return ErrorResponse.internalServerError(Exceptions.toMessageString(e));
+ }
+ }
+
+ private HttpResponse handleGET(HttpRequest request) {
+ String path = request.getUri().getPath();
+ if (path.equals( "/nodes/v2/")) return ResourcesResponse.fromStrings(request.getUri(), "state", "node", "command");
+ if (path.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, nodeRepository);
+ if (path.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, nodeRepository);
+ if (path.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, nodeRepository);
+ if (path.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, nodeRepository);
+ if (path.equals( "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot");
+ return ErrorResponse.notFoundError("Nothing at path '" + request.getUri().getPath() + "'");
+ }
+
+ private HttpResponse handlePUT(HttpRequest request) {
+ String path = request.getUri().getPath();
+ // Check paths to disallow illegal state changes
+ if (path.startsWith("/nodes/v2/state/ready/")) {
+ return new MessageResponse(setNodeReady(path));
+ }
+ else if (path.startsWith("/nodes/v2/state/failed/")) {
+ nodeRepository.fail(lastElement(path));
+ return new MessageResponse("Moved " + lastElement(path) + " to failed");
+ }
+ else if (path.startsWith("/nodes/v2/state/dirty/")) {
+ nodeRepository.deallocate(lastElement(path));
+ return new MessageResponse("Moved " + lastElement(path) + " to dirty");
+ }
+ else if (path.startsWith("/nodes/v2/state/active/")) {
+ nodeRepository.unfail(lastElement(path));
+ return new MessageResponse("Moved " + lastElement(path) + " to active");
+ }
+ else {
+ return ErrorResponse.notFoundError("Cannot put to path '" + request.getUri().getPath() + "'");
+ }
+ }
+
+ private HttpResponse handlePATCH(HttpRequest request) {
+ String path = request.getUri().getPath();
+ if ( ! path.startsWith("/nodes/v2/node/")) return ErrorResponse.notFoundError("Nothing at '" + path + "'");
+ Node node = nodeFromRequest(request);
+ nodeRepository.write(new NodePatcher(nodeFlavors, request.getData(), node).apply());
+ return new MessageResponse("Updated " + node.hostname());
+ }
+
+ private HttpResponse handlePOST(HttpRequest request) {
+ switch (request.getUri().getPath()) {
+ case "/nodes/v2/command/restart" :
+ int restartCount = nodeRepository.restart(toNodeFilter(request)).size();
+ return new MessageResponse("Scheduled restart of " + restartCount + " matching nodes");
+ case "/nodes/v2/command/reboot" :
+ int rebootCount = nodeRepository.reboot(toNodeFilter(request)).size();
+ return new MessageResponse("Scheduled reboot of " + rebootCount + " matching nodes");
+ case "/nodes/v2/node" :
+ int addedNodes = addNodes(request.getData());
+ return new MessageResponse("Added " + addedNodes + " nodes to the provisioned state");
+ default:
+ return ErrorResponse.notFoundError("Nothing at path '" + request.getUri().getPath() + "'");
+ }
+ }
+
+ private HttpResponse handleDELETE(HttpRequest request) {
+ String path = request.getUri().getPath();
+ if (path.startsWith("/nodes/v2/node/")) {
+ String hostname = lastElement(path);
+ if (nodeRepository.remove(hostname))
+ return new MessageResponse("Removed " + hostname);
+ else
+ return ErrorResponse.notFoundError("No node in the failed state with hostname " + hostname);
+ }
+ else {
+ return ErrorResponse.notFoundError("Nothing at path '" + request.getUri().getPath() + "'");
+ }
+ }
+
+ private Node nodeFromRequest(HttpRequest request) {
+ // TODO: The next 4 lines can be a oneliner when updateNodeAttribute is removed (as we won't allow path suffixes)
+ String path = request.getUri().getPath();
+ String prefixString = "/nodes/v2/node/";
+ int beginIndex = path.indexOf(prefixString) + prefixString.length();
+ int endIndex = path.indexOf("/", beginIndex);
+ if (endIndex < 0) endIndex = path.length(); // path ends by ip
+ String hostname = path.substring(beginIndex, endIndex);
+
+ Optional<Node> node = nodeRepository.getNode(hostname);
+ if ( ! node.isPresent()) throw new NotFoundException("No node found with hostname " + hostname);
+ return node.get();
+ }
+
+ public int addNodes(InputStream jsonStream) {
+ List<Node> nodes = createNodesFromSlime(getSlimeFromInputStream(jsonStream).get());
+ return nodeRepository.addNodes(nodes).size();
+ }
+
+ private static Slime getSlimeFromInputStream(InputStream jsonStream) {
+ try {
+ byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000);
+ return SlimeUtils.jsonToSlime(jsonBytes);
+ } catch (IOException e) {
+ throw new RuntimeException();
+ }
+ }
+
+ private List<Node> createNodesFromSlime(Inspector object) {
+ List<Node> nodes = new ArrayList<>();
+ object.traverse((ArrayTraverser) (int i, Inspector item) -> nodes.add(createNode(item)));
+ return nodes;
+ }
+
+ private Node createNode(Inspector inspector) {
+ Optional<String> parentHostname = optionalString(inspector.field("parentHostname"));
+
+ return nodeRepository.createNode(
+ inspector.field("openStackId").asString(),
+ inspector.field("hostname").asString(),
+ parentHostname,
+ new Configuration(nodeFlavors.getFlavorOrThrow(inspector.field("flavor").asString())));
+ }
+
+ // TODO: Move most of this to node repo
+ public String setNodeReady(String path) {
+ String hostname = lastElement(path);
+ if ( nodeRepository.getNode(Node.State.ready, hostname).isPresent())
+ return "Nothing done; " + hostname + " is already ready";
+
+ Optional<Node> node = nodeRepository.getNode(Node.State.provisioned, hostname);
+ if ( ! node.isPresent())
+ node = nodeRepository.getNode(Node.State.dirty, hostname);
+ if ( ! node.isPresent())
+ node = nodeRepository.getNode(Node.State.failed, hostname);
+ if ( ! node.isPresent())
+ throw new IllegalArgumentException("Could not set " + hostname + " ready: Not registered as provisioned, dirty or failed");
+
+ nodeRepository.setReady(Collections.singletonList(node.get()));
+ return "Moved " + hostname + " to ready";
+ }
+
+ public static NodeFilter toNodeFilter(HttpRequest request) {
+ NodeFilter filter = NodeHostFilter.from(HostFilter.from(request.getProperty("hostname"),
+ request.getProperty("flavor"),
+ request.getProperty("clusterType"),
+ request.getProperty("clusterId")));
+ filter = ApplicationFilter.from(request.getProperty("application"), filter);
+ filter = StateFilter.from(request.getProperty("state"), filter);
+ filter = ParentHostFilter.from(request.getProperty("parentHost"), filter);
+ return filter;
+ }
+
+ private String lastElement(String path) {
+ if (path.endsWith("/"))
+ path = path.substring(0, path.length()-1);
+ int lastSlash = path.lastIndexOf("/");
+ if (lastSlash < 0) return path;
+ return path.substring(lastSlash + 1, path.length());
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
new file mode 100644
index 00000000000..4e8fbe6099b
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
@@ -0,0 +1,214 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v2;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter;
+import com.yahoo.vespa.hosted.provision.restapi.NodeStateSerializer;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.List;
+import java.util.Optional;
+
+/**
+* @author bratseth
+*/
+class NodesResponse extends HttpResponse {
+
+ /** The responses this can create */
+ public enum ResponseType { nodeList, stateList, nodesInStateList, singleNode }
+
+ /** The request url minus parameters, with a trailing slash added if missing */
+ private final String parentUrl;
+
+ /** The parent url of nodes */
+ private final String nodeParentUrl;
+
+ private final NodeFilter filter;
+ private final boolean recursive;
+ private final NodeRepository nodeRepository;
+
+ private final Slime slime;
+
+ public NodesResponse(ResponseType type, HttpRequest request, NodeRepository nodeRepository) {
+ super(200);
+ this.parentUrl = toParentUrl(request);
+ this.nodeParentUrl = toNodeParentUrl(request);
+ filter = NodesApiHandler.toNodeFilter(request);
+ this.recursive = request.getBooleanProperty("recursive");
+ this.nodeRepository = nodeRepository;
+
+ slime = new Slime();
+ Cursor root = slime.setObject();
+ switch (type) {
+ case nodeList: nodesToSlime(root); break;
+ case stateList : statesToSlime(root); break;
+ case nodesInStateList: nodesToSlime(stateFromString(lastElement(parentUrl)), root); break;
+ case singleNode : nodeToSlime(lastElement(parentUrl), root); break;
+ default: throw new IllegalArgumentException();
+ }
+ }
+
+ private String toParentUrl(HttpRequest request) {
+ URI uri = request.getUri();
+ String parentUrl = uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + uri.getPath();
+ if ( ! parentUrl.endsWith("/"))
+ parentUrl = parentUrl + "/";
+ return parentUrl;
+ }
+
+ private String toNodeParentUrl(HttpRequest request) {
+ URI uri = request.getUri();
+ return uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + "/nodes/v2/node/";
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ stream.write(toJson());
+ }
+
+ @Override
+ public String getContentType() {
+ return "application/json";
+ }
+
+ private byte[] toJson() throws IOException {
+ return SlimeUtils.toJsonBytes(slime);
+ }
+
+ private void statesToSlime(Cursor root) {
+ Cursor states = root.setObject("states");
+ for (Node.State state : Node.State.values())
+ toSlime(state, states.setObject(NodeStateSerializer.wireNameOf(state)));
+ }
+
+ private void toSlime(Node.State state, Cursor object) {
+ object.setString("url", parentUrl + NodeStateSerializer.wireNameOf(state));
+ if (recursive)
+ nodesToSlime(state, object);
+ }
+
+ /** Outputs the nodes in the given state to a node array */
+ private void nodesToSlime(Node.State state, Cursor parentObject) {
+ Cursor nodeArray = parentObject.setArray("nodes");
+ toSlime(nodeRepository.getNodes(state), nodeArray);
+ }
+
+ /** Outputs all the nodes to a node array */
+ private void nodesToSlime(Cursor parentObject) {
+ Cursor nodeArray = parentObject.setArray("nodes");
+ for (Node.State state : Node.State.values())
+ toSlime(nodeRepository.getNodes(state), nodeArray);
+ }
+
+ private void toSlime(List<Node> nodes, Cursor array) {
+ for (Node node : nodes) {
+ if ( ! filter.matches(node)) continue;
+ toSlime(node, recursive, array.addObject());
+ }
+ }
+
+ private void nodeToSlime(String hostname, Cursor object) {
+ Optional<Node> node = nodeRepository.getNode(hostname);
+ if (! node.isPresent())
+ throw new IllegalArgumentException("No node with hostname '" + hostname + "'");
+ toSlime(node.get(), true, object);
+ }
+
+ private void toSlime(Node node, boolean allFields, Cursor object) {
+ object.setString("url", nodeParentUrl + node.hostname());
+ if ( ! allFields) return;
+ object.setString("id", node.id());
+ object.setString("state", NodeStateSerializer.wireNameOf(node.state()));
+ object.setString("hostname", node.hostname());
+ if (node.parentHostname().isPresent()) {
+ object.setString("parentHostname", node.parentHostname().get());
+ }
+ object.setString("openStackId", node.openStackId());
+ object.setString("flavor", node.configuration().flavor().name());
+ if (node.configuration().flavor().getMinDiskAvailableGb() > 0) {
+ object.setDouble("minDiskAvailableGb", node.configuration().flavor().getMinDiskAvailableGb());
+ }
+ if (node.configuration().flavor().getMinMainMemoryAvailableGb() > 0) {
+ object.setDouble("minMainMemoryAvailableGb", node.configuration().flavor().getMinMainMemoryAvailableGb());
+ }
+ if (node.configuration().flavor().getDescription() != null && ! node.configuration().flavor().getDescription().isEmpty()) {
+ object.setString("description", node.configuration().flavor().getDescription());
+ }
+ if (node.configuration().flavor().getMinCpuCores() > 0) {
+ object.setDouble("minCpuCores", node.configuration().flavor().getMinCpuCores());
+ }
+ object.setString("canonicalFlavor", node.configuration().flavor().canonicalName());
+ if (node.configuration().flavor().cost() > 0) {
+ object.setLong("cost", node.configuration().flavor().cost());
+ }
+ if (node.configuration().flavor().getEnvironment() != null && ! node.configuration().flavor().getEnvironment().isEmpty()) {
+ object.setString("environment", node.configuration().flavor().getEnvironment());
+ }
+ Optional<Allocation> allocation = node.allocation();
+ if (allocation.isPresent()) {
+ toSlime(allocation.get().owner(), object.setObject("owner"));
+ toSlime(allocation.get().membership(), object.setObject("membership"));
+ object.setLong("restartGeneration", allocation.get().restartGeneration().wanted());
+ object.setLong("currentRestartGeneration", allocation.get().restartGeneration().current());
+ allocation.get().membership().cluster().dockerImage().ifPresent(
+ image -> object.setString("wantedDockerImage", image));
+ }
+ object.setLong("rebootGeneration", node.status().reboot().wanted());
+ object.setLong("currentRebootGeneration", node.status().reboot().current());
+ node.status().vespaVersion().ifPresent(version -> object.setString("vespaVersion", version.toString()));
+ node.status().hostedVersion().ifPresent(version -> object.setString("hostedVersion", version.toString()));
+ node.status().dockerImage().ifPresent(image -> object.setString("currentDockerImage", image));
+ node.status().stateVersion().ifPresent(version -> object.setString("convergedStateVersion", version));
+ object.setLong("failCount", node.status().failCount());
+ object.setBool("hardwareFailure", node.status().hardwareFailure());
+ toSlime(node.history(), object.setArray("history"));
+ }
+
+ private void toSlime(ApplicationId id, Cursor object) {
+ object.setString("tenant", id.tenant().value());
+ object.setString("application", id.application().value());
+ object.setString("instance", id.instance().value());
+ }
+
+ private void toSlime(ClusterMembership membership, Cursor object) {
+ object.setString("clustertype", membership.cluster().type().name());
+ object.setString("clusterid", membership.cluster().id().value());
+ object.setString("group", membership.cluster().group().get().value());
+ object.setLong("index", membership.index());
+ object.setBool("retired", membership.retired());
+ }
+
+ private void toSlime(History history, Cursor array) {
+ for (History.Event event : history.events()) {
+ Cursor object = array.addObject();
+ object.setString("event", event.type().name());
+ object.setLong("at", event.at().toEpochMilli());
+ }
+ }
+
+ private String lastElement(String path) {
+ if (path.endsWith("/"))
+ path = path.substring(0, path.length()-1);
+ int lastSlash = path.lastIndexOf("/");
+ if (lastSlash < 0) return path;
+ return path.substring(lastSlash+1, path.length());
+ }
+
+ private static Node.State stateFromString(String stateString) {
+ return NodeStateSerializer.fromWireName(stateString)
+ .orElseThrow(() -> new RuntimeException("Node state '" + stateString + "' is not known"));
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java
new file mode 100644
index 00000000000..92403d7588b
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v2;
+
+/**
+ * Thrown when a resource is not found
+ *
+ * @author bratseth
+ */
+public class NotFoundException extends RuntimeException {
+
+ public NotFoundException(String message) { super(message); }
+
+ public NotFoundException(String message, Throwable cause) { super(message, cause); }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ResourcesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ResourcesResponse.java
new file mode 100644
index 00000000000..ff0c8fdb7b1
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ResourcesResponse.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v2;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+
+/** A response which lists a set of subresources as full urls */
+public class ResourcesResponse extends HttpResponse {
+
+ private final URI parentUrl;
+
+ private final String[] subResources;
+
+ public ResourcesResponse(URI parentUrl, String[] subResources) {
+ super(200);
+ this.parentUrl = parentUrl;
+ this.subResources = subResources;
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ String parentUrlString = parentUrl.toString();
+ if ( ! parentUrlString.endsWith("/"))
+ parentUrlString = parentUrlString + "/";
+
+ Slime slime = new Slime();
+ Cursor root = slime.setObject();
+ Cursor array = root.setArray("resources");
+ for (String subResource : subResources) {
+ array.addObject().setString("url", parentUrlString + subResource + "/");
+ }
+ new JsonFormat(true).encode(stream, slime);
+ }
+
+ @Override
+ public String getContentType() { return "application/json"; }
+
+ public static ResourcesResponse fromStrings(URI parentUrl, String ... subResources) {
+ return new ResourcesResponse(parentUrl, subResources);
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
new file mode 100644
index 00000000000..990051b2317
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.testutils;
+
+/**
+ * For running NodeRepository API with some mocked data.
+ * This is used by both NodeAdmin and NodeRepository tests.
+ *
+ * @author dybdahl
+ */
+public class ContainerConfig {
+ public static final String servicesXmlV2(int port) {
+ return
+ "<jdisc version=\"1.0\">" +
+ " <component id=\"com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors\"/>" +
+ " <component id=\"com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository\"/>" +
+ " <handler id=\"com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler\">" +
+ " <binding>http://*/nodes/v2/*</binding>" +
+ " </handler>" +
+ " <http>\n" +
+ " <server id='myServer' port='" + port + "' />\n" +
+ " </http>" +
+ "</jdisc>";
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/FlavorConfigBuilder.java
new file mode 100644
index 00000000000..65c8facc094
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/FlavorConfigBuilder.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.testutils;
+
+import com.yahoo.vespa.config.nodes.NodeRepositoryConfig;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+
+/**
+ * Sinmplifies creation of a node-repository config containing flavors.
+ * This is needed because the config builder API is inconvenient.
+ *
+ * @author bratseth
+ */
+public class FlavorConfigBuilder {
+
+ private NodeRepositoryConfig.Builder builder = new NodeRepositoryConfig.Builder();
+
+ public NodeRepositoryConfig build() {
+ return new NodeRepositoryConfig(builder);
+ }
+
+ public NodeRepositoryConfig.Flavor.Builder addFlavor(String flavorName, double cpu, double mem, double disk, String environment) {
+ NodeRepositoryConfig.Flavor.Builder flavor = new NodeRepositoryConfig.Flavor.Builder();
+ flavor.name(flavorName);
+ flavor.description("Flavor-name-is-" + flavorName);
+ flavor.minDiskAvailableGb(disk);
+ flavor.minCpuCores(cpu);
+ flavor.minMainMemoryAvailableGb(mem);
+ flavor.environment(environment);
+ builder.flavor(flavor);
+ return flavor;
+ }
+
+ public void addReplaces(String replaces, NodeRepositoryConfig.Flavor.Builder flavor) {
+ NodeRepositoryConfig.Flavor.Replaces.Builder flavorReplaces = new NodeRepositoryConfig.Flavor.Replaces.Builder();
+ flavorReplaces.name(replaces);
+ flavor.replaces(flavorReplaces);
+ }
+
+ public void addCost(int cost, NodeRepositoryConfig.Flavor.Builder flavor) {
+ flavor.cost(cost);
+ }
+
+ /** Convenience method which creates a node flavors instance from a list of flavor names */
+ public static NodeFlavors createDummies(String... flavors) {
+
+ FlavorConfigBuilder flavorConfigBuilder = new FlavorConfigBuilder();
+ for (String flavorName : flavors) {
+ flavorConfigBuilder.addFlavor(flavorName, 1. /* cpu*/ , 3. /* mem GB*/, 2. /*disk GB*/, "foo" /* env*/);
+ }
+ return new NodeFlavors(flavorConfigBuilder.build());
+ }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeFlavors.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeFlavors.java
new file mode 100644
index 00000000000..e5e9bd27cd8
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeFlavors.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.testutils;
+
+import com.yahoo.vespa.config.nodes.NodeRepositoryConfig;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+
+/**
+ * A mock repository prepopulated with flavors, to avoid having config.
+ * Instantiated by DI from application package above.
+ */
+public class MockNodeFlavors extends NodeFlavors {
+
+ public MockNodeFlavors() {
+ super(createConfig());
+ }
+
+ private static NodeRepositoryConfig createConfig() {
+ FlavorConfigBuilder b = new FlavorConfigBuilder();
+ b.addFlavor("default", 2., 16., 400, "env");
+ b.addFlavor("medium-disk", 6., 12., 56, "foo");
+ b.addFlavor("large", 4., 32., 1600, "env");
+ b.addFlavor("docker", 0.2, 0.5, 100, "docker");
+ NodeRepositoryConfig.Flavor.Builder largeVariant = b.addFlavor("large-variant", 64, 128, 2000, "env");
+ b.addReplaces("large", largeVariant);
+ NodeRepositoryConfig.Flavor.Builder expensiveFlavor = b.addFlavor("expensive", 0, 0, 0, "");
+ b.addReplaces("default", expensiveFlavor);
+ b.addCost(200, expensiveFlavor);
+
+ return b.build();
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
new file mode 100644
index 00000000000..d60e43cebed
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.testutils;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.node.Status;
+import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A mock repository prepopulated with some applications.
+ * Instantiated by DI from application package above.
+ */
+public class MockNodeRepository extends NodeRepository {
+
+ private final NodeFlavors flavors;
+
+ /**
+ * Constructor
+ * @param flavors flavors to have in node repo
+ */
+ public MockNodeRepository(NodeFlavors flavors) throws Exception {
+ super(flavors, new MockCurator(), Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")));
+ this.flavors = flavors;
+ populate();
+ }
+
+ private void populate() {
+ NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, flavors, Zone.defaultZone());
+
+ List<Node> nodes = new ArrayList<>();
+ nodes.add(createNode("node1", "host1.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default"))));
+ nodes.add(createNode("node2", "host2.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default"))));
+ nodes.add(createNode("node3", "host3.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("expensive"))));
+
+ // TODO: Use docker flavor
+ Node node4 = createNode("node4", "host4.yahoo.com", Optional.of("dockerhost4"), new Configuration(flavors.getFlavorOrThrow("default")));
+ node4 = node4.setStatus(node4.status().setDockerImage("image-12"));
+ nodes.add(node4);
+
+ Node node5 = createNode("node5", "host5.yahoo.com", Optional.of("dockerhost"), new Configuration(flavors.getFlavorOrThrow("default")));
+ nodes.add(node5.setStatus(node5.status().setDockerImage("image-123")));
+
+ nodes.add(createNode("node6", "host6.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default"))));
+ nodes.add(createNode("node7", "host7.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default"))));
+ // 8 and 9 are added by web service calls
+ Node node10 = createNode("node10", "host10.yahoo.com", Optional.of("parent.yahoo.com"), new Configuration(flavors.getFlavorOrThrow("default")));
+ Status node10newStatus = node10.status();
+ node10newStatus = node10newStatus
+ .setVespaVersion(Version.fromString("5.104.142"))
+ .setHostedVersion(Version.fromString("2.1.2408"))
+ .setStateVersion("5.104.142-2.1.2408");
+ node10 = node10.setStatus(node10newStatus);
+ nodes.add(node10);
+ nodes = addNodes(nodes);
+ nodes.remove(6);
+ setReady(nodes);
+ fail("host5.yahoo.com");
+
+ ApplicationId app1 = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1"));
+ ClusterSpec cluster1 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Optional.empty(), Optional.of("image-123"));
+ provisioner.prepare(app1, cluster1, Capacity.fromNodeCount(2), 1, null);
+
+ ApplicationId app2 = ApplicationId.from(TenantName.from("tenant2"), ApplicationName.from("application2"), InstanceName.from("instance2"));
+ ClusterSpec cluster2 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("id2"), Optional.empty());
+ activate(provisioner.prepare(app2, cluster2, Capacity.fromNodeCount(2), 1, null), app2, provisioner);
+
+ ApplicationId app3 = ApplicationId.from(TenantName.from("tenant3"), ApplicationName.from("application3"), InstanceName.from("instance3"));
+ ClusterSpec cluster3 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("id3"), Optional.empty());
+ activate(provisioner.prepare(app3, cluster3, Capacity.fromNodeCount(2), 1, null), app3, provisioner);
+ }
+
+ private void activate(List<HostSpec> hosts, ApplicationId application, NodeRepositoryProvisioner provisioner) {
+ NestedTransaction transaction = new NestedTransaction();
+ provisioner.activate(transaction, application, hosts);
+ transaction.commit();
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md
new file mode 100644
index 00000000000..0ea723fe80e
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md
@@ -0,0 +1,3 @@
+The test resources are used by both NoadAdmin and NodeRepository
+tests to verify APIs. So when modifying this test data
+remember to check tests for both NodeAdmin and NodeRepository. \ No newline at end of file
diff --git a/node-repository/src/main/resources/configdefinitions/node-repository.def b/node-repository/src/main/resources/configdefinitions/node-repository.def
new file mode 100644
index 00000000000..cd053adca61
--- /dev/null
+++ b/node-repository/src/main/resources/configdefinitions/node-repository.def
@@ -0,0 +1,34 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+# Configuration of the node repository
+namespace=vespa.config.nodes
+
+# A node flavor which (may) be available in this zone.
+# This is to allow configuration per flavor.
+# If a certain flavor has no config it is not necessary to list it here to use it.
+flavor[].name string
+
+# Names of other flavors (whether mentioned in this config or not) which this flavor
+# is a replacement for: If one of these flavor names are requested, this flavor may
+# be assigned instead.
+# Replacements are transitive: If flavor a replaces b replaces c, then a request for flavor
+# c may be satisfied by assigning nodes of flavor a.
+flavor[].replaces[].name string
+
+# The monthly Total Cost of Ownership (TCO) in USD. Typically calculated as TCO divered by
+# the expected lifetime of the node (usually three years).
+flavor[].cost int default=0
+
+# The type of node (e.g. bare metal, docker..).
+flavor[].environment string default="undefined"
+
+# The minimum number of CPU cores available.
+flavor[].minCpuCores double default=0.0
+
+# The minimum amount of main memory available.
+flavor[].minMainMemoryAvailableGb double default=0.0
+
+# The minimum amount of disk available.
+flavor[].minDiskAvailableGb double default=0.0
+
+# Human readable free text for description of node.
+flavor[].description string default=""
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeList.java
new file mode 100644
index 00000000000..2d587b12ddf
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeList.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.config.provision.ClusterSpec;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * A filterable node list
+ *
+ * @author bratseth
+ */
+public class NodeList {
+
+ private final List<Node> nodes;
+
+ public NodeList(List<Node> nodes) {
+ this.nodes = ImmutableList.copyOf(nodes);
+ }
+
+ /** Returns the subset of nodes which are retired */
+ public NodeList retired() {
+ return new NodeList(nodes.stream().filter(node -> node.allocation().get().membership().retired()).collect(Collectors.toList()));
+ }
+
+ /** Returns the subset of nodes which are not retired */
+ public NodeList nonretired() {
+ return new NodeList(nodes.stream().filter(node -> ! node.allocation().get().membership().retired()).collect(Collectors.toList()));
+ }
+
+ /** Returns the subset of nodes of the given flavor */
+ public NodeList flavor(String flavor) {
+ return new NodeList(nodes.stream().filter(node -> node.configuration().flavor().name().equals(flavor)).collect(Collectors.toList()));
+ }
+
+ /** Returns the subset of nodes which does not have the given flavor */
+ public NodeList notFlavor(String flavor) {
+ return new NodeList(nodes.stream().filter(node -> ! node.configuration().flavor().name().equals(flavor)).collect(Collectors.toList()));
+ }
+
+ /** Returns the subset of nodes assigned to the given cluster type */
+ public NodeList type(ClusterSpec.Type type) {
+ return new NodeList(nodes.stream().filter(node -> node.allocation().get().membership().cluster().type().equals(type)).collect(Collectors.toList()));
+ }
+
+ public int size() { return nodes.size(); }
+
+ /** Returns the immutable list of nodes in this */
+ public List<Node> asList() { return nodes; }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClientTest.java
new file mode 100644
index 00000000000..9cbe17dd718
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClientTest.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.assimilate;
+
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import org.junit.Test;
+
+import java.time.Clock;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+/**
+ * @author vegard
+ */
+public class PopulateClientTest {
+
+ final static String servicesXmlFilename = "src/test/resources/services.xml";
+ final static String hostsXmlFilename = "src/test/resources/hosts.xml";
+
+ final List<String> hostnames = Arrays.asList("hostname1", "hostname2", "hostname3", "hostname4", "hostname5", "hostname6");
+ final List<String> clusterTypes = Arrays.asList("container", "container", "content", "content", "content", "content");
+ final List<String> clusterIds = Arrays.asList("default", "default", "default", "default", "mycontent", "mycontent");
+ final List<Integer> nodeIndices = Arrays.asList(0, 1, 99, 42, 0, 1);
+
+ final String tenantId = "vegard";
+ final String applicationId = "killer-app";
+ final String instanceId = "default";
+
+ final Map<String, String> flavorSpec = ImmutableMap.of(
+ "container.default", "vanilla",
+ "content.default", "strawberry",
+ "content.mycontent", "chocolate"
+ );
+
+ NodeFlavors flavors = FlavorConfigBuilder.createDummies(flavorSpec.values().stream().collect(Collectors.toList()).toArray(new String[flavorSpec.size()]));
+
+ @Test
+ public void testCorrectDataIsWrittenToZooKeeper() {
+ Curator curator = new MockCurator();
+ CuratorDatabaseClient curatorDatabaseClient = new CuratorDatabaseClient(flavors, curator, Clock.systemUTC());
+
+ PopulateClient populateClient = new PopulateClient(curator, flavors, tenantId, applicationId, instanceId, servicesXmlFilename, hostsXmlFilename, flavorSpec, false);
+ populateClient.populate(PopulateClient.CONTAINER_CLUSTER_TYPE);
+ populateClient.populate(PopulateClient.CONTENT_CLUSTER_TYPE);
+
+ List<Node> nodes = curatorDatabaseClient.getNodes(ApplicationId.from(
+ TenantName.from(tenantId),
+ ApplicationName.from(applicationId),
+ InstanceName.from(instanceId)));
+
+ assertThat("Zookeeper is populated", nodes.size(), is(hostnames.size()));
+
+ nodes.stream().forEach(node -> {
+ assertThat("Node has allocation", node.allocation(), notNullValue());
+
+ final Allocation allocation = node.allocation().get();
+ assertThat("Application id must match", allocation.owner().application().toString(), is(applicationId));
+ assertThat("Tenant id must match", allocation.owner().tenant().toString(), is(tenantId));
+ assertThat("Instance id must match", allocation.owner().instance().toString(), is(instanceId));
+
+ final int index = hostnames.indexOf(node.hostname());
+ assertThat("Hostname must be one the hostnames", index, is(not(-1)));
+
+ final String clusterType = allocation.membership().cluster().type().name();
+ assertThat("Cluster type must match", clusterType, is(clusterTypes.get(index)));
+
+ final String clusterId = allocation.membership().cluster().id().value();
+ assertThat("Cluster id must match", clusterId, is(clusterIds.get(index)));
+
+ assertThat("Flavor must match", node.configuration().flavor().name(), is(flavorSpec.get(clusterType + "." + clusterId)));
+ assertThat("Node index must match", node.allocation().get().membership().index(), is(nodeIndices.get(index)));
+ });
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testNotSpecifyingAFlavorThrowsException() {
+ Map<String, String> myFlavorSpec = ImmutableMap.of(
+ "container.default", "vanilla",
+ "content.default", "strawberry"
+ // missing content.mycontent
+ );
+
+ new PopulateClient(new MockCurator(), flavors, tenantId, applicationId, instanceId, servicesXmlFilename, hostsXmlFilename, myFlavorSpec, false);
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java
new file mode 100644
index 00000000000..99c6c7d9294
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java
@@ -0,0 +1,148 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.test.ManualClock;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author bratseth
+ */
+public class ApplicationMaintainerTest {
+
+ private Curator curator = new MockCurator();
+
+ @Test
+ public void test_application_maintenance() throws InterruptedException {
+ ManualClock clock = new ManualClock();
+ Zone zone = new Zone(Environment.prod, RegionName.from("us-east"));
+ NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default");
+ NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock);
+
+ createReadyNodes(15, nodeRepository, nodeFlavors);
+
+ Fixture fixture = new Fixture(zone, nodeRepository, nodeFlavors);
+
+ // Create applications
+ fixture.activate();
+
+ // Fail some nodes
+ nodeRepository.fail(nodeRepository.getNodes(fixture.app1).get(3).hostname());
+ nodeRepository.fail(nodeRepository.getNodes(fixture.app2).get(0).hostname());
+ nodeRepository.fail(nodeRepository.getNodes(fixture.app2).get(4).hostname());
+ int failedInApp1 = 1;
+ int failedInApp2 = 2;
+ assertEquals(fixture.wantedNodesApp1 - failedInApp1, nodeRepository.getNodes(fixture.app1, Node.State.active).size());
+ assertEquals(fixture.wantedNodesApp2 - failedInApp2, nodeRepository.getNodes(fixture.app2, Node.State.active).size());
+ assertEquals(failedInApp1 + failedInApp2, nodeRepository.getNodes(Node.State.failed).size());
+ assertEquals(3, nodeRepository.getNodes(Node.State.ready).size());
+
+ // Cause maintenance deployment which will allocate replacement nodes
+ fixture.runApplicationMaintainer();
+ assertEquals(fixture.wantedNodesApp1, nodeRepository.getNodes(fixture.app1, Node.State.active).size());
+ assertEquals(fixture.wantedNodesApp2, nodeRepository.getNodes(fixture.app2, Node.State.active).size());
+ assertEquals(0, nodeRepository.getNodes(Node.State.ready).size());
+
+ // Unfail the previously failed nodes
+ nodeRepository.unfail(nodeRepository.getNodes(Node.State.failed).get(0).hostname());
+ nodeRepository.unfail(nodeRepository.getNodes(Node.State.failed).get(0).hostname());
+ nodeRepository.unfail(nodeRepository.getNodes(Node.State.failed).get(0).hostname());
+ int unfailedInApp1 = 1;
+ int unfailedInApp2 = 2;
+ assertEquals(0, nodeRepository.getNodes(Node.State.failed).size());
+ assertEquals(fixture.wantedNodesApp1 + unfailedInApp1, nodeRepository.getNodes(fixture.app1, Node.State.active).size());
+ assertEquals(fixture.wantedNodesApp2 + unfailedInApp2, nodeRepository.getNodes(fixture.app2, Node.State.active).size());
+ assertEquals("The unfailed nodes are now active but not part of the application",
+ 0, fixture.getNodes(Node.State.active).retired().size());
+
+ // Cause maintenance deployment which will update the applications with the re-activated nodes
+ fixture.runApplicationMaintainer();
+ assertEquals("Superflous content nodes are retired",
+ unfailedInApp2, fixture.getNodes(Node.State.active).retired().size());
+ assertEquals("Superflous container nodes are deactivated (this makes little point for container nodes)",
+ unfailedInApp1, fixture.getNodes(Node.State.inactive).size());
+ }
+
+ private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) {
+ List<Node> nodes = new ArrayList<>(count);
+ for (int i = 0; i < count; i++)
+ nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))));
+ nodes = nodeRepository.addNodes(nodes);
+ nodeRepository.setReady(nodes);
+ }
+
+ private class Fixture {
+
+ final NodeRepository nodeRepository;
+ final NodeRepositoryProvisioner provisioner;
+
+ final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz"));
+ final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz"));
+ final ClusterSpec clusterApp1 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Optional.empty());
+ final ClusterSpec clusterApp2 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty());
+ final int wantedNodesApp1 = 5;
+ final int wantedNodesApp2 = 7;
+
+ Fixture(Zone zone, NodeRepository nodeRepository, NodeFlavors flavors) {
+ this.nodeRepository = nodeRepository;
+ this.provisioner = new NodeRepositoryProvisioner(nodeRepository, flavors, zone);
+ }
+
+ void activate() {
+ activate(app1, clusterApp1, wantedNodesApp1, provisioner);
+ activate(app2, clusterApp2, wantedNodesApp2, provisioner);
+ assertEquals(wantedNodesApp1, nodeRepository.getNodes(app1, Node.State.active).size());
+ assertEquals(wantedNodesApp2, nodeRepository.getNodes(app2, Node.State.active).size());
+ }
+
+ private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodeCount, NodeRepositoryProvisioner provisioner) {
+ List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(nodeCount), 1, null);
+ NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
+ provisioner.activate(transaction, applicationId, hosts);
+ transaction.commit();
+ }
+
+ void runApplicationMaintainer() {
+ Map<ApplicationId, MockDeployer.ApplicationContext> apps = new HashMap<>();
+ apps.put(app1, new MockDeployer.ApplicationContext(app1, clusterApp1, wantedNodesApp1, Optional.of("default"), 1));
+ apps.put(app2, new MockDeployer.ApplicationContext(app2, clusterApp2, wantedNodesApp2, Optional.of("default"), 1));
+ MockDeployer deployer = new MockDeployer(provisioner, apps);
+ new ApplicationMaintainer(deployer, nodeRepository, Duration.ofMinutes(30)).run();
+ }
+
+ NodeList getNodes(Node.State ... states) {
+ return new NodeList(nodeRepository.getNodes(states));
+ }
+
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
new file mode 100644
index 00000000000..278a9b704b9
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
@@ -0,0 +1,119 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.test.ManualClock;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author bratseth
+ */
+public class FailedExpirerTest {
+
+ private Curator curator = new MockCurator();
+
+ @Test
+ public void ensure_failed_nodes_are_deallocated_in_prod() throws InterruptedException {
+ NodeRepository nodeRepository = failureScenarioIn(Environment.prod);
+
+ assertEquals(2, nodeRepository.getNodes(Node.State.failed).size());
+ assertEquals(1, nodeRepository.getNodes(Node.State.dirty).size());
+ assertEquals("node3", nodeRepository.getNodes(Node.State.dirty).get(0).hostname());
+ }
+
+ @Test
+ public void ensure_failed_nodes_are_deallocated_in_dev() throws InterruptedException {
+ NodeRepository nodeRepository = failureScenarioIn(Environment.dev);
+
+ assertEquals(1, nodeRepository.getNodes(Node.State.failed).size());
+ assertEquals(2, nodeRepository.getNodes(Node.State.dirty).size());
+ assertEquals("node2", nodeRepository.getNodes(Node.State.failed).get(0).hostname());
+ }
+
+ private NodeRepository failureScenarioIn(Environment environment) {
+ ManualClock clock = new ManualClock();
+ NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default");
+ NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock);
+ NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone(), clock);
+
+ List<Node> nodes = new ArrayList<>(3);
+ nodes.add(nodeRepository.createNode("node1", "node1", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))));
+ nodes.add(nodeRepository.createNode("node2", "node2", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))));
+ nodes.add(nodeRepository.createNode("node3", "node3", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))));
+ nodeRepository.addNodes(nodes);
+
+ // Set node1 to have failed 4 times before
+ Node node1 = nodeRepository.getNode("node1").get();
+ node1 = node1.setStatus(node1.status().increaseFailCount());
+ node1 = node1.setStatus(node1.status().increaseFailCount());
+ node1 = node1.setStatus(node1.status().increaseFailCount());
+ node1 = node1.setStatus(node1.status().increaseFailCount());
+ nodeRepository.write(node1);
+
+ // Set node2 to have a detected hardware failure
+ Node node2 = nodeRepository.getNode("node2").get();
+ node2 = node2.setStatus(node2.status().setHardwareFailure(true));
+ nodeRepository.write(node2);
+
+ // Allocate the nodes
+ nodeRepository.setReady(nodeRepository.getNodes(Node.State.provisioned));
+ ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz"));
+ ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty());
+ provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(3), 1, null);
+ NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
+ provisioner.activate(transaction, applicationId, asHosts(nodes));
+ transaction.commit();
+ assertEquals(3, nodeRepository.getNodes(Node.State.active).size());
+
+ // Fail the nodes
+ nodeRepository.fail("node1");
+ nodeRepository.fail("node2");
+ nodeRepository.fail("node3");
+ assertEquals(3, nodeRepository.getNodes(Node.State.failed).size());
+
+ // Failure times out
+ clock.advance(Duration.ofDays(5));
+ new FailedExpirer(nodeRepository, new Zone(environment, RegionName.from("us-west-1")), clock, Duration.ofDays(4)).run();
+
+ return nodeRepository;
+ }
+
+ private Set<HostSpec> asHosts(List<Node> nodes) {
+ Set<HostSpec> hosts = new HashSet<>(nodes.size());
+ for (Node node : nodes)
+ hosts.add(new HostSpec(node.hostname(),
+ node.allocation().isPresent() ? Optional.of(node.allocation().get().membership()) :
+ Optional.empty()));
+ return hosts;
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
new file mode 100644
index 00000000000..460ab3906ed
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.test.ManualClock;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author bratseth
+ */
+public class InactiveAndFailedExpirerTest {
+
+ private Curator curator = new MockCurator();
+
+ @Test
+ public void ensure_inactive_and_failed_times_out() throws InterruptedException {
+ ManualClock clock = new ManualClock();
+ NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default");
+ NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock);
+ NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone(), clock);
+
+ List<Node> nodes = new ArrayList<>(2);
+ nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))));
+ nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))));
+ nodeRepository.addNodes(nodes);
+
+ // Allocate then deallocate 2 nodes
+ nodeRepository.setReady(nodes);
+ ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz"));
+ ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty());
+ provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(2), 1, null);
+ NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
+ provisioner.activate(transaction, applicationId, asHosts(nodes));
+ transaction.commit();
+ assertEquals(2, nodeRepository.getNodes(Node.State.active).size());
+ nodeRepository.deactivate(applicationId);
+ assertEquals(2, nodeRepository.getNodes(Node.State.inactive).size());
+
+ // Inactive times out
+ clock.advance(Duration.ofMinutes(14));
+ new InactiveExpirer(nodeRepository, clock, Duration.ofMinutes(10)).run();
+
+ assertEquals(0, nodeRepository.getNodes(Node.State.inactive).size());
+ List<Node> dirty = nodeRepository.getNodes(Node.State.dirty);
+ assertEquals(2, dirty.size());
+ assertFalse(dirty.get(0).allocation().isPresent());
+ assertFalse(dirty.get(1).allocation().isPresent());
+
+ // One node is set back to ready
+ Node ready = nodeRepository.setReady(Collections.singletonList(dirty.get(0))).get(0);
+ assertEquals("Allocated history is removed on readying", 1, ready.history().events().size());
+ assertEquals(History.Event.Type.readied, ready.history().events().iterator().next().type());
+
+ // Dirty times out for the other one
+ clock.advance(Duration.ofMinutes(14));
+ new DirtyExpirer(nodeRepository, clock, Duration.ofMinutes(10)).run();
+ assertEquals(0, nodeRepository.getNodes(Node.State.dirty).size());
+ List<Node> failed = nodeRepository.getNodes(Node.State.failed);
+ assertEquals(1, failed.size());
+ assertEquals(1, failed.get(0).status().failCount());
+ }
+
+ private Set<HostSpec> asHosts(List<Node> nodes) {
+ Set<HostSpec> hosts = new HashSet<>(nodes.size());
+ for (Node node : nodes)
+ hosts.add(new HostSpec(node.hostname(),
+ node.allocation().isPresent() ? Optional.of(node.allocation().get().membership()) :
+ Optional.empty()));
+ return hosts;
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MockDeployer.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MockDeployer.java
new file mode 100644
index 00000000000..7e884b35e16
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MockDeployer.java
@@ -0,0 +1,104 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Deployer;
+import com.yahoo.config.provision.Deployment;
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * @author bratseth
+ */
+public class MockDeployer implements Deployer {
+
+ private final NodeRepositoryProvisioner provisioner;
+ private final Map<ApplicationId, ApplicationContext> applications;
+
+ /** The number of redeployments done to this */
+ public int redeployments = 0;
+
+ /**
+ * Create a mock deployer which contains a substitute for an application repository, sufficient to
+ * be able to call provision with the right parameters.
+ */
+ public MockDeployer(NodeRepositoryProvisioner provisioner, Map<ApplicationId, ApplicationContext> applications) {
+ this.provisioner = provisioner;
+ this.applications = applications;
+ }
+
+ @Override
+ public Optional<Deployment> deployFromLocalActive(ApplicationId id, Duration timeout) {
+ return Optional.of(new MockDeployment(provisioner, applications.get(id)));
+ }
+
+ public class MockDeployment implements Deployment {
+
+ private final NodeRepositoryProvisioner provisioner;
+ private final ApplicationContext application;
+
+ /** The list of hosts prepared in this. Only set after prepare is called (and a provisioner is assigned) */
+ private List<HostSpec> preparedHosts;
+
+ private MockDeployment(NodeRepositoryProvisioner provisioner, ApplicationContext application) {
+ this.provisioner = provisioner;
+ this.application = application;
+ }
+
+ @Override
+ public void prepare() {
+ preparedHosts = application.prepare(provisioner);
+ }
+
+ @Override
+ public void activate() {
+ redeployments++;
+ try (NestedTransaction t = new NestedTransaction()) {
+ provisioner.activate(t, application.id(), preparedHosts);
+ t.commit();
+ }
+ }
+
+ @Override
+ public void restart(HostFilter filter) {}
+
+ }
+
+ /** An application context which substitutes for an application repository */
+ public static class ApplicationContext {
+
+ private ApplicationId id;
+ private ClusterSpec cluster;
+ private int wantedNodes;
+ private Optional<String> flavor;
+ private int groups;
+
+ public ApplicationContext(ApplicationId id, ClusterSpec cluster, int wantedNodes, Optional<String> flavor, int groups) {
+ this.id = id;
+ this.cluster = cluster;
+ this.wantedNodes = wantedNodes;
+ this.flavor = flavor;
+ this.groups = groups;
+ }
+
+ public ApplicationId id() { return id; }
+
+ /** Returns the spec of the cluster of this application. Only a single cluster per application is supported */
+ public ClusterSpec cluster() { return cluster; }
+
+ private List<HostSpec> prepare(NodeRepositoryProvisioner provisioner) {
+ return provisioner.prepare(id, cluster, Capacity.fromNodeCount(wantedNodes, flavor), groups, null);
+ }
+
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java
new file mode 100644
index 00000000000..39a72ef16e8
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java
@@ -0,0 +1,350 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.test.ManualClock;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.ClusterId;
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceCluster;
+import com.yahoo.vespa.applicationmodel.ServiceInstance;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.applicationmodel.TenantId;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException;
+import com.yahoo.vespa.orchestrator.ApplicationStateChangeDeniedException;
+import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException;
+import com.yahoo.vespa.orchestrator.BatchInternalErrorException;
+import com.yahoo.vespa.orchestrator.HostNameNotFoundException;
+import com.yahoo.vespa.orchestrator.Orchestrator;
+import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException;
+import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
+import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
+import com.yahoo.vespa.orchestrator.status.HostStatus;
+import com.yahoo.vespa.service.monitor.ServiceMonitor;
+import com.yahoo.vespa.service.monitor.ServiceMonitorStatus;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests automatic failing of nodes.
+ *
+ * @author bratseth
+ */
+public class NodeFailerTest {
+
+ // Immutable components
+ private static final Zone ZONE = new Zone(Environment.prod, RegionName.from("us-east"));
+ private static final NodeFlavors NODE_FLAVORS = FlavorConfigBuilder.createDummies("default");
+ private static final ApplicationId APP_1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz"));
+ private static final ApplicationId APP_2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz"));
+ private static final Duration DOWNTIME_LIMIT_ONE_HOUR = Duration.ofMinutes(60);
+
+ // Components with state
+ private ManualClock clock;
+ private Curator curator;
+ private ServiceMonitorStub serviceMonitor;
+ private MockDeployer deployer;
+ private NodeRepository nodeRepository;
+ private Orchestrator orchestrator;
+ private NodeFailer failer;
+
+ @Before
+ public void setup() {
+ clock = new ManualClock();
+ curator = new MockCurator();
+ nodeRepository = new NodeRepository(NODE_FLAVORS, curator, clock);
+ NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, NODE_FLAVORS, ZONE);
+
+ createReadyNodes(14, nodeRepository, NODE_FLAVORS);
+
+ // Create applications
+ ClusterSpec clusterApp1 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Optional.empty());
+ ClusterSpec clusterApp2 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty());
+ int wantedNodesApp1 = 5;
+ int wantedNodesApp2 = 7;
+ activate(APP_1, clusterApp1, wantedNodesApp1, provisioner);
+ activate(APP_2, clusterApp2, wantedNodesApp2, provisioner);
+ assertEquals(wantedNodesApp1, nodeRepository.getNodes(APP_1, Node.State.active).size());
+ assertEquals(wantedNodesApp2, nodeRepository.getNodes(APP_2, Node.State.active).size());
+
+ // Create a deployer ...
+ Map<ApplicationId, MockDeployer.ApplicationContext> apps = new HashMap<>();
+ apps.put(APP_1, new MockDeployer.ApplicationContext(APP_1, clusterApp1, wantedNodesApp1, Optional.of("default"), 1));
+ apps.put(APP_2, new MockDeployer.ApplicationContext(APP_2, clusterApp2, wantedNodesApp2, Optional.of("default"), 1));
+ deployer = new MockDeployer(provisioner, apps);
+ // ... and a service monitor
+ serviceMonitor = new ServiceMonitorStub(apps, nodeRepository);
+
+ orchestrator = new OrchestratorMock();
+
+ failer = new NodeFailer(deployer, serviceMonitor, nodeRepository,
+ DOWNTIME_LIMIT_ONE_HOUR, clock, orchestrator);
+ }
+
+ @Test
+ public void nodes_for_suspended_applications_are_not_failed() throws ApplicationStateChangeDeniedException, ApplicationIdNotFoundException {
+ orchestrator.suspend(APP_1);
+
+ // Set two nodes down (one for each application) and wait 65 minutes
+ String host_from_suspended_app = nodeRepository.getNodes(APP_1, Node.State.active).get(1).hostname();
+ String host_from_normal_app = nodeRepository.getNodes(APP_2, Node.State.active).get(3).hostname();
+ serviceMonitor.setHostDown(host_from_suspended_app);
+ serviceMonitor.setHostDown(host_from_normal_app);
+ failer.run();
+ clock.advance(Duration.ofMinutes(65));
+ failer.run();
+
+ assertEquals(Node.State.failed, nodeRepository.getNode(host_from_normal_app).get().state());
+ assertEquals(Node.State.active, nodeRepository.getNode(host_from_suspended_app).get().state());
+ }
+
+ @Test
+ public void test_node_failing() throws InterruptedException {
+ // For a day all nodes work so nothing happens
+ for (int minutes = 0; minutes < 24 * 60; minutes +=5 ) {
+ failer.run();
+ clock.advance(Duration.ofMinutes(5));
+ assertEquals( 0, deployer.redeployments);
+ assertEquals(12, nodeRepository.getNodes(Node.State.active).size());
+ assertEquals( 0, nodeRepository.getNodes(Node.State.failed).size());
+ assertEquals( 2, nodeRepository.getNodes(Node.State.ready).size());
+ }
+
+ String downHost1 = nodeRepository.getNodes(APP_1, Node.State.active).get(1).hostname();
+ String downHost2 = nodeRepository.getNodes(APP_2, Node.State.active).get(3).hostname();
+ serviceMonitor.setHostDown(downHost1);
+ serviceMonitor.setHostDown(downHost2);
+ // nothing happens the first 45 minutes
+ for (int minutes = 0; minutes < 45; minutes +=5 ) {
+ failer.run();
+ clock.advance(Duration.ofMinutes(5));
+ assertEquals( 0, deployer.redeployments);
+ assertEquals(12, nodeRepository.getNodes(Node.State.active).size());
+ assertEquals( 0, nodeRepository.getNodes(Node.State.failed).size());
+ assertEquals( 2, nodeRepository.getNodes(Node.State.ready).size());
+ }
+ serviceMonitor.setHostUp(downHost1);
+ for (int minutes = 0; minutes < 30; minutes +=5 ) {
+ failer.run();
+ clock.advance(Duration.ofMinutes(5));
+ }
+
+ // downHost2 should now be failed and replaced, but not downHost1
+ assertEquals( 1, deployer.redeployments);
+ assertEquals(12, nodeRepository.getNodes(Node.State.active).size());
+ assertEquals( 1, nodeRepository.getNodes(Node.State.failed).size());
+ assertEquals( 1, nodeRepository.getNodes(Node.State.ready).size());
+ assertEquals(downHost2, nodeRepository.getNodes(Node.State.failed).get(0).hostname());
+
+ // downHost1 fails again
+ serviceMonitor.setHostDown(downHost1);
+ failer.run();
+ clock.advance(Duration.ofMinutes(5));
+ // the system goes down and do not have updated information when coming back
+ clock.advance(Duration.ofMinutes(120));
+ serviceMonitor.setStatusIsKnown(false);
+ failer.run();
+ // due to this, nothing is failed
+ assertEquals( 1, deployer.redeployments);
+ assertEquals(12, nodeRepository.getNodes(Node.State.active).size());
+ assertEquals( 1, nodeRepository.getNodes(Node.State.failed).size());
+ assertEquals( 1, nodeRepository.getNodes(Node.State.ready).size());
+ // when status becomes known, and the host is still down, it is failed
+ clock.advance(Duration.ofMinutes(5));
+ serviceMonitor.setStatusIsKnown(true);
+ failer.run();
+ assertEquals( 2, deployer.redeployments);
+ assertEquals(12, nodeRepository.getNodes(Node.State.active).size());
+ assertEquals( 2, nodeRepository.getNodes(Node.State.failed).size());
+ assertEquals( 0, nodeRepository.getNodes(Node.State.ready).size());
+
+ // the last host goes down
+ Node lastNode = highestIndex(nodeRepository.getNodes(APP_1, Node.State.active));
+ serviceMonitor.setHostDown(lastNode.hostname());
+ // it is not failed because there are no ready nodes to replace it
+ for (int minutes = 0; minutes < 75; minutes +=5 ) {
+ failer.run();
+ clock.advance(Duration.ofMinutes(5));
+ assertEquals( 2, deployer.redeployments);
+ assertEquals(12, nodeRepository.getNodes(Node.State.active).size());
+ assertEquals( 2, nodeRepository.getNodes(Node.State.failed).size());
+ assertEquals( 0, nodeRepository.getNodes(Node.State.ready).size());
+ }
+
+ // A new node is available
+ createReadyNodes(1, 14, nodeRepository, NODE_FLAVORS);
+ failer.run();
+ // The node is now failed
+ assertEquals( 3, deployer.redeployments);
+ assertEquals(12, nodeRepository.getNodes(Node.State.active).size());
+ assertEquals( 3, nodeRepository.getNodes(Node.State.failed).size());
+ assertEquals( 0, nodeRepository.getNodes(Node.State.ready).size());
+ assertTrue("The index of the last failed node is not reused",
+ highestIndex(nodeRepository.getNodes(APP_1, Node.State.active)).allocation().get().membership().index()
+ >
+ lastNode.allocation().get().membership().index());
+ }
+
+ private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) {
+ createReadyNodes(count, 0, nodeRepository, nodeFlavors);
+ }
+
+ private void createReadyNodes(int count, int startIndex, NodeRepository nodeRepository, NodeFlavors nodeFlavors) {
+ List<Node> nodes = new ArrayList<>(count);
+ for (int i = startIndex; i < startIndex + count; i++)
+ nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))));
+ nodes = nodeRepository.addNodes(nodes);
+ nodeRepository.setReady(nodes);
+ }
+
+ private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodeCount, NodeRepositoryProvisioner provisioner) {
+ List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(nodeCount), 1, null);
+ NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
+ provisioner.activate(transaction, applicationId, hosts);
+ transaction.commit();
+ }
+
+ /** Returns the node with the highest membership index from the given set of allocated nodes */
+ private Node highestIndex(List<Node> nodes) {
+ Node highestIndex = null;
+ for (Node node : nodes) {
+ if (highestIndex == null || node.allocation().get().membership().index() >
+ highestIndex.allocation().get().membership().index())
+ highestIndex = node;
+ }
+ return highestIndex;
+ }
+
+ private static class ServiceMonitorStub implements ServiceMonitor {
+
+ private final Map<ApplicationId, MockDeployer.ApplicationContext> apps;
+ private final NodeRepository nodeRepository;
+
+ private Set<String> downHosts = new HashSet<>();
+ private boolean statusIsKnown = true;
+
+ /** Create a service monitor where all nodes are initially up */
+ public ServiceMonitorStub(Map<ApplicationId, MockDeployer.ApplicationContext> apps, NodeRepository nodeRepository) {
+ this.apps = apps;
+ this.nodeRepository = nodeRepository;
+ }
+
+ public void setHostDown(String hostname) {
+ downHosts.add(hostname);
+ }
+
+ public void setHostUp(String hostname) {
+ downHosts.remove(hostname);
+ }
+
+ public void setStatusIsKnown(boolean statusIsKnown) {
+ this.statusIsKnown = statusIsKnown;
+ }
+
+ private ServiceMonitorStatus getHostStatus(String hostname) {
+ if ( ! statusIsKnown) return ServiceMonitorStatus.NOT_CHECKED;
+ if (downHosts.contains(hostname)) return ServiceMonitorStatus.DOWN;
+ return ServiceMonitorStatus.UP;
+ }
+
+ @Override
+ public Map<ApplicationInstanceReference, ApplicationInstance<ServiceMonitorStatus>> queryStatusOfAllApplicationInstances() {
+ // Convert apps information to the response payload to return
+ Map<ApplicationInstanceReference, ApplicationInstance<ServiceMonitorStatus>> status = new HashMap<>();
+ for (Map.Entry<ApplicationId, MockDeployer.ApplicationContext> app : apps.entrySet()) {
+ Set<ServiceInstance<ServiceMonitorStatus>> serviceInstances = new HashSet<>();
+ for (Node node : nodeRepository.getNodes(app.getValue().id(), Node.State.active)) {
+ serviceInstances.add(new ServiceInstance<>(new ConfigId("configid"),
+ new HostName(node.hostname()),
+ getHostStatus(node.hostname())));
+ }
+ Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters = new HashSet<>();
+ serviceClusters.add(new ServiceCluster<>(new ClusterId(app.getValue().cluster().id().value()),
+ new ServiceType("serviceType"),
+ serviceInstances));
+ TenantId tenantId = new TenantId(app.getKey().tenant().value());
+ ApplicationInstanceId applicationInstanceId = new ApplicationInstanceId(app.getKey().application().value());
+ status.put(new ApplicationInstanceReference(tenantId, applicationInstanceId),
+ new ApplicationInstance<>(tenantId, applicationInstanceId, serviceClusters));
+ }
+ return status;
+ }
+
+ }
+
+ class OrchestratorMock implements Orchestrator {
+
+ Set<ApplicationId> suspendedApplications = new HashSet<>();
+
+ @Override
+ public HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException {
+ return null;
+ }
+
+ @Override
+ public void resume(HostName hostName) throws HostStateChangeDeniedException, HostNameNotFoundException {}
+
+ @Override
+ public void suspend(HostName hostName) throws HostStateChangeDeniedException, HostNameNotFoundException {}
+
+ @Override
+ public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationId appId) throws ApplicationIdNotFoundException {
+ return suspendedApplications.contains(appId) ? ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN :
+ ApplicationInstanceStatus.NO_REMARKS;
+ }
+
+ @Override
+ public Set<ApplicationId> getAllSuspendedApplications() {
+ return null;
+ }
+
+ @Override
+ public void resume(ApplicationId appId) throws ApplicationStateChangeDeniedException, ApplicationIdNotFoundException {
+ suspendedApplications.remove(appId);
+ }
+
+ @Override
+ public void suspend(ApplicationId appId) throws ApplicationStateChangeDeniedException, ApplicationIdNotFoundException {
+ suspendedApplications.add(appId);
+ }
+
+ @Override
+ public void suspendAll(HostName parentHostname, List<HostName> hostNames) throws BatchInternalErrorException, BatchHostStateChangeDeniedException, BatchHostNameNotFoundException {
+ throw new RuntimeException("Not implemented");
+ }
+ }
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
new file mode 100644
index 00000000000..10e685e3f96
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
+import java.time.Duration;
+
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author bratseth
+ */
+public class ReservationExpirerTest {
+
+ private Curator curator = new MockCurator();
+
+ @Test
+ public void ensure_reservation_times_out() throws InterruptedException {
+ ManualClock clock = new ManualClock();
+ NodeFlavors flavors = FlavorConfigBuilder.createDummies("default");
+ NodeRepository nodeRepository = new NodeRepository(flavors, curator, clock);
+ NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, flavors, Zone.defaultZone(), clock);
+
+ List<Node> nodes = new ArrayList<>(2);
+ nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default"))));
+ nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default"))));
+ nodes = nodeRepository.addNodes(nodes);
+
+ // Reserve 2 nodes
+ assertEquals(2, nodeRepository.getNodes(Node.State.provisioned).size());
+ nodeRepository.setReady(nodes);
+ ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("bar").instanceName("fuz").build();
+ ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty());
+ provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(2), 1, null);
+ assertEquals(2, nodeRepository.getNodes(Node.State.reserved).size());
+
+ // Reservation times out
+ clock.advance(Duration.ofMinutes(14)); // Reserved but not used time out
+ new ReservationExpirer(nodeRepository, clock, Duration.ofMinutes(10)).run();
+
+ // Assert nothing is reserved
+ assertEquals(0, nodeRepository.getNodes(Node.State.reserved).size());
+ List<Node> dirty = nodeRepository.getNodes(Node.State.dirty);
+ assertEquals(2, dirty.size());
+ assertFalse(dirty.get(0).allocation().isPresent());
+ assertFalse(dirty.get(1).allocation().isPresent());
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
new file mode 100644
index 00000000000..f6f26aeced6
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
@@ -0,0 +1,128 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.test.ManualClock;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author bratseth
+ */
+public class RetiredExpirerTest {
+
+ private Curator curator = new MockCurator();
+
+ @Test
+ public void ensure_retired_nodes_time_out() throws InterruptedException {
+ ManualClock clock = new ManualClock();
+ Zone zone = new Zone(Environment.prod, RegionName.from("us-east"));
+ NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default");
+ NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock);
+ NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone);
+
+ createReadyNodes(7, nodeRepository, nodeFlavors);
+
+ ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz"));
+
+ // Allocate content cluster of sizes 7 -> 2 -> 3:
+ // Should end up with 3 nodes in the cluster (one previously retired), and 3 retired
+ ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty());
+ int wantedNodes;
+ activate(applicationId, cluster, wantedNodes=7, 1, provisioner);
+ activate(applicationId, cluster, wantedNodes=2, 1, provisioner);
+ activate(applicationId, cluster, wantedNodes=3, 1, provisioner);
+ assertEquals(7, nodeRepository.getNodes(applicationId, Node.State.active).size());
+ assertEquals(0, nodeRepository.getNodes(applicationId, Node.State.inactive).size());
+
+ // Cause inactivation of retired nodes
+ clock.advance(Duration.ofHours(30)); // Retire period spent
+ MockDeployer deployer =
+ new MockDeployer(provisioner,
+ Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, wantedNodes, Optional.of("default"), 1)));
+ new RetiredExpirer(nodeRepository, deployer, clock, Duration.ofHours(12)).run();
+ assertEquals(3, nodeRepository.getNodes(applicationId, Node.State.active).size());
+ assertEquals(4, nodeRepository.getNodes(applicationId, Node.State.inactive).size());
+ assertEquals(1, deployer.redeployments);
+
+ // inactivated nodes are not retired
+ for (Node node : nodeRepository.getNodes(applicationId, Node.State.inactive))
+ assertFalse(node.allocation().get().membership().retired());
+ }
+
+ @Test
+ public void ensure_retired_groups_time_out() throws InterruptedException {
+ ManualClock clock = new ManualClock();
+ Zone zone = new Zone(Environment.prod, RegionName.from("us-east"));
+ NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default");
+ NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock);
+ NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone);
+
+ createReadyNodes(8, nodeRepository, nodeFlavors);
+
+ ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz"));
+
+ ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty());
+ activate(applicationId, cluster, 8, 8, provisioner);
+ activate(applicationId, cluster, 1, 1, provisioner);
+ assertEquals(8, nodeRepository.getNodes(applicationId, Node.State.active).size());
+ assertEquals(0, nodeRepository.getNodes(applicationId, Node.State.inactive).size());
+
+ // Cause inactivation of retired nodes
+ clock.advance(Duration.ofHours(30)); // Retire period spent
+ MockDeployer deployer =
+ new MockDeployer(provisioner,
+ Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, 1, Optional.of("default"), 1)));
+ new RetiredExpirer(nodeRepository, deployer, clock, Duration.ofHours(12)).run();
+ assertEquals(1, nodeRepository.getNodes(applicationId, Node.State.active).size());
+ assertEquals(7, nodeRepository.getNodes(applicationId, Node.State.inactive).size());
+ assertEquals(1, deployer.redeployments);
+
+ // inactivated nodes are not retired
+ for (Node node : nodeRepository.getNodes(applicationId, Node.State.inactive))
+ assertFalse(node.allocation().get().membership().retired());
+ }
+
+ private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodes, int groups, NodeRepositoryProvisioner provisioner) {
+ List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(nodes), groups, null);
+ NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
+ provisioner.activate(transaction, applicationId, hosts);
+ transaction.commit();
+ }
+
+ private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) {
+ List<Node> nodes = new ArrayList<>(count);
+ for (int i = 0; i < count; i++)
+ nodes.add(nodeRepository.createNode("node" + i, "node" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))));
+ nodes = nodeRepository.addNodes(nodes);
+ nodeRepository.setReady(nodes);
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java
new file mode 100644
index 00000000000..d2092e5be13
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.monitoring;
+
+import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * @author oyving
+ */
+public class ProvisionMetricsTest {
+
+ @Test(timeout = 10_000L)
+ public void test_registered_metric() throws InterruptedException {
+ final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default");
+ final Curator curator = new MockCurator();
+ final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator);
+ final Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")));
+ nodeRepository.addNodes(Collections.singletonList(node));
+
+ final Map<String, Number> expectedMetrics = new HashMap<>();
+ expectedMetrics.put("hostedVespa.provisionedHosts", 1);
+ expectedMetrics.put("hostedVespa.readyHosts", 0);
+ expectedMetrics.put("hostedVespa.reservedHosts", 0);
+ expectedMetrics.put("hostedVespa.activeHosts", 0);
+ expectedMetrics.put("hostedVespa.inactiveHosts", 0);
+ expectedMetrics.put("hostedVespa.dirtyHosts", 0);
+ expectedMetrics.put("hostedVespa.failedHosts", 0);
+
+ final TestMetric metric = new TestMetric(expectedMetrics.size());
+ final ProvisionMetrics provisionMetrics = new ProvisionMetrics(metric, nodeRepository);
+
+ metric.latch.await();
+ assertEquals(expectedMetrics, metric.values);
+
+ provisionMetrics.deconstruct();
+ }
+
+ private static class TestMetric implements Metric {
+ public CountDownLatch latch;
+ public Map<String, Number> values = new HashMap<>();
+ public Map<String, Context> context = new HashMap<>();
+
+ public TestMetric(int latchNumber) {
+ this.latch = new CountDownLatch(latchNumber);
+ }
+
+ @Override
+ public void set(String key, Number val, Context ctx) {
+ values.put(key, val);
+ context.put(key, ctx);
+ countDownAboveZero();
+ }
+
+ @Override
+ public void add(String key, Number val, Context ctx) {
+ values.put(key, val);
+ context.put(key, ctx);
+ countDownAboveZero();
+ }
+
+ @Override
+ public Context createContext(Map<String, ?> properties) {
+ return null;
+ }
+
+ private void countDownAboveZero() {
+ if (latch.getCount() == 0) {
+ throw new AssertionError("Countdown latch too low - check metric.set metric.add calls");
+ }
+
+ latch.countDown();
+ }
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/NodeFlavorsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/NodeFlavorsTest.java
new file mode 100644
index 00000000000..d2934187a44
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/NodeFlavorsTest.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node;
+
+import com.yahoo.vespa.config.nodes.NodeRepositoryConfig;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.*;
+
+
+public class NodeFlavorsTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void testReplacesWithBadValue() {
+ NodeRepositoryConfig.Builder builder = new NodeRepositoryConfig.Builder();
+ List<NodeRepositoryConfig.Flavor.Builder> flavorBuilderList = new ArrayList<>();
+ NodeRepositoryConfig.Flavor.Builder flavorBuilder = new NodeRepositoryConfig.Flavor.Builder();
+ NodeRepositoryConfig.Flavor.Replaces.Builder flavorReplacesBuilder = new NodeRepositoryConfig.Flavor.Replaces.Builder();
+ flavorReplacesBuilder.name("non-existing-config");
+ flavorBuilder.name("strawberry").cost(2).replaces.add(flavorReplacesBuilder);
+ flavorBuilderList.add(flavorBuilder);
+ builder.flavor(flavorBuilderList);
+ NodeRepositoryConfig config = new NodeRepositoryConfig(builder);
+ exception.expect(IllegalStateException.class);
+ exception.expectMessage("Replaces for strawberry pointing to a non existing flavor: non-existing-config");
+ new NodeFlavors(config);
+ }
+
+ @Test
+ public void testConfigParsing() {
+ NodeRepositoryConfig.Builder builder = new NodeRepositoryConfig.Builder();
+ List<NodeRepositoryConfig.Flavor.Builder> flavorBuilderList = new ArrayList<>();
+ {
+ NodeRepositoryConfig.Flavor.Builder flavorBuilder = new NodeRepositoryConfig.Flavor.Builder();
+ NodeRepositoryConfig.Flavor.Replaces.Builder flavorReplacesBuilder = new NodeRepositoryConfig.Flavor.Replaces.Builder();
+ flavorReplacesBuilder.name("banana");
+ flavorBuilder.name("strawberry").cost(2).replaces.add(flavorReplacesBuilder);
+ flavorBuilderList.add(flavorBuilder);
+ }
+ {
+ NodeRepositoryConfig.Flavor.Builder flavorBuilder = new NodeRepositoryConfig.Flavor.Builder();
+ flavorBuilder.name("banana").cost(3);
+ flavorBuilderList.add(flavorBuilder);
+ }
+ builder.flavor(flavorBuilderList);
+ NodeRepositoryConfig config = new NodeRepositoryConfig(builder);
+ NodeFlavors nodeFlavors = new NodeFlavors(config);
+ assertThat(nodeFlavors.getFlavor("banana").get().cost(), is(3));
+ }
+} \ No newline at end of file
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java
new file mode 100644
index 00000000000..50d0e56d999
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import org.junit.Test;
+
+import java.time.Clock;
+import java.util.List;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author Oyvind Gronnesby
+ */
+public class CuratorDatabaseClientTest {
+
+ private Curator curator = new MockCurator();
+ private CuratorDatabaseClient zkClient = new CuratorDatabaseClient(FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC());
+
+ @Test
+ public void ensure_can_read_stored_host_with_instance_information() throws Exception {
+ String zkline = "{\"hostname\":\"oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com\",\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"configuration\":{\"flavor\":\"default\"},\"created\":1421054425159,\"allocated\":1421057746687,\"instance\":{\"tenantId\":\"by_mortent\",\"applicationId\":\"music\",\"instanceId\":\"default\",\"serviceId\":\"container/default/0/0\"}}";
+
+ curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/allocated/oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com", zkline.getBytes());
+
+ List<Node> allocatedNodes = zkClient.getNodes(Node.State.active);
+ assertEquals(1, allocatedNodes.size());
+ assertEquals("container/default/0/0", allocatedNodes.get(0).allocation().get().membership().stringValue());
+ }
+
+ @Test
+ public void ensure_can_read_stored_host_information() throws Exception {
+ String zkline = "{\"hostname\":\"oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com\",\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"configuration\":{\"flavor\":\"default\"},\"created\":1421054425159}";
+ curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/ready/oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com", zkline.getBytes());
+
+ List<Node> allocatedNodes = zkClient.getNodes(Node.State.ready);
+ assertEquals(1, allocatedNodes.size());
+ }
+
+ /** Test that locks can be acquired and released */
+ @Test
+ public void testLocking() {
+ ApplicationId app = ApplicationId.from(TenantName.from("testTenant"), ApplicationName.from("testApp"), InstanceName.from("testInstance"));
+
+ try (CuratorMutex mutex1 = zkClient.lock(app)) {
+ mutex1.toString(); // reference to avoid warning
+ throw new RuntimeException();
+ }
+ catch (RuntimeException expected) {
+ }
+
+ try (CuratorMutex mutex2 = zkClient.lock(app)) {
+ mutex2.toString(); // reference to avoid warning
+ }
+
+ try (CuratorMutex mutex3 = zkClient.lock(app)) {
+ mutex3.toString(); // reference to avoid warning
+ }
+
+ }
+
+ }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java
new file mode 100644
index 00000000000..f675a857bab
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java
@@ -0,0 +1,119 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.yahoo.path.Path;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.curator.transaction.CuratorOperations;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
+import org.junit.Test;
+
+import java.util.List;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests the curator db directly.
+ * This verifies the details of the current implementation of the database, not just its API;
+ * breaking this does not necessarily mean that a change is wrong.
+ *
+ * @author bratseth
+ */
+public class CuratorDatabaseTest {
+
+ @Test
+ public void testTransactionsIncreaseTimer() throws Exception {
+ MockCurator curator = new MockCurator();
+ CuratorDatabase database = new CuratorDatabase(curator, Path.fromString("/"), true);
+
+ assertEquals(0L, (long)curator.counter("/changeCounter").get().get().postValue());
+
+ commitCreate("/1", database);
+ commitCreate("/2", database);
+ commitCreate("/1/1", database);
+ commitCreate("/2/1", database);
+ assertEquals(4L, (long)curator.counter("/changeCounter").get().get().postValue());
+
+ List<String> children1Call0 = database.getChildren(Path.fromString("/1")); // prime the db; this call returns a different instance
+ List<String> children1Call1 = database.getChildren(Path.fromString("/1"));
+ List<String> children1Call2 = database.getChildren(Path.fromString("/1"));
+ assertTrue("We reuse cached data when there are no commits", children1Call1 == children1Call2);
+ assertEquals(1, database.getChildren(Path.fromString("/2")).size());
+ commitCreate("/2/2", database);
+ List<String> children1Call3 = database.getChildren(Path.fromString("/1"));
+ assertEquals(2, database.getChildren(Path.fromString("/2")).size());
+ assertFalse("We do not reuse cached data in different parts of the tree when there are commits",
+ children1Call3 == children1Call2);
+ }
+
+ @Test
+ public void testTransactionsWithDeactivatedCache() throws Exception {
+ MockCurator curator = new MockCurator();
+ CuratorDatabase database = new CuratorDatabase(curator, Path.fromString("/"), false);
+
+ assertEquals(0L, (long)curator.counter("/changeCounter").get().get().postValue());
+
+ commitCreate("/1", database);
+ commitCreate("/2", database);
+ commitCreate("/1/1", database);
+ commitCreate("/2/1", database);
+ assertEquals(4L, (long)curator.counter("/changeCounter").get().get().postValue());
+
+ List<String> children1Call0 = database.getChildren(Path.fromString("/1")); // prime the db; this call returns a different instance
+ List<String> children1Call1 = database.getChildren(Path.fromString("/1"));
+ List<String> children1Call2 = database.getChildren(Path.fromString("/1"));
+ assertTrue("No cache, no reused data", children1Call1 != children1Call2);
+ }
+
+ @Test
+ public void testThatCounterIncreasesAlsoOnCommitFailure() throws Exception {
+ MockCurator curator = new MockCurator();
+ CuratorDatabase database = new CuratorDatabase(curator, Path.fromString("/"), true);
+
+ assertEquals(0L, (long)curator.counter("/changeCounter").get().get().postValue());
+
+ try {
+ commitCreate("/1/2", database); // fail as parent does not exist
+ fail("Expected exception");
+ }
+ catch (Exception expected) {
+ // expected because the parent does not exist
+ }
+ assertEquals(1L, (long)curator.counter("/changeCounter").get().get().postValue());
+ }
+
+ @Test
+ public void testThatCounterIncreasesAlsoOnCommitFailureFromExistingTransaction() throws Exception {
+ MockCurator curator = new MockCurator();
+ CuratorDatabase database = new CuratorDatabase(curator, Path.fromString("/"), true);
+
+ assertEquals(0L, (long)curator.counter("/changeCounter").get().get().postValue());
+
+ try {
+ NestedTransaction t = new NestedTransaction();
+ CuratorTransaction separateC = new CuratorTransaction(curator);
+ separateC.add(CuratorOperations.create("/1/2")); // fail as parent does not exist
+ t.add(separateC);
+
+ CuratorTransaction c = database.newCuratorTransactionIn(t);
+ c.add(CuratorOperations.create("/1")); // does not fail
+
+ t.commit();
+ fail("Expected exception");
+ }
+ catch (Exception expected) {
+ // expected because the parent does not exist
+ }
+ assertEquals(1L, (long)curator.counter("/changeCounter").get().get().postValue());
+ }
+
+ private void commitCreate(String path, CuratorDatabase database) {
+ NestedTransaction t = new NestedTransaction();
+ CuratorTransaction c = database.newCuratorTransactionIn(t);
+ c.add(CuratorOperations.create(path));
+ t.commit();
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
new file mode 100644
index 00000000000..7dc52148e8b
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
@@ -0,0 +1,240 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.test.ManualClock;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.Node.State;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.Generation;
+import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.time.Duration;
+import java.util.Optional;
+
+/**
+ * @author bratseth
+ */
+public class SerializationTest {
+
+ private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", "large", "ugccloud-container");
+ private final NodeSerializer nodeSerializer = new NodeSerializer(nodeFlavors);
+ private final ManualClock clock = new ManualClock();
+
+ @Test
+ public void testProvisionedNodeSerialization() {
+ Node node = createNode();
+
+ Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node));
+ assertEquals(node.id(), copy.id());
+ assertEquals(node.hostname(), copy.hostname());
+ assertEquals(node.state(), copy.state());
+ assertFalse(copy.allocation().isPresent());
+ assertEquals(0, copy.history().events().size());
+ }
+
+ @Test
+ public void testReservedNodeSerialization() {
+ Node node = createNode();
+
+ clock.advance(Duration.ofMinutes(3));
+ assertEquals(0, node.history().events().size());
+ node = node.allocate(ApplicationId.from(TenantName.from("myTenant"),
+ ApplicationName.from("myApplication"),
+ InstanceName.from("myInstance")),
+ ClusterMembership.from("content/myId/0/0", Optional.empty()),
+ clock.instant());
+ assertEquals(1, node.history().events().size());
+ node = node.setRestart(new Generation(1, 2));
+ node = node.setReboot(new Generation(3, 4));
+ node = node.setFlavor(FlavorConfigBuilder.createDummies("large").getFlavorOrThrow("large"));
+ node = node.setStatus(node.status().setVespaVersion(Version.fromString("1.2.3")));
+ node = node.setStatus(node.status().increaseFailCount().increaseFailCount());
+ node = node.setStatus(node.status().setHardwareFailure(true));
+ Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node));
+
+ assertEquals(node.id(), copy.id());
+ assertEquals(node.hostname(), copy.hostname());
+ assertEquals(node.state(), copy.state());
+ assertEquals(1, copy.allocation().get().restartGeneration().wanted());
+ assertEquals(2, copy.allocation().get().restartGeneration().current());
+ assertEquals(3, copy.status().reboot().wanted());
+ assertEquals(4, copy.status().reboot().current());
+ assertEquals("large", copy.configuration().flavor().name());
+ assertEquals("1.2.3", copy.status().vespaVersion().get().toString());
+ assertEquals(2, copy.status().failCount());
+ assertEquals(true, copy.status().hardwareFailure());
+ assertEquals(node.allocation().get().owner(), copy.allocation().get().owner());
+ assertEquals(node.allocation().get().membership(), copy.allocation().get().membership());
+ assertEquals(node.allocation().get().removable(), copy.allocation().get().removable());
+ assertEquals(1, copy.history().events().size());
+ assertEquals(clock.instant(), copy.history().event(History.Event.Type.reserved).get().at());
+ }
+
+ @Test
+ public void testRebootAndRestartNoCurrentValuesSerialization() {
+ String nodeData = "{\n" +
+ " \"rebootGeneration\" : 0,\n" +
+ " \"configuration\" : {\n" +
+ " \"flavor\" : \"default\"\n" +
+ " },\n" +
+ " \"history\" : [\n" +
+ " {\n" +
+ " \"type\" : \"reserved\",\n" +
+ " \"at\" : 1444391402611\n" +
+ " }\n" +
+ " ],\n" +
+ " \"instance\" : {\n" +
+ " \"applicationId\" : \"myApplication\",\n" +
+ " \"tenantId\" : \"myTenant\",\n" +
+ " \"instanceId\" : \"myInstance\",\n" +
+ " \"serviceId\" : \"content/myId/0\",\n" +
+ " \"restartGeneration\" : 0,\n" +
+ " \"removable\" : false\n" +
+ " },\n" +
+ " \"openStackId\" : \"myId\",\n" +
+ " \"hostname\" : \"myHostname\"\n" +
+ "}";
+
+ Node node = nodeSerializer.fromJson(Node.State.provisioned, Utf8.toBytes(nodeData));
+
+ assertEquals(0, node.status().reboot().wanted());
+ assertEquals(0, node.status().reboot().current());
+ assertEquals(0, node.allocation().get().restartGeneration().wanted());
+ assertEquals(0, node.allocation().get().restartGeneration().current());
+ }
+
+ @Test
+ public void testRetiredNodeSerialization() {
+ Node node = createNode();
+
+ clock.advance(Duration.ofMinutes(3));
+ assertEquals(0, node.history().events().size());
+ node = node.allocate(ApplicationId.from(TenantName.from("myTenant"),
+ ApplicationName.from("myApplication"),
+ InstanceName.from("myInstance")),
+ ClusterMembership.from("content/myId/0", Optional.empty()),
+ clock.instant());
+ assertEquals(1, node.history().events().size());
+ clock.advance(Duration.ofMinutes(2));
+ node = node.retireByApplication(clock.instant());
+ Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node));
+ assertEquals(2, copy.history().events().size());
+ assertEquals(clock.instant(), copy.history().event(History.Event.Type.retired).get().at());
+ assertEquals(History.RetiredEvent.Agent.application,
+ ((History.RetiredEvent) copy.history().event(History.Event.Type.retired).get()).agent());
+ assertTrue(copy.allocation().get().membership().retired());
+
+ Node removable = copy.setAllocation(node.allocation().get().makeRemovable());
+ Node removableCopy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(removable));
+ assertTrue(removableCopy.allocation().get().removable());
+ }
+
+ @Test
+ public void testAssimilatedDeserialization() {
+ Node node = nodeSerializer.fromJson(Node.State.active, "{\"hostname\":\"assimilate2.vespahosted.corp.bf1.yahoo.com\",\"openStackId\":\"\",\"configuration\":{\"flavor\":\"ugccloud-container\"},\"instance\":{\"tenantId\":\"by_mortent\",\"applicationId\":\"ugc-assimilate\",\"instanceId\":\"default\",\"serviceId\":\"container/ugccloud-container/0/0\",\"restartGeneration\":0}}\n".getBytes());
+ assertEquals(0, node.history().events().size());
+ assertTrue(node.allocation().isPresent());
+ assertEquals("ugccloud-container", node.allocation().get().membership().cluster().id().value());
+ assertEquals("container", node.allocation().get().membership().cluster().type().name());
+ assertEquals("0", node.allocation().get().membership().cluster().group().get().value());
+ Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node));
+ assertEquals(0, copy.history().events().size());
+ }
+
+ @Test
+ public void testSetFailCount() {
+ Node node = createNode();
+ node = node.allocate(ApplicationId.from(TenantName.from("myTenant"),
+ ApplicationName.from("myApplication"),
+ InstanceName.from("myInstance")),
+ ClusterMembership.from("content/myId/0/0", Optional.empty()),
+ clock.instant());
+
+ node = node.setStatus(node.status().setFailCount(0));
+ Node copy2 = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node));
+
+ assertEquals(0, copy2.status().failCount());
+ }
+
+ @Test
+ public void serialize_docker_image() {
+ Node node = createNode();
+
+ Optional<String> dockerImage = Optional.of("my-docker-image");
+ ClusterMembership clusterMembership = ClusterMembership.from("content/myId/0", dockerImage);
+
+ Node nodeWithAllocation = node.setAllocation(
+ new Allocation(
+ ApplicationId.from(TenantName.from("myTenant"),
+ ApplicationName.from("myApplication"),
+ InstanceName.from("myInstance")),
+ clusterMembership,
+ new Generation(0, 0),
+ false));
+
+ Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(nodeWithAllocation));
+ assertEquals(dockerImage, deserializedNode.allocation().get().membership().cluster().dockerImage());
+ }
+
+ @Test
+ public void serialize_parentHostname() {
+ final String parentHostname = "parent.yahoo.com";
+ Node node = Node.create("myId", "myHostname", Optional.of(parentHostname), new Configuration(nodeFlavors.getFlavorOrThrow("default")));
+
+ Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node));
+ assertEquals(parentHostname, deserializedNode.parentHostname().get());
+ }
+
+ // TODO: Remove when 5.120 is released everywhere
+ @Test
+ public void serialize_parentHostname_from_dockerHostHostName() {
+ final String parentHostname = "parent.yahoo.com";
+ String nodeData = "{\n" +
+ " \"rebootGeneration\" : 0,\n" +
+ " \"configuration\" : {\n" +
+ " \"flavor\" : \"default\"\n" +
+ " },\n" +
+ " \"history\" : [\n" +
+ " {\n" +
+ " \"type\" : \"reserved\",\n" +
+ " \"at\" : 1444391402611\n" +
+ " }\n" +
+ " ],\n" +
+ " \"instance\" : {\n" +
+ " \"applicationId\" : \"myApplication\",\n" +
+ " \"tenantId\" : \"myTenant\",\n" +
+ " \"instanceId\" : \"myInstance\",\n" +
+ " \"serviceId\" : \"content/myId/0\",\n" +
+ " \"restartGeneration\" : 0,\n" +
+ " \"removable\" : false\n" +
+ " },\n" +
+ " \"openStackId\" : \"fooId\",\n" +
+ " \"hostname\" : \"fooHost\",\n" +
+ " \"dockerHostHostName\" : \"" + parentHostname + "\"\n" +
+ "}";
+ // No parent hostname, but dockerHostHostName is set, so parent hostname should be set after deserialization
+
+ Node deserializedNode2 = nodeSerializer.fromJson(State.provisioned, Utf8.toBytes(nodeData));
+ assertEquals(parentHostname, deserializedNode2.parentHostname().get());
+ }
+
+ private Node createNode() {
+ return Node.create("myId", "myHostname", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")));
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
new file mode 100644
index 00000000000..4d7a70fc915
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests deployment to docker images which share the same physical host.
+ *
+ * @author bratseth
+ */
+public class DockerProvisioningTest {
+ private static final String dockerFlavor = "docker1";
+
+ @Test
+ public void docker_application_deployment() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+ ApplicationId application1 = tester.makeApplicationId();
+
+ for (int i = 1; i < 10; i++) {
+ tester.makeReadyDockerNodes(1, dockerFlavor, "dockerHost" + i);
+ }
+
+ List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")), 7, 1, dockerFlavor);
+ tester.activate(application1, new HashSet<>(hosts));
+
+ final NodeList nodes = tester.getNodes(application1, Node.State.active);
+ assertEquals(7, nodes.size());
+ assertEquals(dockerFlavor, nodes.asList().get(0).configuration().flavor().canonicalName());
+ }
+
+ // In dev, test and staging you get nodes with default flavor, but we should get specified flavor for docker nodes
+ @Test
+ public void get_specified_flavor_not_default_flavor_for_docker() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.test, RegionName.from("corp-us-east-1")));
+ ApplicationId application1 = tester.makeApplicationId();
+ tester.makeReadyDockerNodes(1, dockerFlavor, "dockerHost");
+
+ List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")), 1, 1, dockerFlavor);
+ tester.activate(application1, new HashSet<>(hosts));
+
+ final NodeList nodes = tester.getNodes(application1, Node.State.active);
+ assertEquals(1, nodes.size());
+ assertEquals(dockerFlavor, nodes.asList().get(0).configuration().flavor().canonicalName());
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
new file mode 100644
index 00000000000..7d9fb1e42d6
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
@@ -0,0 +1,196 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.maintenance.MockDeployer;
+import com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author bratseth
+ */
+public class MultigroupProvisioningTest {
+
+ @Test
+ public void test_provisioning_of_multiple_groups() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application1 = tester.makeApplicationId();
+
+ tester.makeReadyNodes(21, "default");
+
+ deploy(application1, 6, 1, tester);
+ deploy(application1, 6, 2, tester);
+ deploy(application1, 6, 3, tester);
+ deploy(application1, 6, 6, tester);
+ deploy(application1, 6, 1, tester);
+ deploy(application1, 6, 6, tester);
+ deploy(application1, 6, 6, tester);
+ deploy(application1, 6, 2, tester);
+ deploy(application1, 8, 2, tester);
+ deploy(application1, 9, 3, tester);
+ deploy(application1, 9, 3, tester);
+ deploy(application1, 9, 3, tester);
+ deploy(application1,12, 4, tester);
+ deploy(application1, 8, 4, tester);
+ deploy(application1,12, 4, tester);
+ deploy(application1, 8, 2, tester);
+ deploy(application1, 6, 3, tester);
+ }
+
+ /**
+ * This demonstrates a case where we end up provisioning new nodes rather than reusing retired nodes
+ * due to asymmetric group sizes after step 2 (second group has 3 additional retired nodes).
+ * We probably need to switch to a multipass group allocation procedure to fix this case.
+ */
+ @Test @Ignore
+ public void test_provisioning_of_groups_with_asymmetry() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application1 = tester.makeApplicationId();
+
+ tester.makeReadyNodes(21, "default");
+
+ deploy(application1, 12, 2, tester);
+ deploy(application1, 9, 3, tester);
+ deploy(application1,12, 3, tester);
+ }
+
+ @Test
+ public void test_provisioning_of_multiple_groups_after_flavor_migration() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application1 = tester.makeApplicationId();
+
+ tester.makeReadyNodes(10, "small");
+ tester.makeReadyNodes(10, "large");
+
+ deploy(application1, 8, 1, "small", tester);
+ deploy(application1, 8, 1, "large", tester);
+ deploy(application1, 8, 8, "large", tester);
+ }
+
+ @Test
+ public void test_one_node_and_group_to_two() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application1 = tester.makeApplicationId();
+
+ tester.makeReadyNodes(10, "small");
+
+ deploy(application1, Capacity.fromRequiredNodeCount(1, "small"), 1, tester);
+ deploy(application1, Capacity.fromRequiredNodeCount(2, "small"), 2, tester);
+ }
+
+ @Test
+ public void test_one_node_and_group_to_two_with_flavor_migration() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application1 = tester.makeApplicationId();
+
+ tester.makeReadyNodes(10, "small");
+ tester.makeReadyNodes(10, "large");
+
+ deploy(application1, Capacity.fromRequiredNodeCount(1, "small"), 1, tester);
+ deploy(application1, Capacity.fromRequiredNodeCount(2, "large"), 2, tester);
+ }
+
+ @Test
+ public void test_provisioning_of_multiple_groups_after_flavor_migration_and_exiration() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application1 = tester.makeApplicationId();
+
+ tester.makeReadyNodes(10, "small");
+ tester.makeReadyNodes(10, "large");
+
+ deploy(application1, 8, 1, "small", tester);
+ deploy(application1, 8, 1, "large", tester);
+
+ // Expire small nodes
+ tester.advanceTime(Duration.ofDays(7));
+ MockDeployer deployer =
+ new MockDeployer(tester.provisioner(),
+ Collections.singletonMap(application1, new MockDeployer.ApplicationContext(application1, cluster(), 8, Optional.of("large"), 1)));
+ new RetiredExpirer(tester.nodeRepository(), deployer, tester.clock(), Duration.ofHours(12)).run();
+
+ assertEquals(8, tester.getNodes(application1, Node.State.inactive).flavor("small").size());
+ deploy(application1, 8, 8, "large", tester);
+ }
+
+ private void deploy(ApplicationId application, int nodeCount, int groupCount, String flavor, ProvisioningTester tester) {
+ deploy(application, Capacity.fromNodeCount(nodeCount, Optional.of(flavor)), groupCount, tester);
+ }
+ private void deploy(ApplicationId application, int nodeCount, int groupCount, ProvisioningTester tester) {
+ deploy(application, Capacity.fromNodeCount(nodeCount, "default"), groupCount, tester);
+ }
+ private void deploy(ApplicationId application, Capacity capacity, int groupCount, ProvisioningTester tester) {
+ int nodeCount = capacity.nodeCount();
+ String flavor = capacity.flavor().get();
+
+ int previousActiveNodeCount = tester.getNodes(application, Node.State.active).flavor(flavor).size();
+
+ tester.activate(application, prepare(application, capacity, groupCount, tester));
+
+ assertEquals("Superfluous nodes are retired, but no others - went from " + previousActiveNodeCount + " to " + nodeCount + " nodes",
+ Math.max(0, previousActiveNodeCount - capacity.nodeCount()),
+ tester.getNodes(application, Node.State.active).retired().flavor(flavor).size());
+ assertEquals("Other flavors are retired",
+ 0, tester.getNodes(application, Node.State.active).nonretired().notFlavor(capacity.flavor().get()).size());
+
+ // Check invariants for all nodes
+ Set<Integer> allIndexes = new HashSet<>();
+ for (Node node : tester.getNodes(application, Node.State.active).asList()) {
+ // Node indexes must be unique
+ int index = node.allocation().get().membership().index();
+ assertFalse("Node indexes are unique", allIndexes.contains(index));
+ allIndexes.add(index);
+
+ assertTrue(node.allocation().get().membership().cluster().group().isPresent());
+ }
+
+ // Count unretired nodes and groups of the requested flavor
+ Set<Integer> indexes = new HashSet<>();
+ Map<ClusterSpec.Group, Integer> groups = new HashMap<>();
+ for (Node node : tester.getNodes(application, Node.State.active).nonretired().flavor(flavor).asList()) {
+ indexes.add(node.allocation().get().membership().index());
+
+ ClusterSpec.Group group = node.allocation().get().membership().cluster().group().get();
+ groups.put(group, groups.getOrDefault(group, 0) + 1);
+
+ if (groupCount > 1)
+ assertTrue(Integer.parseInt(group.value()) < groupCount);
+ }
+ assertEquals("Total nonretired nodes", nodeCount, indexes.size());
+ assertEquals("Total nonretired groups", groupCount, groups.size());
+ for (Integer groupSize : groups.values())
+ assertEquals("Group size", (long)nodeCount / groupCount, (long)groupSize);
+ }
+
+ private ClusterSpec cluster() { return ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test")); }
+
+ private Set<HostSpec> prepare(ApplicationId application, Capacity capacity, int groupCount, ProvisioningTester tester) {
+ return new HashSet<>(tester.prepare(application, cluster(), capacity, groupCount));
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionTest.java
new file mode 100644
index 00000000000..0d6a2cd6b9e
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionTest.java
@@ -0,0 +1,525 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.OutOfCapacityException;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.provision.Node;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Various allocation sequence scenarios
+ *
+ * @author bratseth
+ */
+public class ProvisionTest {
+
+ @Test
+ public void application_deployment_constant_application_size() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application1 = tester.makeApplicationId();
+ ApplicationId application2 = tester.makeApplicationId();
+
+ tester.makeReadyNodes(21, "default");
+
+ // deploy
+ SystemState state1 = prepare(application1, 2, 2, 3, 3, "default", tester);
+ tester.activate(application1, state1.allHosts);
+
+ // redeploy
+ SystemState state2 = prepare(application1, 2, 2, 3, 3, "default", tester);
+ state2.assertEquals(state1);
+ tester.activate(application1, state2.allHosts);
+
+ // deploy another application
+ SystemState state1App2 = prepare(application2, 2, 2, 3, 3, "default", tester);
+ assertFalse("Hosts to different apps are disjunct", state1App2.allHosts.removeAll(state1.allHosts));
+ tester.activate(application2, state1App2.allHosts);
+
+ // prepare twice
+ SystemState state3 = prepare(application1, 2, 2, 3, 3, "default", tester);
+ SystemState state4 = prepare(application1, 2, 2, 3, 3, "default", tester);
+ state3.assertEquals(state2);
+ state4.assertEquals(state3);
+ tester.activate(application1, state4.allHosts);
+
+ // remove nodes before deploying
+ SystemState state5 = prepare(application1, 2, 2, 3, 3, "default", tester);
+ HostSpec removed = tester.removeOne(state5.allHosts);
+ tester.activate(application1, state5.allHosts);
+ assertEquals(removed.hostname(), tester.nodeRepository().getNodes(application1, Node.State.inactive).get(0).hostname());
+
+ // remove some of the clusters
+ SystemState state6 = prepare(application1, 0, 2, 0, 3, "default", tester);
+ tester.activate(application1, state6.allHosts);
+ assertEquals(5, tester.getNodes(application1, Node.State.active).size());
+ assertEquals(5, tester.getNodes(application1, Node.State.inactive).size());
+
+ // delete app
+ tester.provisioner().removed(application1);
+ assertEquals(tester.toHostNames(state1.allHosts), tester.toHostNames(tester.nodeRepository().getNodes(application1, Node.State.inactive)));
+ assertEquals(0, tester.getNodes(application1, Node.State.active).size());
+
+ // other application is unaffected
+ assertEquals(state1App2.hostNames(), tester.toHostNames(tester.nodeRepository().getNodes(application2, Node.State.active)));
+
+ // fail a node from app2 and make sure it does not get inactive nodes from first
+ HostSpec failed = tester.removeOne(state1App2.allHosts);
+ tester.fail(failed);
+ assertEquals(9, tester.getNodes(application2, Node.State.active).size());
+ SystemState state2App2 = prepare(application2, 2, 2, 3, 3, "default", tester);
+ assertFalse("Hosts to different apps are disjunct", state2App2.allHosts.removeAll(state1.allHosts));
+ assertEquals("A new node was reserved to replace the failed one", 10, state2App2.allHosts.size());
+ assertFalse("The new host is not the failed one", state2App2.allHosts.contains(failed));
+ tester.activate(application2, state2App2.allHosts);
+
+ // deploy first app again
+ SystemState state7 = prepare(application1, 2, 2, 3, 3, "default", tester);
+ state7.assertEquals(state1);
+ tester.activate(application1, state7.allHosts);
+ assertEquals(0, tester.getNodes(application1, Node.State.inactive).size());
+
+ // restart
+ HostFilter allFilter = HostFilter.all();
+ HostFilter hostFilter = HostFilter.hostname(state6.allHosts.iterator().next().hostname());
+ HostFilter clusterTypeFilter = HostFilter.clusterType(ClusterSpec.Type.container);
+ HostFilter clusterIdFilter = HostFilter.clusterId(ClusterSpec.Id.from("container1"));
+
+ tester.provisioner().restart(application1, allFilter);
+ tester.provisioner().restart(application1, hostFilter);
+ tester.provisioner().restart(application1, clusterTypeFilter);
+ tester.provisioner().restart(application1, clusterIdFilter);
+ tester.assertRestartCount(application1, allFilter, hostFilter, clusterTypeFilter, clusterIdFilter);
+ }
+
+ @Test
+ public void application_deployment_variable_application_size() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application1 = tester.makeApplicationId();
+
+ tester.makeReadyNodes(24, "default");
+
+ // deploy
+ SystemState state1 = prepare(application1, 2, 2, 3, 3, "default", tester);
+ tester.activate(application1, state1.allHosts);
+
+ // redeploy with increased sizes
+ SystemState state2 = prepare(application1, 3, 4, 4, 5, "default", tester);
+ state2.assertExtends(state1);
+ assertEquals("New nodes are reserved", 6, tester.getNodes(application1, Node.State.reserved).size());
+ tester.activate(application1, state2.allHosts);
+
+ // decrease again
+ SystemState state3 = prepare(application1, 2, 2, 3, 3, "default", tester);
+ tester.activate(application1, state3.allHosts);
+ assertEquals("Superfluous container nodes are deactivated",
+ 3-2 + 4-2, tester.getNodes(application1, Node.State.inactive).size());
+ assertEquals("Superfluous content nodes are retired",
+ 4-3 + 5-3, tester.getNodes(application1, Node.State.active).retired().size());
+
+ // increase even more, and remove one node before deploying
+ SystemState state4 = prepare(application1, 4, 5, 5, 6, "default", tester);
+ assertEquals("Inactive nodes are reused", 0, tester.getNodes(application1, Node.State.inactive).size());
+ assertEquals("Earlier retired nodes are not unretired before activate",
+ 4-3 + 5-3, tester.getNodes(application1, Node.State.active).retired().size());
+ state4.assertExtends(state2);
+ assertEquals("New and inactive nodes are reserved", 4 + 3, tester.getNodes(application1, Node.State.reserved).size());
+ HostSpec removed = tester.removeOne(state4.allHosts);
+ tester.activate(application1, state4.allHosts);
+ assertEquals(removed.hostname(), tester.getNodes(application1, Node.State.inactive).asList().get(0).hostname());
+ assertEquals("Earlier retired nodes are unretired on activate",
+ 0, tester.getNodes(application1, Node.State.active).retired().size());
+
+ // decrease again
+ SystemState state5 = prepare(application1, 2, 2, 3, 3, "default", tester);
+ tester.activate(application1, state5.allHosts);
+ assertEquals("Superfluous container nodes are deactivated",
+ 4-2 + 5-2, tester.getNodes(application1, Node.State.inactive).size());
+ assertEquals("Superfluous content nodes are retired",
+ 5-3 + 6-3, tester.getNodes(application1, Node.State.active).retired().size());
+
+ // increase content slightly
+ SystemState state6 = prepare(application1, 2, 2, 4, 3, "default", tester);
+ tester.activate(application1, state6.allHosts);
+ assertEquals("One content node is unretired",
+ 5-4 + 6-3, tester.getNodes(application1, Node.State.active).retired().size());
+
+ // Then reserve more
+ SystemState state7 = prepare(application1, 8, 2, 2, 2, "default", tester);
+
+ // delete app
+ tester.provisioner().removed(application1);
+ assertEquals(0, tester.getNodes(application1, Node.State.active).size());
+ assertEquals(0, tester.getNodes(application1, Node.State.reserved).size());
+ }
+
+ @Test
+ public void application_deployment_multiple_flavors() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application1 = tester.makeApplicationId();
+
+ tester.makeReadyNodes(12, "small");
+ tester.makeReadyNodes(16, "large");
+
+ // deploy
+ SystemState state1 = prepare(application1, 2, 2, 4, 4, "small", tester);
+ tester.activate(application1, state1.allHosts);
+
+ // redeploy with reduced size (to cause us to have retired nodes before switching flavor)
+ SystemState state2 = prepare(application1, 2, 2, 3, 3, "small", tester);
+ tester.activate(application1, state2.allHosts);
+
+ // redeploy with increased sizes and new flavor
+ SystemState state3 = prepare(application1, 3, 4, 4, 5, "large", tester);
+ assertEquals("New nodes are reserved", 16, tester.nodeRepository().getNodes(application1, Node.State.reserved).size());
+ tester.activate(application1, state3.allHosts);
+ assertEquals("'small' container nodes are retired because we are swapping the entire cluster",
+ 2 + 2, tester.getNodes(application1, Node.State.active).retired().type(ClusterSpec.Type.container).flavor("small").size());
+ assertEquals("'small' content nodes are retired",
+ 4 + 4, tester.getNodes(application1, Node.State.active).retired().type(ClusterSpec.Type.content).flavor("small").size());
+ assertEquals("No 'large' content nodes are retired",
+ 0, tester.getNodes(application1, Node.State.active).retired().flavor("large").size());
+ }
+
+ @Test
+ public void application_deployment_multiple_flavors_default_per_type() {
+ ConfigserverConfig.Builder config = new ConfigserverConfig.Builder();
+ config.environment("prod");
+ config.region("us-east");
+ config.defaultFlavor("not-used");
+ config.defaultContainerFlavor("small");
+ config.defaultContentFlavor("large");
+ ProvisioningTester tester = new ProvisioningTester(new Zone(new ConfigserverConfig(config)));
+
+ ApplicationId application1 = tester.makeApplicationId();
+
+ tester.makeReadyNodes(10, "small");
+ tester.makeReadyNodes(9, "large");
+
+ // deploy
+ SystemState state1 = prepare(application1, 2, 3, 4, 5, null, tester);
+ tester.activate(application1, state1.allHosts);
+ assertEquals("'small' nodes are used for containers",
+ 2 + 3, tester.getNodes(application1, Node.State.active).flavor("small").size());
+ assertEquals("'large' nodes are used for content",
+ 4 + 5, tester.getNodes(application1, Node.State.active).flavor("large").size());
+ }
+
+ @Test
+ public void application_deployment_multiple_flavors_with_replacement() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application1 = tester.makeApplicationId();
+
+ tester.makeReadyNodes(8, "large");
+ tester.makeReadyNodes(8, "large-variant");
+
+ // deploy with flavor which will be fulfilled by some old and new nodes
+ SystemState state1 = prepare(application1, 2, 2, 4, 4, "old-large1", tester);
+ tester.activate(application1, state1.allHosts);
+
+ // redeploy with increased sizes, this will map to the remaining old/new nodes
+ SystemState state2 = prepare(application1, 3, 4, 4, 5, "old-large2", tester);
+ assertEquals("New nodes are reserved", 4, tester.getNodes(application1, Node.State.reserved).size());
+ tester.activate(application1, state2.allHosts);
+ assertEquals("All nodes are used",
+ 16, tester.getNodes(application1, Node.State.active).size());
+ assertEquals("No nodes are retired",
+ 0, tester.getNodes(application1, Node.State.active).retired().size());
+
+ // This is a noop as we are already using large nodes and nodes which replace large
+ SystemState state3 = prepare(application1, 3, 4, 4, 5, "large", tester);
+ assertEquals("Noop", 0, tester.getNodes(application1, Node.State.reserved).size());
+ tester.activate(application1, state3.allHosts);
+
+ try {
+ SystemState state4 = prepare(application1, 3, 4, 4, 5, "large-variant", tester);
+ org.junit.Assert.fail("Should fail as we don't have that many large-variant nodes");
+ }
+ catch (OutOfCapacityException expected) {
+ }
+
+ // make enough nodes to complete the switch to large-variant
+ tester.makeReadyNodes(8, "large-variant");
+ SystemState state4 = prepare(application1, 3, 4, 4, 5, "large-variant", tester);
+ assertEquals("New 'large-variant' nodes are reserved", 8, tester.getNodes(application1, Node.State.reserved).size());
+ tester.activate(application1, state4.allHosts);
+ // (we can not check for the precise state here without carrying over from earlier as the distribution of
+ // old and new on different clusters is unknown)
+ }
+
+ @Test
+ public void application_deployment_above_then_at_capacity_limit() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application1 = tester.makeApplicationId();
+
+ tester.makeReadyNodes(5, "default");
+
+ // deploy
+ SystemState state1 = prepare(application1, 2, 0, 3, 0, "default", tester);
+ tester.activate(application1, state1.allHosts);
+
+ // redeploy a too large application
+ try {
+ SystemState state2 = prepare(application1, 3, 0, 3, 0, "default", tester);
+ org.junit.Assert.fail("Expected out of capacity exception");
+ }
+ catch (OutOfCapacityException expected) {
+ }
+
+ // deploy first state again
+ SystemState state3 = prepare(application1, 2, 0, 3, 0, "default", tester);
+ tester.activate(application1, state3.allHosts);
+ }
+
+ @Test
+ public void dev_deployment_size() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.dev, RegionName.from("us-east")));
+
+ ApplicationId application = tester.makeApplicationId();
+ tester.makeReadyNodes(4, "default");
+ SystemState state = prepare(application, 2, 2, 3, 3, "default", tester);
+ assertEquals(4, state.allHosts.size());
+ tester.activate(application, state.allHosts);
+ }
+
+ @Test
+ public void test_deployment_size() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.test, RegionName.from("us-east")));
+
+ ApplicationId application = tester.makeApplicationId();
+ tester.makeReadyNodes(4, "default");
+ SystemState state = prepare(application, 2, 2, 3, 3, "default", tester);
+ assertEquals(4, state.allHosts.size());
+ tester.activate(application, state.allHosts);
+ }
+
+ @Ignore // TODO: Re-activate when the check is reactivate in CapacityPolicies
+ @Test(expected = IllegalArgumentException.class)
+ public void prod_deployment_requires_redundancy() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application = tester.makeApplicationId();
+ tester.makeReadyNodes(10, "default");
+ prepare(application, 1, 2, 3, 3, "default", tester);
+ }
+
+ /** Dev always uses the zone default flavor */
+ @Test
+ public void dev_deployment_flavor() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.dev, RegionName.from("us-east")));
+
+ ApplicationId application = tester.makeApplicationId();
+ tester.makeReadyNodes(4, "default");
+ SystemState state = prepare(application, 2, 2, 3, 3, "large", tester);
+ assertEquals(4, state.allHosts.size());
+ tester.activate(application, state.allHosts);
+ }
+
+ /** Test always uses the zone default flavor */
+ @Test
+ public void test_deployment_flavor() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.test, RegionName.from("us-east")));
+
+ ApplicationId application = tester.makeApplicationId();
+ tester.makeReadyNodes(4, "default");
+ SystemState state = prepare(application, 2, 2, 3, 3, "large", tester);
+ assertEquals(4, state.allHosts.size());
+ tester.activate(application, state.allHosts);
+ }
+
+ @Test
+ public void staging_deployment_size() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.staging, RegionName.from("us-east")));
+
+ ApplicationId application = tester.makeApplicationId();
+ tester.makeReadyNodes(14, "default");
+ SystemState state = prepare(application, 1, 1, 1, 64, "default", tester); // becomes 1, 1, 1, 6
+ assertEquals(9, state.allHosts.size());
+ tester.activate(application, state.allHosts);
+ }
+
+ @Test
+ public void activate_after_reservation_timeout() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ tester.makeReadyNodes(10, "default");
+ ApplicationId application = tester.makeApplicationId();
+ SystemState state = prepare(application, 2, 2, 3, 3, "default", tester);
+
+ // Simulate expiry
+ tester.nodeRepository().deactivate(application);
+
+ try {
+ tester.activate(application, state.allHosts);
+ org.junit.Assert.fail("Expected exception");
+ }
+ catch (IllegalArgumentException e) {
+ assertTrue(e.getMessage().startsWith("Activation of " + application + " failed"));
+ }
+ }
+
+ @Test
+ public void out_of_capacity() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ tester.makeReadyNodes(9, "default"); // need 2+2+3+3=10
+ ApplicationId application = tester.makeApplicationId();
+ try {
+ prepare(application, 2, 2, 3, 3, "default", tester);
+ org.junit.Assert.fail("Expected exception");
+ }
+ catch (OutOfCapacityException e) {
+ assertTrue(e.getMessage().startsWith("Could not satisfy request"));
+ }
+ }
+
+ @Test
+ public void out_of_desired_flavor() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ tester.makeReadyNodes(10, "small"); // need 2+2+3+3=10
+ tester.makeReadyNodes( 9, "large"); // need 2+2+3+3=10
+ ApplicationId application = tester.makeApplicationId();
+ try {
+ prepare(application, 2, 2, 3, 3, "large", tester);
+ org.junit.Assert.fail("Expected exception");
+ }
+ catch (OutOfCapacityException e) {
+ assertTrue(e.getMessage().startsWith("Could not satisfy request for 3 nodes of flavor 'large'"));
+ }
+ }
+
+ @Test
+ public void nonexisting_flavor() {
+ ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+
+ ApplicationId application = tester.makeApplicationId();
+ try {
+ prepare(application, 2, 2, 3, 3, "nonexisting", tester);
+ org.junit.Assert.fail("Expected exception");
+ }
+ catch (IllegalArgumentException e) {
+ assertEquals("Unknown flavor 'nonexisting' Flavors are [default, docker1, large, old-large1, old-large2, small, v-4-8-100]", e.getMessage());
+ }
+ }
+
+ private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int group0Size, int group1Size, String flavor, ProvisioningTester tester) {
+ // "deploy prepare" with a two container clusters and a storage cluster having of two groups
+ ClusterSpec containerCluster0 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("container0"), Optional.empty());
+ ClusterSpec containerCluster1 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), Optional.empty());
+ ClusterSpec contentGroup0 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.of(ClusterSpec.Group.from("g0")));
+ ClusterSpec contentGroup1 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.of(ClusterSpec.Group.from("g1")));
+
+ Set<HostSpec> container0 = new HashSet<>(tester.prepare(application, containerCluster0, container0Size, 1, flavor));
+ Set<HostSpec> container1 = new HashSet<>(tester.prepare(application, containerCluster1, container1Size, 1, flavor));
+ Set<HostSpec> group0 = new HashSet<>(tester.prepare(application, contentGroup0, group0Size, 1, flavor));
+ Set<HostSpec> group1 = new HashSet<>(tester.prepare(application, contentGroup1, group1Size, 1, flavor));
+
+ Set<HostSpec> allHosts = new HashSet<>();
+ allHosts.addAll(container0);
+ allHosts.addAll(container1);
+ allHosts.addAll(group0);
+ allHosts.addAll(group1);
+
+ int expectedContainer0Size = tester.capacityPolicies().decideSize(Capacity.fromNodeCount(container0Size));
+ int expectedContainer1Size = tester.capacityPolicies().decideSize(Capacity.fromNodeCount(container1Size));
+ int expectedGroup0Size = tester.capacityPolicies().decideSize(Capacity.fromNodeCount(group0Size));
+ int expectedGroup1Size = tester.capacityPolicies().decideSize(Capacity.fromNodeCount(group1Size));
+
+ assertEquals("Hosts in each group cluster is disjunct and the total number of unretired nodes is correct",
+ expectedContainer0Size + expectedContainer1Size + expectedGroup0Size + expectedGroup1Size,
+ tester.nonretired(allHosts).size());
+ // Check cluster/group sizes
+ assertEquals(expectedContainer0Size, tester.nonretired(container0).size());
+ assertEquals(expectedContainer1Size, tester.nonretired(container1).size());
+ assertEquals(expectedGroup0Size, tester.nonretired(group0).size());
+ assertEquals(expectedGroup1Size, tester.nonretired(group1).size());
+ // Check cluster membership
+ tester.assertMembersOf(containerCluster0, container0);
+ tester.assertMembersOf(containerCluster1, container1);
+ tester.assertMembersOf(contentGroup0, group0);
+ tester.assertMembersOf(contentGroup1, group1);
+
+ return new SystemState(allHosts, container0, container1, group0, group1);
+ }
+
+ private static class SystemState {
+
+ private Set<HostSpec> allHosts;
+ private Set<HostSpec> container1;
+ private Set<HostSpec> container2;
+ private Set<HostSpec> group1;
+ private Set<HostSpec> group2;
+
+ public SystemState(Set<HostSpec> allHosts,
+ Set<HostSpec> container1,
+ Set<HostSpec> container2,
+ Set<HostSpec> group1,
+ Set<HostSpec> group2) {
+ this.allHosts = allHosts;
+ this.container1 = container1;
+ this.container2 = container2;
+ this.group1 = group1;
+ this.group2 = group2;
+ }
+
+ public Set<String> hostNames() {
+ return allHosts.stream().map(HostSpec::hostname).collect(Collectors.toSet());
+ }
+
+ public void assertExtends(SystemState other) {
+ assertTrue(this.allHosts.containsAll(other.allHosts));
+ assertExtends(this.container1, other.container1);
+ assertExtends(this.container2, other.container2);
+ assertExtends(this.group1, other.group1);
+ assertExtends(this.group2, other.group2);
+ }
+
+ private void assertExtends(Set<HostSpec> extension,
+ Set<HostSpec> original) {
+ for (HostSpec originalHost : original) {
+ HostSpec newHost = findHost(originalHost.hostname(), extension);
+ org.junit.Assert.assertEquals(newHost.membership(), originalHost.membership());
+ }
+ }
+
+ private HostSpec findHost(String hostName, Set<HostSpec> hosts) {
+ for (HostSpec host : hosts)
+ if (host.hostname().equals(hostName))
+ return host;
+ return null;
+ }
+
+ public void assertEquals(SystemState other) {
+ org.junit.Assert.assertEquals(this.allHosts, other.allHosts);
+ org.junit.Assert.assertEquals(this.container1, other.container1);
+ org.junit.Assert.assertEquals(this.container2, other.container2);
+ org.junit.Assert.assertEquals(this.group1, other.group1);
+ org.junit.Assert.assertEquals(this.group2, other.group2);
+ }
+
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
new file mode 100644
index 00000000000..01d8a0eeabf
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -0,0 +1,249 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.ProvisionLogger;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.test.ManualClock;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.config.nodes.NodeRepositoryConfig;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
+
+import java.io.IOException;
+import java.time.temporal.TemporalAmount;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.logging.Level;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * A test utility for provisioning tests.
+ *
+ * @author bratseth
+ */
+public class ProvisioningTester implements AutoCloseable {
+
+ private Curator curator = new MockCurator();
+ private NodeFlavors nodeFlavors;
+ private ManualClock clock;
+ private NodeRepository nodeRepository;
+ private NodeRepositoryProvisioner provisioner;
+ private CapacityPolicies capacityPolicies;
+ private ProvisionLogger provisionLogger;
+
+ public ProvisioningTester(Zone zone) {
+ try {
+ nodeFlavors = new NodeFlavors(createConfig());
+ clock = new ManualClock();
+ nodeRepository = new NodeRepository(nodeFlavors, curator, clock);
+ provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, clock);
+ capacityPolicies = new CapacityPolicies(zone, nodeFlavors);
+ provisionLogger = new NullProvisionLogger();
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private NodeRepositoryConfig createConfig() {
+ FlavorConfigBuilder b = new FlavorConfigBuilder();
+ b.addFlavor("default", 2., 4., 100, "BARE_METAL");
+ b.addFlavor("small", 1., 2., 50, "BARE_METAL");
+ b.addFlavor("docker1", 1., 1., 10, "DOCKER_CONTAINER");
+ b.addFlavor("v-4-8-100", 4., 8., 100, "VIRTUAL_MACHINE");
+ b.addFlavor("old-large1", 2., 4., 100, "BARE_METAL");
+ b.addFlavor("old-large2", 2., 5., 100, "BARE_METAL");
+ NodeRepositoryConfig.Flavor.Builder large = b.addFlavor("large", 4., 8., 100, "BARE_METAL");
+ b.addReplaces("old-large1", large);
+ b.addReplaces("old-large2", large);
+ NodeRepositoryConfig.Flavor.Builder largeVariant = b.addFlavor("large-variant", 3., 9., 101, "BARE_METAL");
+ b.addReplaces("large", largeVariant);
+ NodeRepositoryConfig.Flavor.Builder largeVariantVariant = b.addFlavor("large-variant-variant", 4., 9., 101, "BARE_METAL");
+ b.addReplaces("large-variant", largeVariantVariant);
+ return b.build();
+ }
+
+ private NodeRepositoryConfig.Flavor.Builder addFlavor(String flavorName, NodeRepositoryConfig.Builder b) {
+ NodeRepositoryConfig.Flavor.Builder flavor = new NodeRepositoryConfig.Flavor.Builder();
+ flavor.name(flavorName);
+ b.flavor(flavor);
+ return flavor;
+ }
+
+ private void addReplaces(String replaces, NodeRepositoryConfig.Flavor.Builder flavor) {
+ NodeRepositoryConfig.Flavor.Replaces.Builder flavorReplaces = new NodeRepositoryConfig.Flavor.Replaces.Builder();
+ flavorReplaces.name(replaces);
+ flavor.replaces(flavorReplaces);
+ }
+
+ @Override
+ public void close() throws IOException {
+ //testingServer.close();
+ }
+
+ public void advanceTime(TemporalAmount duration) { clock.advance(duration); }
+ public NodeRepository nodeRepository() { return nodeRepository; }
+ public ManualClock clock() { return clock; }
+ public NodeRepositoryProvisioner provisioner() { return provisioner; }
+ public CapacityPolicies capacityPolicies() { return capacityPolicies; }
+ public NodeList getNodes(ApplicationId id, Node.State ... inState) { return new NodeList(nodeRepository.getNodes(id, inState)); }
+
+ public void patchNode(Node node) { nodeRepository.write(node); }
+
+ public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups, String flavor) {
+ return prepare(application, cluster, Capacity.fromNodeCount(nodeCount, Optional.ofNullable(flavor)), groups);
+ }
+ public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, int groups) {
+ if (capacity.nodeCount() == 0) return Collections.emptyList();
+ Set<String> reservedBefore = toHostNames(nodeRepository.getNodes(application, Node.State.reserved));
+ Set<String> inactiveBefore = toHostNames(nodeRepository.getNodes(application, Node.State.inactive));
+ // prepare twice to ensure idempotence
+ List<HostSpec> hosts1 = provisioner.prepare(application, cluster, capacity, groups, provisionLogger);
+ List<HostSpec> hosts2 = provisioner.prepare(application, cluster, capacity, groups, provisionLogger);
+ assertEquals(hosts1, hosts2);
+ Set<String> newlyActivated = toHostNames(nodeRepository.getNodes(application, Node.State.reserved));
+ newlyActivated.removeAll(reservedBefore);
+ newlyActivated.removeAll(inactiveBefore);
+ return hosts2;
+ }
+
+ public void activate(ApplicationId application, Set<HostSpec> hosts) {
+ NestedTransaction transaction = new NestedTransaction();
+ transaction.add(new CuratorTransaction(curator));
+ provisioner.activate(transaction, application, hosts);
+ transaction.commit();
+ assertEquals(toHostNames(hosts), toHostNames(nodeRepository.getNodes(application, Node.State.active)));
+ }
+
+ public Set<String> toHostNames(Set<HostSpec> hosts) {
+ return hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet());
+ }
+
+ public Set<String> toHostNames(List<Node> nodes) {
+ return nodes.stream().map(Node::hostname).collect(Collectors.toSet());
+ }
+
+ /**
+ * Asserts that each active node in this application has a restart count equaling the
+ * number of matches to the given filters
+ */
+ public void assertRestartCount(ApplicationId application, HostFilter... filters) {
+ for (Node node : nodeRepository.getNodes(application, Node.State.active)) {
+ int expectedRestarts = 0;
+ for (HostFilter filter : filters)
+ if (NodeHostFilter.from(filter).matches(node))
+ expectedRestarts++;
+ assertEquals(expectedRestarts, node.allocation().get().restartGeneration().wanted());
+ }
+ }
+
+ public void fail(HostSpec host) {
+ int beforeFailCount = nodeRepository.getNode(Node.State.active, host.hostname()).get().status().failCount();
+ Node failedNode = nodeRepository.fail(host.hostname());
+ assertTrue(nodeRepository.getNodes(Node.State.failed).contains(failedNode));
+ assertEquals(beforeFailCount + 1, failedNode.status().failCount());
+ }
+
+ public void assertMembersOf(ClusterSpec requestedCluster, Collection<HostSpec> hosts) {
+ Set<Integer> indices = new HashSet<>();
+ for (HostSpec host : hosts) {
+ ClusterSpec nodeCluster = host.membership().get().cluster();
+ assertTrue(requestedCluster.equalsIgnoringGroup(nodeCluster));
+ if (requestedCluster.group().isPresent())
+ assertEquals(requestedCluster.group(), nodeCluster.group());
+ else
+ assertEquals("0", nodeCluster.group().get().value());
+
+ indices.add(host.membership().get().index());
+ }
+ assertEquals("Indexes in " + requestedCluster + " are disjunct", hosts.size(), indices.size());
+ }
+
+ public HostSpec removeOne(Set<HostSpec> hosts) {
+ Iterator<HostSpec> i = hosts.iterator();
+ HostSpec removed = i.next();
+ i.remove();
+ return removed;
+ }
+
+ public ApplicationId makeApplicationId() {
+ return ApplicationId.from(
+ TenantName.from(UUID.randomUUID().toString()),
+ ApplicationName.from(UUID.randomUUID().toString()),
+ InstanceName.from(UUID.randomUUID().toString()));
+ }
+
+ public List<Node> makeReadyNodes(int n, String flavor) {
+ List<Node> nodes = new ArrayList<>(n);
+ for (int i = 0; i < n; i++)
+ nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(),
+ UUID.randomUUID().toString(),
+ Optional.empty(),
+ new Configuration(nodeFlavors.getFlavorOrThrow(flavor))));
+ nodes = nodeRepository.addNodes(nodes);
+ nodeRepository.setReady(nodes);
+ return nodes;
+ }
+
+ /** Creates a set of virtual docker nodes on a single docker host */
+ public List<Node> makeReadyDockerNodes(int n, String flavor, String dockerHostId) {
+ return makeReadyVirtualNodes(n, flavor, Optional.of(dockerHostId));
+ }
+
+ /** Creates a set of virtual nodes on a single parent host */
+ public List<Node> makeReadyVirtualNodes(int n, String flavor, Optional<String> parentHostId) {
+ List<Node> nodes = new ArrayList<>(n);
+ for (int i = 0; i < n; i++) {
+ final String hostname = UUID.randomUUID().toString();
+ nodes.add(nodeRepository.createNode("openstack-id", hostname, parentHostId,
+ new Configuration(nodeFlavors.getFlavorOrThrow(flavor))));
+ }
+ nodes = nodeRepository.addNodes(nodes);
+ nodeRepository.setReady(nodes);
+ return nodes;
+ }
+
+ public List<Node> makeReadyVirtualNodes(int n, String flavor, String parentHostId) {
+ return makeReadyVirtualNodes(n, flavor, Optional.of(parentHostId));
+ }
+
+ /** Returns the hosts from the input list which are not retired */
+ public List<HostSpec> nonretired(Collection<HostSpec> hosts) {
+ return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList());
+ }
+
+ private static class NullProvisionLogger implements ProvisionLogger {
+
+ @Override
+ public void log(Level level, String message) {
+ }
+
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java
new file mode 100644
index 00000000000..a79123959cc
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java
@@ -0,0 +1,302 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.OutOfCapacityException;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.provision.Node;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Tests provisioning of virtual nodes
+ *
+ * @author musum
+ * @author mpolden
+ */
+public class VirtualNodeProvisioningTest {
+ private static final String flavor = "v-4-8-100";
+ private static final ClusterSpec contentClusterSpec = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"));
+ private static final ClusterSpec containerClusterSpec = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"));
+
+
+ private ProvisioningTester tester;
+ private ApplicationId applicationId;
+
+ @Before
+ public void setup() {
+ tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
+ applicationId = tester.makeApplicationId();
+ }
+
+ @Test
+ public void optimize_sorts_by_more_vms() {
+ List<Node> readyList = new ArrayList<Node>();
+ readyList.addAll(tester.makeReadyNodes(2, flavor));
+ readyList.addAll(tester.makeReadyVirtualNodes(2, flavor, "parentHost1"));
+ readyList.addAll(tester.makeReadyNodes(2, flavor));
+ readyList.addAll(tester.makeReadyVirtualNodes(3, flavor, "parentHost2"));
+ readyList.addAll(tester.makeReadyVirtualNodes(2, flavor, "parentHost3"));
+ readyList.addAll(tester.makeReadyVirtualNodes(1, flavor, "parentHost4"));
+ readyList.addAll(tester.makeReadyNodes(3, flavor));
+ assertEquals(15, readyList.size());
+ List<Node> optimized = GroupPreparer.optimize(readyList);
+ assertEquals(15, optimized.size());
+ assertEquals("parentHost2", optimized.get(0).parentHostname().get());
+ assertEquals("parentHost4", optimized.get(3).parentHostname().get());
+ assertEquals("parentHost2", optimized.get(4).parentHostname().get());
+ assertEquals("parentHost2", optimized.get(7).parentHostname().get());
+ assertEquals(false, optimized.get(8).parentHostname().isPresent());
+ assertEquals(false, optimized.get(9).parentHostname().isPresent());
+ assertEquals(false, optimized.get(10).parentHostname().isPresent());
+ assertEquals(false, optimized.get(11).parentHostname().isPresent());
+ assertEquals(false, optimized.get(12).parentHostname().isPresent());
+ assertEquals(false, optimized.get(13).parentHostname().isPresent());
+ assertEquals(false, optimized.get(14).parentHostname().isPresent());
+ }
+
+ @Test
+ public void distinct_parent_host_for_each_node_in_a_cluster() {
+ tester.makeReadyVirtualNodes(2, flavor, "parentHost1");
+ tester.makeReadyVirtualNodes(2, flavor, "parentHost2");
+ tester.makeReadyVirtualNodes(2, flavor, "parentHost3");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost4");
+
+ final int containerNodeCount = 4;
+ final int contentNodeCount = 3;
+ final int groups = 1;
+ List<HostSpec> containerHosts = prepare(containerClusterSpec, containerNodeCount, groups);
+ List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups);
+ activate(containerHosts, contentHosts);
+
+ final List<Node> nodes = getNodes(applicationId);
+ assertEquals(contentNodeCount + containerNodeCount, nodes.size());
+ assertDistinctParentHosts(nodes, ClusterSpec.Type.container, containerNodeCount);
+ assertDistinctParentHosts(nodes, ClusterSpec.Type.content, contentNodeCount);
+
+ // Go down to 3 nodes in container cluster
+ List<HostSpec> containerHosts2 = prepare(containerClusterSpec, containerNodeCount - 1, groups);
+ activate(containerHosts2);
+ final List<Node> nodes2 = getNodes(applicationId);
+ assertDistinctParentHosts(nodes2, ClusterSpec.Type.container, containerNodeCount - 1);
+
+ // Go up to 4 nodes again in container cluster
+ List<HostSpec> containerHosts3 = prepare(containerClusterSpec, containerNodeCount, groups);
+ activate(containerHosts3);
+ final List<Node> nodes3 = getNodes(applicationId);
+ assertDistinctParentHosts(nodes3, ClusterSpec.Type.container, containerNodeCount);
+ }
+
+ @Test
+ public void will_retire_clashing_active() {
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost1");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost2");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost3");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost4");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost5");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost6");
+
+ final int containerNodeCount = 2;
+ final int contentNodeCount = 2;
+ final int groups = 1;
+ List<HostSpec> containerHosts = prepare(containerClusterSpec, containerNodeCount, groups);
+ List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups);
+ activate(containerHosts, contentHosts);
+
+ List<Node> nodes = getNodes(applicationId);
+ assertEquals(4, nodes.size());
+ assertDistinctParentHosts(nodes, ClusterSpec.Type.container, containerNodeCount);
+ assertDistinctParentHosts(nodes, ClusterSpec.Type.content, contentNodeCount);
+
+ for (Node n : nodes) {
+ tester.patchNode(n.setParentHostname("clashing"));
+ }
+ containerHosts = prepare(containerClusterSpec, containerNodeCount, groups);
+ contentHosts = prepare(contentClusterSpec, contentNodeCount, groups);
+ activate(containerHosts, contentHosts);
+
+ nodes = getNodes(applicationId);
+ assertEquals(6, nodes.size());
+ assertEquals(2, nodes.stream().filter(n -> n.allocation().get().membership().retired()).count());
+ }
+
+ @Test
+ public void fail_when_all_hosts_become_clashing() {
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost1");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost2");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost3");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost4");
+
+ final int containerNodeCount = 2;
+ final int contentNodeCount = 2;
+ final int groups = 1;
+ List<HostSpec> containerHosts = prepare(containerClusterSpec, containerNodeCount, groups);
+ List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups);
+ activate(containerHosts, contentHosts);
+
+ List<Node> nodes = getNodes(applicationId);
+ assertEquals(4, nodes.size());
+ assertDistinctParentHosts(nodes, ClusterSpec.Type.container, containerNodeCount);
+ assertDistinctParentHosts(nodes, ClusterSpec.Type.content, contentNodeCount);
+
+ for (Node n : nodes) {
+ tester.patchNode(n.setParentHostname("clashing"));
+ }
+ OutOfCapacityException expected = null;
+ try {
+ containerHosts = prepare(containerClusterSpec, containerNodeCount, groups);
+ } catch (OutOfCapacityException e) {
+ expected = e;
+ }
+ assertNotNull(expected);
+ }
+
+ @Test(expected = OutOfCapacityException.class)
+ // TODO Should fail with something else than OutOfCapacityException
+ public void fail_when_too_few_distinct_parent_hosts() {
+ tester.makeReadyVirtualNodes(2, flavor, "parentHost1");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost2");
+
+ final int contentNodeCount = 3;
+ List<HostSpec> hosts = prepare(contentClusterSpec, contentNodeCount, 1);
+ activate(hosts);
+
+ final List<Node> nodes = getNodes(applicationId);
+ assertDistinctParentHosts(nodes, ClusterSpec.Type.content, contentNodeCount);
+ }
+
+ @Test
+ public void incomplete_parent_hosts_has_distinct_distribution() {
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost1");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost2");
+ tester.makeReadyVirtualNodes(1, flavor, Optional.empty());
+
+ final int contentNodeCount = 3;
+ final int groups = 1;
+ final List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups);
+ activate(contentHosts);
+ assertEquals(3, getNodes(applicationId).size());
+
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost1");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost2");
+
+ assertEquals(contentHosts, prepare(contentClusterSpec, contentNodeCount, groups));
+ }
+
+ @Test
+ public void indistinct_distribution_with_known_ready_nodes() {
+ tester.makeReadyVirtualNodes(3, flavor, Optional.empty());
+
+ final int contentNodeCount = 3;
+ final int groups = 1;
+ final List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups);
+ activate(contentHosts);
+
+ List<Node> nodes = getNodes(applicationId);
+ assertEquals(3, nodes.size());
+
+ // Set indistinct parents
+ tester.patchNode(nodes.get(0).setParentHostname("parentHost1"));
+ tester.patchNode(nodes.get(1).setParentHostname("parentHost1"));
+ tester.patchNode(nodes.get(2).setParentHostname("parentHost2"));
+ nodes = getNodes(applicationId);
+ assertEquals(3, nodes.stream().filter(n -> n.parentHostname().isPresent()).count());
+
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost1");
+ tester.makeReadyVirtualNodes(2, flavor, "parentHost2");
+
+ OutOfCapacityException expectedException = null;
+ try {
+ prepare(contentClusterSpec, contentNodeCount, groups);
+ } catch (OutOfCapacityException e) {
+ expectedException = e;
+ }
+ assertNotNull(expectedException);
+ }
+
+ @Test
+ public void unknown_distribution_with_known_ready_nodes() {
+ tester.makeReadyVirtualNodes(3, flavor, Optional.empty());
+
+ final int contentNodeCount = 3;
+ final int groups = 1;
+ final List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups);
+ activate(contentHosts);
+ assertEquals(3, getNodes(applicationId).size());
+
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost1");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost2");
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost3");
+ assertEquals(contentHosts, prepare(contentClusterSpec, contentNodeCount, groups));
+ }
+
+ @Test
+ public void unknown_distribution_with_known_and_unknown_ready_nodes() {
+ tester.makeReadyVirtualNodes(3, flavor, Optional.empty());
+
+ final int contentNodeCount = 3;
+ final int groups = 1;
+ final List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups);
+ activate(contentHosts);
+ assertEquals(3, getNodes(applicationId).size());
+
+ tester.makeReadyVirtualNodes(1, flavor, "parentHost1");
+ tester.makeReadyVirtualNodes(1, flavor, Optional.empty());
+ assertEquals(contentHosts, prepare(contentClusterSpec, contentNodeCount, groups));
+ }
+
+ private void assertDistinctParentHosts(List<Node> nodes, ClusterSpec.Type clusterType, int expectedCount) {
+ List<String> parentHosts = getParentHostsFromNodes(nodes, Optional.of(clusterType));
+
+ assertEquals(expectedCount, parentHosts.size());
+ assertEquals(expectedCount, getDistinctParentHosts(parentHosts).size());
+ }
+
+ private List<String> getParentHostsFromNodes(List<Node> nodes, Optional<ClusterSpec.Type> clusterType) {
+ List<String> parentHosts = new ArrayList<>();
+ for (Node node : nodes) {
+ if (node.parentHostname().isPresent() && (clusterType.isPresent() && clusterType.get() == node.allocation().get().membership().cluster().type())) {
+ parentHosts.add(node.parentHostname().get());
+ }
+ }
+ return parentHosts;
+ }
+
+ private Set<String> getDistinctParentHosts(List<String> hostnames) {
+ return hostnames.stream()
+ .distinct()
+ .collect(Collectors.<String>toSet());
+ }
+
+ private List<Node> getNodes(ApplicationId applicationId) {
+ return tester.getNodes(applicationId, Node.State.active).asList();
+ }
+
+ private List<HostSpec> prepare(ClusterSpec clusterSpec, int nodeCount, int groups) {
+ return tester.prepare(applicationId, clusterSpec, nodeCount, groups, flavor);
+ }
+
+ @SafeVarargs
+ private final void activate(List<HostSpec>... hostLists) {
+ HashSet<HostSpec> hosts = new HashSet<>();
+ for (List<HostSpec> h : hostLists) {
+ hosts.addAll(h);
+ }
+ tester.activate(applicationId, hosts);
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializerTest.java
new file mode 100644
index 00000000000..5987b295dbe
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializerTest.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi;
+
+import com.yahoo.vespa.hosted.provision.Node;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author bakksjo
+ */
+public class NodeStateSerializerTest {
+ @Test
+ public void allStatesHaveASerializedForm() {
+ for (Node.State nodeState : Node.State.values()) {
+ assertThat(NodeStateSerializer.wireNameOf(nodeState), is(notNullValue()));
+ }
+ }
+
+ @Test
+ public void wireNamesDoNotOverlap() {
+ final Set<String> wireNames = new HashSet<>();
+ for (Node.State nodeState : Node.State.values()) {
+ wireNames.add(NodeStateSerializer.wireNameOf(nodeState));
+ }
+ assertThat(wireNames.size(), is(Node.State.values().length));
+ }
+
+ @Test
+ public void serializationAndDeserializationIsSymmetric() {
+ for (Node.State nodeState : Node.State.values()) {
+ final String serialized = NodeStateSerializer.wireNameOf(nodeState);
+ final Node.State deserialized = NodeStateSerializer.fromWireName(serialized)
+ .orElseThrow(() -> new RuntimeException(
+ "Cannot deserialize '" + serialized + "', serialized form of " + nodeState.name()));
+ assertThat(deserialized, is(nodeState));
+ }
+ }
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResourceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResourceTest.java
new file mode 100644
index 00000000000..56c2e755c21
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResourceTest.java
@@ -0,0 +1,148 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.legacy;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mortent
+ */
+public class ProvisionResourceTest {
+
+ NodeRepository nodeRepository;
+ NodeFlavors nodeFlavors;
+ ProvisionResource provisionResource;
+ Curator curator;
+ int capacity = 2;
+ ApplicationId application;
+ private NodeRepositoryProvisioner provisioner;
+
+ @Before
+ public void setUpTest() throws Exception {
+ curator = new MockCurator();
+ nodeFlavors = FlavorConfigBuilder.createDummies("default");
+ nodeRepository = new NodeRepository(nodeFlavors, curator);
+ provisionResource = new ProvisionResource(nodeRepository, nodeFlavors);
+ application = ApplicationId.from(TenantName.from("myTenant"), ApplicationName.from("myApplication"), InstanceName.from("default"));
+ provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone());
+ }
+
+ private void createNodesInRepository(int readyCount, int provisionedCount) {
+ List<Node> readyNodes = new ArrayList<>();
+ for (HostInfo hostInfo : createHostInfos(readyCount, 0))
+ readyNodes.add(nodeRepository.createNode(hostInfo.openStackId, hostInfo.hostname,
+ Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))));
+ readyNodes = nodeRepository.addNodes(readyNodes);
+ nodeRepository.setReady(readyNodes);
+
+ List<Node> provisionedNodes = new ArrayList<>();
+ for (HostInfo hostInfo : createHostInfos(provisionedCount, readyCount))
+ provisionedNodes.add(nodeRepository.createNode(hostInfo.openStackId, hostInfo.hostname,
+ Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))));
+ nodeRepository.addNodes(provisionedNodes);
+ }
+
+ @Test
+ public void test_node_allocation() {
+ createNodesInRepository(10, 0);
+ List<Node> assignments = assignNode(application, capacity);
+ assertEquals(2, assignments.size());
+ }
+
+ @Test
+ public void test_node_reallocation() {
+ createNodesInRepository(10, 0);
+ List<Node> assignments1 = assignNode(application, capacity);
+ List<Node> assignments2 = assignNode(application, capacity);
+
+ assertEquals(assignments2.size(), assignments1.size());
+ }
+
+ @Test
+ public void test_node_reallocation_add_hostalias() {
+ createNodesInRepository(5, 0);
+
+ List<Node> assignments1 = assignNode(application, 2);
+ List<Node> assignments2 = assignNode(application, 3);
+
+ assertEquals(assignments2.size(), assignments1.size() + 1);
+ }
+
+ @Test
+ public void test_node_allocation_remove_hostalias() {
+ createNodesInRepository(10, 0);
+
+ List<Node> assignments1 = assignNode(application, 3, ClusterSpec.Type.container);
+ List<Node> assignments2 = assignNode(application, 2, ClusterSpec.Type.container);
+
+ assertEquals(assignments2.size(), assignments1.size() - 1);
+ ProvisionStatus provisionStatus = provisionResource.getStatus();
+ assertEquals(1, provisionStatus.decomissionNodes.size());
+ }
+
+ @Test
+ public void test_recycle_deallocated() {
+ createNodesInRepository(2, 0);
+ assignNode(application, 2);
+ nodeRepository.deactivate(application);
+ List<Node> nodes = nodeRepository.deallocate(nodeRepository.getNodes(application, Node.State.inactive));
+ assertEquals(0, nodeRepository.getNodes(Node.State.ready).size());
+ assertEquals(2, nodeRepository.getNodes(Node.State.dirty).size());
+ provisionResource.setReady(nodes.get(0).hostname());
+ provisionResource.setReady(nodes.get(1).hostname());
+ assertEquals(2, nodeRepository.getNodes(Node.State.ready).size());
+ assertEquals(0, nodeRepository.getNodes(Node.State.dirty).size());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void test_ready_node_unknown() {
+ provisionResource.setReady("does.not.exist");
+ }
+
+ private List<HostInfo> createHostInfos(int count, int startIndex) {
+ String format = "node%d";
+ List<HostInfo> hostInfos = new ArrayList<>();
+ for (int i = 0; i < count; ++i)
+ hostInfos.add(HostInfo.createHostInfo(String.format(format, i + startIndex), UUID.randomUUID().toString(), "medium"));
+ return hostInfos;
+ }
+
+ private List<Node> assignNode(ApplicationId applicationId, int capacity) {
+ return assignNode(applicationId, capacity, ClusterSpec.Type.content);
+ }
+
+ private List<Node> assignNode(ApplicationId applicationId, int capacity, ClusterSpec.Type type) {
+ ClusterSpec cluster = ClusterSpec.from(type, ClusterSpec.Id.from("test"), Optional.empty());
+ List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(capacity), 1, null);
+ NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
+ provisioner.activate(transaction, applicationId, hosts);
+ transaction.commit();
+ return nodeRepository.getNodes(applicationId, Node.State.active);
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java
new file mode 100644
index 00000000000..9afb14f632a
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java
@@ -0,0 +1,113 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v1;
+
+import com.yahoo.application.container.JDisc;
+import com.yahoo.application.Networking;
+
+import com.yahoo.application.container.handler.Request;
+import com.yahoo.application.container.handler.Response;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Configuration;
+import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
+import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
+import org.junit.Test;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author bratseth
+ */
+public class RestApiTest {
+
+ private static final String servicesXml =
+ "<jdisc version=\"1.0\">" +
+ " <component id=\"com.yahoo.vespa.hosted.provision.restapi.v1.RestApiTest$MockNodeRepository\"/>" +
+ " <handler id=\"com.yahoo.vespa.hosted.provision.restapi.v1.NodesApiHandler\">" +
+ " <binding>http://*/nodes/v1/</binding>" +
+ " </handler>" +
+ "</jdisc>";
+
+ @Test
+ public void testTopLevelRequest() throws Exception {
+ try (JDisc container = JDisc.fromServicesXml(servicesXml, Networking.disable)) {
+ Response response = container.handleRequest(new Request("http://localhost:8080/nodes/v1/"));
+ assertEquals("{\"provisioned\":[{\"id\":\"node6\",\"hostname\":\"host6.yahoo.com\",\"flavor\":\"default\"}],\"reserved\":[{\"id\":\"node3\",\"hostname\":\"host3.yahoo.com\",\"flavor\":\"default\",\"owner\":{\"tenant\":\"tenant1\",\"application\":\"application1\",\"instance\":\"instance1\"},\"membership\":{\"clustertype\":\"container\",\"clusterid\":\"id1\",\"index\":0,\"retired\":false},\"restartGeneration\":0},{\"id\":\"node2\",\"hostname\":\"host2.yahoo.com\",\"flavor\":\"default\",\"owner\":{\"tenant\":\"tenant1\",\"application\":\"application1\",\"instance\":\"instance1\"},\"membership\":{\"clustertype\":\"container\",\"clusterid\":\"id1\",\"index\":1,\"retired\":false},\"restartGeneration\":0}],\"active\":[{\"id\":\"node1\",\"hostname\":\"host1.yahoo.com\",\"flavor\":\"default\",\"owner\":{\"tenant\":\"tenant2\",\"application\":\"application2\",\"instance\":\"instance2\"},\"membership\":{\"clustertype\":\"content\",\"clusterid\":\"id2\",\"index\":0,\"retired\":false},\"restartGeneration\":0},{\"id\":\"node4\",\"hostname\":\"host4.yahoo.com\",\"flavor\":\"default\",\"owner\":{\"tenant\":\"tenant2\",\"application\":\"application2\",\"instance\":\"instance2\"},\"membership\":{\"clustertype\":\"content\",\"clusterid\":\"id2\",\"index\":1,\"retired\":false},\"restartGeneration\":0}],\"failed\":[{\"id\":\"node5\",\"hostname\":\"host5.yahoo.com\",\"flavor\":\"default\"}]}",
+ response.getBodyAsString());
+ }
+ }
+
+ @Test
+ public void testSingleNodeRequest() throws Exception {
+ try (JDisc container = JDisc.fromServicesXml(servicesXml, Networking.disable)) {
+ Response response1 = container.handleRequest(new Request("http://localhost:8080/nodes/v1/?hostname=host3.yahoo.com"));
+ assertEquals("{\"reserved\":[{\"id\":\"node3\",\"hostname\":\"host3.yahoo.com\",\"flavor\":\"default\",\"owner\":{\"tenant\":\"tenant1\",\"application\":\"application1\",\"instance\":\"instance1\"},\"membership\":{\"clustertype\":\"container\",\"clusterid\":\"id1\",\"index\":0,\"retired\":false},\"restartGeneration\":0}]}",
+ response1.getBodyAsString());
+
+ Response response2 = container.handleRequest(new Request("http://localhost:8080/nodes/v1/?hostname=host6.yahoo.com"));
+ assertEquals("{\"provisioned\":[{\"id\":\"node6\",\"hostname\":\"host6.yahoo.com\",\"flavor\":\"default\"}]}",
+ response2.getBodyAsString());
+
+ Response response3 = container.handleRequest(new Request("http://localhost:8080/nodes/v1/?hostname=nonexisting-host.yahoo.com"));
+ assertEquals("{}",
+ response3.getBodyAsString());
+ }
+ }
+
+ // Instantiated by DI from application package above
+ public static class MockNodeRepository extends NodeRepository {
+
+ private static final NodeFlavors flavors = FlavorConfigBuilder.createDummies("default");
+
+ public MockNodeRepository() throws Exception {
+ super(flavors, new MockCurator(), Clock.systemUTC());
+ populate();
+ }
+
+ private void populate() {
+ NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, flavors, Zone.defaultZone());
+
+ NodeFlavors flavors = FlavorConfigBuilder.createDummies("default");
+ List<Node> nodes = new ArrayList<>();
+ nodes.add(createNode("node1", "host1.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default"))));
+ nodes.add(createNode("node2", "host2.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default"))));
+ nodes.add(createNode("node3", "host3.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default"))));
+ nodes.add(createNode("node4", "host4.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default"))));
+ nodes.add(createNode("node5", "host5.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default"))));
+ nodes.add(createNode("node6", "host6.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default"))));
+ nodes = addNodes(nodes);
+ nodes.remove(5);
+ setReady(nodes);
+ fail("host5.yahoo.com");
+
+ ApplicationId app1 = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1"));
+ ClusterSpec cluster1 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Optional.empty());
+ provisioner.prepare(app1, cluster1, Capacity.fromNodeCount(2), 1, null);
+
+ ApplicationId app2 = ApplicationId.from(TenantName.from("tenant2"), ApplicationName.from("application2"), InstanceName.from("instance2"));
+ ClusterSpec cluster2 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("id2"), Optional.empty());
+ List<HostSpec> hosts = provisioner.prepare(app2, cluster2, Capacity.fromNodeCount(2), 1, null);
+ NestedTransaction transaction = new NestedTransaction();
+ provisioner.activate(transaction, app2, hosts);
+ transaction.commit();
+ }
+
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
new file mode 100644
index 00000000000..922e0038ea5
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -0,0 +1,268 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v2;
+
+import com.yahoo.application.Networking;
+import com.yahoo.application.container.JDisc;
+import com.yahoo.application.container.handler.Request;
+import com.yahoo.application.container.handler.Response;
+import com.yahoo.io.IOUtils;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author bratseth
+ */
+public class RestApiTest {
+
+ private final static String responsesPath = "src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/";
+
+ /** This test gives examples of all the requests that can be made to nodes/v2 */
+ @Test
+ public void testRequests() throws Exception {
+ // GET
+ assertFile(new Request("http://localhost:8080/nodes/v2/"), "root.json");
+ assertFile(new Request("http://localhost:8080/nodes/v2/state/"), "states.json");
+ assertFile(new Request("http://localhost:8080/nodes/v2/state/?recursive=true"), "states-recursive.json");
+ assertFile(new Request("http://localhost:8080/nodes/v2/state/active?recursive=true"), "active-nodes.json");
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/"), "nodes.json");
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true"), "nodes-recursive.json");
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "node2.json");
+
+ // GET with filters
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&hostname=host3.yahoo.com%20host6.yahoo.com"), "application2-nodes.json");
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterType=content"), "active-nodes.json");
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterId=id2"), "application2-nodes.json");
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&application=tenant2.application2.instance2"), "application2-nodes.json");
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&parentHost=parent.yahoo.com,parent.host.yahoo.com"), "parent-nodes.json");
+
+ // POST restart command
+ assertRestart(1, new Request("http://localhost:8080/nodes/v2/command/restart?hostname=host3.yahoo.com",
+ new byte[0], Request.Method.POST));
+ assertRestart(2, new Request("http://localhost:8080/nodes/v2/command/restart?application=tenant2.application2.instance2",
+ new byte[0], Request.Method.POST));
+ assertRestart(4, new Request("http://localhost:8080/nodes/v2/command/restart",
+ new byte[0], Request.Method.POST));
+ assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host3.yahoo.com"),
+ "\"restartGeneration\":3");
+
+ // POST reboot command
+ assertReboot(5, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active",
+ new byte[0], Request.Method.POST));
+ assertReboot(2, new Request("http://localhost:8080/nodes/v2/command/reboot?application=tenant2.application2.instance2",
+ new byte[0], Request.Method.POST));
+ assertReboot(8, new Request("http://localhost:8080/nodes/v2/command/reboot",
+ new byte[0], Request.Method.POST));
+ assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host3.yahoo.com"),
+ "\"rebootGeneration\":3");
+
+ // POST new nodes
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node",
+ ("[" + asNodeJson("host8.yahoo.com", "default") + "," +
+ asNodeJson("host9.yahoo.com", "large-variant") + "," +
+ asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com") + "]").
+ getBytes(StandardCharsets.UTF_8),
+ Request.Method.POST),
+ "{\"message\":\"Added 3 nodes to the provisioned state\"}");
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), "node8.json");
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com"), "node9.json");
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/host11.yahoo.com"), "node11.json");
+
+ // PUT nodes ready
+ assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com",
+ new byte[0], Request.Method.PUT),
+ "{\"message\":\"Moved host8.yahoo.com to ready\"}");
+ assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"),
+ "\"state\":\"ready\"");
+ // calling ready again is a noop:
+ assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com",
+ new byte[0], Request.Method.PUT),
+ "{\"message\":\"Nothing done; host8.yahoo.com is already ready\"}");
+
+ // PUT a node in failed ...
+ assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host8.yahoo.com",
+ new byte[0], Request.Method.PUT),
+ "{\"message\":\"Moved host8.yahoo.com to failed\"}");
+ assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"),
+ "\"state\":\"failed\"");
+ // ... and put it back in active (after fixing). This is useful to restore data when multiple nodes fail.
+ assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host8.yahoo.com",
+ new byte[0], Request.Method.PUT),
+ "{\"message\":\"Moved host8.yahoo.com to active\"}");
+
+ // PUT a node in failed ...
+ assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host8.yahoo.com",
+ new byte[0], Request.Method.PUT),
+ "{\"message\":\"Moved host8.yahoo.com to failed\"}");
+ assertResponseContains(new Request("http://localhost:8080()/nodes/v2/node/host8.yahoo.com"),
+ "\"state\":\"failed\"");
+ // ... and delete it
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com",
+ new byte[0], Request.Method.DELETE),
+ "{\"message\":\"Removed host8.yahoo.com\"}");
+
+ // or, PUT a node in failed ...
+ assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host6.yahoo.com",
+ new byte[0], Request.Method.PUT),
+ "{\"message\":\"Moved host6.yahoo.com to failed\"}");
+ assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"),
+ "\"state\":\"failed\"");
+ // ... and deallocate it such that it moves to dirty and is recycled
+ assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host6.yahoo.com",
+ new byte[0], Request.Method.PUT),
+ "{\"message\":\"Moved host6.yahoo.com to dirty\"}");
+
+ // Update (PATCH) a node (multiple fields can also be sent in one request body)
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+ Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.PATCH),
+ "{\"message\":\"Updated host4.yahoo.com\"}");
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com/currentRebootGeneration",
+ Utf8.toBytes("{\"currentRebootGeneration\": 1}"), Request.Method.PATCH),
+ "{\"message\":\"Updated host4.yahoo.com\"}");
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+ Utf8.toBytes("{\"flavor\": \"medium-disk\"}"), Request.Method.PATCH),
+ "{\"message\":\"Updated host4.yahoo.com\"}");
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+ Utf8.toBytes("{\"currentVespaVersion\": \"5.104.142\"}"), Request.Method.PATCH),
+ "{\"message\":\"Updated host4.yahoo.com\"}");
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+ Utf8.toBytes("{\"currentHostedVersion\": \"2.1.2408\"}"), Request.Method.PATCH),
+ "{\"message\":\"Updated host4.yahoo.com\"}");
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+ Utf8.toBytes("{\"convergedStateVersion\": \"5.104.142-2.1.2408\"}"), Request.Method.PATCH),
+ "{\"message\":\"Updated host4.yahoo.com\"}");
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+ Utf8.toBytes("{\"hardwareFailure\": true}"), Request.Method.PATCH),
+ "{\"message\":\"Updated host4.yahoo.com\"}");
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+ Utf8.toBytes("{\"parentHostname\": \"parent.yahoo.com\"}"), Request.Method.PATCH),
+ "{\"message\":\"Updated host4.yahoo.com\"}");
+
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json");
+ }
+
+ @Test
+ public void testInvalidRequests() throws IOException {
+ // Attempt to DELETE a node which is not put in failed first
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com",
+ new byte[0], Request.Method.DELETE),
+ 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node in the failed state with hostname host8.yahoo.com\"}");
+
+ // PUT current restart generation with string instead of long
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+ Utf8.toBytes("{\"currentRestartGeneration\": \"1\"}"), Request.Method.PATCH),
+ 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Expected a LONG value, got a STRING\"}");
+
+ // PUT flavor with long instead of string
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+ Utf8.toBytes("{\"flavor\": 1}"), Request.Method.PATCH),
+ 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'flavor': Expected a STRING value, got a LONG\"}");
+ }
+
+
+ @Test
+ public void testNodePatching() throws IOException {
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+ Utf8.toBytes("{" +
+ "\"currentRestartGeneration\": 1," +
+ "\"currentRebootGeneration\": 3," +
+ "\"flavor\": \"medium-disk\"," +
+ "\"currentVespaVersion\": \"5.104.142\"," +
+ "\"currentHostedVersion\": \"2.1.2408\"," +
+ "\"hardwareFailure\": true," +
+ "\"failCount\": 0," +
+ "\"parentHostname\": \"parent.yahoo.com\"" +
+ "}"
+ ),
+ Request.Method.PATCH),
+ "{\"message\":\"Updated host4.yahoo.com\"}");
+
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com",
+ Utf8.toBytes("{\"currentRestartGeneration\": 1}"),
+ Request.Method.PATCH),
+ 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Node is not allocated\"}");
+ }
+
+ /** Tests the rendering of each node separately to make it easier to find errors */
+ @Test
+ public void testSingleNodeRendering() throws IOException {
+ for (int i = 1; i <= 10; i++) {
+ if (i == 8 || i == 9) continue; // these nodes are added later
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/host" + i + ".yahoo.com"), "node" + i + ".json");
+ }
+ }
+
+ private JDisc container;
+ @Before
+ public void startContainer() { container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); }
+ @After
+ public void stopContainer() { container.close(); }
+
+ private String asDockerNodeJson(String hostname, String parentHostname) {
+ return "{\"hostname\":\"" + hostname + "\", \"parentHostname\":\"" + parentHostname +
+ "\", \"openStackId\":\"" + hostname + "\",\"flavor\":\"docker\"}";
+ }
+
+ private String asNodeJson(String hostname, String flavor) {
+ return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\",\"flavor\":\"" + flavor + "\"}";
+ }
+
+ /** Asserts a particular response and 200 as response status */
+ private void assertResponse(Request request, String responseMessage) throws IOException {
+ assertResponse(request, 200, responseMessage);
+ }
+
+ private void assertResponse(Request request, int responseStatus, String responseMessage) throws IOException {
+ Response response = container.handleRequest(request);
+ // Compare both status and message at once for easier diagnosis
+ assertEquals("status: " + responseStatus + "\nmessage: " + responseMessage,
+ "status: " + response.getStatus() + "\nmessage: " + response.getBodyAsString());
+ }
+
+ private void assertResponseContains(Request request, String responseSnippet) throws IOException {
+ assertTrue("Response contains " + responseSnippet,
+ container.handleRequest(request).getBodyAsString().contains(responseSnippet));
+ }
+
+ private void assertFile(Request request, String responseFile) throws IOException {
+ String expectedResponse = IOUtils.readFile(new File(responsesPath + responseFile));
+ expectedResponse = include(expectedResponse);
+ expectedResponse = expectedResponse.replaceAll("\\s", "");
+ String responseString = container.handleRequest(request).getBodyAsString();
+ assertEquals(responseFile, expectedResponse, responseString);
+ }
+
+ private void assertRestart(int restartCount, Request request) throws IOException {
+ assertResponse(request, 200, "{\"message\":\"Scheduled restart of " + restartCount + " matching nodes\"}");
+ }
+
+ private void assertReboot(int rebootCount, Request request) throws IOException {
+ assertResponse(request, 200, "{\"message\":\"Scheduled reboot of " + rebootCount + " matching nodes\"}");
+ }
+
+ /** Replace @include(localFile) with the content of the file */
+ private String include(String response) throws IOException {
+ // Please don't look at this code
+ int includeIndex = response.indexOf("@include(");
+ if (includeIndex < 0) return response;
+ String prefix = response.substring(0, includeIndex);
+ String rest = response.substring(includeIndex + "@include(".length());
+ int filenameEnd = rest.indexOf(")");
+ String includeFileName = rest.substring(0, filenameEnd);
+ String includedContent = IOUtils.readFile(new File(responsesPath + includeFileName));
+ includedContent = include(includedContent);
+ String postFix = rest.substring(filenameEnd + 1);
+ postFix = include(postFix);
+ return prefix + includedContent + postFix;
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json
new file mode 100644
index 00000000000..d1df5b83f24
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json
@@ -0,0 +1,8 @@
+{
+ "nodes": [
+ @include(node6.json),
+ @include(node3.json),
+ @include(node2.json),
+ @include(node1.json)
+ ]
+} \ No newline at end of file
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json
new file mode 100644
index 00000000000..f1285766c1b
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json
@@ -0,0 +1,6 @@
+{
+ "nodes": [
+ @include(node6.json),
+ @include(node3.json)
+ ]
+} \ No newline at end of file
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json
new file mode 100644
index 00000000000..734b6702c1e
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json
@@ -0,0 +1,33 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com",
+ "id": "host1.yahoo.com",
+ "state": "active",
+ "hostname": "host1.yahoo.com",
+ "openStackId": "node1",
+ "flavor": "default",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "canonicalFlavor": "default",
+ "environment":"env",
+ "owner": {
+ "tenant": "tenant3",
+ "application": "application3",
+ "instance": "instance3"
+ },
+ "membership": {
+ "clustertype": "content",
+ "clusterid": "id3",
+ "group": "0",
+ "index": 1,
+ "retired": false
+ },
+ "restartGeneration": 0,
+ "currentRestartGeneration": 0,
+ "rebootGeneration": 0,
+ "currentRebootGeneration": 0,
+ "failCount": 0,
+ "hardwareFailure" : false,
+ "history":[{"event":"readied","at":123},{"event":"reserved","at":123},{"event":"activated","at":123}]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json
new file mode 100644
index 00000000000..d412e803bf5
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json
@@ -0,0 +1,38 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com",
+ "id": "host10.yahoo.com",
+ "state": "reserved",
+ "hostname": "host10.yahoo.com",
+ "parentHostname": "parent.yahoo.com",
+ "openStackId": "node10",
+ "flavor": "default",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "canonicalFlavor": "default",
+ "environment":"env",
+ "owner": {
+ "tenant": "tenant1",
+ "application": "application1",
+ "instance": "instance1"
+ },
+ "membership": {
+ "clustertype": "container",
+ "clusterid": "id1",
+ "group": "0",
+ "index": 1,
+ "retired": false
+ },
+ "restartGeneration": 0,
+ "currentRestartGeneration": 0,
+ "wantedDockerImage":"image-123",
+ "rebootGeneration": 0,
+ "currentRebootGeneration": 0,
+ "vespaVersion": "5.104.142",
+ "hostedVersion": "2.1.2408",
+ "convergedStateVersion": "5.104.142-2.1.2408",
+ "failCount": 0,
+ "hardwareFailure" : false,
+ "history":[{"event":"readied","at":123},{"event":"reserved","at":123}]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json
new file mode 100644
index 00000000000..6d1922e7fc0
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json
@@ -0,0 +1,20 @@
+{
+ "url":"http://localhost:8080/nodes/v2/node/host11.yahoo.com",
+ "id":"host11.yahoo.com",
+ "state":"provisioned",
+ "hostname":"host11.yahoo.com",
+ "parentHostname":"parent.host.yahoo.com",
+ "openStackId":"host11.yahoo.com",
+ "flavor":"docker",
+ "minDiskAvailableGb":100.0,
+ "minMainMemoryAvailableGb":0.5,
+ "description":"Flavor-name-is-docker",
+ "minCpuCores":0.2,
+ "canonicalFlavor":"docker",
+ "environment":"docker",
+ "rebootGeneration":0,
+ "currentRebootGeneration":0,
+ "failCount":0,
+ "hardwareFailure":false,
+ "history":[]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json
new file mode 100644
index 00000000000..830c866ae81
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json
@@ -0,0 +1,33 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com",
+ "id": "host2.yahoo.com",
+ "state": "active",
+ "hostname": "host2.yahoo.com",
+ "openStackId": "node2",
+ "flavor": "default",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "canonicalFlavor": "default",
+ "environment":"env",
+ "owner": {
+ "tenant": "tenant3",
+ "application": "application3",
+ "instance": "instance3"
+ },
+ "membership": {
+ "clustertype": "content",
+ "clusterid": "id3",
+ "group": "0",
+ "index": 0,
+ "retired": false
+ },
+ "restartGeneration": 0,
+ "currentRestartGeneration": 0,
+ "rebootGeneration": 0,
+ "currentRebootGeneration": 0,
+ "failCount": 0,
+ "hardwareFailure" : false,
+ "history":[{"event":"readied","at":123},{"event":"reserved","at":123},{"event":"activated","at":123}]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json
new file mode 100644
index 00000000000..5bf8631797a
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json
@@ -0,0 +1,30 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com",
+ "id": "host3.yahoo.com",
+ "state": "active",
+ "hostname": "host3.yahoo.com",
+ "openStackId": "node3",
+ "flavor":"expensive",
+ "description":"Flavor-name-is-expensive",
+ "canonicalFlavor":"default",
+ "cost":200,
+ "owner": {
+ "tenant": "tenant2",
+ "application": "application2",
+ "instance": "instance2"
+ },
+ "membership": {
+ "clustertype": "content",
+ "clusterid": "id2",
+ "group": "0",
+ "index": 1,
+ "retired": false
+ },
+ "restartGeneration": 0,
+ "currentRestartGeneration": 0,
+ "rebootGeneration": 0,
+ "currentRebootGeneration": 0,
+ "failCount": 0,
+ "hardwareFailure" : false,
+ "history":[{"event":"readied","at":123},{"event":"reserved","at":123},{"event":"activated","at":123}]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
new file mode 100644
index 00000000000..ef88154fc5c
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
@@ -0,0 +1,48 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+ "id": "host4.yahoo.com",
+ "state": "reserved",
+ "hostname": "host4.yahoo.com",
+ "parentHostname": "parent.yahoo.com",
+ "openStackId": "node4",
+ "flavor": "medium-disk",
+ "minDiskAvailableGb": 56.0,
+ "minMainMemoryAvailableGb": 12.0,
+ "description": "Flavor-name-is-medium-disk",
+ "minCpuCores": 6.0,
+ "canonicalFlavor": "medium-disk",
+ "environment": "foo",
+ "owner": {
+ "tenant": "tenant1",
+ "application": "application1",
+ "instance": "instance1"
+ },
+ "membership": {
+ "clustertype": "container",
+ "clusterid": "id1",
+ "group": "0",
+ "index": 0,
+ "retired": false
+ },
+ "restartGeneration": 0,
+ "currentRestartGeneration": 1,
+ "wantedDockerImage": "image-123",
+ "rebootGeneration": 1,
+ "currentRebootGeneration": 1,
+ "vespaVersion": "5.104.142",
+ "hostedVersion": "2.1.2408",
+ "currentDockerImage": "image-12",
+ "convergedStateVersion": "5.104.142-2.1.2408",
+ "failCount": 0,
+ "hardwareFailure": true,
+ "history": [
+ {
+ "event": "readied",
+ "at": 123
+ },
+ {
+ "event": "reserved",
+ "at": 123
+ }
+ ]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json
new file mode 100644
index 00000000000..a1bc67705ce
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json
@@ -0,0 +1,36 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+ "id": "host4.yahoo.com",
+ "state": "reserved",
+ "hostname": "host4.yahoo.com",
+ "parentHostname":"dockerhost4",
+ "openStackId": "node4",
+ "flavor": "default",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "canonicalFlavor": "default",
+ "environment":"env",
+ "owner": {
+ "tenant": "tenant1",
+ "application": "application1",
+ "instance": "instance1"
+ },
+ "membership": {
+ "clustertype": "container",
+ "clusterid": "id1",
+ "group": "0",
+ "index": 0,
+ "retired": false
+ },
+ "restartGeneration": 0,
+ "currentRestartGeneration": 0,
+ "wantedDockerImage":"image-123",
+ "rebootGeneration": 0,
+ "currentRebootGeneration": 0,
+ "currentDockerImage":"image-12",
+ "failCount": 0,
+ "hardwareFailure" : false,
+ "history":[{"event":"readied","at":123},{"event":"reserved","at":123}]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json
new file mode 100644
index 00000000000..da4b49280c7
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json
@@ -0,0 +1,21 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com",
+ "id": "host5.yahoo.com",
+ "state": "failed",
+ "hostname": "host5.yahoo.com",
+ "parentHostname":"dockerhost",
+ "openStackId": "node5",
+ "flavor": "default",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "canonicalFlavor": "default",
+ "environment":"env",
+ "rebootGeneration": 0,
+ "currentRebootGeneration": 0,
+ "currentDockerImage":"image-123",
+ "failCount": 1,
+ "hardwareFailure" : false,
+ "history":[{"event":"readied","at":123},{"event":"failed","at":123}]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json
new file mode 100644
index 00000000000..9ef8adb1f07
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json
@@ -0,0 +1,33 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com",
+ "id": "host6.yahoo.com",
+ "state": "active",
+ "hostname": "host6.yahoo.com",
+ "openStackId": "node6",
+ "flavor": "default",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "canonicalFlavor": "default",
+ "environment":"env",
+ "owner": {
+ "tenant": "tenant2",
+ "application": "application2",
+ "instance": "instance2"
+ },
+ "membership": {
+ "clustertype": "content",
+ "clusterid": "id2",
+ "group": "0",
+ "index": 0,
+ "retired": false
+ },
+ "restartGeneration": 0,
+ "currentRestartGeneration": 0,
+ "rebootGeneration": 0,
+ "currentRebootGeneration": 0,
+ "failCount": 0,
+ "hardwareFailure" : false,
+ "history":[{"event":"readied","at":123},{"event":"reserved","at":123},{"event":"activated","at":123}]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json
new file mode 100644
index 00000000000..52f01407b2b
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json
@@ -0,0 +1,19 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com",
+ "id": "host7.yahoo.com",
+ "state": "provisioned",
+ "hostname": "host7.yahoo.com",
+ "openStackId": "node7",
+ "flavor": "default",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "canonicalFlavor": "default",
+ "environment":"env",
+ "rebootGeneration": 0,
+ "currentRebootGeneration": 0,
+ "failCount": 0,
+ "hardwareFailure" : false,
+ "history":[]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json
new file mode 100644
index 00000000000..c00b6ed797c
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json
@@ -0,0 +1,19 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/host8.yahoo.com",
+ "id": "host8.yahoo.com",
+ "state": "provisioned",
+ "hostname": "host8.yahoo.com",
+ "openStackId": "host8.yahoo.com",
+ "flavor": "default",
+ "minDiskAvailableGb":400.0,
+ "minMainMemoryAvailableGb":16.0,
+ "description":"Flavor-name-is-default",
+ "minCpuCores":2.0,
+ "canonicalFlavor": "default",
+ "environment":"env",
+ "rebootGeneration": 0,
+ "currentRebootGeneration": 0,
+ "failCount": 0,
+ "hardwareFailure" : false,
+ "history":[]
+} \ No newline at end of file
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json
new file mode 100644
index 00000000000..73a0eb8a266
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json
@@ -0,0 +1,19 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/host9.yahoo.com",
+ "id": "host9.yahoo.com",
+ "state": "provisioned",
+ "hostname": "host9.yahoo.com",
+ "openStackId": "host9.yahoo.com",
+ "flavor": "large-variant",
+ "minDiskAvailableGb":2000.0,
+ "minMainMemoryAvailableGb":128.0,
+ "description":"Flavor-name-is-large-variant",
+ "minCpuCores":64.0,
+ "canonicalFlavor": "large",
+ "environment":"env",
+ "rebootGeneration": 0,
+ "currentRebootGeneration": 0,
+ "failCount": 0,
+ "hardwareFailure" : false,
+ "history":[]
+} \ No newline at end of file
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json
new file mode 100644
index 00000000000..8ea48599f0e
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json
@@ -0,0 +1,12 @@
+{
+ "nodes": [
+ @include(node7.json),
+ @include(node10.json),
+ @include(node4.json),
+ @include(node6.json),
+ @include(node3.json),
+ @include(node2.json),
+ @include(node1.json),
+ @include(node5.json)
+ ]
+} \ No newline at end of file
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json
new file mode 100644
index 00000000000..73947ded547
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json
@@ -0,0 +1,28 @@
+{
+ "nodes": [
+ {
+ "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com"
+ },
+ {
+ "url":"http://localhost:8080/nodes/v2/node/host10.yahoo.com"
+ },
+ {
+ "url":"http://localhost:8080/nodes/v2/node/host4.yahoo.com"
+ },
+ {
+ "url":"http://localhost:8080/nodes/v2/node/host6.yahoo.com"
+ },
+ {
+ "url":"http://localhost:8080/nodes/v2/node/host3.yahoo.com"
+ },
+ {
+ "url":"http://localhost:8080/nodes/v2/node/host2.yahoo.com"
+ },
+ {
+ "url":"http://localhost:8080/nodes/v2/node/host1.yahoo.com"
+ },
+ {
+ "url":"http://localhost:8080/nodes/v2/node/host5.yahoo.com"
+ }
+ ]
+} \ No newline at end of file
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json
new file mode 100644
index 00000000000..28a17b03cc6
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json
@@ -0,0 +1,5 @@
+{
+ "nodes": [
+ @include(node10.json)
+]
+} \ No newline at end of file
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json
new file mode 100644
index 00000000000..9648c059af6
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json
@@ -0,0 +1,13 @@
+{
+ "resources": [
+ {
+ "url": "http://localhost:8080/nodes/v2/state/"
+ },
+ {
+ "url": "http://localhost:8080/nodes/v2/node/"
+ },
+ {
+ "url": "http://localhost:8080/nodes/v2/command/"
+ }
+ ]
+} \ No newline at end of file
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json
new file mode 100644
index 00000000000..b83616d2e0b
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json
@@ -0,0 +1,47 @@
+{
+ "states": {
+ "provisioned": {
+ "url": "http://localhost:8080/nodes/v2/state/provisioned",
+ "nodes": [
+ @include(node7.json)
+ ]
+ },
+ "ready": {
+ "url": "http://localhost:8080/nodes/v2/state/ready",
+ "nodes": [
+ ]
+ },
+ "reserved": {
+ "url": "http://localhost:8080/nodes/v2/state/reserved",
+ "nodes": [
+ @include(node10.json),
+ @include(node4.json)
+ ]
+ },
+ "active": {
+ "url": "http://localhost:8080/nodes/v2/state/active",
+ "nodes": [
+ @include(node6.json),
+ @include(node3.json),
+ @include(node2.json),
+ @include(node1.json)
+ ]
+ },
+ "inactive": {
+ "url": "http://localhost:8080/nodes/v2/state/inactive",
+ "nodes": [
+ ]
+ },
+ "dirty": {
+ "url": "http://localhost:8080/nodes/v2/state/dirty",
+ "nodes": [
+ ]
+ },
+ "failed": {
+ "url": "http://localhost:8080/nodes/v2/state/failed",
+ "nodes": [
+ @include(node5.json)
+ ]
+ }
+ }
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json
new file mode 100644
index 00000000000..b2d7354a6c9
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json
@@ -0,0 +1,25 @@
+{
+ "states": {
+ "provisioned": {
+ "url": "http://localhost:8080/nodes/v2/state/provisioned"
+ },
+ "ready": {
+ "url": "http://localhost:8080/nodes/v2/state/ready"
+ },
+ "reserved": {
+ "url": "http://localhost:8080/nodes/v2/state/reserved"
+ },
+ "active": {
+ "url": "http://localhost:8080/nodes/v2/state/active"
+ },
+ "inactive": {
+ "url": "http://localhost:8080/nodes/v2/state/inactive"
+ },
+ "dirty": {
+ "url": "http://localhost:8080/nodes/v2/state/dirty"
+ },
+ "failed": {
+ "url": "http://localhost:8080/nodes/v2/state/failed"
+ }
+ }
+} \ No newline at end of file
diff --git a/node-repository/src/test/resources/hosts.xml b/node-repository/src/test/resources/hosts.xml
new file mode 100644
index 00000000000..8f833840c5c
--- /dev/null
+++ b/node-repository/src/test/resources/hosts.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<hosts>
+ <host name="hostname1">
+ <alias>myhostalias1</alias>
+ </host>
+
+ <host name="hostname2">
+ <alias>myhostalias2</alias>
+ </host>
+
+ <host name="hostname3">
+ <alias>myhostalias3</alias>
+ </host>
+
+ <host name="hostname4">
+ <alias>myhostalias4</alias>
+ </host>
+
+ <host name="hostname5">
+ <alias>myhostalias5</alias>
+ </host>
+
+ <host name="hostname6">
+ <alias>myhostalias6</alias>
+ </host>
+</hosts>
diff --git a/node-repository/src/test/resources/services.xml b/node-repository/src/test/resources/services.xml
new file mode 100644
index 00000000000..ac1ecfb02de
--- /dev/null
+++ b/node-repository/src/test/resources/services.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<services version="1.0">
+ <jdisc version="1.0" id="default">
+ <nodes>
+ <node hostalias="myhostalias1"/>
+ <node hostalias="myhostalias2"/>
+ </nodes>
+ </jdisc>
+ <content version="1.0"> <!-- id="default" -->
+ <nodes>
+ <node hostalias="myhostalias3" distribution-key="99"/> <!-- arbitrary distribution keys -->
+ <node hostalias="myhostalias4" distribution-key="42"/>
+ </nodes>
+ </content>
+ <content version="1.0" id="mycontent"> <!-- second content cluster -->
+ <group> <!-- element name is group instead of nodes -->
+ <node hostalias="myhostalias5" distribution-key="0"/>
+ <node hostalias="myhostalias6" distribution-key="1"/>
+ </group>
+ </content>
+</services>