diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /node-repository |
Publish
Diffstat (limited to 'node-repository')
111 files changed, 9740 insertions, 0 deletions
diff --git a/node-repository/OWNERS b/node-repository/OWNERS new file mode 100644 index 00000000000..cc4d4971a75 --- /dev/null +++ b/node-repository/OWNERS @@ -0,0 +1,2 @@ +bratseth +musum diff --git a/node-repository/README b/node-repository/README new file mode 100644 index 00000000000..c6050e560ed --- /dev/null +++ b/node-repository/README @@ -0,0 +1 @@ +Node repository component that manages node (de)allocation in hosted vespa. diff --git a/node-repository/pom.xml b/node-repository/pom.xml new file mode 100644 index 00000000000..da48ff9eb4b --- /dev/null +++ b/node-repository/pom.xml @@ -0,0 +1,137 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>node-repository</artifactId> + <version>6-SNAPSHOT</version> + <packaging>container-plugin</packaging> + <description>Keeps track of node assignment in a multi-application setup.</description> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dev</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>application</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa_jersey2</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + <type>pom</type> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-jersey2</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>application-model</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>service-monitor</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zkfacade</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>testutil</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.datatype</groupId> + <artifactId>jackson-datatype-joda</artifactId> + <version>${jackson2.version}</version> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>orchestrator</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Xlint:-try</arg> + <arg>-Xlint:-serial</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml new file mode 100644 index 00000000000..de9d5da1eef --- /dev/null +++ b/node-repository/src/main/config/node-repository.xml @@ -0,0 +1,21 @@ +<!-- services.xml snippet for the node repository. Included in config server services.xml if the package is installed--> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<component id="com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner" bundle="node-repository" /> +<component id="NodeRepository" class="com.yahoo.vespa.hosted.provision.NodeRepository" bundle="node-repository"/> +<component id="com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance" bundle="node-repository"/> +<component id="com.yahoo.vespa.hosted.provision.monitoring.ProvisionMetrics" bundle="node-repository" /> +<component id="com.yahoo.vespa.hosted.provision.node.NodeFlavors" bundle="node-repository" /> + +<rest-api path="hack" jersey2="true"> + <components bundle="node-repository" /> +</rest-api> + +<handler id="com.yahoo.vespa.hosted.provision.restapi.v1.NodesApiHandler" bundle="node-repository"> + <binding>http://*/nodes/v1/</binding> +</handler> + +<handler id="com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler" bundle="node-repository"> + <binding>http://*/nodes/v2/*</binding> +</handler> + +<preprocess:include file="node-flavors.xml" required="false" /> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java new file mode 100644 index 00000000000..2ab98f0c582 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -0,0 +1,242 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.Flavor; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.Status; +import com.yahoo.vespa.hosted.provision.node.Generation; + +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * A node in the node repository. The identity of a node is given by its id. + * The classes making up the node model are found in the node package. + * This (and hence all classes referenced from it) is immutable. + * + * @author bratseth + */ +public final class Node { + + private final String id; + private final String hostname; + private final String openStackId; + private final Optional<String> parentHostname; + private final Configuration configuration; + private final Status status; + private final State state; + + /** Record of the last event of each type happening to this node */ + private final History history; + + /** The current allocation of this node, if any */ + private Optional<Allocation> allocation; + + /** Creates a node in the initial state (provisioned) */ + public static Node create(String openStackId, String hostname, Optional<String> parentHostname, Configuration configuration) { + return new Node(openStackId, hostname, parentHostname, configuration, Status.initial(), State.provisioned, + Optional.empty(), History.empty()); + } + + /** Do not use. Construct nodes by calling {@link NodeRepository#createNode} */ + public Node(String openStackId, String hostname, Optional<String> parentHostname, + Configuration configuration, Status status, State state, Allocation allocation, History history) { + this(openStackId, hostname, parentHostname, configuration, status, state, Optional.of(allocation), history); + } + + public Node(String openStackId, String hostname, Optional<String> parentHostname, + Configuration configuration, Status status, State state, Optional<Allocation> allocation, History history) { + Objects.requireNonNull(openStackId, "A node must have an openstack id"); + Objects.requireNonNull(hostname, "A node must have a hostname"); + Objects.requireNonNull(parentHostname, "A null parentHostname is not permitted."); + Objects.requireNonNull(configuration, "A node must have a configuration"); + Objects.requireNonNull(status, "A node must have a status"); + Objects.requireNonNull(state, "A null node state is not permitted"); + Objects.requireNonNull(allocation, "A null node allocation is not permitted"); + Objects.requireNonNull(history, "A null node history is not permitted"); + + this.id = hostname; + this.hostname = hostname; + this.parentHostname = parentHostname; + this.openStackId = openStackId; + this.configuration = configuration; + this.status = status; + this.state = state; + this.allocation = allocation; + this.history = history; + } + + /** + * Returns the unique id of this host. + * This may be the host name or some other opaque id which is unique across hosts + */ + public String id() { return id; } + + /** Returns the host name of this node */ + public String hostname() { return hostname; } + + // TODO: Different meaning for vms and docker hosts? + /** Returns the OpenStack id of this node, or of its docker host if this is a virtual node */ + public String openStackId() { return openStackId; } + + /** Returns the parent hostname for this node if this node is a docker container or a VM (i.e. it has a parent host). Otherwise, empty **/ + public Optional<String> parentHostname() { return parentHostname; } + + /** Returns the hardware configuration of this node */ + public Configuration configuration() { return configuration; } + + /** Returns the known information about the nodes ephemeral status */ + public Status status() { return status; } + + /** Returns the current state of this node (in the node state machine) */ + public State state() { return state; } + + /** Returns the current allocation of this, if any */ + public Optional<Allocation> allocation() { return allocation; } + + /** Returns a history of the last events happening to this node */ + public History history() { return history; } + + /** + * Returns a copy of this node which is retired by the application owning it. + * If the node was already retired it is returned as-is. + */ + public Node retireByApplication(Instant retiredAt) { + if (allocation().get().membership().retired()) return this; + return setAllocation(allocation.get().retire()) + .setHistory(history.record(new History.RetiredEvent(retiredAt, History.RetiredEvent.Agent.application))); + } + + /** Returns a copy of this node which is retired by the system */ + // We will use this when we support operators retiring a flavor completely from hosted Vespa + public Node retireBySystem(Instant retiredAt) { + return setAllocation(allocation.get().retire()) + .setHistory(history.record(new History.RetiredEvent(retiredAt, History.RetiredEvent.Agent.system))); + } + + /** Returns a copy of this node which is not retired */ + public Node unretire() { + return setAllocation(allocation.get().unretire()); + } + + /** Returns a copy of this with the current generation set to generation */ + public Node setRestart(Generation generation) { + final Optional<Allocation> allocation = this.allocation; + if ( ! allocation.isPresent()) + throw new IllegalArgumentException("Cannot set restart generation for " + hostname() + ": The node is unallocated"); + + return setAllocation(allocation.get().setRestart(generation)); + } + + /** Returns a node with the status assigned to the given value */ + public Node setStatus(Status status) { + return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history); + } + + /** Returns a node with the hardware configuration assigned to the given value */ + public Node setConfiguration(Configuration configuration) { + return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history); + } + + /** Returns a copy of this with the current generation set to generation */ + public Node setReboot(Generation generation) { + return new Node(openStackId, hostname, parentHostname, configuration, status.setReboot(generation), state, + allocation, history); + } + + /** Returns a copy of this with the flavor set to flavor */ + public Node setFlavor(Flavor flavor) { + return new Node(openStackId, hostname, parentHostname, new Configuration(flavor), status, state, + allocation, history); + } + + /** Returns a copy of this with a history record saying it was detected to be down at this instant */ + public Node setDown(Instant instant) { + return setHistory(history.record(new History.Event(History.Event.Type.down, instant))); + } + + /** Returns a copy of this with any history record saying it has been detected down removed */ + public Node setUp() { + return setHistory(history.clear(History.Event.Type.down)); + } + + /** Returns a copy of this with allocation set as specified. <code>node.state</code> is *not* changed. */ + public Node allocate(ApplicationId owner, ClusterMembership membership, Instant at) { + return setAllocation(new Allocation(owner, membership, new Generation(0, 0), false)) + .setHistory(history.record(new History.Event(History.Event.Type.reserved, at))); + } + + /** + * Returns a copy of this node with the allocation assigned to the given allocation. + * Do not use this to allocate a node. + */ + public Node setAllocation(Allocation allocation) { + return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history); + } + + /** Returns a copy of this node with the parent hostname assigned to the given value. */ + public Node setParentHostname(String parentHostname) { + return new Node(openStackId, hostname, Optional.of(parentHostname), configuration, status, state, allocation, history); + } + + /** Returns a copy of this node with the given history. */ + private Node setHistory(History history) { + return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history); + } + + @Override + public int hashCode() { + return id.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (this == other) return true; + if ( ! other.getClass().equals(this.getClass())) return false; + return ((Node)other).id.equals(this.id); + } + + @Override + public String toString() { + return state + " node " + + (hostname !=null ? hostname : id) + + (allocation.isPresent() ? " " + allocation.get() : "") + + (parentHostname.isPresent() ? " [on: " + parentHostname.get() + "]" : ""); + } + + public enum State { + + /** This node has been requested (from OpenStack) but is not yet read for use */ + provisioned, + + /** This node is free and ready for use */ + ready, + + /** This node has been reserved by an application but is not yet used by it */ + reserved, + + /** This node is in active use by an application */ + active, + + /** This node has been used by an application, is still allocated to it and retains the data needed for its allocated role */ + inactive, + + /** This node is not allocated to an application but may contain data which must be cleaned before it is ready */ + dirty, + + /** This node has failed and must be repaired or removed. The node retains any allocation data for diagnosis. */ + failed; + + /** Returns whether this is a state where the node is assigned to an application */ + public boolean isAllocated() { + return this == reserved || this == active || this == inactive || this == failed; + } + + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java new file mode 100644 index 00000000000..1980b1f5318 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -0,0 +1,318 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision; + +import com.google.inject.Inject; +import com.yahoo.collections.ListMap; +import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.transaction.Mutex; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; +import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter; +import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; + +/** + * The hosted Vespa production node repository, which stores its state in Zookeeper. + * The node repository knows about all nodes in a zone, their states and manages all transitions between + * node states. + * <p> + * Node repo locking: Locks must be acquired before making changes to the set of nodes, or to the content + * of the nodes. + * Unallocated states use a single lock, while application level locks are used for all allocated states + * such that applications can mostly change in parallel. + * If both locks are needed acquire the application lock first, then the unallocated lock. + * <p> + * Changes to the set of active nodes must be accompanied by changes to the config model of the application. + * Such changes are not handled by the node repository but by the classes calling it - see + * {@link com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner} for such changes initiated + * by the application package and {@link com.yahoo.vespa.hosted.provision.maintenance.ApplicationMaintainer} + * for changes initiated by the node repository. + * Refer to {@link com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance} for timing details + * of the node state transitions. + * + * @author bratseth + */ +// Node state transitions: +// 1) (new) - > provisioned -> ready -> reserved -> active -> inactive -> dirty -> ready +// 2) inactive -> reserved +// 3) reserved -> dirty +// 3) * -> failed -> dirty | active | (removed) +// Nodes have an application assigned when in states reserved, active and inactive. +// Nodes might have an application assigned in dirty. +public class NodeRepository extends AbstractComponent { + + private final CuratorDatabaseClient zkClient; + + /** + * Creates a node repository form a zookeeper provider. + * This will use the system time to make time-sensitive decisions + */ + @Inject + public NodeRepository(NodeFlavors flavors, Curator curator) { + this(flavors, curator, Clock.systemUTC()); + } + + /** + * Creates a node repository form a zookeeper provider and a clock instance + * which will be used for time-sensitive decisions. + */ + public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock) { + this.zkClient = new CuratorDatabaseClient(flavors, curator, clock); + + // read and write all nodes to make sure they are stored in the latest version of the serialized format + for (Node.State state : Node.State.values()) + zkClient.writeTo(state, zkClient.getNodes(state)); + } + + // ---------------- Query API ---------------------------------------------------------------- + + /** Finds and returns the node with the given hostname */ + public Optional<Node> getNode(String hostname) { + for (Node.State state : Node.State.values()) { + Optional<Node> node = getNode(state, hostname); + if (node.isPresent()) + return node; + } + return Optional.empty(); + } + + /** Finds and returns the node with the given state and hostname, or empty if not found */ + public Optional<Node> getNode(Node.State state, String hostname) { + return zkClient.getNode(state, hostname); + } + + public List<Node> getNodes(Node.State ... inState) { return zkClient.getNodes(inState); } + public List<Node> getNodes(ApplicationId id, Node.State ... inState) { return zkClient.getNodes(id, inState); } + public List<Node> getInactive() { return zkClient.getNodes(Node.State.inactive); } + public List<Node> getFailed() { return zkClient.getNodes(Node.State.failed); } + + public int getNodeCount(String tenantId, Node.State ... inState) { + return zkClient.getNodes(inState).stream() + .filter(node -> node.allocation().get().owner().tenant().value().equals(tenantId)) + .collect(Collectors.counting()).intValue(); + } + + // ----------------- Node lifecycle ----------------------------------------------------------- + + /** Creates a new node object, without adding it to the node repo */ + public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Configuration configuration) { + return Node.create(openStackId, hostname, parentHostname, configuration); + } + + /** Adds a list of (newly created) nodes to the node repository as <i>provisioned</i> nodes */ + public List<Node> addNodes(List<Node> nodes) { + for (Node node : nodes) { + Optional<Node> existing = getNode(node.hostname()); + if (existing.isPresent()) + throw new IllegalArgumentException("Cannot add " + node.hostname() + ": A node with this name already exists"); + } + try (Mutex lock = lockUnallocated()) { + return zkClient.addNodes(nodes); + } + } + + /** Sets a list of nodes ready and returns the nodes in the ready state */ + public List<Node> setReady(List<Node> nodes) { + for (Node node : nodes) + if (node.state() != Node.State.provisioned && node.state() != Node.State.dirty) + throw new IllegalArgumentException("Can not set " + node + " ready. It is not provisioned or dirty."); + try (Mutex lock = lockUnallocated()) { + return zkClient.writeTo(Node.State.ready, nodes); + } + } + + /** Reserve nodes. This method does <b>not</b> lock the node repository */ + public List<Node> reserve(List<Node> nodes) { return zkClient.writeTo(Node.State.reserved, nodes); } + + /** Activate nodes. This method does <b>not</b> lock the node repository */ + public List<Node> activate(List<Node> nodes, NestedTransaction transaction) { + return zkClient.writeTo(Node.State.active, nodes, transaction); + } + + /** + * Sets a list of nodes to have their allocation removable (active to inactive) in the node repository. + * + * @param application the application the nodes belong to + * @param nodes the nodes to make removable. These nodes MUST be in the active state. + */ + public void setRemovable(ApplicationId application, List<Node> nodes) { + try (Mutex lock = lock(application)) { + List<Node> removableNodes = + nodes.stream().map(node -> node.setAllocation(node.allocation().get().makeRemovable())) + .collect(Collectors.toList()); + write(removableNodes); + } + } + + public void deactivate(ApplicationId application) { + try (Mutex lock = lock(application)) { + zkClient.writeTo(Node.State.inactive, zkClient.getNodes(application, Node.State.reserved, Node.State.active)); + } + } + + /** + * Deactivates these nodes in a transaction and returns + * the nodes in the new state which will hold if the transaction commits. + * This method does <b>not</b> lock + */ + public List<Node> deactivate(List<Node> nodes, NestedTransaction transaction) { + return zkClient.writeTo(Node.State.inactive, nodes, transaction); + } + + /** Deallocates these nodes, causing them to move to the dirty state */ + public List<Node> deallocate(List<Node> nodes) { + return performOn(NodeListFilter.from(nodes), node -> zkClient.writeTo(Node.State.dirty, node)); + } + + /** Deallocate a node which is in the failed state. Use this to recycle failed nodes which have been repaired. */ + public Node deallocate(String hostname) { + Optional<Node> nodeToDeallocate = getNode(Node.State.failed, hostname); + if ( ! nodeToDeallocate.isPresent()) + throw new IllegalArgumentException("Could not deallocate " + hostname + ": Node not found in the failed state"); + return deallocate(Collections.singletonList(nodeToDeallocate.get())).get(0); + } + + /** + * Fails this node and returns it in its new state. + * + * @return the node in its new state + * @throws IllegalArgumentException if the node is not found + */ + public Node fail(String hostname) { + return move(hostname, Node.State.failed); + } + + /** + * Moves a previously failed node back to the active state. + * + * @return the node in its new state + * @throws IllegalArgumentException if the node is not found + */ + public Node unfail(String hostname) { + return move(hostname, Node.State.active); + } + + public Node move(String hostname, Node.State toState) { + Optional<Node> node = getNode(hostname); + if ( ! node.isPresent()) + throw new IllegalArgumentException("Could not move " + hostname + " to " + toState + ": Node not found"); + try (Mutex lock = lock(node.get())) { + return zkClient.writeTo(toState, node.get()); + } + } + + /** + * Removes a node. A node must be in the failed state before it can be removed. + * + * @return true if the node was removed, false if it was not found + */ + public boolean remove(String hostname) { + Optional<Node> nodeToRemove = getNode(Node.State.failed, hostname); + if ( ! nodeToRemove.isPresent()) return false; + try (Mutex lock = lock(nodeToRemove.get())) { + return zkClient.removeNode(Node.State.failed, hostname); + } + } + + /** + * Increases the restart generation of the active nodes matching the filter. + * Returns the nodes in their new state. + */ + public List<Node> restart(NodeFilter filter) { + return performOn(StateFilter.from(Node.State.active, filter), node -> write(node.setRestart(node.allocation().get().restartGeneration().increaseWanted()))); + } + + /** + * Increases the reboot generation of the nodes matching the filter. + * Returns the nodes in their new state. + */ + public List<Node> reboot(NodeFilter filter) { + return performOn(filter, node -> write(node.setReboot(node.status().reboot().increaseWanted()))); + } + + /** + * Writes this node after it has changed some internal state but NOT changed its state field. + * This does NOT lock the node repository. + * + * @return the written node for convenience + */ + public Node write(Node node) { return zkClient.writeTo(node.state(), node); } + + /** + * Writes these nodes after they have changed some internal state but NOT changed their state field. + * This does NOT lock the node repository. + * + * @return the written nodes for convenience + */ + public List<Node> write(List<Node> nodes) { + if (nodes.isEmpty()) return Collections.emptyList(); + + // decide current state and make sure all nodes have it (alternatively we could create a transaction here) + Node.State state = nodes.get(0).state(); + for (Node node : nodes) { + if ( node.state() != state) + throw new IllegalArgumentException("Multiple states: " + node.state() + " and " + state); + } + return zkClient.writeTo(state, nodes); + } + + /** + * Performs an operation requiring locking on all nodes matching some filter. + * + * @param filter the filter determining the set of nodes where the operation will be performed + * @param action the action to perform + * @return the set of nodes on which the action was performed, as they became as a result of the operation + */ + private List<Node> performOn(NodeFilter filter, UnaryOperator<Node> action) { + List<Node> unallocatedNodes = new ArrayList<>(); + ListMap<ApplicationId, Node> allocatedNodes = new ListMap<>(); + + // Group matching nodes by the lock needed + for (Node node : zkClient.getNodes()) { + if ( ! filter.matches(node)) continue; + if (node.allocation().isPresent()) + allocatedNodes.put(node.allocation().get().owner(), node); + else + unallocatedNodes.add(node); + } + + // perform operation while holding locks + List<Node> resultingNodes = new ArrayList<>(); + try (Mutex lock = lockUnallocated()) { + for (Node node : unallocatedNodes) + resultingNodes.add(action.apply(node)); + } + for (Map.Entry<ApplicationId, List<Node>> applicationNodes : allocatedNodes.entrySet()) { + try (Mutex lock = lock(applicationNodes.getKey())) { + for (Node node : applicationNodes.getValue()) + resultingNodes.add(action.apply(node)); + } + } + return resultingNodes; + } + + /** Create a lock which provides exclusive rights to making changes to the given application */ + public Mutex lock(ApplicationId application) { return zkClient.lock(application); } + + /** Create a lock which provides exclusive rights to changing the set of ready nodes */ + public Mutex lockUnallocated() { return zkClient.lockInactive(); } + + /** Acquires the appropriate lock for this node */ + private Mutex lock(Node node) { + return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockUnallocated(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClient.java new file mode 100644 index 00000000000..1ec20f6df0c --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClient.java @@ -0,0 +1,157 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.assimilate; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.config.provision.*; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.Flavor; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.node.Status; +import com.yahoo.vespa.hosted.provision.persistence.NodeSerializer; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +import javax.xml.xpath.XPathConstants; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.time.Clock; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** +* @author vegard +*/ +// TODO Moved here from hosted repo as is, more work to be done before it is usable +public class PopulateClient { + static final Map<String, String> CLUSTER_TYPE_ELEMENT = ImmutableMap.of("container", "jdisc", "content", "content"); + static final String CONTAINER_CLUSTER_TYPE = "container"; + static final String CONTENT_CLUSTER_TYPE = "content"; + + private final String tenantId; + private final String applicationId; + private final String instanceId; + private final Document servicesXml; + private final Map<String, String> flavorSpec; + private final Map<String, String> hostMapping; + private final boolean dryRun; + + private final NodeFlavors nodeFlavors; + + private Clock creationClock = Clock.systemUTC(); + private CuratorDatabaseClient zkClient; + + // TODO NodeFlavors is now based on configuration, so callers need to do some work to get a proper NodeFlavors injected + public PopulateClient(Curator curator, NodeFlavors nodeFlavors, String tenantId, String applicationId, String instanceId, + String servicesXmlFilename, String hostsXmlFilename, Map<String, String> flavorSpec, boolean dryRun) { + this.nodeFlavors = nodeFlavors; + this.tenantId = tenantId; + this.applicationId = applicationId; + this.instanceId = instanceId; + this.servicesXml = XmlUtils.parseXml(servicesXmlFilename); + this.hostMapping = XmlUtils.getHostMapping(XmlUtils.parseXml(hostsXmlFilename)); + this.flavorSpec = flavorSpec; + this.dryRun = dryRun; + this.zkClient = new CuratorDatabaseClient(nodeFlavors, curator, creationClock); + + ensureFlavorIsDefinedForEveryCluster(); + } + + public void populate(String clusterType) { + final List<Node> nodes = getNodesForCluster(clusterType); + + if (dryRun) { + System.out.println("Will populate zookeeper with the following:"); + nodes.stream().forEach(node -> System.out.println(byteArrayToUTF8(new NodeSerializer(nodeFlavors).toJson(node)))); + return; + } + + zkClient.addNodesInState(nodes, Node.State.active); + } + + private Optional<Flavor> getFlavor(String clusterType, String clusterId) { + return nodeFlavors.getFlavor(flavorSpec.get(clusterType + "." + clusterId)); + } + + private boolean hasDefinedFlavor(String clusterType, String clusterId) { + return flavorSpec.containsKey(clusterType + "." + clusterId); + } + + private Node buildNode(String hostname, String clusterType, String clusterId, int nodeIndex) { + return new Node( + hostname, // Id + hostname, // Hostname + Optional.empty(), // parent hostname + new Configuration(getFlavor(clusterType, clusterId).get()), // Flavor + Status.initial(), + Node.State.active, // State = active/allocated + Optional.empty(), // Allocation + History.empty()) // History + + .allocate( + ApplicationId.from( + TenantName.from(tenantId), + ApplicationName.from(applicationId), + InstanceName.from(instanceId)), + ClusterMembership.from( + ClusterSpec.from(ClusterSpec.Type.from(clusterType), + ClusterSpec.Id.from(clusterId)), + nodeIndex), + creationClock.instant()); + } + + private List<Node> getNodesForCluster(String clusterType) { + List<Node> nodes = new ArrayList<>(); + + final String elementName = CLUSTER_TYPE_ELEMENT.get(clusterType); + final NodeList clusterList = (NodeList) XmlUtils.evalXPath(servicesXml, String.format("services/%s", elementName), XPathConstants.NODESET); + for (int i = 0; i < clusterList.getLength(); i++) { + Element cluster = (Element) clusterList.item(i); + String clusterId = XmlUtils.attributeOrDefault(cluster, "id", "default"); + + // An empty cluster id is interpreted as 'default' + final String partialPath = clusterId.equals("default") ? + String.format("services/%s[@id='default' or string-length(@id)=0]", elementName) : + String.format("services/%s[@id='%s']", elementName, clusterId); + + // Get all 'node' elements under 'group' or 'nodes' + final NodeList nodeList = (NodeList) XmlUtils.evalXPath(servicesXml, String.format("%s/group/node | %s/nodes/node", partialPath, partialPath), XPathConstants.NODESET); + + for (int nodeIndex = 0; nodeIndex < nodeList.getLength(); nodeIndex++) { + Element node = (Element) nodeList.item(nodeIndex); + String hostname = hostMapping.get(node.getAttribute("hostalias")); + String indexString = XmlUtils.attributeOrDefault(node, "distribution-key", ""); + + final int index = !indexString.isEmpty() ? Integer.valueOf(indexString) : nodeIndex; + nodes.add(buildNode(hostname, clusterType, clusterId, index)); + } + } + + return nodes; + } + + private void ensureFlavorIsDefinedForEveryCluster() { + CLUSTER_TYPE_ELEMENT.forEach((clusterType, element) -> { + final NodeList clusters = (NodeList) XmlUtils.evalXPath(servicesXml, "/services/" + element, XPathConstants.NODESET); + + for (int i = 0; i < clusters.getLength(); i++) { + String clusterId = XmlUtils.attributeOrDefault((Element) clusters.item(i), "id", "default"); + + if (!hasDefinedFlavor(clusterType, clusterId)) { + throw new RuntimeException(String.format("Flavor is not defined for %s.%s\n", clusterType, clusterId)); + } + } + }); + } + + private static String byteArrayToUTF8(byte[] array) { + return Charset.forName("UTF-8").decode(ByteBuffer.wrap(array)).toString(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/XmlUtils.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/XmlUtils.java new file mode 100644 index 00000000000..931d7dd9878 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/XmlUtils.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.assimilate; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.namespace.QName; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * @author vegard +*/ +class XmlUtils { + static String attributeOrDefault(Element element, String attributeName, String defaultValue) { + String value = element.getAttribute(attributeName); + return !value.isEmpty() ? value : defaultValue; + } + + static Document parseXml(String filename) { + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + return builder.parse(new File(filename)); + } + catch (ParserConfigurationException | SAXException | IOException e) { + throw new RuntimeException(e); + } + } + + static Object evalXPath(Document doc, String xPathExpr, QName returnType) { + try { + XPathFactory xPathfactory = XPathFactory.newInstance(); + XPath xpath = xPathfactory.newXPath(); + XPathExpression expr = xpath.compile(xPathExpr); + return expr.evaluate(doc, returnType); + } + catch (XPathExpressionException e) { + throw new RuntimeException(e); + } + } + + /** + * Reads hosts.xml into a map (alias to hostname) + */ + static Map<String, String> getHostMapping(Document hostsXml) { + Map<String, String> hostMapping = new HashMap<>(); + + NodeList hostList = hostsXml.getElementsByTagName("host"); + for (int i = 0; i < hostList.getLength(); i++) { + Element host = (Element) hostList.item(i); + hostMapping.put(host.getElementsByTagName("alias").item(0).getTextContent(), host.getAttribute("name")); + } + + return hostMapping; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java new file mode 100644 index 00000000000..fd5e229fa1b --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Deployer; +import com.yahoo.config.provision.Deployment; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.yolean.Exceptions; + +import java.time.Duration; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; +import java.util.stream.Collectors; + +/** + * The application maintainer regularly redeploys all applications. + * This is necessary because applications may gain and lose active nodes due to nodes being moved to and from the + * failed state. This is corrected by redeploying the applications periodically. + * It can not (at this point) be done reliably synchronously as part of the fail/unfail call due to the need for this + * to happen at a node having the deployer. + * + * @author bratseth + */ +public class ApplicationMaintainer extends Maintainer { + + private final Deployer deployer; + + public ApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration rate) { + super(nodeRepository, rate); + this.deployer = deployer; + } + + @Override + protected void maintain() { + Set<ApplicationId> applications = + nodeRepository().getNodes(Node.State.active).stream().map(node -> node.allocation().get().owner()).collect(Collectors.toSet()); + + for (ApplicationId application : applications) { + try { + Optional<Deployment> deployment = deployer.deployFromLocalActive(application, Duration.ofMinutes(30)); + if ( ! deployment.isPresent()) continue; // this will be done at another config server + + deployment.get().prepare(); + deployment.get().activate(); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Exception on maintenance redeploy of " + application, e); + } + } + } + + @Override + public String toString() { return "Periodic application redeployer"; } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java new file mode 100644 index 00000000000..4d106098ab5 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.History; + +import java.time.Clock; +import java.time.Duration; +import java.util.List; +import java.util.stream.Collectors; + +/** + * This moves nodes from dirty to failed if they have been in dirty too long + * with the assumption that a node is stuck in dirty because it has failed. + * <p> + * As the nodes are moved back to dirty their failure count is increased, + * and if the count is sufficiently low they will be attempted recycled to dirty again. + * The upshot is nodes may get multiple attempts at clearing through dirty, but they will + * eventually stay in failed. + * + * @author bratseth + */ +public class DirtyExpirer extends Expirer { + + private final NodeRepository nodeRepository; + + public DirtyExpirer(NodeRepository nodeRepository, Clock clock, Duration dirtyTimeout) { + super(Node.State.dirty, History.Event.Type.deallocated, nodeRepository, clock, dirtyTimeout); + this.nodeRepository = nodeRepository; + } + + @Override + protected void expire(List<Node> expired) { + for (Node expiredNode : expired.stream().collect(Collectors.toList())) + nodeRepository.fail(expiredNode.hostname()); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java new file mode 100644 index 00000000000..8333996c23e --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.History; + +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Superclass of expiry tasks which moves nodes from some state to the dirty state. + * These jobs runs at least every 25 minutes. + * + * @author bratseth + */ +public abstract class Expirer extends Maintainer { + + protected static final Logger log = Logger.getLogger(Expirer.class.getName()); + + /** The state to expire from */ + private final Node.State fromState; + + /** The event record type which contains the timestamp to use for expiry */ + private final History.Event.Type eventType; + + private final Clock clock; + + private final Duration expiryTime; + + public Expirer(Node.State fromState, History.Event.Type eventType, NodeRepository nodeRepository, Clock clock, Duration expiryTime) { + super(nodeRepository, min(Duration.ofMinutes(25), expiryTime)); + this.fromState = fromState; + this.eventType = eventType; + this.clock = clock; + this.expiryTime = expiryTime; + } + + private static Duration min(Duration a, Duration b) { + return a.toMillis() < b.toMillis() ? a : b; + } + + @Override + protected void maintain() { + List<Node> expired = new ArrayList<>(); + for (Node node : nodeRepository().getNodes(fromState)) { + Optional<History.Event> event = node.history().event(eventType); + if (event.isPresent() && event.get().at().plus(expiryTime).isBefore(clock.instant())) + expired.add(node); + } + if ( ! expired.isEmpty()) + log.info(fromState + " expirer found " + expired.size() + " expired nodes"); + expire(expired); + } + + @Override + public String toString() { return "Expiry from " + fromState; } + + /** Implement this callback to take action to expire these nodes */ + protected abstract void expire(List<Node> node); + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java new file mode 100644 index 00000000000..ef9c65c8a01 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.History; + +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * This moves nodes from failed back to dirty if + * <ul> + * <li>No hardware failure is known to be detected on the node + * <li>The node has failed less than 5 times OR the environment is dev, test or perf, + * as those environments have no protection against users running bogus applications, so + * we cannot use the node failure count to conclude the node has a failure. + * </ul> + * Failed nodes are typically given a long expiry time to enable us to manually moved them back to + * active to recover data in cases where the node was failed accidentally. + * <p> + * The purpose of the automatic recycling to dirty + fail count is that nodes which were moved + * to failed due to some undetected hardware failure will end up being failed again. + * When that has happened enough they will not be recycled. + * <p> + * The Chef recipe running locally on the node may set the hardwareFailure flag to avoid the node + * being automatically recycled in cases where an error has been positively detected. + * + * @author bratseth + */ +public class FailedExpirer extends Expirer { + + private final NodeRepository nodeRepository; + private final Zone zone; + + public FailedExpirer(NodeRepository nodeRepository, Zone zone, Clock clock, Duration failTimeout) { + super(Node.State.failed, History.Event.Type.failed, nodeRepository, clock, failTimeout); + this.nodeRepository = nodeRepository; + this.zone = zone; + } + + @Override + protected void expire(List<Node> expired) { + List<Node> nodesToRecycle = new ArrayList<>(); + for (Node recycleCandidate : expired) { + if (recycleCandidate.status().hardwareFailure()) continue; + if (failCountIndicatesHwFail(zone) && recycleCandidate.status().failCount() >= 5) continue; + nodesToRecycle.add(recycleCandidate); + } + nodeRepository.deallocate(nodesToRecycle); + } + + private boolean failCountIndicatesHwFail(Zone zone) { + return zone.environment() == Environment.prod || zone.environment() == Environment.staging; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java new file mode 100644 index 00000000000..652a3783d4b --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.History; + +import java.time.Clock; +import java.time.Duration; +import java.util.List; + +/** + * Maintenance job which moves inactive nodes to dirty after timeout. + * The timeout is in place for two reasons: + * <ul> + * <li>To ensure that the new application configuration has time to + * propagate before the node is used for something else + * <li>To provide a grace period in which nodes can be brought back to active + * if they were deactivated in error. As inactive nodes retain their state + * they can be brought back to active and correct state faster than a new node. + * </ul> + * + * @author bratseth + */ +public class InactiveExpirer extends Expirer { + + private final NodeRepository nodeRepository; + + public InactiveExpirer(NodeRepository nodeRepository, Clock clock, Duration inactiveTimeout) { + super(Node.State.inactive, History.Event.Type.deactivated, nodeRepository, clock, inactiveTimeout); + this.nodeRepository = nodeRepository; + } + + @Override + protected void expire(List<Node> expired) { + nodeRepository.deallocate(expired); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java new file mode 100644 index 00000000000..59cd8f5f85c --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.yolean.Exceptions; + +import java.time.Duration; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A maintainer is some job which runs at a fixed rate to perform some maintenance task on the node repo. + * + * @author bratseth + */ +public abstract class Maintainer extends AbstractComponent implements Runnable { + + protected static final Logger log = Logger.getLogger(Maintainer.class.getName()); + + private final NodeRepository nodeRepository; + private final Duration rate; + + private final ScheduledExecutorService service; + + public Maintainer(NodeRepository nodeRepository, Duration rate) { + this.nodeRepository = nodeRepository; + this.rate = rate; + + this.service = new ScheduledThreadPoolExecutor(1); + this.service.scheduleAtFixedRate(this, rate.toMillis(), rate.toMillis(), TimeUnit.MILLISECONDS); + } + + /** Returns the node repository */ + protected NodeRepository nodeRepository() { return nodeRepository; } + + /** Returns the rate at which this job is set to run */ + protected Duration rate() { return rate; } + + @Override + public void run() { + try { + maintain(); + } + catch (RuntimeException e) { + log.log(Level.WARNING, this + " failed. Will retry in " + rate.toMinutes() + " minutes", e); + } + } + + @Override + public void deconstruct() { + this.service.shutdown(); + } + + /** Returns a textual description of this job */ + @Override + public abstract String toString(); + + /** Called once each time this maintenance job should run */ + protected abstract void maintain(); + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java new file mode 100644 index 00000000000..c18aacea284 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -0,0 +1,160 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.Deployer; +import com.yahoo.config.provision.Deployment; +import com.yahoo.log.LogLevel; +import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException; +import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.service.monitor.ServiceMonitor; +import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Maintains information in the node repo about when this node last responded to ping + * and fails nodes which have not responded within the given time limit. + * + * @author bratseth + */ +public class NodeFailer extends Maintainer { + + private static final Logger log = Logger.getLogger(NodeFailer.class.getName()); + + private final Deployer deployer; + private final ServiceMonitor serviceMonitor; + private final Duration downTimeLimit; + private final Clock clock; + private final Orchestrator orchestrator; + + public NodeFailer(Deployer deployer, ServiceMonitor serviceMonitor, NodeRepository nodeRepository, + Duration downTimeLimit, Clock clock, Orchestrator orchestrator) { + // check ping status every five minutes, but at least twice as often as the down time limit + super(nodeRepository, min(downTimeLimit.dividedBy(2), Duration.ofMinutes(5))); + this.deployer = deployer; + this.serviceMonitor = serviceMonitor; + this.downTimeLimit = downTimeLimit; + this.clock = clock; + this.orchestrator = orchestrator; + } + + private static Duration min(Duration d1, Duration d2) { + return d1.toMillis() < d2.toMillis() ? d1 : d2; + } + + @Override + protected void maintain() { + List<Node> downNodes = maintainDownStatus(); + + for (Node node : downNodes) { + // Grace time before failing the node + Instant graceTimeEnd = node.history().event(History.Event.Type.down).get().at().plus(downTimeLimit); + + if (graceTimeEnd.isBefore(clock.instant()) && ! applicationSuspended(node)) + fail(node); + } + } + + private boolean applicationSuspended(Node node) { + try { + return orchestrator.getApplicationInstanceStatus(node.allocation().get().owner()) + == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN; + } catch (ApplicationIdNotFoundException e) { + //Treat it as not suspended and allow to fail the node anyway + return false; + } + } + + /** + * If the node is positively DOWN, and there is no "down" history record, we add it. + * If the node is positively UP we remove any "down" history record. + * + * @return a list of all nodes which are positively currently in the down state + */ + private List<Node> maintainDownStatus() { + List<Node> downNodes = new ArrayList<>(); + for (ApplicationInstance<ServiceMonitorStatus> application : serviceMonitor.queryStatusOfAllApplicationInstances().values()) { + for (ServiceCluster<ServiceMonitorStatus> cluster : application.serviceClusters()) { + for (ServiceInstance<ServiceMonitorStatus> service : cluster.serviceInstances()) { + Optional<Node> node = nodeRepository().getNode(Node.State.active, service.hostName().s()); + if ( ! node.isPresent()) continue; // we also get status from infrastructure nodes, which are not in the repo + + if (service.serviceStatus().equals(ServiceMonitorStatus.DOWN)) + downNodes.add(recordAsDown(node.get())); + else if (service.serviceStatus().equals(ServiceMonitorStatus.UP)) + clearDownRecord(node.get()); + // else: we don't know current status; don't take any action until we have positive information + } + } + } + return downNodes; + } + + /** + * Record a node as down if not already recorded and returns the node in the new state. + * This assumes the node is found in the node + * repo and that the node is allocated. If we get here otherwise something is truly odd. + */ + private Node recordAsDown(Node node) { + if (node.history().event(History.Event.Type.down).isPresent()) return node; // already down: Don't change down timestamp + + try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) { + node = nodeRepository().getNode(Node.State.active, node.hostname()).get(); // re-get inside lock + return nodeRepository().write(node.setDown(clock.instant())); + } + } + + private void clearDownRecord(Node node) { + if ( ! node.history().event(History.Event.Type.down).isPresent()) return; + + try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) { + node = nodeRepository().getNode(Node.State.active, node.hostname()).get(); // re-get inside lock + nodeRepository().write(node.setUp()); + } + } + + /** + * Called when a node should be moved to the failed state: Do that if it seems safe, + * which is when the node repo has available capacity to replace the node. + * Otherwise not replacing the node ensures (by Orchestrator check) that no further action will be taken. + */ + private void fail(Node node) { + Optional<Deployment> deployment = + deployer.deployFromLocalActive(node.allocation().get().owner(), Duration.ofMinutes(30)); + if ( ! deployment.isPresent()) return; // this will be done at another config server + + try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) { + node = nodeRepository().fail(node.hostname()); + try { + deployment.get().prepare(); + deployment.get().activate(); + } + catch (RuntimeException e) { + // The expected reason for deployment to fail here is that there is no capacity available to redeploy. + // In that case we should leave the node in the active state to avoid failing additional nodes. + nodeRepository().unfail(node.hostname()); + log.log(Level.WARNING, "Attempted to fail " + node + " for " + node.allocation().get().owner() + + ", but redeploying without the node failed", e); + } + } + } + + @Override + public String toString() { return "Node failer"; } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java new file mode 100644 index 00000000000..c23a0fa5f56 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.Deployer; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.vespa.service.monitor.ServiceMonitor; + +import java.time.Clock; +import java.time.Duration; +import java.util.Optional; + +/** + * A component which sets up all the node repo maintenance jobs. + * + * @author bratseth + */ +public class NodeRepositoryMaintenance extends AbstractComponent { + + private final NodeFailer nodeFailer; + private final ApplicationMaintainer applicationMaintainer; + private final ReservationExpirer reservationExpirer; + private final InactiveExpirer inactiveExpirer; + private final RetiredExpirer retiredExpirer; + private final FailedExpirer failedExpirer; + private final DirtyExpirer dirtyExpirer; + + @Inject + public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, ServiceMonitor serviceMonitor, Zone zone, Orchestrator orchestrator) { + this(nodeRepository, deployer, serviceMonitor, zone, Clock.systemUTC(), orchestrator); + } + + public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, ServiceMonitor serviceMonitor, Zone zone, Clock clock, Orchestrator orchestrator) { + DefaultTimes defaults = new DefaultTimes(zone.environment()); + nodeFailer = new NodeFailer(deployer, serviceMonitor, nodeRepository, fromEnv("fail_grace").orElse(defaults.failGrace), clock, orchestrator); + applicationMaintainer = new ApplicationMaintainer(deployer, nodeRepository, fromEnv("redeploy_frequency").orElse(defaults.redeployFrequency)); + reservationExpirer = new ReservationExpirer(nodeRepository, clock, fromEnv("reservation_expiry").orElse(defaults.reservationExpiry)); + retiredExpirer = new RetiredExpirer(nodeRepository, deployer, clock, fromEnv("retired_expiry").orElse(defaults.retiredExpiry)); + inactiveExpirer = new InactiveExpirer(nodeRepository, clock, fromEnv("inactive_expiry").orElse(defaults.inactiveExpiry)); + failedExpirer = new FailedExpirer(nodeRepository, zone, clock, fromEnv("failed_expiry").orElse(defaults.failedExpiry)); + dirtyExpirer = new DirtyExpirer(nodeRepository, clock, fromEnv("dirty_expiry").orElse(defaults.dirtyExpiry)); + } + + private Optional<Duration> fromEnv(String envVariable) { + String prefix = "vespa_node_repository__"; + return Optional.ofNullable(System.getenv(prefix + envVariable)).map(Long::parseLong).map(Duration::ofSeconds); + } + + @Override + public void deconstruct() { + nodeFailer.deconstruct(); + applicationMaintainer.deconstruct(); + reservationExpirer.deconstruct(); + inactiveExpirer.deconstruct(); + retiredExpirer.deconstruct(); + failedExpirer.deconstruct(); + dirtyExpirer.deconstruct(); + } + + private static class DefaultTimes { + + /** All applications are redeployed with this frequency */ + private final Duration redeployFrequency; + + /** The time a node must be continuously nonresponsive before it is failed */ + private final Duration failGrace; + + private final Duration reservationExpiry; + private final Duration inactiveExpiry; + private final Duration retiredExpiry; + private final Duration failedExpiry; + private final Duration dirtyExpiry; + + DefaultTimes(Environment environment) { + if (environment.equals(Environment.prod)) { + // These values are to avoid losing data (retired), and to be able to return an application + // back to a previous state fast (inactive) + redeployFrequency = Duration.ofMinutes(30); + failGrace = Duration.ofMinutes(60); + reservationExpiry = Duration.ofMinutes(15); + inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy + retiredExpiry = Duration.ofDays(4); // enough time to migrate data + failedExpiry = Duration.ofDays(4); // enough time to recover data even if it happens friday night + dirtyExpiry = Duration.ofHours(2); // enough time to clean the node + } else { + redeployFrequency = Duration.ofMinutes(30); + failGrace = Duration.ofMinutes(60); + // These values ensure tests and development is not delayed due to nodes staying around + // Use non-null values as these also determine the maintenance interval + reservationExpiry = Duration.ofMinutes(10); // Need to be long enough for deployment to be finished for all config model versions + inactiveExpiry = Duration.ofMinutes(1); + retiredExpiry = Duration.ofMinutes(1); + failedExpiry = Duration.ofMinutes(10); + dirtyExpiry = Duration.ofMinutes(30); + } + } + + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java new file mode 100644 index 00000000000..f45f8ebd086 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.History; + +import java.time.Duration; +import java.time.Clock; +import java.util.List; + +/** + * Maintenance job which moves reserved nodes to dirty after timeout. + * Nodes need to time out in case someone reserves nodes (by calling prepare) but never commits. + * reserved nodes may in some cases come from the inactive state, in which case they are dirty. + * For this reason, all reserved nodes go through the dirty state before going back to ready. + * + * @author bratseth + * @version $Id$ + */ +public class ReservationExpirer extends Expirer { + + private final NodeRepository nodeRepository; + + public ReservationExpirer(NodeRepository nodeRepository, Clock clock, Duration reservationPeriod) { + super(Node.State.reserved, History.Event.Type.reserved, nodeRepository, clock, reservationPeriod); + this.nodeRepository = nodeRepository; + } + + @Override + protected void expire(List<Node> expired) { nodeRepository.deallocate(expired); } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java new file mode 100644 index 00000000000..392d72f9f8e --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.collections.ListMap; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Deployer; +import com.yahoo.config.provision.Deployment; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.History; + +import java.time.Clock; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; + +/** + * Maintenance job which deactivates nodes which has been retired. + * This should take place after the system has been given sufficient time to migrate data to other nodes. + * <p> + * As these nodes are active, and therefore part of the configuration the impacted applications must be + * reconfigured after inactivation. + * + * @author bratseth + * @version $Id$ + */ +public class RetiredExpirer extends Expirer { + + private final NodeRepository nodeRepository; + private final Deployer deployer; + + public RetiredExpirer(NodeRepository nodeRepository, Deployer deployer, Clock clock, Duration retiredDuration) { + super(Node.State.active, History.Event.Type.retired, nodeRepository, clock, retiredDuration); + this.nodeRepository = nodeRepository; + this.deployer = deployer; + } + + @Override + protected void expire(List<Node> expired) { + // Only expire nodes which are retired. Do one application at the time. + ListMap<ApplicationId, Node> applicationNodes = new ListMap<>(); + for (Node node : expired) { + if (node.allocation().isPresent() && node.allocation().get().membership().retired()) + applicationNodes.put(node.allocation().get().owner(), node); + } + + for (Map.Entry<ApplicationId, List<Node>> entry : applicationNodes.entrySet()) { + ApplicationId application = entry.getKey(); + List<Node> nodesToRemove = entry.getValue(); + try { + Optional<Deployment> deployment = deployer.deployFromLocalActive(application, Duration.ofMinutes(30)); + if ( ! deployment.isPresent()) continue; // this will be done at another config server + + nodeRepository.setRemovable(application, nodesToRemove); + + deployment.get().prepare(); + deployment.get().activate(); + + log.info("Redeployed " + application + " to deactivate " + nodesToRemove.size() + " retired nodes"); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Exception trying to remove previously retired nodes " + nodesToRemove + + "from " + application, e); + } + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetrics.java new file mode 100644 index 00000000000..2753f435bde --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetrics.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.monitoring; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.jdisc.Metric; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * @author oyving + */ +public class ProvisionMetrics extends AbstractComponent { + + private static final Logger log = Logger.getLogger(ProvisionMetrics.class.getName()); + private final ScheduledExecutorService executorService; + + // TODO: make report interval configurable + public ProvisionMetrics(Metric metric, NodeRepository nodeRepository) { + this.executorService = new ScheduledThreadPoolExecutor(1); + this.executorService.scheduleAtFixedRate( + new ProvisionMetricsTask(metric, nodeRepository), + 0, // start immediately + 1, // report every minute + TimeUnit.MINUTES + ); + } + + @Override + public void deconstruct() { + this.executorService.shutdown(); + } + + private static class ProvisionMetricsTask implements Runnable { + private final Metric metric; + private final NodeRepository nodeRepository; + + private ProvisionMetricsTask(Metric metric, NodeRepository nodeRepository) { + this.metric = metric; + this.nodeRepository = nodeRepository; + } + + @Override + public void run() { + log.log(LogLevel.DEBUG, "Running provision metrics task"); + try { + for (Node.State state : Node.State.values()) + metric.set("hostedVespa." + state.name() + "Hosts", nodeRepository.getNodes(state).size(), null); + } catch (RuntimeException e) { + log.log(LogLevel.INFO, "Failed gathering metrics data: " + e.getMessage()); + } + } + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java new file mode 100644 index 00000000000..8a15c04a7df --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.vespa.hosted.provision.Node; + +/** + * The allocation of a node + * + * @author bratseth + */ +public class Allocation { + + private final ApplicationId owner; + private final ClusterMembership clusterMembership; + + /** + * Restart generation, see {@link com.yahoo.vespa.hosted.provision.node.Generation}, + * wanted is increased when a restart of the services on the node is needed, current is updated + * when a restart has been done on the node. + */ + private final Generation restartGeneration; + + /** This node can (and should) be removed from the cluster on the next deployment */ + private final boolean removable; + + public Allocation(ApplicationId owner, ClusterMembership clusterMembership, + Generation restartGeneration, boolean removable) { + this.owner = owner; + this.clusterMembership = clusterMembership; + this.restartGeneration = restartGeneration; + this.removable = removable; + } + + /** Returns the id of the application this is allocated to */ + public ApplicationId owner() { return owner; } + + /** Returns the role this node is allocated to */ + public ClusterMembership membership() { return clusterMembership; } + + /** Returns the restart generation (wanted and current) of this */ + public Generation restartGeneration() { return restartGeneration; } + + /** Returns a copy of this which is retired */ + public Allocation retire() { return new Allocation(owner, clusterMembership.retire(), restartGeneration, removable); } + + /** Returns a copy of this which is not retired */ + public Allocation unretire() { return new Allocation(owner, clusterMembership.unretire(), restartGeneration, removable); } + + /** Return whether this node is ready to be removed from the application */ + public boolean removable() { return removable; } + + /** Returns a copy of this with the current restart generation set to generation */ + public Allocation setRestart(Generation generation) { + return new Allocation(owner, clusterMembership, generation, removable); + } + + /** Returns a copy of this allocation where removable is set to true */ + public Allocation makeRemovable() { + return new Allocation(owner, clusterMembership, restartGeneration, true); + } + + public Allocation changeMembership(ClusterMembership newMembership) { + return new Allocation(owner, newMembership, restartGeneration, removable); + } + + @Override + public String toString() { return "allocated to " + owner + " as '" + clusterMembership + "'"; } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Configuration.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Configuration.java new file mode 100644 index 00000000000..9647c3f938b --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Configuration.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node; + +import java.util.Objects; + +/** + * The hardware configuration of a node + * + * @author bratseth + */ +public class Configuration { + + private final Flavor flavor; + + public Configuration(Flavor flavor) { + Objects.requireNonNull(flavor, "A node configuration must have a flavor"); + this.flavor = flavor; + } + + /** Returns the name of this hardware configuration */ + public Flavor flavor() { return flavor; } + + /** Returns a configuration with the flavor set to the given value */ + public Configuration setFlavor(Flavor flavor) { return new Configuration(flavor); } + + @Override + public String toString() { + return flavor.toString(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Flavor.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Flavor.java new file mode 100644 index 00000000000..bbb9a95a155 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Flavor.java @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node; + +import com.google.common.collect.ImmutableList; +import com.yahoo.vespa.config.nodes.NodeRepositoryConfig; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * A host flavor (type). This is a value object where the identity is the name. + * Use {@link NodeFlavors} to create a flavor. + * + * @author bratseth + */ +public class Flavor { + + private final String name; + private final int cost; + private final String environment; + private final double minCpuCores; + private final double minMainMemoryAvailableGb; + private final double minDiskAvailableGb; + private final String description; + private List<Flavor> replacesFlavors; + + /** + * Creates a Flavor, but does not set the replacesFlavors. + * @param flavorConfig config to be used for Flavor. + */ + public Flavor(NodeRepositoryConfig.Flavor flavorConfig) { + this.name = flavorConfig.name(); + this.replacesFlavors = new ArrayList<>(); + this.cost = flavorConfig.cost(); + this.environment = flavorConfig.environment(); + this.minCpuCores = flavorConfig.minCpuCores(); + this.minMainMemoryAvailableGb = flavorConfig.minMainMemoryAvailableGb(); + this.minDiskAvailableGb = flavorConfig.minDiskAvailableGb(); + this.description = flavorConfig.description(); + } + + public String name() { return name; } + + /** + * Get the monthly cost (total cost of ownership) in USD for this flavor, typically total cost + * divided by 36 months. + * @return Monthly cost in USD + */ + public int cost() { return cost; } + + public double getMinMainMemoryAvailableGb() { + return minMainMemoryAvailableGb; + } + + public double getMinDiskAvailableGb() { + return minDiskAvailableGb; + } + + public double getMinCpuCores() { + return minCpuCores; + } + + public String getDescription() { + return description; + } + + public String getEnvironment() { + return environment; + } + + /** + * Returns the canonical name of this flavor - which is the name which should be used as an interface to users. + * The canonical name of this flavor is + * <ul> + * <li>If it replaces one flavor, the canonical name of the flavor it replaces + * <li>If it replaces multiple or no flavors - itself + * </ul> + * + * The logic is that we can use this to capture the gritty details of configurations in exact flavor names + * but also encourage users to refer to them by a common name by letting such flavor variants declare that they + * replace the caninical name we want. However, if a node replaces multiple names, it means that a former + * flavor distinction has become obsolete so this name becomes one of the canonical names users should refer to. + */ + public String canonicalName() { + return replacesFlavors.size() == 1 ? replacesFlavors.get(0).canonicalName() : name; + } + + /** + * The flavors this (directly) replaces. + * This is immutable if this is frozen, and a mutable list otherwise. + */ + public List<Flavor> replaces() { return replacesFlavors; } + + /** + * Returns whether this flavor satisfies the requested flavor, either directly + * (by being the same), or by directly or indirectly replacing it + */ + public boolean satisfies(Flavor flavor) { + if (this.equals(flavor)) return true; + for (Flavor replaces : replacesFlavors) + if (replaces.satisfies(flavor)) + return true; + return false; + } + + /** Irreversibly freezes the content of this */ + public void freeze() { + replacesFlavors = ImmutableList.copyOf(replacesFlavors); + } + + @Override + public int hashCode() { return name.hashCode(); } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof Flavor)) return false; + return ((Flavor)other).name.equals(this.name); + } + + @Override + public String toString() { return "flavor '" + name + "'"; } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Generation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Generation.java new file mode 100644 index 00000000000..b1be9da62fe --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Generation.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node; + +import javax.annotation.concurrent.Immutable; + +/** + * An immutable generation, with wanted and current generation fields. Wanted generation + * is increased when an action (restart services or reboot are the available + * actions) is wanted, current is updated when the action has been done on the node. + * + * @author musum + */ +@Immutable +public class Generation { + + private final long wanted; + private final long current; + + public Generation(long wanted, long current) { + this.wanted = wanted; + this.current = current; + } + + public long wanted() { + return wanted; + } + + public long current() { + return current; + } + + public Generation increaseWanted() { + return new Generation(wanted + 1, current); + } + + public Generation setCurrent(long newValue) { + return new Generation(wanted, newValue); + } + + @Override + public String toString() { + return "current generation: " + current + ", wanted: " + wanted; + } + + /** Returns the initial generation (0, 0) */ + public static Generation inital() { return new Generation(0, 0); } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java new file mode 100644 index 00000000000..42134d082b7 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.vespa.hosted.provision.Node; + +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * An immutable record of the last event of each type happening to this node. + * Note that the history cannot be used to find the nodes current state - it will have a record of some + * event happening in the past even if that event is later undone. + * + * @author bratseth + */ +public class History { + + private final ImmutableMap<Event.Type, Event> events; + + public History(Collection<Event> events) { + this(toImmutableMap(events)); + } + + private History(ImmutableMap<Event.Type, Event> events) { + this.events = events; + } + + private static ImmutableMap<Event.Type, Event> toImmutableMap(Collection<Event> events) { + ImmutableMap.Builder<Event.Type, Event> builder = new ImmutableMap.Builder<>(); + for (Event event : events) + builder.put(event.type(), event); + return builder.build(); + } + + /** Returns this event if it is present in this history */ + public Optional<Event> event(Event.Type type) { return Optional.ofNullable(events.get(type)); } + + public Collection<Event> events() { return events.values(); } + + /** Returns a copy of this history with the given event added */ + public History record(Event event) { + ImmutableMap.Builder<Event.Type, Event> builder = builderWithout(event.type()); + builder.put(event.type(), event); + return new History(builder.build()); + } + + /** Returns a copy of this history with the given event type removed (or an identical if it was not present) */ + public History clear(Event.Type type) { + return new History(builderWithout(type).build()); + } + + private ImmutableMap.Builder<Event.Type, Event> builderWithout(Event.Type type) { + ImmutableMap.Builder<Event.Type, Event> builder = new ImmutableMap.Builder<>(); + for (Event event : events.values()) + if (event.type() != type) + builder.put(event.type(), event); + return builder; + } + + /** Returns a copy of this history with a record of this state transition added, if applicable */ + public History recordStateTransition(Node.State from, Node.State to, Instant at) { + if (from == to) return this; + switch (to) { + case ready: return record(new Event(Event.Type.readied, at)); + case active: return record(new Event(Event.Type.activated, at)); + case inactive: return record(new Event(Event.Type.deactivated, at)); + case reserved: return record(new Event(Event.Type.reserved, at)); + case failed: return record(new Event(Event.Type.failed, at)); + case dirty: return record(new Event(Event.Type.deallocated, at)); + default: return this; + } + } + + /** Returns the empty history */ + public static History empty() { return new History(Collections.emptyList()); } + + /** An event which may happen to a node */ + public static class Event { + + private final Instant at; + private final Event.Type type; + + public Event(Event.Type type, Instant at) { + this.type = type; + this.at = at; + } + + /** Returns the type of event */ + public Event.Type type() { return type; } + + /** Returns the instant this even took place */ + public Instant at() { return at; } + + public enum Type { readied, reserved, activated, retired, deactivated, failed, deallocated, down } + + @Override + public String toString() { return type + " event at " + at; } + + } + + /** A retired event includes additional information about the causing agent. */ + public static class RetiredEvent extends Event { + + private final RetiredEvent.Agent agent; + + public RetiredEvent(Instant at, RetiredEvent.Agent agent) { + super(Type.retired, at); + this.agent = agent; + } + + /** Returns the agent which caused retirement */ + public RetiredEvent.Agent agent() { return agent; } + + public enum Agent { system, application } + + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeFlavors.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeFlavors.java new file mode 100644 index 00000000000..9a07be3f86c --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeFlavors.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; +import com.yahoo.vespa.config.nodes.NodeRepositoryConfig; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * All the available node flavors. + * + * @author bratseth + */ +public class NodeFlavors { + + /** Flavors <b>which are configured</b> in this zone */ + private final ImmutableMap<String, Flavor> flavors; + + @Inject + public NodeFlavors(NodeRepositoryConfig config) { + ImmutableMap.Builder<String, Flavor> b = new ImmutableMap.Builder<>(); + for (Flavor flavor : toFlavors(config)) + b.put(flavor.name(), flavor); + this.flavors = b.build(); + } + + /** Returns a flavor by name, or empty if there is no flavor with this name. */ + public Optional<Flavor> getFlavor(String name) { + return Optional.ofNullable(flavors.get(name)); + } + + /** Returns the flavor with the given name or throws an IllegalArgumentException if it does not exist */ + public Flavor getFlavorOrThrow(String flavorName) { + Optional<Flavor> flavor = getFlavor(flavorName); + if ( flavor.isPresent()) return flavor.get(); + throw new IllegalArgumentException("Unknown flavor '" + flavorName + "' Flavors are " + canonicalFlavorNames()); + } + + private List<String> canonicalFlavorNames() { + return flavors.values().stream().map(Flavor::canonicalName).distinct().sorted().collect(Collectors.toList()); + } + + private static Collection<Flavor> toFlavors(NodeRepositoryConfig config) { + Map<String, Flavor> flavors = new HashMap<>(); + // First pass, create all flavors, but do not include flavorReplacesConfig. + for (NodeRepositoryConfig.Flavor flavorConfig : config.flavor()) { + flavors.put(flavorConfig.name(), new Flavor(flavorConfig)); + } + // Second pass, set flavorReplacesConfig to point to correct flavor. + for (NodeRepositoryConfig.Flavor flavorConfig : config.flavor()) { + Flavor flavor = flavors.get(flavorConfig.name()); + for (NodeRepositoryConfig.Flavor.Replaces flavorReplacesConfig : flavorConfig.replaces()) { + if (! flavors.containsKey(flavorReplacesConfig.name())) { + throw new IllegalStateException("Replaces for " + + flavor.name() + " pointing to a non existing flavor: " + flavorReplacesConfig.name()); + } + flavor.replaces().add(flavors.get(flavorReplacesConfig.name())); + } + flavor.freeze(); + } + return flavors.values(); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java new file mode 100644 index 00000000000..a26f02a53dc --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node; + +import com.yahoo.component.Version; + +import javax.annotation.concurrent.Immutable; +import java.util.Optional; + +/** + * Information about current status of a node + * + * @author bratseth + */ +@Immutable +public class Status { + + private final Generation reboot; + private final Optional<Version> vespaVersion; + private final Optional<Version> hostedVersion; + private final Optional<String> stateVersion; + private final Optional<String> dockerImage; + private final int failCount; + private final boolean hardwareFailure; + + public Status(Generation generation, + Optional<Version> vespaVersion, + Optional<Version> hostedVersion, + Optional<String> stateVersion, + Optional<String> dockerImage, + int failCount, + boolean hardwareFailure) { + this.reboot = generation; + this.vespaVersion = vespaVersion; + this.hostedVersion = hostedVersion; + this.stateVersion = stateVersion; + this.dockerImage = dockerImage; + this.failCount = failCount; + this.hardwareFailure = hardwareFailure; + } + + /** Returns a copy of this with the reboot generation changed */ + public Status setReboot(Generation reboot) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, dockerImage, failCount, hardwareFailure); } + + /** Returns the reboot generation of this node */ + public Generation reboot() { return reboot; } + + /** Returns a copy of this with the vespa version changed */ + public Status setVespaVersion(Version version) { return new Status(reboot, Optional.of(version), hostedVersion, stateVersion, dockerImage, failCount, hardwareFailure); } + + /** Returns the Vespa version installed on the node, if known */ + public Optional<Version> vespaVersion() { return vespaVersion; } + + /** Returns a copy of this with the hosted version changed */ + public Status setHostedVersion(Version version) { return new Status(reboot, vespaVersion, Optional.of(version), stateVersion, dockerImage, failCount, hardwareFailure); } + + /** Returns the hosted version installed on the node, if known */ + public Optional<Version> hostedVersion() { return hostedVersion; } + + /** Returns a copy of this with the state version changed */ + public Status setStateVersion(String version) { return new Status(reboot, vespaVersion, hostedVersion, Optional.of(version), dockerImage, failCount, hardwareFailure); } + + /** + * Returns the state version the node last successfully converged with. + * The state version contains the version-specific parts in identifying state + * files on dist, and is of the form VESPAVERSION-HOSTEDVERSION in CI, or otherwise HOSTEDVERSION. + * It's also used to uniquely identify a hosted Vespa release. + */ + public Optional<String> stateVersion() { return stateVersion; } + + /** Returns a copy of this with the docker image changed */ + public Status setDockerImage(String dockerImage) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, Optional.of(dockerImage), failCount, hardwareFailure); } + + /** Returns the current docker image the node is running, if known. */ + public Optional<String> dockerImage() { return dockerImage; } + + public Status increaseFailCount() { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, dockerImage, failCount+1, hardwareFailure); } + + public Status decreaseFailCount() { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, dockerImage, failCount-1, hardwareFailure); } + + public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, dockerImage, value, hardwareFailure); } + + /** Returns how many times this node has been moved to the failed state. */ + public int failCount() { return failCount; } + + /** Returns whether a hardware failure has been detected on this node */ + public boolean hardwareFailure() { return hardwareFailure; } + + public Status setHardwareFailure(boolean hardwareFailure) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, dockerImage, failCount, hardwareFailure); } + + /** Returns the initial status of a newly provisioned node */ + public static Status initial() { return new Status(Generation.inital(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), 0, false); } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java new file mode 100644 index 00000000000..b728b862686 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node.filter; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.provision.Node; + +import java.util.Collections; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A node filter which matches a set of applications. + * + * @author bratseth + */ +public class ApplicationFilter extends NodeFilter { + + private final Set<ApplicationId> applications; + + /** Creates a node filter which filters using the given host filter */ + private ApplicationFilter(Set<ApplicationId> applications, NodeFilter next) { + super(next); + Objects.requireNonNull(applications, "Applications set cannot be null, use an empty set"); + this.applications = applications; + } + + @Override + public boolean matches(Node node) { + if ( ! applications.isEmpty() && ! (node.allocation().isPresent() && applications.contains(node.allocation().get().owner()))) return false; + return nextMatches(node); + } + + public static ApplicationFilter from(ApplicationId applicationId, NodeFilter next) { + return new ApplicationFilter(ImmutableSet.of(applicationId), next); + } + + public static ApplicationFilter from(Set<ApplicationId> applicationIds, NodeFilter next) { + return new ApplicationFilter(ImmutableSet.copyOf(applicationIds), next); + } + + public static ApplicationFilter from(String applicationIds, NodeFilter next) { + return new ApplicationFilter(HostFilter.split(applicationIds).stream().map(ApplicationFilter::toApplicationId).collect(Collectors.toSet()), next); + } + + public static ApplicationId toApplicationId(String applicationIdString) { + String[] parts = applicationIdString.split("\\."); + if (parts.length != 3) + throw new IllegalArgumentException("Application id must be on the form tenant.application.instance, got '" + + applicationIdString + "'"); + return ApplicationId.from(TenantName.from(parts[0]), ApplicationName.from(parts[1]), InstanceName.from(parts[2])); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java new file mode 100644 index 00000000000..6a53d935311 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node.filter; + +import com.yahoo.vespa.hosted.provision.Node; + +import java.util.Objects; + +/** + * A chainable node filter + * + * @author bratseth + */ +public abstract class NodeFilter { + + private final NodeFilter next; + + /** Creates a node filter with a nchained filter, or null if this is the last filter */ + protected NodeFilter(NodeFilter next) { + this.next = next; + } + + /** Returns whether this node matches this filter */ + public abstract boolean matches(Node node); + + /** Returns whether this is a match according to the chained filter */ + protected final boolean nextMatches(Node node) { + if (next == null) return true; + return next.matches(node); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeHostFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeHostFilter.java new file mode 100644 index 00000000000..1753461afea --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeHostFilter.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node.filter; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.provision.Node; + +import java.util.Collections; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A node filter adaption of a host filter + * + * @author bratseth + */ +public class NodeHostFilter extends NodeFilter { + + private final HostFilter filter; + + /** Creates a node filter which filters using the given host filter */ + private NodeHostFilter(HostFilter filter, NodeFilter next) { + super(next); + Objects.requireNonNull(filter, "filter cannot be null, use HostFilter.all()"); + this.filter = filter; + } + + @Override + public boolean matches(Node node) { + if ( ! filter.matches(node.hostname(), node.configuration().flavor().name(), membership(node))) return false; + return nextMatches(node); + } + + private Optional<ClusterMembership> membership(Node node) { + if (node.allocation().isPresent()) + return Optional.of(node.allocation().get().membership()); + else + return Optional.empty(); + } + + public static NodeHostFilter from(HostFilter hostFilter) { + return new NodeHostFilter(hostFilter, null); + } + + public static NodeHostFilter from(HostFilter hostFilter, NodeFilter next) { + return new NodeHostFilter(hostFilter, next); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java new file mode 100644 index 00000000000..63e0493d53d --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node.filter; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.vespa.hosted.provision.Node; + +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * A node filter which matches a particular list of nodes + * + * @author bratseth + */ +public class NodeListFilter extends NodeFilter { + + private final Set<Node> nodes; + + private NodeListFilter(List<Node> nodes, NodeFilter next) { + super(next); + this.nodes = ImmutableSet.copyOf(nodes); + } + + @Override + public boolean matches(Node node) { + return nodes.contains(node); + } + + public static NodeListFilter from(List<Node> nodes) { + return new NodeListFilter(nodes, null); + } + + public static NodeListFilter from(List<Node> nodes, NodeFilter next) { + return new NodeListFilter(nodes, next); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java new file mode 100644 index 00000000000..3e79acffce3 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node.filter; + +import com.yahoo.config.provision.HostFilter; +import com.yahoo.vespa.hosted.provision.Node; + +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Filter based on the parent host value (for virtualized nodes). + * @author dybdahl + */ +public class ParentHostFilter extends NodeFilter { + + private final Set<String> parentHostNames; + + /** Creates a node filter which filters using the given parent host name */ + private ParentHostFilter(Set<String> parentHostNames, NodeFilter next) { + super(next); + Objects.requireNonNull(parentHostNames, "parentHostNames cannot be null."); + this.parentHostNames = parentHostNames; + } + + @Override + public boolean matches(Node node) { + if (! parentHostNames.isEmpty() && ( + ! node.parentHostname().isPresent() || ! parentHostNames.contains(node.parentHostname().get()))) + return false; + return nextMatches(node); + } + + /** Returns a copy of the given filter which only matches for the given parent */ + public static ParentHostFilter from(String parentNames, NodeFilter filter) { + return new ParentHostFilter(HostFilter.split(parentNames).stream().collect(Collectors.toSet()), filter); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java new file mode 100644 index 00000000000..a605835c11c --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node.filter; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.provision.Node; + +import java.util.Collections; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A node filter which filters on node states. + * + * @author bratseth + */ +public class StateFilter extends NodeFilter { + + private final Set<Node.State> states; + + /** Creates a node filter which filters using the given host filter */ + private StateFilter(Set<Node.State> states, NodeFilter next) { + super(next); + Objects.requireNonNull(states, "state cannot be null, use an empty optional"); + this.states = states; + } + + @Override + public boolean matches(Node node) { + if ( ! states.isEmpty() && ! states.contains(node.state())) return false; + return nextMatches(node); + } + + /** Returns a copy of the given filter which only matches for the given state */ + public static StateFilter from(Node.State state, NodeFilter filter) { + return new StateFilter(Collections.singleton(state), filter); + } + + /** Returns a node filter which matches a comma or space-separated list of states */ + public static StateFilter from(String states, NodeFilter next) { + return new StateFilter(HostFilter.split(states).stream().map(Node.State::valueOf).collect(Collectors.toSet()), next); + } + + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java new file mode 100644 index 00000000000..be45c0df8e0 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.provision; + +import com.yahoo.osgi.annotation.ExportPackage; + +/** The node repository controls and allocates the nodes available in a hosted Vespa zone */
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java new file mode 100644 index 00000000000..de7513de242 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.recipes.CuratorCounter; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; + +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A curator transaction which increases a change counter on commit. + * As this only ever does a single thing it needs no operations. + */ +class CountingCuratorTransaction implements Transaction { + + private final CuratorCounter counter; + + public CountingCuratorTransaction(CuratorCounter counter) { + this.counter = counter; + } + + @Override + public Transaction add(Operation operation) { return this; } + + @Override + public Transaction add(List<Operation> operation) { return this; } + + @Override + public List<Operation> operations() { return Collections.emptyList(); } + + @Override + public void prepare() { + // Increase the counter also if there are prepare errors to throw away the cached state + // in case that state leads to the rollback + counter.next(); + } + + @Override + public void rollbackOrLog() { } + + @Override + public void close() { } + + @Override + public void commit() { } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java new file mode 100644 index 00000000000..c7134de5ae6 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java @@ -0,0 +1,197 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.google.common.collect.ImmutableList; +import com.yahoo.path.Path; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.recipes.CuratorCounter; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Thius encapsulated the curator database of the node repo. + * It serves reads from an in-memory cache of the content which is invalidated when changed on another node + * using a global, shared counter. The counter is updated on all write operations, ensured by wrapping write + * operations in a 2pc transaction containing the counter update. + * + * @author bratseth + */ +public class CuratorDatabase { + + private final Curator curator; + + /** A shared atomic counter which is incremented every time we write to the curator database */ + private final CuratorCounter changeGenerationCounter; + + /** A partial cache of the Curator database, which is only valid if generations match */ + private final AtomicReference<CuratorDatabaseCache> cache = new AtomicReference<>(); + + /** Whether we should return data from the cache or always read fro ZooKeeper */ + private final boolean useCache; + + /** + * All keys, to allow reentrancy. + * This will grow forever with the number of applications seen, but this should be too slow to be a problem. + */ + private final ConcurrentHashMap<Path, CuratorMutex> locks = new ConcurrentHashMap<>(); + + /** + * Creates a curator database + * + * @param curator the curator instance + * @param root the file system root of the db + */ + public CuratorDatabase(Curator curator, Path root, boolean useCache) { + this.useCache = useCache; + this.curator = curator; + changeGenerationCounter = new CuratorCounter(curator, root.append("changeCounter").getAbsolute()); + cache.set(newCache(changeGenerationCounter.get())); + } + + /** Create a reentrant lock */ + // Locks are not cached in the in-memory state + public CuratorMutex lock(Path path) { + CuratorMutex lock = locks.computeIfAbsent(path, (pathArg) -> new CuratorMutex(pathArg.getAbsolute(), curator.framework())); + lock.acquire(); + return lock; + + } + + // --------- Write operations ------------------------------------------------------------------------------ + // These must either create a nested transaction ending in a counter increment or not depend on prior state + + /** + * Creates a new curator transaction against this database and adds it to the given nested transaction. + * Important: It is the nested transaction which must be committed - never the curator transaction directly. + */ + public CuratorTransaction newCuratorTransactionIn(NestedTransaction transaction) { + // Add a counting transaction first, to make sure we always invalidate the current state on any transaction commit + transaction.add(new CountingCuratorTransaction(changeGenerationCounter), CuratorTransaction.class); + CuratorTransaction curatorTransaction = new CuratorTransaction(curator); + transaction.add(curatorTransaction); + return curatorTransaction; + } + + /** Creates a path in curator and all its parents as necessary. If the path already exists this does nothing. */ + // As this operation does not depend on the prior state we do not need to increment the write counter + public void create(Path path) { + curator.create(path); + } + + // --------- Read operations ------------------------------------------------------------------------------- + // These can read from the memory file system, which accurately mirrors the ZooKeeper content IF + + /** Returns the immediate, local names of the children under this node in any order */ + public List<String> getChildren(Path path) { + CuratorDatabaseCache cache = getCache(); + List<String> children = cache.children(path); + if (children == null) { // children are not in this cache - get and add + children = curator.getChildren(path); + cache.addChildren(path, children); + } + return children; + } + + public Optional<byte[]> getData(Path path) { + CuratorDatabaseCache cache = getCache(); + Optional<byte[]> data = cache.data(path); + if (data == null) { // data is not in this cache - get and add + data = curator.getData(path); + cache.addData(path, data); + } + return data; + } + + private CuratorDatabaseCache getCache() { + CuratorDatabaseCache cache = this.cache.get(); + long currentCuratorGeneration = changeGenerationCounter.get(); + if (currentCuratorGeneration != cache.generation()) { // current cache is invalid - start new + cache = newCache(currentCuratorGeneration); + this.cache.set(cache); + } + return cache; + } + + /** Caches must only be instantiated using this method */ + private CuratorDatabaseCache newCache(long generation) { + return useCache ? new CuratorDatabaseCache(generation) : new DeactivatedCache(generation); + } + + /** + * A thread safe partial snapshot of the curator database content with a given generation. + * Note that a snapshot is not necessarily consistent - consistency is handled by pessimistic and optimistic locking + * in other layers. + * This is merely what Curator returned at various points in time it had the counter at this generation. + */ + private static class CuratorDatabaseCache { + + private final long generation; + + // The data of this partial state mirror. The amount of curator state mirrored in this may grow + // over time by multiple threads. Growing is the only operation permitted by this. + // The content of the map is immutable. + private final Map<Path, List<String>> children = new ConcurrentHashMap<>(); + private final Map<Path, Optional<byte[]>> data = new ConcurrentHashMap<>(); + + /** Create an empty snapshot at a given generation (as empty snapshot is a valid "partial snapshot" */ + public CuratorDatabaseCache(long generation) { + this.generation = generation; + } + + public long generation() { return generation; } + + /** + * Returns the children of this path, which may be empty. + * Returns null only if it is not present in this state mirror + */ + public List<String> children(Path path) { return children.get(path); } + + public void addChildren(Path path, List<String> childrenAtPath) { + if (children.containsKey(path)) throw new RuntimeException("Programming error"); + children.put(path, ImmutableList.copyOf(childrenAtPath)); + } + + /** + * Returns the content of this child - which may be empty. + * Returns null only if it is not present in this state mirror + */ + public Optional<byte[]> data(Path path) { + Optional<byte[]> dataAtPath = data.get(path); + if (dataAtPath == null) return null; + return dataAtPath.map(d -> Arrays.copyOf(d, d.length)); + } + + public void addData(Path path, Optional<byte[]> dataAtPath) { + if (data.containsKey(path)) throw new RuntimeException("Programming error"); + data.put(path, dataAtPath); + } + + } + + /** An implementation of the curator database cache which does no caching */ + private static class DeactivatedCache extends CuratorDatabaseCache { + + public DeactivatedCache(long generation) { super(generation); } + + @Override + public List<String> children(Path path) { return null; } + + @Override + public void addChildren(Path path, List<String> childrenAtPath) {} + + @Override + public Optional<byte[]> data(Path path) { return null; } + + @Override + public void addData(Path path, Optional<byte[]> dataAtPath) {} + + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java new file mode 100644 index 00000000000..5ff1f41272a --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -0,0 +1,256 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.joda.JodaModule; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; + +import com.yahoo.vespa.curator.transaction.CuratorOperations; +import com.yahoo.vespa.hosted.provision.node.Status; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Client which reads and writes nodes to a curator database. + * Nodes are stored in files named <code>/provision/v1/[nodestate]/[hostname]</code>. + * The responsibility of this class is to turn operations on the level of node states, applications and nodes + * into operations on the level of file paths and bytes. + * + * @author bratseth + */ +public class CuratorDatabaseClient { + + private static final Logger log = Logger.getLogger(CuratorDatabaseClient.class.getName()); + + private static final Path root = Path.fromString("/provision/v1"); + + private final NodeSerializer nodeSerializer; + + private final CuratorDatabase curatorDatabase; + + /** Used to serialize and de-serialize JSON data stored in ZK */ + private final ObjectMapper jsonMapper = new ObjectMapper(); + + private final Clock clock; + + public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock) { + this.nodeSerializer = new NodeSerializer(flavors); + jsonMapper.registerModule(new JodaModule()); + this.curatorDatabase = new CuratorDatabase(curator, root, /* useCache: */ false); + this.clock = clock; + initZK(); + } + + private void initZK() { + curatorDatabase.create(root); + for (Node.State state : Node.State.values()) + curatorDatabase.create(toPath(state)); + } + + /** + * Adds a set of nodes. Rollbacks/fails transaction if any node is not in the expected state. + */ + public List<Node> addNodesInState(List<Node> nodes, Node.State expectedState) { + NestedTransaction transaction = new NestedTransaction(); + CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + for (Node node : nodes) { + if (node.state() != expectedState) + throw new IllegalArgumentException(node + " is not in the " + node.state() + " state"); + curatorTransaction.add(CuratorOperations.create(toPath(node).getAbsolute(), nodeSerializer.toJson(node))); + } + transaction.commit(); + + for (Node node : nodes) + log.log(LogLevel.INFO, "Added " + node); + + return nodes; + } + + /** + * Adds a set of nodes in the initial, provisioned state. + * + * @return the given nodes for convenience. + */ + public List<Node> addNodes(List<Node> nodes) { + return addNodesInState(nodes, Node.State.provisioned); + } + + /** + * Removes a node. + * + * @param state the current state of the node + * @param hostName the host name of the node to remove + * @return true if the node was removed, false if it was not found + */ + public boolean removeNode(Node.State state, String hostName) { + Path path = toPath(state, hostName); + NestedTransaction transaction = new NestedTransaction(); + CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + curatorTransaction.add(CuratorOperations.delete(path.getAbsolute())); + transaction.commit(); + log.log(LogLevel.INFO, "Removed: " + state + " node " + hostName); + return true; + } + + /** + * Writes the given nodes to the given state (whether or not they are already in this state or another), + * and returns a copy of the incoming nodes in their persisted state. + * + * @param toState the state to write the nodes to + * @param nodes the list of nodes to write + * @return the nodes in their persisted state + */ + public List<Node> writeTo(Node.State toState, List<Node> nodes) { + try (NestedTransaction nestedTransaction = new NestedTransaction()) { + List<Node> writtenNodes = writeTo(toState, nodes, nestedTransaction); + nestedTransaction.commit(); + return writtenNodes; + } + } + public Node writeTo(Node.State toState, Node node) { + return writeTo(toState, Collections.singletonList(node)).get(0); + } + + /** + * Adds to the given transaction operations to write the given nodes to the given state, + * and returns a copy of the nodes in the state they will have if the transaction is committed. + * + * @param toState the state to write the nodes to + * @param nodes the list of nodes to write + * @param transaction the transaction to which write operations are added by this + * @return the nodes in their state as it will be written if committed + */ + public List<Node> writeTo(Node.State toState, List<Node> nodes, NestedTransaction transaction) { + if (nodes.isEmpty()) return nodes; + + List<Node> writtenNodes = new ArrayList<>(nodes.size()); + + CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + for (Node node : nodes) { + Node newNode = new Node(node.openStackId(), node.hostname(), node.parentHostname(), node.configuration(), + newNodeStatus(node, toState), + toState, + toState.isAllocated() ? node.allocation() : Optional.empty(), + newNodeHistory(node, toState)); + curatorTransaction.add(CuratorOperations.delete(toPath(node).getAbsolute())) + .add(CuratorOperations.create(toPath(toState, newNode.hostname()).getAbsolute(), nodeSerializer.toJson(newNode))); + writtenNodes.add(newNode); + } + + transaction.onCommitted(() -> { // schedule logging on commit of nodes which changed state + for (Node node : nodes) { + if (toState != node.state()) + log.log(LogLevel.INFO, "Moved to " + toState + ": " + node); + } + }); + return writtenNodes; + } + + private Status newNodeStatus(Node node, Node.State toState) { + if (node.state() != Node.State.failed && toState == Node.State.failed) return node.status().increaseFailCount(); + if (node.state() == Node.State.failed && toState == Node.State.active) return node.status().decreaseFailCount(); // fail undo + return node.status(); + } + + private History newNodeHistory(Node node, Node.State toState) { + History history = node.history(); + + // wipe history to avoid expiring based on events under the previous allocation + if (toState == Node.State.ready) + history = History.empty(); + + return history.recordStateTransition(node.state(), toState, clock.instant()); + } + + /** + * Returns all nodes which are in one of the given states. + * If no states are given this returns all nodes. + */ + public List<Node> getNodes(Node.State ... states) { + List<Node> nodes = new ArrayList<>(); + if (states.length == 0) + states = Node.State.values(); + for (Node.State state : states) { + for (String hostname : curatorDatabase.getChildren(toPath(state))) { + final Optional<Node> node = getNode(state, hostname); + if (node.isPresent()) nodes.add(node.get()); // node might disappear between getChildren and getNode + } + } + return nodes; + } + + /** Returns all nodes allocated to the given application which are in one of the given states */ + public List<Node> getNodes(ApplicationId applicationId, Node.State ... states) { + List<Node> nodes = getNodes(states); + nodes.removeIf(node -> ! node.allocation().isPresent() || ! node.allocation().get().owner().equals(applicationId)); + return nodes; + } + + /** Returns a particular node, or empty if there is no such node in this state */ + public Optional<Node> getNode(Node.State state, String hostname) { + return curatorDatabase.getData(toPath(state, hostname)).map((data) -> nodeSerializer.fromJson(state, data)); + } + + private Path toPath(Node.State nodeState) { return root.append(toDir(nodeState)); } + + private Path toPath(Node node) { + return root.append(toDir(node.state())).append(node.hostname()); + } + + private Path toPath(Node.State nodeState, String nodeName) { + return root.append(toDir(nodeState)).append(nodeName); + } + + /** Creates an returns the path to the lock for this application */ + private Path lockPath(ApplicationId application) { + Path lockPath = root.append("locks") + .append(application.tenant().value()) + .append(application.application().value()) + .append(application.instance().value()); + curatorDatabase.create(lockPath); + return lockPath; + } + + private String toDir(Node.State state) { + switch (state) { + case provisioned: return "provisioned"; + case ready: return "ready"; + case reserved: return "reserved"; + case active: return "allocated"; // legacy name + case inactive: return "deallocated"; // legacy name + case dirty: return "dirty"; + case failed: return "failed"; + default: throw new RuntimeException("Node state " + state + " does not map to a directory name"); + } + } + + /** Acquires the single cluster-global, reentrant lock for all non-active nodes */ + public CuratorMutex lockInactive() { + return lock(root.append("locks").append("unallocatedLock")); + } + + /** Acquires the single cluster-global, reentrant lock for active nodes of this application */ + public CuratorMutex lock(ApplicationId application) { + return lock(lockPath(application)); + } + + /** Acquires the single cluster-global, reentrant lock for all non-active nodes */ + public CuratorMutex lock(Path path) { + return curatorDatabase.lock(path); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorMutex.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorMutex.java new file mode 100644 index 00000000000..6f8a7aae3d5 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorMutex.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.transaction.Mutex; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * A cluster-wide reentrant mutex which is released on (the last symmetric) close + * + * @author bratseth + */ +public class CuratorMutex implements Mutex { + + private final InterProcessMutex mutex; + private final String lockPath; + + public CuratorMutex(String lockPath, CuratorFramework curator) { + this.lockPath = lockPath; + mutex = new InterProcessMutex(curator, lockPath); + } + + /** Take the lock. This may be called multiple times from the same thread - each matched by a close */ + public void acquire() { + try { + boolean acquired = mutex.acquire(60, TimeUnit.SECONDS); + if ( ! acquired) { + throw new TimeoutException("Timed out after waiting 60 seconds"); + } + } + catch (Exception e) { + throw new RuntimeException("Exception acquiring lock '" + lockPath + "'", e); + } + } + + @Override + public void close() { + try { + mutex.release(); + } + catch (Exception e) { + throw new RuntimeException("Exception releasing lock '" + lockPath + "'"); + } + } + +} + + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java new file mode 100644 index 00000000000..9e0a26be308 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -0,0 +1,273 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.Generation; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.node.Status; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static com.yahoo.vespa.config.SlimeUtils.optionalString; + +/** + * Serializes a node to/from JSON. + * Instances of this are multithread safe and can be reused + * + * @author bratseth + */ +public class NodeSerializer { + + /** The configured node flavors */ + private final NodeFlavors flavors; + + // Node fields + private static final String hostnameKey = "hostname"; + private static final String openStackIdKey = "openStackId"; + // TODO Legacy name. Remove when 5.120 is released everywhere + private static final String dockerHostHostNameKey = "dockerHostHostName"; + private static final String parentHostnameKey = "parentHostname"; + private static final String configurationKey ="configuration"; + private static final String historyKey = "history"; + private static final String instanceKey = "instance"; // legacy name, TODO: change to allocation with backwards compat + private static final String rebootGenerationKey = "rebootGeneration"; + private static final String currentRebootGenerationKey = "currentRebootGeneration"; + private static final String vespaVersionKey = "vespaVersion"; + private static final String hostedVersionKey = "hostedVersion"; + private static final String stateVersionKey = "stateVersion"; + private static final String failCountKey = "failCount"; + private static final String hardwareFailureKey = "hardwareFailure"; + + // Configuration fields + private static final String flavorKey = "flavor"; + + // Allocation fields + private static final String tenantIdKey = "tenantId"; + private static final String applicationIdKey = "applicationId"; + private static final String instanceIdKey = "instanceId"; + private static final String serviceIdKey = "serviceId"; // legacy name, TODO: change to membership with backwards compat + private static final String restartGenerationKey = "restartGeneration"; + private static final String currentRestartGenerationKey = "currentRestartGeneration"; + private static final String removableKey = "removable"; + //Saved as part of allocation instead of serviceId, since serviceId serialized form is not easily extendable. + private static final String dockerImageKey = "dockerImage"; + + // History event fields + private static final String typeKey = "type"; + private static final String atKey = "at"; + private static final String agentKey = "agent"; // retired events only + + // ---------------- Serialization ---------------------------------------------------- + + public NodeSerializer(NodeFlavors flavors) { + this.flavors = flavors; + } + + public byte[] toJson(Node node) { + try { + Slime slime = new Slime(); + toSlime(node, slime.setObject()); + return SlimeUtils.toJsonBytes(slime); + } + catch (IOException e) { + throw new RuntimeException("Serialization of " + node + " to json failed", e); + } + } + + private void toSlime(Node node, Cursor object) { + object.setString(hostnameKey, node.hostname()); + object.setString(openStackIdKey, node.openStackId()); + node.parentHostname().ifPresent(hostname -> object.setString(parentHostnameKey, hostname)); + toSlime(node.configuration(), object.setObject(configurationKey)); + object.setLong(rebootGenerationKey, node.status().reboot().wanted()); + object.setLong(currentRebootGenerationKey, node.status().reboot().current()); + node.status().vespaVersion().ifPresent(version -> object.setString(vespaVersionKey, version.toString())); + node.status().hostedVersion().ifPresent(version -> object.setString(hostedVersionKey, version.toString())); + node.status().stateVersion().ifPresent(version -> object.setString(stateVersionKey, version.toString())); + node.status().dockerImage().ifPresent(image -> object.setString(dockerImageKey, image)); + object.setLong(failCountKey, node.status().failCount()); + object.setBool(hardwareFailureKey, node.status().hardwareFailure()); + node.allocation().ifPresent(allocation -> toSlime(allocation, object.setObject(instanceKey))); + toSlime(node.history(), object.setArray(historyKey)); + } + + private void toSlime(Configuration configuration, Cursor object) { + object.setString(flavorKey, configuration.flavor().name()); + } + + private void toSlime(Allocation allocation, Cursor object) { + object.setString(tenantIdKey, allocation.owner().tenant().value()); + object.setString(applicationIdKey, allocation.owner().application().value()); + object.setString(instanceIdKey, allocation.owner().instance().value()); + object.setString(serviceIdKey, allocation.membership().stringValue()); + object.setLong(restartGenerationKey, allocation.restartGeneration().wanted()); + object.setLong(currentRestartGenerationKey, allocation.restartGeneration().current()); + object.setBool(removableKey, allocation.removable()); + allocation.membership().cluster().dockerImage().ifPresent( dockerImage -> + object.setString(dockerImageKey, dockerImage)); + } + + private void toSlime(History history, Cursor array) { + for (History.Event event : history.events()) + toSlime(event, array.addObject()); + } + + private void toSlime(History.Event event, Cursor object) { + object.setString(typeKey, toString(event.type())); + object.setLong(atKey, event.at().toEpochMilli()); + if (event instanceof History.RetiredEvent) + object.setString(agentKey, toString(((History.RetiredEvent)event).agent())); + } + + // ---------------- Deserialization -------------------------------------------------- + + public Node fromJson(Node.State state, byte[] data) { + return nodeFromSlime(state, SlimeUtils.jsonToSlime(data).get()); + } + + private Node nodeFromSlime(Node.State state, Inspector object) { + return new Node(object.field(openStackIdKey).asString(), + object.field(hostnameKey).asString(), + parentHostnameFromSlime(object), + configurationFromSlime(object.field(configurationKey)), + statusFromSlime(object), + state, + allocationFromSlime(object.field(instanceKey)), + historyFromSlime(object.field(historyKey))); + } + + private Status statusFromSlime(Inspector object) { + return new Status( + generationFromSlime(object, rebootGenerationKey, currentRebootGenerationKey), + softwareVersionFromSlime(object.field(vespaVersionKey)), + softwareVersionFromSlime(object.field(hostedVersionKey)), + optionalString(object.field(stateVersionKey)), + optionalString(object.field(dockerImageKey)), + (int)object.field(failCountKey).asLong(), + object.field(hardwareFailureKey).asBool()); + } + + private Configuration configurationFromSlime(Inspector object) { + return new Configuration(flavors.getFlavorOrThrow(object.field(flavorKey).asString())); + } + + private Optional<Allocation> allocationFromSlime(Inspector object) { + if ( ! object.valid()) return Optional.empty(); + return Optional.of(new Allocation( + applicationIdFromSlime(object), + ClusterMembership.from(object.field(serviceIdKey).asString(), optionalString(object.field(dockerImageKey))), + generationFromSlime(object, restartGenerationKey, currentRestartGenerationKey), + object.field(removableKey).asBool())); + } + + private ApplicationId applicationIdFromSlime(Inspector object) { + return ApplicationId.from(TenantName.from(object.field(tenantIdKey).asString()), + ApplicationName.from(object.field(applicationIdKey).asString()), + InstanceName.from(object.field(instanceIdKey).asString())); + } + + private History historyFromSlime(Inspector array) { + List<History.Event> events = new ArrayList<>(); + array.traverse((ArrayTraverser) (int i, Inspector item) -> { + History.Event event = eventFromSlime(item); + if (event != null) + events.add(event); + }); + return new History(events); + } + + private History.Event eventFromSlime(Inspector object) { + History.Event.Type type = eventTypeFromString(object.field(typeKey).asString()); + if (type == null) return null; + Instant at = Instant.ofEpochMilli(object.field(atKey).asLong()); + if (type.equals(History.Event.Type.retired)) + return new History.RetiredEvent(at, eventAgentFromString(object.field(agentKey).asString())); + else + return new History.Event(type, at); + + } + + private Generation generationFromSlime(Inspector object, String wantedField, String currentField) { + Inspector current = object.field(currentField); + return new Generation(object.field(wantedField).asLong(), current.asLong()); + } + + private Optional<Version> softwareVersionFromSlime(Inspector object) { + if ( ! object.valid()) return Optional.empty(); + return Optional.of(Version.fromString(object.asString())); + } + + private Optional<String> parentHostnameFromSlime(Inspector object) { + if (object.field(parentHostnameKey).valid()) + return Optional.of(object.field(parentHostnameKey).asString()); + // TODO Remove when 5.120 is released everywhere + else if (object.field(dockerHostHostNameKey).valid()) + return Optional.of(object.field(dockerHostHostNameKey).asString()); + else + return Optional.empty(); + } + + /** Returns the event type, or null if this event type should be ignored */ + private History.Event.Type eventTypeFromString(String eventTypeString) { + switch (eventTypeString) { + case "readied" : return History.Event.Type.readied; + case "reserved" : return History.Event.Type.reserved; + case "activated" : return History.Event.Type.activated; + case "retired" : return History.Event.Type.retired; + case "deactivated" : return History.Event.Type.deactivated; + case "failed" : return History.Event.Type.failed; + case "deallocated" : return History.Event.Type.deallocated; + case "down" : return History.Event.Type.down; + } + throw new IllegalArgumentException("Unknown node event type '" + eventTypeString + "'"); + } + + private String toString(History.Event.Type nodeEventType) { + switch (nodeEventType) { + case readied : return "readied"; + case reserved : return "reserved"; + case activated : return "activated"; + case retired : return "retired"; + case deactivated : return "deactivated"; + case failed : return "failed"; + case deallocated : return "deallocated"; + case down : return "down"; + } + throw new IllegalArgumentException("Serialized form of '" + nodeEventType + "' not defined"); + } + + private History.RetiredEvent.Agent eventAgentFromString(String eventAgentString) { + switch (eventAgentString) { + case "application" : return History.RetiredEvent.Agent.application; + case "system" : return History.RetiredEvent.Agent.system; + } + throw new IllegalArgumentException("Unknown node event agent '" + eventAgentString + "'"); + } + + private String toString(History.RetiredEvent.Agent agent) { + switch (agent) { + case application : return "application"; + case system : return "system"; + } + throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined"); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java new file mode 100644 index 00000000000..3e288bd7785 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.transaction.Mutex; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Performs activation of nodes for an application + * + * @author bratseth + */ +class Activator { + + private final NodeRepository nodeRepository; + private final Clock clock; + + public Activator(NodeRepository nodeRepository, Clock clock) { + this.nodeRepository = nodeRepository; + this.clock = clock; + } + + /** + * Add operations to activates nodes for an application to the given transaction. + * The operations are not effective until the transaction is committed. + * <p> + * Pre condition: The application has a possibly empty set of nodes in each of reserved and active. + * <p> + * Post condition: Nodes in reserved which are present in <code>hosts</code> are moved to active. + * Nodes in active which are not present in <code>hosts</code> are moved to inactive. + * + * @param transaction Transaction with operations to commit together with any operations done within the repository. + * @param application the application to allocate nodes for + * @param hosts the hosts to make the set of active nodes of this + */ + public void activate(ApplicationId application, Collection<HostSpec> hosts, NestedTransaction transaction) { + try (Mutex lock = nodeRepository.lock(application)) { + Set<String> hostnames = hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); + + List<Node> reserved = nodeRepository.getNodes(application, Node.State.reserved); + List<Node> reservedToActivate = retainHostsInList(hostnames, reserved); + List<Node> active = nodeRepository.getNodes(application, Node.State.active); + List<Node> continuedActive = retainHostsInList(hostnames, active); + List<Node> allActive = new ArrayList<>(continuedActive); + allActive.addAll(reservedToActivate); + if ( ! containsAll(hostnames, allActive)) + throw new IllegalArgumentException("Activation of " + application + " failed. " + + "Could not find all requested hosts." + + "\nRequested: " + hosts + + "\nReserved: " + toHostNames(reserved) + + "\nActive: " + toHostNames(active)); + + List<Node> activeToRemove = removeHostsFromList(hostnames, active); + activeToRemove = activeToRemove.stream().map(Node::unretire).collect(Collectors.toList()); // only active nodes can be retired + nodeRepository.deactivate(activeToRemove, transaction); + nodeRepository.activate(updateFrom(hosts, continuedActive), transaction); // update active with any changes + nodeRepository.activate(reservedToActivate, transaction); + } + } + + private List<Node> retainHostsInList(Set<String> hosts, List<Node> nodes) { + return nodes.stream().filter(node -> hosts.contains(node.hostname())).collect(Collectors.toList()); + } + + private List<Node> removeHostsFromList(Set<String> hosts, List<Node> nodes) { + return nodes.stream().filter(node -> ! hosts.contains(node.hostname())).collect(Collectors.toList()); + } + + private Set<String> toHostNames(List<Node> nodes) { + return nodes.stream().map(Node::hostname).collect(Collectors.toSet()); + } + + private boolean containsAll(Set<String> hosts, List<Node> nodes) { + Set<String> notFoundHosts = new HashSet<>(hosts); + for (Node node : nodes) + notFoundHosts.remove(node.hostname()); + return notFoundHosts.isEmpty(); + } + + /** + * Returns the input nodes with the changes resulting from applying the settings in hosts to the given list of nodes. + */ + private List<Node> updateFrom(Collection<HostSpec> hosts, List<Node> nodes) { + List<Node> updated = new ArrayList<>(); + for (Node node : nodes) { + HostSpec hostSpec = getHost(node.hostname(), hosts); + node = hostSpec.membership().get().retired() ? node.retireByApplication(clock.instant()) : node.unretire(); + node = node.setAllocation(node.allocation().get().changeMembership(hostSpec.membership().get())); + updated.add(node); + } + return updated; + } + + private HostSpec getHost(String hostname, Collection<HostSpec> fromHosts) { + for (HostSpec host : fromHosts) + if (host.hostname().equals(hostname)) + return host; + return null; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java new file mode 100644 index 00000000000..c96a7c6dab4 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.node.Flavor; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; + +import java.util.Optional; + +/** + * Defines the policies for assigning cluster capacity in various environments + * + * @author bratseth + */ +public class CapacityPolicies { + + private final Zone zone; + private final NodeFlavors flavors; + + public CapacityPolicies(Zone zone, NodeFlavors flavors) { + this.zone = zone; + this.flavors = flavors; + } + + /** provides capacity defaults for various environments */ + public int decideSize(Capacity requestedCapacity) { + int requestedNodes = requestedCapacity.nodeCount(); + if (requestedCapacity.isRequired()) return requestedNodes; + + switch(zone.environment()) { + case dev : case test : return 1; + case perf : return Math.min(requestedCapacity.nodeCount(), 3); + case staging: return requestedNodes <= 1 ? requestedNodes : Math.max(2, requestedNodes / 10); + case prod : return ensureRedundancy(requestedCapacity.nodeCount()); + default : throw new IllegalArgumentException("Unsupported environment " + zone.environment()); + } + } + + public Flavor decideFlavor(Capacity requestedCapacity, ClusterSpec cluster) { + // for now, always use requested docker flavor when requested + final Optional<String> requestedFlavor = requestedCapacity.flavor(); + if (requestedFlavor.isPresent() && flavors.getFlavorOrThrow(requestedFlavor.get()).getEnvironment().equals("DOCKER_CONTAINER")) + return flavors.getFlavorOrThrow(requestedFlavor.get()); + + switch(zone.environment()) { + case dev : case test : case staging : return flavors.getFlavorOrThrow(zone.defaultFlavor(cluster.type())); + default : return flavors.getFlavorOrThrow(requestedFlavor.orElse(zone.defaultFlavor(cluster.type()))); + } + } + + /** + * Throw if the node count is 1 + + * @return the argument node count + * @throws IllegalArgumentException if only one node is requested + */ + private int ensureRedundancy(int nodeCount) { + // TODO: Reactivate this check when we have sufficient capacity in ap-northeast + // if (nodeCount == 1) + // throw new IllegalArgumentException("Deployments to prod require at least 2 nodes per cluster for redundancy"); + return nodeCount; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java new file mode 100644 index 00000000000..6b5d37d812a --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -0,0 +1,345 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.lang.MutableInteger; +import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Flavor; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Performs preparation of node activation changes for a single host group in an application. + * + * @author bratseth + */ +class GroupPreparer { + + private final NodeRepository nodeRepository; + private final Clock clock; + + private static final boolean canChangeGroup = true; + + public GroupPreparer(NodeRepository nodeRepository, Clock clock) { + this.nodeRepository = nodeRepository; + this.clock = clock; + } + + /** + * Ensure sufficient nodes are reserved or active for the given application, group and cluster + * + * @param application the application we are allocating to + * @param cluster the cluster and group we are allocating to + * @param nodeCount the desired number of nodes to return + * @param flavor the desired flavor of those nodes + * @param surplusActiveNodes currently active nodes which are available to be assigned to this group. + * This method will remove from this list if it finds it needs additional nodes + * @param highestIndex the current highest node index among all active nodes in this cluster. + * This method will increase this number when it allocates new nodes to the cluster. + * @return the list of nodes this cluster group will have allocated if activated + */ + // Note: This operation may make persisted changes to the set of reserved and inactive nodes, + // but it may not change the set of active nodes, as the active nodes must stay in sync with the + // active config model which is changed on activate + public List<Node> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, Flavor flavor, List<Node> surplusActiveNodes, MutableInteger highestIndex) { + try (Mutex lock = nodeRepository.lock(application)) { + NodeList nodeList = new NodeList(application, cluster, nodeCount, flavor, highestIndex); + + // Use active nodes + nodeList.offer(nodeRepository.getNodes(application, Node.State.active), !canChangeGroup); + if (nodeList.satisfied()) return nodeList.finalNodes(surplusActiveNodes); + + // Use active nodes that will otherwise be retired + List<Node> accepted = nodeList.offer(surplusActiveNodes, canChangeGroup); + surplusActiveNodes.removeAll(accepted); + if (nodeList.satisfied()) return nodeList.finalNodes(surplusActiveNodes); + + // Use previously reserved nodes + nodeList.offer(nodeRepository.getNodes(application, Node.State.reserved), !canChangeGroup); + if (nodeList.satisfied()) return nodeList.finalNodes(surplusActiveNodes); + + // Use inactive nodes + accepted = nodeList.offer(nodeRepository.getNodes(application, Node.State.inactive), !canChangeGroup); + nodeList.update(nodeRepository.reserve(accepted)); + if (nodeList.satisfied()) return nodeList.finalNodes(surplusActiveNodes); + + // Use new, ready nodes. Need to lock ready pool to ensure that nodes are not grabbed by others. + try (Mutex readyLock = nodeRepository.lockUnallocated()) { + List<Node> readyNodes = nodeRepository.getNodes(Node.State.ready); + accepted = nodeList.offer(optimize(readyNodes), !canChangeGroup); + nodeList.update(nodeRepository.reserve(accepted)); + } + if (nodeList.satisfied()) return nodeList.finalNodes(surplusActiveNodes); + + if (nodeList.whatAboutUsingRetiredNodes()) { + throw new OutOfCapacityException("Could not satisfy request for " + nodeCount + + " nodes of " + flavor + " for " + cluster + + " because we want to retire existing nodes."); + } + if (nodeList.whatAboutUsingVMs()) { + throw new OutOfCapacityException("Could not satisfy request for " + nodeCount + + " nodes of " + flavor + " for " + cluster + + " because too many have same parentHost."); + } + throw new OutOfCapacityException("Could not satisfy request for " + nodeCount + + " nodes of " + flavor + " for " + cluster + "."); + } + } + + // optimize based on parent hosts + static List<Node> optimize(List<Node> input) { + int cnt = input.size(); + List<Node> output = new ArrayList<Node>(cnt); + + // first deal with VMs. + long vms = input.stream() + .filter(n -> n.parentHostname().isPresent()) + .collect(Collectors.counting()); + if (vms > 0) { + // Make a map where each parenthost maps to a list of VMs: + Map<String, List<Node>> byParentHosts = input.stream() + .filter(n -> n.parentHostname().isPresent()) + .collect(Collectors.groupingBy(n -> n.parentHostname().get())); + + // sort keys, those parent hosts with the most (remaining) ready VMs first + List<String> sortedParentHosts = byParentHosts + .keySet().stream() + .sorted((k1, k2) -> byParentHosts.get(k2).size() - byParentHosts.get(k1).size()) + .collect(Collectors.toList()); + while (vms > 0) { + // take one VM from each parent host, round-robin. + for (String k : sortedParentHosts) { + List<Node> leftFromHost = byParentHosts.get(k); + if (! leftFromHost.isEmpty()) { + output.add(leftFromHost.remove(0)); + --vms; + } + } + } + } + + // now add non-VMs (nodes without a parent): + input.stream() + .filter(n -> ! n.parentHostname().isPresent()) + .forEach(n -> output.add(n)); + return output; + } + + /** Used to manage a list of nodes during the node reservation process */ + private class NodeList { + + /** The application this list is for */ + private final ApplicationId application; + + /** The cluster this list is for */ + private final ClusterSpec cluster; + + /** The requested capacity of the list */ + private final int requestedNodes; + + /** The requested node flavor */ + private final Flavor requestedFlavor; + + /** The nodes this has accepted so far */ + private final Set<Node> nodes = new LinkedHashSet<>(); + + /** The number of nodes in the accepted nodes which are of the requested flavor */ + private int acceptedOfRequestedFlavor = 0; + + /** The number of nodes rejected because of clashing parentHostname */ + private int rejectedWithClashingParentHost = 0; + + /** The number of nodes that just now was changed to retired */ + private int wasRetiredJustNow = 0; + + /** The node indexes to verify uniqueness of each members index */ + private Set<Integer> indexes = new HashSet<>(); + + /** The next membership index to assign to a new node */ + private MutableInteger highestIndex; + + public NodeList(ApplicationId application, ClusterSpec cluster, int requestedNodes, Flavor requestedFlavor, MutableInteger highestIndex) { + this.application = application; + this.cluster = cluster; + this.requestedNodes = requestedNodes; + this.requestedFlavor = requestedFlavor; + this.highestIndex = highestIndex; + } + + /** + * Offer some nodes to this. The nodes may have an allocation to a different application or cluster, + * an allocation to this cluster, or no current allocation (in which case one is assigned). + * <p> + * Note that if unallocated nodes are offered before allocated nodes, this will unnecessarily + * reject allocated nodes due to index duplicates. + * + * @param offeredNodes the nodes which are potentially on offer. These may belong to a different application etc. + * @param canChangeGroup whether it is ok to change the group the offered node is to belong to if necessary + * @return the subset of offeredNodes which was accepted, with the correct allocation assigned + */ + public List<Node> offer(List<Node> offeredNodes, boolean canChangeGroup) { + List<Node> accepted = new ArrayList<>(); + for (Node offered : offeredNodes) { + boolean wantToRetireNode = false; + if (offered.allocation().isPresent()) { + ClusterMembership membership = offered.allocation().get().membership(); + if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application + if ( ! membership.cluster().equalsIgnoringGroup(cluster)) continue; // wrong cluster id/type + if ( (! canChangeGroup || satisfied()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it + if ( offered.allocation().get().removable()) continue; // don't accept; causes removal + if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) + + // conditions on which we want to retire nodes that were allocated previously + if ( offeredNodeHasParentHostnameAlreadyAccepted(this.nodes, offered)) wantToRetireNode = true; + if ( !hasCompatibleFlavor(offered)) wantToRetireNode = true; + + if ( ( !satisfied() && hasCompatibleFlavor(offered)) || acceptToRetire(offered) ) + accepted.add(acceptNode(offered, wantToRetireNode)); + } + else if (! satisfied() && hasCompatibleFlavor(offered)) { + if ( offeredNodeHasParentHostnameAlreadyAccepted(this.nodes, offered)) { + ++rejectedWithClashingParentHost; + continue; + } + Node alloc = offered.allocate(application, ClusterMembership.from(cluster, highestIndex.add(1)), clock.instant()); + accepted.add(acceptNode(alloc, wantToRetireNode)); + } + } + + return accepted; + } + + private boolean offeredNodeHasParentHostnameAlreadyAccepted(Collection<Node> accepted, Node offered) { + for (Node acceptedNode : accepted) { + if (acceptedNode.parentHostname().isPresent() && offered.parentHostname().isPresent() && + acceptedNode.parentHostname().get().equals(offered.parentHostname().get())) { + return true; + } + } + return false; + } + + /** + * Returns whether this node should be accepted into the cluster even if it is not currently desired + * (already enough nodes, or wrong flavor). + * Such nodes will be marked retired during finalization of the list of accepted nodes. + * The conditions for this are + * <ul> + * <li>This is a content node. These must always be retired before being removed to allow the cluster to + * migrate away data. + * <li>This is a container node and it is not desired due to having the wrong flavor. In this case this + * will (normally) obtain for all the current nodes in the cluster and so retiring before removing must + * be used to avoid removing all the current nodes at once, before the newly allocated replacements are + * initialized. (In the other case, where a container node is not desired because we have enough nodes we + * do want to remove it immediately to get immediate feedback on how the size reduction works out.) + * </ul> + */ + private boolean acceptToRetire(Node node) { + if (node.state() != Node.State.active) return false; + if (! node.allocation().get().membership().cluster().group().equals(cluster.group())) return false; + + return (cluster.type() == ClusterSpec.Type.content) || + (cluster.type() == ClusterSpec.Type.container && ! hasCompatibleFlavor(node)); + } + + private boolean hasCompatibleFlavor(Node node) { + return node.configuration().flavor().satisfies(requestedFlavor); + } + + /** Updates the state of some existing nodes in this list by replacing them by id with the given instances. */ + public void update(List<Node> updatedNodes) { + nodes.removeAll(updatedNodes); + nodes.addAll(updatedNodes); + } + + private Node acceptNode(Node node, boolean wantToRetire) { + if (! wantToRetire) { + if ( ! node.state().equals(Node.State.active)) { + // reactivated node - make sure its not retired + node = node.unretire(); + } + acceptedOfRequestedFlavor++; + } else { + ++wasRetiredJustNow; + // retire nodes which are of an unwanted flavor + // or have an overlapping parent host + node = node.retireByApplication(clock.instant()); + } + if ( ! node.allocation().get().membership().cluster().equals(cluster)) { + // group may be different + node = setCluster(cluster, node); + } + indexes.add(node.allocation().get().membership().index()); + highestIndex.set(Math.max(highestIndex.get(), node.allocation().get().membership().index())); + nodes.add(node); + return node; + } + + private Node setCluster(ClusterSpec cluster, Node node) { + ClusterMembership membership = node.allocation().get().membership().changeCluster(cluster); + return node.setAllocation(node.allocation().get().changeMembership(membership)); + } + + /** Returns true if we have accepted at least the requested number of nodes of the requested flavor */ + public boolean satisfied() { + return acceptedOfRequestedFlavor >= requestedNodes; + } + + public boolean whatAboutUsingRetiredNodes() { + return acceptedOfRequestedFlavor + wasRetiredJustNow >= requestedNodes; + } + + public boolean whatAboutUsingVMs() { + return acceptedOfRequestedFlavor + rejectedWithClashingParentHost >= requestedNodes; + } + + /** + * Make the number of <i>non-retired</i> nodes in the list equal to the requested number + * of nodes, and retire the rest of the list. Only retire currently active nodes. + * Prefer to retire nodes of the wrong flavor. + * Make as few changes to the retired set as possible. + * + * @return the final list of nodes + */ + public List<Node> finalNodes(List<Node> surplusNodes) { + long currentRetired = nodes.stream().filter(node -> node.allocation().get().membership().retired()).count(); + long surplus = nodes.size() - requestedNodes - currentRetired; + + List<Node> changedNodes = new ArrayList<>(); + if (surplus > 0) { // retire until surplus is 0 + for (Node node : nodes) { + if ( ! node.allocation().get().membership().retired() && node.state().equals(Node.State.active)) { + changedNodes.add(node.retireByApplication(clock.instant())); + surplusNodes.add(node); // will be used in another group or retired + if (--surplus == 0) break; + } + } + } + else if (surplus < 0) { // unretire until surplus is 0 + for (Node node : nodes) { + if ( node.allocation().get().membership().retired() && hasCompatibleFlavor(node)) { + changedNodes.add(node.unretire()); + if (++surplus == 0) break; + } + } + } + update(changedNodes); + return new ArrayList<>(nodes); + } + + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java new file mode 100644 index 00000000000..e2ff947de80 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.google.inject.Inject; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.ProvisionLogger; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.Zone; +import com.yahoo.log.LogLevel; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Flavor; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; +import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Implementation of the host provisioner API for hosted Vespa, using the node repository to allocate nodes. + * Does not allocate hosts for the routing application, see VespaModelFactory.createHostProvisioner + * + * @author bratseth + */ +public class NodeRepositoryProvisioner implements Provisioner { + + private static Logger log = Logger.getLogger(NodeRepositoryProvisioner.class.getName()); + + private final NodeRepository nodeRepository; + private final CapacityPolicies capacityPolicies; + private final Zone zone; + private final Preparer preparer; + private final Activator activator; + + @Inject + public NodeRepositoryProvisioner(NodeRepository nodeRepository, NodeFlavors flavors, Zone zone) { + this(nodeRepository, flavors, zone, Clock.systemUTC()); + } + + public NodeRepositoryProvisioner(NodeRepository nodeRepository, NodeFlavors flavors, Zone zone, Clock clock) { + this.nodeRepository = nodeRepository; + this.capacityPolicies = new CapacityPolicies(zone, flavors); + this.zone = zone; + this.preparer = new Preparer(nodeRepository, clock); + this.activator = new Activator(nodeRepository, clock); + } + + /** + * Returns a list of nodes in the prepared or active state, matching the given constraints. + * The nodes are ordered by increasing index number. + */ + @Override + public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity requestedCapacity, int groups, ProvisionLogger logger) { + log.log(LogLevel.DEBUG, () -> "Received deploy prepare request for " + requestedCapacity + " in " + + groups + " groups for application " + application + ", cluster " + cluster); + + Flavor flavor = capacityPolicies.decideFlavor(requestedCapacity, cluster); + int nodeCount = capacityPolicies.decideSize(requestedCapacity); + int effectiveGroups = groups > nodeCount ? nodeCount : groups; // cannot have more groups than nodes + + if (zone.environment().isManuallyDeployed() && nodeCount < requestedCapacity.nodeCount()) + logger.log(Level.WARNING, "Requested " + requestedCapacity.nodeCount() + " nodes for " + cluster + + ", downscaling to " + nodeCount + " nodes in " + zone.environment()); + + return asSortedHosts(preparer.prepare(application, cluster, nodeCount, flavor, effectiveGroups)); + } + + @Override + public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { + activator.activate(application, hosts, transaction); + } + + @Override + public void restart(ApplicationId application, HostFilter filter) { + nodeRepository.restart(ApplicationFilter.from(application, NodeHostFilter.from(filter))); + } + + @Override + public void removed(ApplicationId application) { + nodeRepository.deactivate(application); + } + + private List<HostSpec> asSortedHosts(List<Node> nodes) { + nodes.sort(Comparator.comparingInt((Node node) -> node.allocation().get().membership().index())); + List<HostSpec> hosts = new ArrayList<>(nodes.size()); + for (Node node : nodes) + hosts.add(new HostSpec(node.hostname(), + node.allocation().orElseThrow(IllegalStateException::new).membership())); + return hosts; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java new file mode 100644 index 00000000000..52eeb7e536f --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.lang.MutableInteger; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Flavor; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Performs preparation of node activation changes for an application. + * + * @author bratseth + */ +class Preparer { + + private final NodeRepository nodeRepository; + private final Clock clock; + private final GroupPreparer groupPreparer; + + public Preparer(NodeRepository nodeRepository, Clock clock) { + this.nodeRepository = nodeRepository; + this.clock = clock; + groupPreparer = new GroupPreparer(nodeRepository, clock); + } + + /** + * Ensure sufficient nodes are reserved or active for the given application and cluster + * + * @return the list of nodes this cluster will have allocated if activated + */ + // Note: This operation may make persisted changes to the set of reserved and inactive nodes, + // but it may not change the set of active nodes, as the active nodes must stay in sync with the + // active config model which is changed on activate + public List<Node> prepare(ApplicationId application, ClusterSpec cluster, int nodes, Flavor flavor, int groups) { + if (cluster.group().isPresent() && groups > 1) + throw new IllegalArgumentException("Cannot specify both a particular group and request multiple groups"); + if (nodes > 0 && nodes % groups != 0) + throw new IllegalArgumentException("Requested " + nodes + " nodes in " + groups + " groups, " + + "which doesn't allow the nodes to be divided evenly into groups"); + + // no group -> this asks for the entire cluster -> we are free to remove groups we won't need + List<Node> surplusActiveNodes = + cluster.group().isPresent() ? new ArrayList<>() : findNodesInRemovableGroups(application, cluster, groups); + + MutableInteger highestIndex = new MutableInteger(findHighestIndex(application, cluster)); + List<Node> acceptedNodes = new ArrayList<>(); + for (int groupIndex = 0; groupIndex < groups; groupIndex++) { + // Generated groups always have contiguous indexes starting from 0 + ClusterSpec clusterGroup = + cluster.group().isPresent() ? cluster : cluster.changeGroup(Optional.of(ClusterSpec.Group.from(String.valueOf(groupIndex)))); + + List<Node> accepted = groupPreparer.prepare(application, clusterGroup, nodes/groups, flavor, surplusActiveNodes, highestIndex); + replace(acceptedNodes, accepted); + } + replace(acceptedNodes, retire(surplusActiveNodes)); + return acceptedNodes; + } + + /** + * Returns a list of the nodes which are + * in groups with index number above or equal the group count + */ + private List<Node> findNodesInRemovableGroups(ApplicationId application, ClusterSpec requestedCluster, int groups) { + List<Node> surplusActiveNodes = new ArrayList<>(0); + for (Node node : nodeRepository.getNodes(application, Node.State.active)) { + ClusterSpec nodeCluster = node.allocation().get().membership().cluster(); + if ( ! nodeCluster.id().equals(requestedCluster.id())) continue; + if ( ! nodeCluster.type().equals(requestedCluster.type())) continue; + if (Integer.parseInt(nodeCluster.group().get().value()) >= groups) + surplusActiveNodes.add(node); + } + return surplusActiveNodes; + } + + private List<Node> replace(List<Node> list, List<Node> changed) { + list.removeAll(changed); + list.addAll(changed); + return list; + } + + /** + * Returns the highest index number of all active and failed nodes in this cluster, or -1 if there are no nodes. + * We include failed nodes to avoid reusing the index of the failed node in the case where the failed node is the + * node with the highest index. + */ + private int findHighestIndex(ApplicationId application, ClusterSpec cluster) { + int highestIndex = -1; + for (Node node : nodeRepository.getNodes(application, Node.State.active, Node.State.failed)) { + ClusterSpec nodeCluster = node.allocation().get().membership().cluster(); + if ( ! nodeCluster.id().equals(cluster.id())) continue; + if ( ! nodeCluster.type().equals(cluster.type())) continue; + + highestIndex = Math.max(node.allocation().get().membership().index(), highestIndex); + } + return highestIndex; + } + + /** Returns retired copies of the given nodes, unless they are removable */ + private List<Node> retire(List<Node> nodes) { + List<Node> retired = new ArrayList<>(nodes.size()); + for (Node node : nodes) { + if ( ! node.allocation().get().removable()) + retired.add(node.retireByApplication(clock.instant())); + } + return retired; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java new file mode 100644 index 00000000000..f85d8ae2924 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.osgi.annotation.ExportPackage; + +/** Implements the provisioning API to perform node provisioning form a node repository */
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializer.java new file mode 100644 index 00000000000..321a75421a2 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializer.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.vespa.hosted.provision.Node; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * Converts {@link Node.State} to/from serialized form in REST APIs. + * + * @author bakksjo + */ +public class NodeStateSerializer { + + private static final Map<Node.State, String> serializationMap = new HashMap<>(); + private static final Map<String, Node.State> deserializationMap = new HashMap<>(); + + private static void addMapping(final Node.State nodeState, final String wireName) { + serializationMap.put(nodeState, wireName); + deserializationMap.put(wireName, nodeState); + } + + static { + // Alphabetical order. No cheating, please - don't use .name(), .toString(), reflection etc. to get wire name. + addMapping(Node.State.active, "active"); + addMapping(Node.State.dirty, "dirty"); + addMapping(Node.State.failed, "failed"); + addMapping(Node.State.inactive, "inactive"); + addMapping(Node.State.provisioned, "provisioned"); + addMapping(Node.State.ready, "ready"); + addMapping(Node.State.reserved, "reserved"); + } + + private NodeStateSerializer() {} // Utility class, no instances. + + public static Optional<Node.State> fromWireName(final String wireName) { + return Optional.ofNullable(deserializationMap.get(wireName)); + } + + public static String wireNameOf(final Node.State nodeState) { + final String wireName = serializationMap.get(nodeState); + if (wireName == null) { + throw new RuntimeException("Bug: Unknown serialization form of node state " + nodeState.name()); + } + return wireName; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ContainersForHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ContainersForHost.java new file mode 100644 index 00000000000..a501e7f5a0a --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ContainersForHost.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.legacy; + +import java.util.List; + +/** + * Represents the JSON reply for getContainersForHost. + * Serialized by jackson, and therefore uses public fields to avoid writing cruft. + * + * @author tonytv + */ +public class ContainersForHost { + + public List<DockerContainer> dockerContainers; + + public static class DockerContainer { + public String containerHostname; + public String dockerImage; + public String nodeState; + public long wantedRestartGeneration; + public long currentRestartGeneration; + + public DockerContainer( + String containerHostname, + String dockerImage, + String nodeState, + long wantedRestartGeneration, + long currentRestartGeneration) { + this.containerHostname = containerHostname; + this.dockerImage = dockerImage; + this.nodeState = nodeState; + this.wantedRestartGeneration = wantedRestartGeneration; + this.currentRestartGeneration = currentRestartGeneration; + } + } + + public ContainersForHost(List<DockerContainer> dockerContainers) { + this.dockerContainers = dockerContainers; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/HostInfo.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/HostInfo.java new file mode 100644 index 00000000000..81211f978f7 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/HostInfo.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.legacy; + +/** + * Value class used to automatically convert to/from JSON. + * + * @author Oyvind Gronnesby + */ +class HostInfo { + + public String hostname; + public String openStackId; + public String flavor; + + public static HostInfo createHostInfo(String hostname, String openStackId, String flavor) { + HostInfo hostInfo = new HostInfo(); + hostInfo.hostname = hostname; + hostInfo.openStackId = openStackId; + hostInfo.flavor = flavor; + return hostInfo; + } + + public String toString(){ + return String.format("%s/%s", openStackId, hostname); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionEndpoint.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionEndpoint.java new file mode 100644 index 00000000000..caef4630544 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionEndpoint.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.legacy; + +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +/** + * To avoid duplication of URI construction. + * This class should be deleted when there's a provision client configured in services xml. + * @author tonytv + */ +public class ProvisionEndpoint { + + public static final int configServerPort = 19071; + + public static URI provisionUri(String configServerHostName, int port) { + try { + return new URL("http", configServerHostName, port, "/hack/provision").toURI(); + } catch (URISyntaxException | MalformedURLException e) { + throw new IllegalArgumentException("Failed creating provisionUri from " + configServerHostName, e); + } + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResource.java new file mode 100644 index 00000000000..f4c52010415 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResource.java @@ -0,0 +1,151 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.legacy; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jaxrs.annotation.Component; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.Node.State; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.restapi.NodeStateSerializer; +import com.yahoo.vespa.hosted.provision.restapi.legacy.ContainersForHost.DockerContainer; + +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import java.util.*; +import java.util.function.Predicate; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * The provisioning web service used by the provisioning controller to provide nodes to a node repository. + * + * @author mortent + */ +@Path("/provision") +@Produces(MediaType.APPLICATION_JSON) +public class ProvisionResource { + private static final Logger log = Logger.getLogger(ProvisionResource.class.getName()); + + private final NodeRepository nodeRepository; + + private final NodeFlavors nodeFlavors; + + public ProvisionResource(@Component NodeRepository nodeRepository, @Component NodeFlavors nodeFlavors) { + super(); + this.nodeRepository = nodeRepository; + this.nodeFlavors = nodeFlavors; + } + + + @POST + @Path("/node") + @Consumes(MediaType.APPLICATION_JSON) + public void addNodes(List<HostInfo> hostInfoList) { + List<Node> nodes = new ArrayList<>(); + for (HostInfo hostInfo : hostInfoList) + nodes.add(nodeRepository.createNode(hostInfo.openStackId, hostInfo.hostname, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow(hostInfo.flavor)))); + nodeRepository.addNodes(nodes); + } + + @GET + @Path("/node/required") + public ProvisionStatus getStatus() { + ProvisionStatus provisionStatus = new ProvisionStatus(); + provisionStatus.requiredNodes = 0; // This concept has no meaning any more ... + provisionStatus.decomissionNodes = toHostInfo(nodeRepository.getInactive()); + provisionStatus.failedNodes = toHostInfo(nodeRepository.getFailed()); + + return provisionStatus; + } + + private List<HostInfo> toHostInfo(List<Node> nodes) { + List<HostInfo> hostInfoList = new ArrayList<>(nodes.size()); + for (Node node : nodes) + hostInfoList.add(HostInfo.createHostInfo(node.hostname(), node.openStackId(), "medium")); + return hostInfoList; + } + + + @PUT + @Path("/node/ready") + public void setReady(String hostName) { + if ( nodeRepository.getNode(Node.State.ready, hostName).isPresent()) return; // node already 'ready' + + Optional<Node> node = nodeRepository.getNode(Node.State.provisioned, hostName); + if ( ! node.isPresent()) + node = nodeRepository.getNode(Node.State.dirty, hostName); + if ( ! node.isPresent()) + throw new IllegalArgumentException("Could not set " + hostName + " ready: Not registered as provisioned or dirty"); + + nodeRepository.setReady(Collections.singletonList(node.get())); + } + + @GET + @Path("/node/usage/{tenantId}") + public TenantStatus getTenantUsage(@PathParam("tenantId") String tenantId) { + TenantStatus ts = new TenantStatus(); + ts.tenantId = tenantId; + ts.allocated = nodeRepository.getNodeCount(tenantId, Node.State.active); + ts.reserved = nodeRepository.getNodeCount(tenantId, Node.State.reserved); + + Map<String, TenantStatus.ApplicationUsage> appinstanceUsageMap = new HashMap<>(); + + nodeRepository.getNodes(Node.State.active).stream() + .filter(node -> { + return node.allocation().get().owner().tenant().value().equals(tenantId); + }) + .forEach(node -> { + ApplicationId owner = node.allocation().get().owner(); + appinstanceUsageMap.merge( + String.format("%s:%s", owner.application().value(), owner.instance().value()), + TenantStatus.ApplicationUsage.create(owner.application().value(), owner.instance().value(), 1), + (a, b) -> { + a.usage += b.usage; + return a; + } + ); + }); + + ts.applications = new ArrayList<>(appinstanceUsageMap.values()); + return ts; + } + + //TODO: move this to nodes/v2/ when the spec for this has been nailed. + @GET + @Path("/dockerhost/{hostname}") + public ContainersForHost getContainersForHost(@PathParam("hostname") String hostname) { + List<DockerContainer> dockerContainersForHost = + nodeRepository.getNodes(State.active, State.inactive).stream() + .filter(runsOnDockerHost(hostname)) + .flatMap(ProvisionResource::toDockerContainer) + .collect(Collectors.toList()); + + return new ContainersForHost(dockerContainersForHost); + } + + //returns stream since there is no conversion from optional to stream in java. + private static Stream<DockerContainer> toDockerContainer(Node node) { + try { + String dockerImage = node.allocation().get().membership().cluster().dockerImage().orElseThrow(() -> + new Exception("Docker image not set for node " + node)); + + return Stream.of(new DockerContainer( + node.hostname(), + dockerImage, + NodeStateSerializer.wireNameOf(node.state()), + node.allocation().get().restartGeneration().wanted(), + node.allocation().get().restartGeneration().current())); + } catch (Exception e) { + log.log(LogLevel.ERROR, "Ignoring docker container.", e); + return Stream.empty(); + } + } + + private static Predicate<Node> runsOnDockerHost(String hostname) { + return node -> node.parentHostname().map(hostname::equals).orElse(false); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionStatus.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionStatus.java new file mode 100644 index 00000000000..7e0eb41627f --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionStatus.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.legacy; + +import java.util.List; + +/** + * Value class used to convert to/from JSON. + * + * @author mortent + */ +class ProvisionStatus { + + public int requiredNodes; + public List<HostInfo> decomissionNodes; + public List<HostInfo> failedNodes; + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/TenantStatus.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/TenantStatus.java new file mode 100644 index 00000000000..4f20670fa12 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/TenantStatus.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.legacy; + +import java.util.List; + +/** + * Value class used to convert to/from JSON. + * + * @author Oyvind Gronnesby + */ +class TenantStatus { + + public String tenantId; + public int allocated; + public int reserved; + public List<ApplicationUsage> applications; + + public static class ApplicationUsage { + public String application; + public String instance; + public int usage; + + public static ApplicationUsage create(String applicationId, String instanceId, int usage) { + ApplicationUsage appUsage = new ApplicationUsage(); + appUsage.application = applicationId; + appUsage.instance = instanceId; + appUsage.usage = usage; + return appUsage; + } + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/package-info.java new file mode 100644 index 00000000000..75ffa3e240e --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.provision.restapi.legacy; + +import com.yahoo.osgi.annotation.ExportPackage; + +/** + * Rest API which allows nodes to be added and removed from this node repository + * This API, aptly named "hack" will be removed once the dependencies are off it - Jon, March 2015 + */
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v1/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v1/NodesApiHandler.java new file mode 100644 index 00000000000..00e232dcfd3 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v1/NodesApiHandler.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v1; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Response; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Allocation; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; + +/** + * The implementation of the /state/v1 API. + * This dumps the content of the node repository on request, possibly with a host filter to return just the single + * matching node. + * + * @author bratseth + */ +public class + NodesApiHandler extends LoggingRequestHandler { + + private final NodeRepository nodeRepository; + + public NodesApiHandler(Executor executor, AccessLog accessLog, NodeRepository nodeRepository) { + super(executor, accessLog); + this.nodeRepository = nodeRepository; + } + + @Override + public HttpResponse handle(HttpRequest request) { + return new NodesResponse(Response.Status.OK, + Optional.ofNullable(request.getProperty("hostname")), nodeRepository); + } + + private static class NodesResponse extends HttpResponse { + + /** If present only the node with this hostname will be present in the response */ + private final Optional<String> hostnameFilter; + private final NodeRepository nodeRepository; + + public NodesResponse(int status, Optional<String> hostnameFilter, NodeRepository nodeRepository) { + super(status); + this.hostnameFilter = hostnameFilter; + this.nodeRepository = nodeRepository; + } + + @Override + public void render(OutputStream stream) throws IOException { + stream.write(toJson()); + } + + @Override + public String getContentType() { + return "application/json"; + } + + private byte[] toJson() throws IOException { + Slime slime = new Slime(); + toSlime(slime.setObject()); + return SlimeUtils.toJsonBytes(slime); + } + + private void toSlime(Cursor root) { + for (Node.State state : Node.State.values()) + toSlime(state, root); + } + + private void toSlime(Node.State state, Cursor object) { + List<Node> nodes = nodeRepository.getNodes(state); + Cursor nodeArray = null; // create if there are nodes + for (Node node : nodes) { + if (hostnameFilter.isPresent() && ! node.hostname().equals(hostnameFilter.get())) continue; + if (nodeArray == null) + nodeArray = object.setArray(state.name()); + toSlime(node, nodeArray.addObject()); + } + } + + private void toSlime(Node node, Cursor object) { + object.setString("id", node.openStackId()); + object.setString("hostname", node.hostname()); + object.setString("flavor", node.configuration().flavor().name()); + Optional<Allocation> allocation = node.allocation(); + if (! allocation.isPresent()) return; + toSlime(allocation.get().owner(), object.setObject("owner")); + toSlime(allocation.get().membership(), object.setObject("membership")); + object.setLong("restartGeneration", allocation.get().restartGeneration().wanted()); + } + + private void toSlime(ApplicationId id, Cursor object) { + object.setString("tenant", id.tenant().value()); + object.setString("application", id.application().value()); + object.setString("instance", id.instance().value()); + } + + private void toSlime(ClusterMembership membership, Cursor object) { + object.setString("clustertype", membership.cluster().type().name()); + object.setString("clusterid", membership.cluster().id().value()); + object.setLong("index", membership.index()); + object.setBool("retired", membership.retired()); + } + + } + +}
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ErrorResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ErrorResponse.java new file mode 100644 index 00000000000..7c5a1fffbc0 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ErrorResponse.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.slime.JsonFormat; + +import java.io.IOException; +import java.io.OutputStream; + +import static com.yahoo.jdisc.Response.Status.*; + +public class ErrorResponse extends HttpResponse { + + private final Slime slime = new Slime(); + + public ErrorResponse(int code, String errorType, String message) { + super(code); + Cursor root = slime.setObject(); + root.setString("error-code", errorType); + root.setString("message", message); + } + + public enum errorCodes { + NOT_FOUND, + BAD_REQUEST, + METHOD_NOT_ALLOWED, + INTERNAL_SERVER_ERROR, + INVALID_APPLICATION_PACKAGE, + UNKNOWN_VESPA_VERSION + } + + public static ErrorResponse notFoundError(String message) { + return new ErrorResponse(NOT_FOUND, errorCodes.NOT_FOUND.name(), message); + } + + public static ErrorResponse internalServerError(String message) { + return new ErrorResponse(INTERNAL_SERVER_ERROR, errorCodes.INTERNAL_SERVER_ERROR.name(), message); + } + + public static ErrorResponse badRequest(String message) { + return new ErrorResponse(BAD_REQUEST, errorCodes.BAD_REQUEST.name(), message); + } + + public static ErrorResponse methodNotAllowed(String message) { + return new ErrorResponse(METHOD_NOT_ALLOWED, errorCodes.METHOD_NOT_ALLOWED.name(), message); + } + + @Override + public void render(OutputStream stream) throws IOException { + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { return "application/json"; } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/MessageResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/MessageResponse.java new file mode 100644 index 00000000000..0c91efa823b --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/MessageResponse.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import java.io.IOException; +import java.io.OutputStream; + +import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; + +/** + * A 200 ok response with a message in JSON + * + * @author bratseth + */ +public class MessageResponse extends HttpResponse { + + private final Slime slime = new Slime(); + + public MessageResponse(String message) { + super(200); + slime.setObject().setString("message", message); + } + + @Override + public void render(OutputStream stream) throws IOException { + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { return "application/json"; } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java new file mode 100644 index 00000000000..5d1b8a65b3c --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.component.Version; +import com.yahoo.io.IOUtils; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Type; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; + +/** + * A class which can take a partial JSON node/v2 node JSON structure and apply it to a node object. + * This is a one-time use object. + * + * @author bratseth + */ +public class NodePatcher { + + private final NodeFlavors nodeFlavors; + + private final Inspector inspector; + private Node node; + + public NodePatcher(NodeFlavors nodeFlavors, InputStream json, Node node) { + try { + inspector = SlimeUtils.jsonToSlime(IOUtils.readBytes(json, 1000 * 1000)).get(); + this.node = node; + this.nodeFlavors = nodeFlavors; + } + catch (IOException e) { + throw new RuntimeException("Error reading request body", e); + } + } + + /** + * Apply the json to the node and return the resulting node + */ + public Node apply() { + inspector.traverse((String name, Inspector value) -> { + try { + node = applyField(name, value); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Could not set field '" + name + "'", e); + } + } ); + return node; + } + + private Node applyField(String name, Inspector value) { + switch (name) { + case "convergedStateVersion" : + return node.setStatus(node.status().setStateVersion(asString(value))); + case "currentRebootGeneration" : + return node.setStatus(node.status().setReboot(node.status().reboot().setCurrent(asLong(value)))); + case "currentRestartGeneration" : + return patchCurrentRestartGeneration(asLong(value)); + case "currentDockerImage" : + return node.setStatus(node.status().setDockerImage(asString(value))); + case "currentVespaVersion" : + return node.setStatus(node.status().setVespaVersion(Version.fromString(asString(value)))); + case "currentHostedVersion" : + return node.setStatus(node.status().setHostedVersion(Version.fromString(asString(value)))); + case "failCount" : + return node.setStatus(node.status().setFailCount(asLong(value).intValue())); + case "flavor" : + return node.setConfiguration(node.configuration().setFlavor(nodeFlavors.getFlavorOrThrow(asString(value)))); + case "hardwareFailure" : + return node.setStatus(node.status().setHardwareFailure(asBoolean(value))); + case "parentHostname" : + return node.setParentHostname(asString(value)); + default : + throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); + } + } + + private Node patchCurrentRestartGeneration(Long value) { + Optional<Allocation> allocation = node.allocation(); + if (allocation.isPresent()) + return node.setAllocation(allocation.get().setRestart(allocation.get().restartGeneration().setCurrent(value))); + else + throw new IllegalArgumentException("Node is not allocated"); + } + + private Long asLong(Inspector field) { + if ( ! field.type().equals(Type.LONG)) + throw new IllegalArgumentException("Expected a LONG value, got a " + field.type()); + return field.asLong(); + } + + private String asString(Inspector field) { + if ( ! field.type().equals(Type.STRING)) + throw new IllegalArgumentException("Expected a STRING value, got a " + field.type()); + return field.asString(); + } + + private boolean asBoolean(Inspector field) { + if ( ! field.type().equals(Type.BOOL)) + throw new IllegalArgumentException("Expected a BOOL value, got a " + field.type()); + return field.asBool(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java new file mode 100644 index 00000000000..9981602e4d0 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -0,0 +1,232 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.config.provision.HostFilter; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.io.IOUtils; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; +import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; +import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; +import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; +import com.yahoo.vespa.hosted.provision.restapi.v2.NodesResponse.ResponseType; +import com.yahoo.yolean.Exceptions; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.logging.Level; + +import static com.yahoo.vespa.config.SlimeUtils.optionalString; + +/** + * The implementation of the /state/v2 API. + * See RestApiTest for documentation. + * + * @author bratseth + */ +public class NodesApiHandler extends LoggingRequestHandler { + + private final NodeRepository nodeRepository; + private final NodeFlavors nodeFlavors; + + public NodesApiHandler(Executor executor, AccessLog accessLog, NodeRepository nodeRepository, NodeFlavors flavors) { + super(executor, accessLog); + this.nodeRepository = nodeRepository; + this.nodeFlavors = flavors; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: return handleGET(request); + case PUT: return handlePUT(request); + case POST: return handlePOST(request); + case DELETE: return handleDELETE(request); + case PATCH: return handlePATCH(request); + default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + } + } + catch (NotFoundException e) { + return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); + } + catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } + catch (RuntimeException e) { + e.printStackTrace(); + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse handleGET(HttpRequest request) { + String path = request.getUri().getPath(); + if (path.equals( "/nodes/v2/")) return ResourcesResponse.fromStrings(request.getUri(), "state", "node", "command"); + if (path.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, nodeRepository); + if (path.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, nodeRepository); + if (path.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, nodeRepository); + if (path.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, nodeRepository); + if (path.equals( "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot"); + return ErrorResponse.notFoundError("Nothing at path '" + request.getUri().getPath() + "'"); + } + + private HttpResponse handlePUT(HttpRequest request) { + String path = request.getUri().getPath(); + // Check paths to disallow illegal state changes + if (path.startsWith("/nodes/v2/state/ready/")) { + return new MessageResponse(setNodeReady(path)); + } + else if (path.startsWith("/nodes/v2/state/failed/")) { + nodeRepository.fail(lastElement(path)); + return new MessageResponse("Moved " + lastElement(path) + " to failed"); + } + else if (path.startsWith("/nodes/v2/state/dirty/")) { + nodeRepository.deallocate(lastElement(path)); + return new MessageResponse("Moved " + lastElement(path) + " to dirty"); + } + else if (path.startsWith("/nodes/v2/state/active/")) { + nodeRepository.unfail(lastElement(path)); + return new MessageResponse("Moved " + lastElement(path) + " to active"); + } + else { + return ErrorResponse.notFoundError("Cannot put to path '" + request.getUri().getPath() + "'"); + } + } + + private HttpResponse handlePATCH(HttpRequest request) { + String path = request.getUri().getPath(); + if ( ! path.startsWith("/nodes/v2/node/")) return ErrorResponse.notFoundError("Nothing at '" + path + "'"); + Node node = nodeFromRequest(request); + nodeRepository.write(new NodePatcher(nodeFlavors, request.getData(), node).apply()); + return new MessageResponse("Updated " + node.hostname()); + } + + private HttpResponse handlePOST(HttpRequest request) { + switch (request.getUri().getPath()) { + case "/nodes/v2/command/restart" : + int restartCount = nodeRepository.restart(toNodeFilter(request)).size(); + return new MessageResponse("Scheduled restart of " + restartCount + " matching nodes"); + case "/nodes/v2/command/reboot" : + int rebootCount = nodeRepository.reboot(toNodeFilter(request)).size(); + return new MessageResponse("Scheduled reboot of " + rebootCount + " matching nodes"); + case "/nodes/v2/node" : + int addedNodes = addNodes(request.getData()); + return new MessageResponse("Added " + addedNodes + " nodes to the provisioned state"); + default: + return ErrorResponse.notFoundError("Nothing at path '" + request.getUri().getPath() + "'"); + } + } + + private HttpResponse handleDELETE(HttpRequest request) { + String path = request.getUri().getPath(); + if (path.startsWith("/nodes/v2/node/")) { + String hostname = lastElement(path); + if (nodeRepository.remove(hostname)) + return new MessageResponse("Removed " + hostname); + else + return ErrorResponse.notFoundError("No node in the failed state with hostname " + hostname); + } + else { + return ErrorResponse.notFoundError("Nothing at path '" + request.getUri().getPath() + "'"); + } + } + + private Node nodeFromRequest(HttpRequest request) { + // TODO: The next 4 lines can be a oneliner when updateNodeAttribute is removed (as we won't allow path suffixes) + String path = request.getUri().getPath(); + String prefixString = "/nodes/v2/node/"; + int beginIndex = path.indexOf(prefixString) + prefixString.length(); + int endIndex = path.indexOf("/", beginIndex); + if (endIndex < 0) endIndex = path.length(); // path ends by ip + String hostname = path.substring(beginIndex, endIndex); + + Optional<Node> node = nodeRepository.getNode(hostname); + if ( ! node.isPresent()) throw new NotFoundException("No node found with hostname " + hostname); + return node.get(); + } + + public int addNodes(InputStream jsonStream) { + List<Node> nodes = createNodesFromSlime(getSlimeFromInputStream(jsonStream).get()); + return nodeRepository.addNodes(nodes).size(); + } + + private static Slime getSlimeFromInputStream(InputStream jsonStream) { + try { + byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000); + return SlimeUtils.jsonToSlime(jsonBytes); + } catch (IOException e) { + throw new RuntimeException(); + } + } + + private List<Node> createNodesFromSlime(Inspector object) { + List<Node> nodes = new ArrayList<>(); + object.traverse((ArrayTraverser) (int i, Inspector item) -> nodes.add(createNode(item))); + return nodes; + } + + private Node createNode(Inspector inspector) { + Optional<String> parentHostname = optionalString(inspector.field("parentHostname")); + + return nodeRepository.createNode( + inspector.field("openStackId").asString(), + inspector.field("hostname").asString(), + parentHostname, + new Configuration(nodeFlavors.getFlavorOrThrow(inspector.field("flavor").asString()))); + } + + // TODO: Move most of this to node repo + public String setNodeReady(String path) { + String hostname = lastElement(path); + if ( nodeRepository.getNode(Node.State.ready, hostname).isPresent()) + return "Nothing done; " + hostname + " is already ready"; + + Optional<Node> node = nodeRepository.getNode(Node.State.provisioned, hostname); + if ( ! node.isPresent()) + node = nodeRepository.getNode(Node.State.dirty, hostname); + if ( ! node.isPresent()) + node = nodeRepository.getNode(Node.State.failed, hostname); + if ( ! node.isPresent()) + throw new IllegalArgumentException("Could not set " + hostname + " ready: Not registered as provisioned, dirty or failed"); + + nodeRepository.setReady(Collections.singletonList(node.get())); + return "Moved " + hostname + " to ready"; + } + + public static NodeFilter toNodeFilter(HttpRequest request) { + NodeFilter filter = NodeHostFilter.from(HostFilter.from(request.getProperty("hostname"), + request.getProperty("flavor"), + request.getProperty("clusterType"), + request.getProperty("clusterId"))); + filter = ApplicationFilter.from(request.getProperty("application"), filter); + filter = StateFilter.from(request.getProperty("state"), filter); + filter = ParentHostFilter.from(request.getProperty("parentHost"), filter); + return filter; + } + + private String lastElement(String path) { + if (path.endsWith("/")) + path = path.substring(0, path.length()-1); + int lastSlash = path.lastIndexOf("/"); + if (lastSlash < 0) return path; + return path.substring(lastSlash + 1, path.length()); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java new file mode 100644 index 00000000000..4e8fbe6099b --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -0,0 +1,214 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; +import com.yahoo.vespa.hosted.provision.restapi.NodeStateSerializer; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.util.List; +import java.util.Optional; + +/** +* @author bratseth +*/ +class NodesResponse extends HttpResponse { + + /** The responses this can create */ + public enum ResponseType { nodeList, stateList, nodesInStateList, singleNode } + + /** The request url minus parameters, with a trailing slash added if missing */ + private final String parentUrl; + + /** The parent url of nodes */ + private final String nodeParentUrl; + + private final NodeFilter filter; + private final boolean recursive; + private final NodeRepository nodeRepository; + + private final Slime slime; + + public NodesResponse(ResponseType type, HttpRequest request, NodeRepository nodeRepository) { + super(200); + this.parentUrl = toParentUrl(request); + this.nodeParentUrl = toNodeParentUrl(request); + filter = NodesApiHandler.toNodeFilter(request); + this.recursive = request.getBooleanProperty("recursive"); + this.nodeRepository = nodeRepository; + + slime = new Slime(); + Cursor root = slime.setObject(); + switch (type) { + case nodeList: nodesToSlime(root); break; + case stateList : statesToSlime(root); break; + case nodesInStateList: nodesToSlime(stateFromString(lastElement(parentUrl)), root); break; + case singleNode : nodeToSlime(lastElement(parentUrl), root); break; + default: throw new IllegalArgumentException(); + } + } + + private String toParentUrl(HttpRequest request) { + URI uri = request.getUri(); + String parentUrl = uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + uri.getPath(); + if ( ! parentUrl.endsWith("/")) + parentUrl = parentUrl + "/"; + return parentUrl; + } + + private String toNodeParentUrl(HttpRequest request) { + URI uri = request.getUri(); + return uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + "/nodes/v2/node/"; + } + + @Override + public void render(OutputStream stream) throws IOException { + stream.write(toJson()); + } + + @Override + public String getContentType() { + return "application/json"; + } + + private byte[] toJson() throws IOException { + return SlimeUtils.toJsonBytes(slime); + } + + private void statesToSlime(Cursor root) { + Cursor states = root.setObject("states"); + for (Node.State state : Node.State.values()) + toSlime(state, states.setObject(NodeStateSerializer.wireNameOf(state))); + } + + private void toSlime(Node.State state, Cursor object) { + object.setString("url", parentUrl + NodeStateSerializer.wireNameOf(state)); + if (recursive) + nodesToSlime(state, object); + } + + /** Outputs the nodes in the given state to a node array */ + private void nodesToSlime(Node.State state, Cursor parentObject) { + Cursor nodeArray = parentObject.setArray("nodes"); + toSlime(nodeRepository.getNodes(state), nodeArray); + } + + /** Outputs all the nodes to a node array */ + private void nodesToSlime(Cursor parentObject) { + Cursor nodeArray = parentObject.setArray("nodes"); + for (Node.State state : Node.State.values()) + toSlime(nodeRepository.getNodes(state), nodeArray); + } + + private void toSlime(List<Node> nodes, Cursor array) { + for (Node node : nodes) { + if ( ! filter.matches(node)) continue; + toSlime(node, recursive, array.addObject()); + } + } + + private void nodeToSlime(String hostname, Cursor object) { + Optional<Node> node = nodeRepository.getNode(hostname); + if (! node.isPresent()) + throw new IllegalArgumentException("No node with hostname '" + hostname + "'"); + toSlime(node.get(), true, object); + } + + private void toSlime(Node node, boolean allFields, Cursor object) { + object.setString("url", nodeParentUrl + node.hostname()); + if ( ! allFields) return; + object.setString("id", node.id()); + object.setString("state", NodeStateSerializer.wireNameOf(node.state())); + object.setString("hostname", node.hostname()); + if (node.parentHostname().isPresent()) { + object.setString("parentHostname", node.parentHostname().get()); + } + object.setString("openStackId", node.openStackId()); + object.setString("flavor", node.configuration().flavor().name()); + if (node.configuration().flavor().getMinDiskAvailableGb() > 0) { + object.setDouble("minDiskAvailableGb", node.configuration().flavor().getMinDiskAvailableGb()); + } + if (node.configuration().flavor().getMinMainMemoryAvailableGb() > 0) { + object.setDouble("minMainMemoryAvailableGb", node.configuration().flavor().getMinMainMemoryAvailableGb()); + } + if (node.configuration().flavor().getDescription() != null && ! node.configuration().flavor().getDescription().isEmpty()) { + object.setString("description", node.configuration().flavor().getDescription()); + } + if (node.configuration().flavor().getMinCpuCores() > 0) { + object.setDouble("minCpuCores", node.configuration().flavor().getMinCpuCores()); + } + object.setString("canonicalFlavor", node.configuration().flavor().canonicalName()); + if (node.configuration().flavor().cost() > 0) { + object.setLong("cost", node.configuration().flavor().cost()); + } + if (node.configuration().flavor().getEnvironment() != null && ! node.configuration().flavor().getEnvironment().isEmpty()) { + object.setString("environment", node.configuration().flavor().getEnvironment()); + } + Optional<Allocation> allocation = node.allocation(); + if (allocation.isPresent()) { + toSlime(allocation.get().owner(), object.setObject("owner")); + toSlime(allocation.get().membership(), object.setObject("membership")); + object.setLong("restartGeneration", allocation.get().restartGeneration().wanted()); + object.setLong("currentRestartGeneration", allocation.get().restartGeneration().current()); + allocation.get().membership().cluster().dockerImage().ifPresent( + image -> object.setString("wantedDockerImage", image)); + } + object.setLong("rebootGeneration", node.status().reboot().wanted()); + object.setLong("currentRebootGeneration", node.status().reboot().current()); + node.status().vespaVersion().ifPresent(version -> object.setString("vespaVersion", version.toString())); + node.status().hostedVersion().ifPresent(version -> object.setString("hostedVersion", version.toString())); + node.status().dockerImage().ifPresent(image -> object.setString("currentDockerImage", image)); + node.status().stateVersion().ifPresent(version -> object.setString("convergedStateVersion", version)); + object.setLong("failCount", node.status().failCount()); + object.setBool("hardwareFailure", node.status().hardwareFailure()); + toSlime(node.history(), object.setArray("history")); + } + + private void toSlime(ApplicationId id, Cursor object) { + object.setString("tenant", id.tenant().value()); + object.setString("application", id.application().value()); + object.setString("instance", id.instance().value()); + } + + private void toSlime(ClusterMembership membership, Cursor object) { + object.setString("clustertype", membership.cluster().type().name()); + object.setString("clusterid", membership.cluster().id().value()); + object.setString("group", membership.cluster().group().get().value()); + object.setLong("index", membership.index()); + object.setBool("retired", membership.retired()); + } + + private void toSlime(History history, Cursor array) { + for (History.Event event : history.events()) { + Cursor object = array.addObject(); + object.setString("event", event.type().name()); + object.setLong("at", event.at().toEpochMilli()); + } + } + + private String lastElement(String path) { + if (path.endsWith("/")) + path = path.substring(0, path.length()-1); + int lastSlash = path.lastIndexOf("/"); + if (lastSlash < 0) return path; + return path.substring(lastSlash+1, path.length()); + } + + private static Node.State stateFromString(String stateString) { + return NodeStateSerializer.fromWireName(stateString) + .orElseThrow(() -> new RuntimeException("Node state '" + stateString + "' is not known")); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java new file mode 100644 index 00000000000..92403d7588b --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +/** + * Thrown when a resource is not found + * + * @author bratseth + */ +public class NotFoundException extends RuntimeException { + + public NotFoundException(String message) { super(message); } + + public NotFoundException(String message, Throwable cause) { super(message, cause); } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ResourcesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ResourcesResponse.java new file mode 100644 index 00000000000..ff0c8fdb7b1 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ResourcesResponse.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; + +/** A response which lists a set of subresources as full urls */ +public class ResourcesResponse extends HttpResponse { + + private final URI parentUrl; + + private final String[] subResources; + + public ResourcesResponse(URI parentUrl, String[] subResources) { + super(200); + this.parentUrl = parentUrl; + this.subResources = subResources; + } + + @Override + public void render(OutputStream stream) throws IOException { + String parentUrlString = parentUrl.toString(); + if ( ! parentUrlString.endsWith("/")) + parentUrlString = parentUrlString + "/"; + + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor array = root.setArray("resources"); + for (String subResource : subResources) { + array.addObject().setString("url", parentUrlString + subResource + "/"); + } + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { return "application/json"; } + + public static ResourcesResponse fromStrings(URI parentUrl, String ... subResources) { + return new ResourcesResponse(parentUrl, subResources); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java new file mode 100644 index 00000000000..990051b2317 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.testutils; + +/** + * For running NodeRepository API with some mocked data. + * This is used by both NodeAdmin and NodeRepository tests. + * + * @author dybdahl + */ +public class ContainerConfig { + public static final String servicesXmlV2(int port) { + return + "<jdisc version=\"1.0\">" + + " <component id=\"com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors\"/>" + + " <component id=\"com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository\"/>" + + " <handler id=\"com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler\">" + + " <binding>http://*/nodes/v2/*</binding>" + + " </handler>" + + " <http>\n" + + " <server id='myServer' port='" + port + "' />\n" + + " </http>" + + "</jdisc>"; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/FlavorConfigBuilder.java new file mode 100644 index 00000000000..65c8facc094 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/FlavorConfigBuilder.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.testutils; + +import com.yahoo.vespa.config.nodes.NodeRepositoryConfig; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; + +/** + * Sinmplifies creation of a node-repository config containing flavors. + * This is needed because the config builder API is inconvenient. + * + * @author bratseth + */ +public class FlavorConfigBuilder { + + private NodeRepositoryConfig.Builder builder = new NodeRepositoryConfig.Builder(); + + public NodeRepositoryConfig build() { + return new NodeRepositoryConfig(builder); + } + + public NodeRepositoryConfig.Flavor.Builder addFlavor(String flavorName, double cpu, double mem, double disk, String environment) { + NodeRepositoryConfig.Flavor.Builder flavor = new NodeRepositoryConfig.Flavor.Builder(); + flavor.name(flavorName); + flavor.description("Flavor-name-is-" + flavorName); + flavor.minDiskAvailableGb(disk); + flavor.minCpuCores(cpu); + flavor.minMainMemoryAvailableGb(mem); + flavor.environment(environment); + builder.flavor(flavor); + return flavor; + } + + public void addReplaces(String replaces, NodeRepositoryConfig.Flavor.Builder flavor) { + NodeRepositoryConfig.Flavor.Replaces.Builder flavorReplaces = new NodeRepositoryConfig.Flavor.Replaces.Builder(); + flavorReplaces.name(replaces); + flavor.replaces(flavorReplaces); + } + + public void addCost(int cost, NodeRepositoryConfig.Flavor.Builder flavor) { + flavor.cost(cost); + } + + /** Convenience method which creates a node flavors instance from a list of flavor names */ + public static NodeFlavors createDummies(String... flavors) { + + FlavorConfigBuilder flavorConfigBuilder = new FlavorConfigBuilder(); + for (String flavorName : flavors) { + flavorConfigBuilder.addFlavor(flavorName, 1. /* cpu*/ , 3. /* mem GB*/, 2. /*disk GB*/, "foo" /* env*/); + } + return new NodeFlavors(flavorConfigBuilder.build()); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeFlavors.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeFlavors.java new file mode 100644 index 00000000000..e5e9bd27cd8 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeFlavors.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.testutils; + +import com.yahoo.vespa.config.nodes.NodeRepositoryConfig; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; + +/** + * A mock repository prepopulated with flavors, to avoid having config. + * Instantiated by DI from application package above. + */ +public class MockNodeFlavors extends NodeFlavors { + + public MockNodeFlavors() { + super(createConfig()); + } + + private static NodeRepositoryConfig createConfig() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("default", 2., 16., 400, "env"); + b.addFlavor("medium-disk", 6., 12., 56, "foo"); + b.addFlavor("large", 4., 32., 1600, "env"); + b.addFlavor("docker", 0.2, 0.5, 100, "docker"); + NodeRepositoryConfig.Flavor.Builder largeVariant = b.addFlavor("large-variant", 64, 128, 2000, "env"); + b.addReplaces("large", largeVariant); + NodeRepositoryConfig.Flavor.Builder expensiveFlavor = b.addFlavor("expensive", 0, 0, 0, ""); + b.addReplaces("default", expensiveFlavor); + b.addCost(200, expensiveFlavor); + + return b.build(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java new file mode 100644 index 00000000000..d60e43cebed --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.testutils; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.node.Status; +import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * A mock repository prepopulated with some applications. + * Instantiated by DI from application package above. + */ +public class MockNodeRepository extends NodeRepository { + + private final NodeFlavors flavors; + + /** + * Constructor + * @param flavors flavors to have in node repo + */ + public MockNodeRepository(NodeFlavors flavors) throws Exception { + super(flavors, new MockCurator(), Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z"))); + this.flavors = flavors; + populate(); + } + + private void populate() { + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, flavors, Zone.defaultZone()); + + List<Node> nodes = new ArrayList<>(); + nodes.add(createNode("node1", "host1.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes.add(createNode("node2", "host2.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes.add(createNode("node3", "host3.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("expensive")))); + + // TODO: Use docker flavor + Node node4 = createNode("node4", "host4.yahoo.com", Optional.of("dockerhost4"), new Configuration(flavors.getFlavorOrThrow("default"))); + node4 = node4.setStatus(node4.status().setDockerImage("image-12")); + nodes.add(node4); + + Node node5 = createNode("node5", "host5.yahoo.com", Optional.of("dockerhost"), new Configuration(flavors.getFlavorOrThrow("default"))); + nodes.add(node5.setStatus(node5.status().setDockerImage("image-123"))); + + nodes.add(createNode("node6", "host6.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes.add(createNode("node7", "host7.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + // 8 and 9 are added by web service calls + Node node10 = createNode("node10", "host10.yahoo.com", Optional.of("parent.yahoo.com"), new Configuration(flavors.getFlavorOrThrow("default"))); + Status node10newStatus = node10.status(); + node10newStatus = node10newStatus + .setVespaVersion(Version.fromString("5.104.142")) + .setHostedVersion(Version.fromString("2.1.2408")) + .setStateVersion("5.104.142-2.1.2408"); + node10 = node10.setStatus(node10newStatus); + nodes.add(node10); + nodes = addNodes(nodes); + nodes.remove(6); + setReady(nodes); + fail("host5.yahoo.com"); + + ApplicationId app1 = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1")); + ClusterSpec cluster1 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Optional.empty(), Optional.of("image-123")); + provisioner.prepare(app1, cluster1, Capacity.fromNodeCount(2), 1, null); + + ApplicationId app2 = ApplicationId.from(TenantName.from("tenant2"), ApplicationName.from("application2"), InstanceName.from("instance2")); + ClusterSpec cluster2 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("id2"), Optional.empty()); + activate(provisioner.prepare(app2, cluster2, Capacity.fromNodeCount(2), 1, null), app2, provisioner); + + ApplicationId app3 = ApplicationId.from(TenantName.from("tenant3"), ApplicationName.from("application3"), InstanceName.from("instance3")); + ClusterSpec cluster3 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("id3"), Optional.empty()); + activate(provisioner.prepare(app3, cluster3, Capacity.fromNodeCount(2), 1, null), app3, provisioner); + } + + private void activate(List<HostSpec> hosts, ApplicationId application, NodeRepositoryProvisioner provisioner) { + NestedTransaction transaction = new NestedTransaction(); + provisioner.activate(transaction, application, hosts); + transaction.commit(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md new file mode 100644 index 00000000000..0ea723fe80e --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md @@ -0,0 +1,3 @@ +The test resources are used by both NoadAdmin and NodeRepository +tests to verify APIs. So when modifying this test data +remember to check tests for both NodeAdmin and NodeRepository.
\ No newline at end of file diff --git a/node-repository/src/main/resources/configdefinitions/node-repository.def b/node-repository/src/main/resources/configdefinitions/node-repository.def new file mode 100644 index 00000000000..cd053adca61 --- /dev/null +++ b/node-repository/src/main/resources/configdefinitions/node-repository.def @@ -0,0 +1,34 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Configuration of the node repository +namespace=vespa.config.nodes + +# A node flavor which (may) be available in this zone. +# This is to allow configuration per flavor. +# If a certain flavor has no config it is not necessary to list it here to use it. +flavor[].name string + +# Names of other flavors (whether mentioned in this config or not) which this flavor +# is a replacement for: If one of these flavor names are requested, this flavor may +# be assigned instead. +# Replacements are transitive: If flavor a replaces b replaces c, then a request for flavor +# c may be satisfied by assigning nodes of flavor a. +flavor[].replaces[].name string + +# The monthly Total Cost of Ownership (TCO) in USD. Typically calculated as TCO divered by +# the expected lifetime of the node (usually three years). +flavor[].cost int default=0 + +# The type of node (e.g. bare metal, docker..). +flavor[].environment string default="undefined" + +# The minimum number of CPU cores available. +flavor[].minCpuCores double default=0.0 + +# The minimum amount of main memory available. +flavor[].minMainMemoryAvailableGb double default=0.0 + +# The minimum amount of disk available. +flavor[].minDiskAvailableGb double default=0.0 + +# Human readable free text for description of node. +flavor[].description string default="" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeList.java new file mode 100644 index 00000000000..2d587b12ddf --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision; + +import com.google.common.collect.ImmutableList; +import com.yahoo.config.provision.ClusterSpec; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * A filterable node list + * + * @author bratseth + */ +public class NodeList { + + private final List<Node> nodes; + + public NodeList(List<Node> nodes) { + this.nodes = ImmutableList.copyOf(nodes); + } + + /** Returns the subset of nodes which are retired */ + public NodeList retired() { + return new NodeList(nodes.stream().filter(node -> node.allocation().get().membership().retired()).collect(Collectors.toList())); + } + + /** Returns the subset of nodes which are not retired */ + public NodeList nonretired() { + return new NodeList(nodes.stream().filter(node -> ! node.allocation().get().membership().retired()).collect(Collectors.toList())); + } + + /** Returns the subset of nodes of the given flavor */ + public NodeList flavor(String flavor) { + return new NodeList(nodes.stream().filter(node -> node.configuration().flavor().name().equals(flavor)).collect(Collectors.toList())); + } + + /** Returns the subset of nodes which does not have the given flavor */ + public NodeList notFlavor(String flavor) { + return new NodeList(nodes.stream().filter(node -> ! node.configuration().flavor().name().equals(flavor)).collect(Collectors.toList())); + } + + /** Returns the subset of nodes assigned to the given cluster type */ + public NodeList type(ClusterSpec.Type type) { + return new NodeList(nodes.stream().filter(node -> node.allocation().get().membership().cluster().type().equals(type)).collect(Collectors.toList())); + } + + public int size() { return nodes.size(); } + + /** Returns the immutable list of nodes in this */ + public List<Node> asList() { return nodes; } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClientTest.java new file mode 100644 index 00000000000..9cbe17dd718 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClientTest.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.assimilate; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import java.time.Clock; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * @author vegard + */ +public class PopulateClientTest { + + final static String servicesXmlFilename = "src/test/resources/services.xml"; + final static String hostsXmlFilename = "src/test/resources/hosts.xml"; + + final List<String> hostnames = Arrays.asList("hostname1", "hostname2", "hostname3", "hostname4", "hostname5", "hostname6"); + final List<String> clusterTypes = Arrays.asList("container", "container", "content", "content", "content", "content"); + final List<String> clusterIds = Arrays.asList("default", "default", "default", "default", "mycontent", "mycontent"); + final List<Integer> nodeIndices = Arrays.asList(0, 1, 99, 42, 0, 1); + + final String tenantId = "vegard"; + final String applicationId = "killer-app"; + final String instanceId = "default"; + + final Map<String, String> flavorSpec = ImmutableMap.of( + "container.default", "vanilla", + "content.default", "strawberry", + "content.mycontent", "chocolate" + ); + + NodeFlavors flavors = FlavorConfigBuilder.createDummies(flavorSpec.values().stream().collect(Collectors.toList()).toArray(new String[flavorSpec.size()])); + + @Test + public void testCorrectDataIsWrittenToZooKeeper() { + Curator curator = new MockCurator(); + CuratorDatabaseClient curatorDatabaseClient = new CuratorDatabaseClient(flavors, curator, Clock.systemUTC()); + + PopulateClient populateClient = new PopulateClient(curator, flavors, tenantId, applicationId, instanceId, servicesXmlFilename, hostsXmlFilename, flavorSpec, false); + populateClient.populate(PopulateClient.CONTAINER_CLUSTER_TYPE); + populateClient.populate(PopulateClient.CONTENT_CLUSTER_TYPE); + + List<Node> nodes = curatorDatabaseClient.getNodes(ApplicationId.from( + TenantName.from(tenantId), + ApplicationName.from(applicationId), + InstanceName.from(instanceId))); + + assertThat("Zookeeper is populated", nodes.size(), is(hostnames.size())); + + nodes.stream().forEach(node -> { + assertThat("Node has allocation", node.allocation(), notNullValue()); + + final Allocation allocation = node.allocation().get(); + assertThat("Application id must match", allocation.owner().application().toString(), is(applicationId)); + assertThat("Tenant id must match", allocation.owner().tenant().toString(), is(tenantId)); + assertThat("Instance id must match", allocation.owner().instance().toString(), is(instanceId)); + + final int index = hostnames.indexOf(node.hostname()); + assertThat("Hostname must be one the hostnames", index, is(not(-1))); + + final String clusterType = allocation.membership().cluster().type().name(); + assertThat("Cluster type must match", clusterType, is(clusterTypes.get(index))); + + final String clusterId = allocation.membership().cluster().id().value(); + assertThat("Cluster id must match", clusterId, is(clusterIds.get(index))); + + assertThat("Flavor must match", node.configuration().flavor().name(), is(flavorSpec.get(clusterType + "." + clusterId))); + assertThat("Node index must match", node.allocation().get().membership().index(), is(nodeIndices.get(index))); + }); + } + + @Test(expected = RuntimeException.class) + public void testNotSpecifyingAFlavorThrowsException() { + Map<String, String> myFlavorSpec = ImmutableMap.of( + "container.default", "vanilla", + "content.default", "strawberry" + // missing content.mycontent + ); + + new PopulateClient(new MockCurator(), flavors, tenantId, applicationId, instanceId, servicesXmlFilename, hostsXmlFilename, myFlavorSpec, false); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java new file mode 100644 index 00000000000..99c6c7d9294 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java @@ -0,0 +1,148 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.test.ManualClock; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class ApplicationMaintainerTest { + + private Curator curator = new MockCurator(); + + @Test + public void test_application_maintenance() throws InterruptedException { + ManualClock clock = new ManualClock(); + Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); + NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock); + + createReadyNodes(15, nodeRepository, nodeFlavors); + + Fixture fixture = new Fixture(zone, nodeRepository, nodeFlavors); + + // Create applications + fixture.activate(); + + // Fail some nodes + nodeRepository.fail(nodeRepository.getNodes(fixture.app1).get(3).hostname()); + nodeRepository.fail(nodeRepository.getNodes(fixture.app2).get(0).hostname()); + nodeRepository.fail(nodeRepository.getNodes(fixture.app2).get(4).hostname()); + int failedInApp1 = 1; + int failedInApp2 = 2; + assertEquals(fixture.wantedNodesApp1 - failedInApp1, nodeRepository.getNodes(fixture.app1, Node.State.active).size()); + assertEquals(fixture.wantedNodesApp2 - failedInApp2, nodeRepository.getNodes(fixture.app2, Node.State.active).size()); + assertEquals(failedInApp1 + failedInApp2, nodeRepository.getNodes(Node.State.failed).size()); + assertEquals(3, nodeRepository.getNodes(Node.State.ready).size()); + + // Cause maintenance deployment which will allocate replacement nodes + fixture.runApplicationMaintainer(); + assertEquals(fixture.wantedNodesApp1, nodeRepository.getNodes(fixture.app1, Node.State.active).size()); + assertEquals(fixture.wantedNodesApp2, nodeRepository.getNodes(fixture.app2, Node.State.active).size()); + assertEquals(0, nodeRepository.getNodes(Node.State.ready).size()); + + // Unfail the previously failed nodes + nodeRepository.unfail(nodeRepository.getNodes(Node.State.failed).get(0).hostname()); + nodeRepository.unfail(nodeRepository.getNodes(Node.State.failed).get(0).hostname()); + nodeRepository.unfail(nodeRepository.getNodes(Node.State.failed).get(0).hostname()); + int unfailedInApp1 = 1; + int unfailedInApp2 = 2; + assertEquals(0, nodeRepository.getNodes(Node.State.failed).size()); + assertEquals(fixture.wantedNodesApp1 + unfailedInApp1, nodeRepository.getNodes(fixture.app1, Node.State.active).size()); + assertEquals(fixture.wantedNodesApp2 + unfailedInApp2, nodeRepository.getNodes(fixture.app2, Node.State.active).size()); + assertEquals("The unfailed nodes are now active but not part of the application", + 0, fixture.getNodes(Node.State.active).retired().size()); + + // Cause maintenance deployment which will update the applications with the re-activated nodes + fixture.runApplicationMaintainer(); + assertEquals("Superflous content nodes are retired", + unfailedInApp2, fixture.getNodes(Node.State.active).retired().size()); + assertEquals("Superflous container nodes are deactivated (this makes little point for container nodes)", + unfailedInApp1, fixture.getNodes(Node.State.inactive).size()); + } + + private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { + List<Node> nodes = new ArrayList<>(count); + for (int i = 0; i < count; i++) + nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodes = nodeRepository.addNodes(nodes); + nodeRepository.setReady(nodes); + } + + private class Fixture { + + final NodeRepository nodeRepository; + final NodeRepositoryProvisioner provisioner; + + final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz")); + final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz")); + final ClusterSpec clusterApp1 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Optional.empty()); + final ClusterSpec clusterApp2 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty()); + final int wantedNodesApp1 = 5; + final int wantedNodesApp2 = 7; + + Fixture(Zone zone, NodeRepository nodeRepository, NodeFlavors flavors) { + this.nodeRepository = nodeRepository; + this.provisioner = new NodeRepositoryProvisioner(nodeRepository, flavors, zone); + } + + void activate() { + activate(app1, clusterApp1, wantedNodesApp1, provisioner); + activate(app2, clusterApp2, wantedNodesApp2, provisioner); + assertEquals(wantedNodesApp1, nodeRepository.getNodes(app1, Node.State.active).size()); + assertEquals(wantedNodesApp2, nodeRepository.getNodes(app2, Node.State.active).size()); + } + + private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodeCount, NodeRepositoryProvisioner provisioner) { + List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(nodeCount), 1, null); + NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); + provisioner.activate(transaction, applicationId, hosts); + transaction.commit(); + } + + void runApplicationMaintainer() { + Map<ApplicationId, MockDeployer.ApplicationContext> apps = new HashMap<>(); + apps.put(app1, new MockDeployer.ApplicationContext(app1, clusterApp1, wantedNodesApp1, Optional.of("default"), 1)); + apps.put(app2, new MockDeployer.ApplicationContext(app2, clusterApp2, wantedNodesApp2, Optional.of("default"), 1)); + MockDeployer deployer = new MockDeployer(provisioner, apps); + new ApplicationMaintainer(deployer, nodeRepository, Duration.ofMinutes(30)).run(); + } + + NodeList getNodes(Node.State ... states) { + return new NodeList(nodeRepository.getNodes(states)); + } + + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java new file mode 100644 index 00000000000..278a9b704b9 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.test.ManualClock; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author bratseth + */ +public class FailedExpirerTest { + + private Curator curator = new MockCurator(); + + @Test + public void ensure_failed_nodes_are_deallocated_in_prod() throws InterruptedException { + NodeRepository nodeRepository = failureScenarioIn(Environment.prod); + + assertEquals(2, nodeRepository.getNodes(Node.State.failed).size()); + assertEquals(1, nodeRepository.getNodes(Node.State.dirty).size()); + assertEquals("node3", nodeRepository.getNodes(Node.State.dirty).get(0).hostname()); + } + + @Test + public void ensure_failed_nodes_are_deallocated_in_dev() throws InterruptedException { + NodeRepository nodeRepository = failureScenarioIn(Environment.dev); + + assertEquals(1, nodeRepository.getNodes(Node.State.failed).size()); + assertEquals(2, nodeRepository.getNodes(Node.State.dirty).size()); + assertEquals("node2", nodeRepository.getNodes(Node.State.failed).get(0).hostname()); + } + + private NodeRepository failureScenarioIn(Environment environment) { + ManualClock clock = new ManualClock(); + NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone(), clock); + + List<Node> nodes = new ArrayList<>(3); + nodes.add(nodeRepository.createNode("node1", "node1", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodes.add(nodeRepository.createNode("node2", "node2", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodes.add(nodeRepository.createNode("node3", "node3", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodeRepository.addNodes(nodes); + + // Set node1 to have failed 4 times before + Node node1 = nodeRepository.getNode("node1").get(); + node1 = node1.setStatus(node1.status().increaseFailCount()); + node1 = node1.setStatus(node1.status().increaseFailCount()); + node1 = node1.setStatus(node1.status().increaseFailCount()); + node1 = node1.setStatus(node1.status().increaseFailCount()); + nodeRepository.write(node1); + + // Set node2 to have a detected hardware failure + Node node2 = nodeRepository.getNode("node2").get(); + node2 = node2.setStatus(node2.status().setHardwareFailure(true)); + nodeRepository.write(node2); + + // Allocate the nodes + nodeRepository.setReady(nodeRepository.getNodes(Node.State.provisioned)); + ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); + ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty()); + provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(3), 1, null); + NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); + provisioner.activate(transaction, applicationId, asHosts(nodes)); + transaction.commit(); + assertEquals(3, nodeRepository.getNodes(Node.State.active).size()); + + // Fail the nodes + nodeRepository.fail("node1"); + nodeRepository.fail("node2"); + nodeRepository.fail("node3"); + assertEquals(3, nodeRepository.getNodes(Node.State.failed).size()); + + // Failure times out + clock.advance(Duration.ofDays(5)); + new FailedExpirer(nodeRepository, new Zone(environment, RegionName.from("us-west-1")), clock, Duration.ofDays(4)).run(); + + return nodeRepository; + } + + private Set<HostSpec> asHosts(List<Node> nodes) { + Set<HostSpec> hosts = new HashSet<>(nodes.size()); + for (Node node : nodes) + hosts.add(new HostSpec(node.hostname(), + node.allocation().isPresent() ? Optional.of(node.allocation().get().membership()) : + Optional.empty())); + return hosts; + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java new file mode 100644 index 00000000000..460ab3906ed --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.test.ManualClock; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author bratseth + */ +public class InactiveAndFailedExpirerTest { + + private Curator curator = new MockCurator(); + + @Test + public void ensure_inactive_and_failed_times_out() throws InterruptedException { + ManualClock clock = new ManualClock(); + NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone(), clock); + + List<Node> nodes = new ArrayList<>(2); + nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodeRepository.addNodes(nodes); + + // Allocate then deallocate 2 nodes + nodeRepository.setReady(nodes); + ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); + ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty()); + provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(2), 1, null); + NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); + provisioner.activate(transaction, applicationId, asHosts(nodes)); + transaction.commit(); + assertEquals(2, nodeRepository.getNodes(Node.State.active).size()); + nodeRepository.deactivate(applicationId); + assertEquals(2, nodeRepository.getNodes(Node.State.inactive).size()); + + // Inactive times out + clock.advance(Duration.ofMinutes(14)); + new InactiveExpirer(nodeRepository, clock, Duration.ofMinutes(10)).run(); + + assertEquals(0, nodeRepository.getNodes(Node.State.inactive).size()); + List<Node> dirty = nodeRepository.getNodes(Node.State.dirty); + assertEquals(2, dirty.size()); + assertFalse(dirty.get(0).allocation().isPresent()); + assertFalse(dirty.get(1).allocation().isPresent()); + + // One node is set back to ready + Node ready = nodeRepository.setReady(Collections.singletonList(dirty.get(0))).get(0); + assertEquals("Allocated history is removed on readying", 1, ready.history().events().size()); + assertEquals(History.Event.Type.readied, ready.history().events().iterator().next().type()); + + // Dirty times out for the other one + clock.advance(Duration.ofMinutes(14)); + new DirtyExpirer(nodeRepository, clock, Duration.ofMinutes(10)).run(); + assertEquals(0, nodeRepository.getNodes(Node.State.dirty).size()); + List<Node> failed = nodeRepository.getNodes(Node.State.failed); + assertEquals(1, failed.size()); + assertEquals(1, failed.get(0).status().failCount()); + } + + private Set<HostSpec> asHosts(List<Node> nodes) { + Set<HostSpec> hosts = new HashSet<>(nodes.size()); + for (Node node : nodes) + hosts.add(new HostSpec(node.hostname(), + node.allocation().isPresent() ? Optional.of(node.allocation().get().membership()) : + Optional.empty())); + return hosts; + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MockDeployer.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MockDeployer.java new file mode 100644 index 00000000000..7e884b35e16 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MockDeployer.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Deployer; +import com.yahoo.config.provision.Deployment; +import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * @author bratseth + */ +public class MockDeployer implements Deployer { + + private final NodeRepositoryProvisioner provisioner; + private final Map<ApplicationId, ApplicationContext> applications; + + /** The number of redeployments done to this */ + public int redeployments = 0; + + /** + * Create a mock deployer which contains a substitute for an application repository, sufficient to + * be able to call provision with the right parameters. + */ + public MockDeployer(NodeRepositoryProvisioner provisioner, Map<ApplicationId, ApplicationContext> applications) { + this.provisioner = provisioner; + this.applications = applications; + } + + @Override + public Optional<Deployment> deployFromLocalActive(ApplicationId id, Duration timeout) { + return Optional.of(new MockDeployment(provisioner, applications.get(id))); + } + + public class MockDeployment implements Deployment { + + private final NodeRepositoryProvisioner provisioner; + private final ApplicationContext application; + + /** The list of hosts prepared in this. Only set after prepare is called (and a provisioner is assigned) */ + private List<HostSpec> preparedHosts; + + private MockDeployment(NodeRepositoryProvisioner provisioner, ApplicationContext application) { + this.provisioner = provisioner; + this.application = application; + } + + @Override + public void prepare() { + preparedHosts = application.prepare(provisioner); + } + + @Override + public void activate() { + redeployments++; + try (NestedTransaction t = new NestedTransaction()) { + provisioner.activate(t, application.id(), preparedHosts); + t.commit(); + } + } + + @Override + public void restart(HostFilter filter) {} + + } + + /** An application context which substitutes for an application repository */ + public static class ApplicationContext { + + private ApplicationId id; + private ClusterSpec cluster; + private int wantedNodes; + private Optional<String> flavor; + private int groups; + + public ApplicationContext(ApplicationId id, ClusterSpec cluster, int wantedNodes, Optional<String> flavor, int groups) { + this.id = id; + this.cluster = cluster; + this.wantedNodes = wantedNodes; + this.flavor = flavor; + this.groups = groups; + } + + public ApplicationId id() { return id; } + + /** Returns the spec of the cluster of this application. Only a single cluster per application is supported */ + public ClusterSpec cluster() { return cluster; } + + private List<HostSpec> prepare(NodeRepositoryProvisioner provisioner) { + return provisioner.prepare(id, cluster, Capacity.fromNodeCount(wantedNodes, flavor), groups, null); + } + + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java new file mode 100644 index 00000000000..39a72ef16e8 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java @@ -0,0 +1,350 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.test.ManualClock; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException; +import com.yahoo.vespa.orchestrator.ApplicationStateChangeDeniedException; +import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException; +import com.yahoo.vespa.orchestrator.BatchInternalErrorException; +import com.yahoo.vespa.orchestrator.HostNameNotFoundException; +import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; +import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; +import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.service.monitor.ServiceMonitor; +import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests automatic failing of nodes. + * + * @author bratseth + */ +public class NodeFailerTest { + + // Immutable components + private static final Zone ZONE = new Zone(Environment.prod, RegionName.from("us-east")); + private static final NodeFlavors NODE_FLAVORS = FlavorConfigBuilder.createDummies("default"); + private static final ApplicationId APP_1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz")); + private static final ApplicationId APP_2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz")); + private static final Duration DOWNTIME_LIMIT_ONE_HOUR = Duration.ofMinutes(60); + + // Components with state + private ManualClock clock; + private Curator curator; + private ServiceMonitorStub serviceMonitor; + private MockDeployer deployer; + private NodeRepository nodeRepository; + private Orchestrator orchestrator; + private NodeFailer failer; + + @Before + public void setup() { + clock = new ManualClock(); + curator = new MockCurator(); + nodeRepository = new NodeRepository(NODE_FLAVORS, curator, clock); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, NODE_FLAVORS, ZONE); + + createReadyNodes(14, nodeRepository, NODE_FLAVORS); + + // Create applications + ClusterSpec clusterApp1 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Optional.empty()); + ClusterSpec clusterApp2 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty()); + int wantedNodesApp1 = 5; + int wantedNodesApp2 = 7; + activate(APP_1, clusterApp1, wantedNodesApp1, provisioner); + activate(APP_2, clusterApp2, wantedNodesApp2, provisioner); + assertEquals(wantedNodesApp1, nodeRepository.getNodes(APP_1, Node.State.active).size()); + assertEquals(wantedNodesApp2, nodeRepository.getNodes(APP_2, Node.State.active).size()); + + // Create a deployer ... + Map<ApplicationId, MockDeployer.ApplicationContext> apps = new HashMap<>(); + apps.put(APP_1, new MockDeployer.ApplicationContext(APP_1, clusterApp1, wantedNodesApp1, Optional.of("default"), 1)); + apps.put(APP_2, new MockDeployer.ApplicationContext(APP_2, clusterApp2, wantedNodesApp2, Optional.of("default"), 1)); + deployer = new MockDeployer(provisioner, apps); + // ... and a service monitor + serviceMonitor = new ServiceMonitorStub(apps, nodeRepository); + + orchestrator = new OrchestratorMock(); + + failer = new NodeFailer(deployer, serviceMonitor, nodeRepository, + DOWNTIME_LIMIT_ONE_HOUR, clock, orchestrator); + } + + @Test + public void nodes_for_suspended_applications_are_not_failed() throws ApplicationStateChangeDeniedException, ApplicationIdNotFoundException { + orchestrator.suspend(APP_1); + + // Set two nodes down (one for each application) and wait 65 minutes + String host_from_suspended_app = nodeRepository.getNodes(APP_1, Node.State.active).get(1).hostname(); + String host_from_normal_app = nodeRepository.getNodes(APP_2, Node.State.active).get(3).hostname(); + serviceMonitor.setHostDown(host_from_suspended_app); + serviceMonitor.setHostDown(host_from_normal_app); + failer.run(); + clock.advance(Duration.ofMinutes(65)); + failer.run(); + + assertEquals(Node.State.failed, nodeRepository.getNode(host_from_normal_app).get().state()); + assertEquals(Node.State.active, nodeRepository.getNode(host_from_suspended_app).get().state()); + } + + @Test + public void test_node_failing() throws InterruptedException { + // For a day all nodes work so nothing happens + for (int minutes = 0; minutes < 24 * 60; minutes +=5 ) { + failer.run(); + clock.advance(Duration.ofMinutes(5)); + assertEquals( 0, deployer.redeployments); + assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); + assertEquals( 0, nodeRepository.getNodes(Node.State.failed).size()); + assertEquals( 2, nodeRepository.getNodes(Node.State.ready).size()); + } + + String downHost1 = nodeRepository.getNodes(APP_1, Node.State.active).get(1).hostname(); + String downHost2 = nodeRepository.getNodes(APP_2, Node.State.active).get(3).hostname(); + serviceMonitor.setHostDown(downHost1); + serviceMonitor.setHostDown(downHost2); + // nothing happens the first 45 minutes + for (int minutes = 0; minutes < 45; minutes +=5 ) { + failer.run(); + clock.advance(Duration.ofMinutes(5)); + assertEquals( 0, deployer.redeployments); + assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); + assertEquals( 0, nodeRepository.getNodes(Node.State.failed).size()); + assertEquals( 2, nodeRepository.getNodes(Node.State.ready).size()); + } + serviceMonitor.setHostUp(downHost1); + for (int minutes = 0; minutes < 30; minutes +=5 ) { + failer.run(); + clock.advance(Duration.ofMinutes(5)); + } + + // downHost2 should now be failed and replaced, but not downHost1 + assertEquals( 1, deployer.redeployments); + assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); + assertEquals( 1, nodeRepository.getNodes(Node.State.failed).size()); + assertEquals( 1, nodeRepository.getNodes(Node.State.ready).size()); + assertEquals(downHost2, nodeRepository.getNodes(Node.State.failed).get(0).hostname()); + + // downHost1 fails again + serviceMonitor.setHostDown(downHost1); + failer.run(); + clock.advance(Duration.ofMinutes(5)); + // the system goes down and do not have updated information when coming back + clock.advance(Duration.ofMinutes(120)); + serviceMonitor.setStatusIsKnown(false); + failer.run(); + // due to this, nothing is failed + assertEquals( 1, deployer.redeployments); + assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); + assertEquals( 1, nodeRepository.getNodes(Node.State.failed).size()); + assertEquals( 1, nodeRepository.getNodes(Node.State.ready).size()); + // when status becomes known, and the host is still down, it is failed + clock.advance(Duration.ofMinutes(5)); + serviceMonitor.setStatusIsKnown(true); + failer.run(); + assertEquals( 2, deployer.redeployments); + assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); + assertEquals( 2, nodeRepository.getNodes(Node.State.failed).size()); + assertEquals( 0, nodeRepository.getNodes(Node.State.ready).size()); + + // the last host goes down + Node lastNode = highestIndex(nodeRepository.getNodes(APP_1, Node.State.active)); + serviceMonitor.setHostDown(lastNode.hostname()); + // it is not failed because there are no ready nodes to replace it + for (int minutes = 0; minutes < 75; minutes +=5 ) { + failer.run(); + clock.advance(Duration.ofMinutes(5)); + assertEquals( 2, deployer.redeployments); + assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); + assertEquals( 2, nodeRepository.getNodes(Node.State.failed).size()); + assertEquals( 0, nodeRepository.getNodes(Node.State.ready).size()); + } + + // A new node is available + createReadyNodes(1, 14, nodeRepository, NODE_FLAVORS); + failer.run(); + // The node is now failed + assertEquals( 3, deployer.redeployments); + assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); + assertEquals( 3, nodeRepository.getNodes(Node.State.failed).size()); + assertEquals( 0, nodeRepository.getNodes(Node.State.ready).size()); + assertTrue("The index of the last failed node is not reused", + highestIndex(nodeRepository.getNodes(APP_1, Node.State.active)).allocation().get().membership().index() + > + lastNode.allocation().get().membership().index()); + } + + private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { + createReadyNodes(count, 0, nodeRepository, nodeFlavors); + } + + private void createReadyNodes(int count, int startIndex, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { + List<Node> nodes = new ArrayList<>(count); + for (int i = startIndex; i < startIndex + count; i++) + nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodes = nodeRepository.addNodes(nodes); + nodeRepository.setReady(nodes); + } + + private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodeCount, NodeRepositoryProvisioner provisioner) { + List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(nodeCount), 1, null); + NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); + provisioner.activate(transaction, applicationId, hosts); + transaction.commit(); + } + + /** Returns the node with the highest membership index from the given set of allocated nodes */ + private Node highestIndex(List<Node> nodes) { + Node highestIndex = null; + for (Node node : nodes) { + if (highestIndex == null || node.allocation().get().membership().index() > + highestIndex.allocation().get().membership().index()) + highestIndex = node; + } + return highestIndex; + } + + private static class ServiceMonitorStub implements ServiceMonitor { + + private final Map<ApplicationId, MockDeployer.ApplicationContext> apps; + private final NodeRepository nodeRepository; + + private Set<String> downHosts = new HashSet<>(); + private boolean statusIsKnown = true; + + /** Create a service monitor where all nodes are initially up */ + public ServiceMonitorStub(Map<ApplicationId, MockDeployer.ApplicationContext> apps, NodeRepository nodeRepository) { + this.apps = apps; + this.nodeRepository = nodeRepository; + } + + public void setHostDown(String hostname) { + downHosts.add(hostname); + } + + public void setHostUp(String hostname) { + downHosts.remove(hostname); + } + + public void setStatusIsKnown(boolean statusIsKnown) { + this.statusIsKnown = statusIsKnown; + } + + private ServiceMonitorStatus getHostStatus(String hostname) { + if ( ! statusIsKnown) return ServiceMonitorStatus.NOT_CHECKED; + if (downHosts.contains(hostname)) return ServiceMonitorStatus.DOWN; + return ServiceMonitorStatus.UP; + } + + @Override + public Map<ApplicationInstanceReference, ApplicationInstance<ServiceMonitorStatus>> queryStatusOfAllApplicationInstances() { + // Convert apps information to the response payload to return + Map<ApplicationInstanceReference, ApplicationInstance<ServiceMonitorStatus>> status = new HashMap<>(); + for (Map.Entry<ApplicationId, MockDeployer.ApplicationContext> app : apps.entrySet()) { + Set<ServiceInstance<ServiceMonitorStatus>> serviceInstances = new HashSet<>(); + for (Node node : nodeRepository.getNodes(app.getValue().id(), Node.State.active)) { + serviceInstances.add(new ServiceInstance<>(new ConfigId("configid"), + new HostName(node.hostname()), + getHostStatus(node.hostname()))); + } + Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters = new HashSet<>(); + serviceClusters.add(new ServiceCluster<>(new ClusterId(app.getValue().cluster().id().value()), + new ServiceType("serviceType"), + serviceInstances)); + TenantId tenantId = new TenantId(app.getKey().tenant().value()); + ApplicationInstanceId applicationInstanceId = new ApplicationInstanceId(app.getKey().application().value()); + status.put(new ApplicationInstanceReference(tenantId, applicationInstanceId), + new ApplicationInstance<>(tenantId, applicationInstanceId, serviceClusters)); + } + return status; + } + + } + + class OrchestratorMock implements Orchestrator { + + Set<ApplicationId> suspendedApplications = new HashSet<>(); + + @Override + public HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException { + return null; + } + + @Override + public void resume(HostName hostName) throws HostStateChangeDeniedException, HostNameNotFoundException {} + + @Override + public void suspend(HostName hostName) throws HostStateChangeDeniedException, HostNameNotFoundException {} + + @Override + public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationId appId) throws ApplicationIdNotFoundException { + return suspendedApplications.contains(appId) ? ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN : + ApplicationInstanceStatus.NO_REMARKS; + } + + @Override + public Set<ApplicationId> getAllSuspendedApplications() { + return null; + } + + @Override + public void resume(ApplicationId appId) throws ApplicationStateChangeDeniedException, ApplicationIdNotFoundException { + suspendedApplications.remove(appId); + } + + @Override + public void suspend(ApplicationId appId) throws ApplicationStateChangeDeniedException, ApplicationIdNotFoundException { + suspendedApplications.add(appId); + } + + @Override + public void suspendAll(HostName parentHostname, List<HostName> hostNames) throws BatchInternalErrorException, BatchHostStateChangeDeniedException, BatchHostNameNotFoundException { + throw new RuntimeException("Not implemented"); + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java new file mode 100644 index 00000000000..10e685e3f96 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Zone; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import java.time.Duration; + +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author bratseth + */ +public class ReservationExpirerTest { + + private Curator curator = new MockCurator(); + + @Test + public void ensure_reservation_times_out() throws InterruptedException { + ManualClock clock = new ManualClock(); + NodeFlavors flavors = FlavorConfigBuilder.createDummies("default"); + NodeRepository nodeRepository = new NodeRepository(flavors, curator, clock); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, flavors, Zone.defaultZone(), clock); + + List<Node> nodes = new ArrayList<>(2); + nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes = nodeRepository.addNodes(nodes); + + // Reserve 2 nodes + assertEquals(2, nodeRepository.getNodes(Node.State.provisioned).size()); + nodeRepository.setReady(nodes); + ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("bar").instanceName("fuz").build(); + ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty()); + provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(2), 1, null); + assertEquals(2, nodeRepository.getNodes(Node.State.reserved).size()); + + // Reservation times out + clock.advance(Duration.ofMinutes(14)); // Reserved but not used time out + new ReservationExpirer(nodeRepository, clock, Duration.ofMinutes(10)).run(); + + // Assert nothing is reserved + assertEquals(0, nodeRepository.getNodes(Node.State.reserved).size()); + List<Node> dirty = nodeRepository.getNodes(Node.State.dirty); + assertEquals(2, dirty.size()); + assertFalse(dirty.get(0).allocation().isPresent()); + assertFalse(dirty.get(1).allocation().isPresent()); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java new file mode 100644 index 00000000000..f6f26aeced6 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.test.ManualClock; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author bratseth + */ +public class RetiredExpirerTest { + + private Curator curator = new MockCurator(); + + @Test + public void ensure_retired_nodes_time_out() throws InterruptedException { + ManualClock clock = new ManualClock(); + Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); + NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone); + + createReadyNodes(7, nodeRepository, nodeFlavors); + + ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); + + // Allocate content cluster of sizes 7 -> 2 -> 3: + // Should end up with 3 nodes in the cluster (one previously retired), and 3 retired + ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty()); + int wantedNodes; + activate(applicationId, cluster, wantedNodes=7, 1, provisioner); + activate(applicationId, cluster, wantedNodes=2, 1, provisioner); + activate(applicationId, cluster, wantedNodes=3, 1, provisioner); + assertEquals(7, nodeRepository.getNodes(applicationId, Node.State.active).size()); + assertEquals(0, nodeRepository.getNodes(applicationId, Node.State.inactive).size()); + + // Cause inactivation of retired nodes + clock.advance(Duration.ofHours(30)); // Retire period spent + MockDeployer deployer = + new MockDeployer(provisioner, + Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, wantedNodes, Optional.of("default"), 1))); + new RetiredExpirer(nodeRepository, deployer, clock, Duration.ofHours(12)).run(); + assertEquals(3, nodeRepository.getNodes(applicationId, Node.State.active).size()); + assertEquals(4, nodeRepository.getNodes(applicationId, Node.State.inactive).size()); + assertEquals(1, deployer.redeployments); + + // inactivated nodes are not retired + for (Node node : nodeRepository.getNodes(applicationId, Node.State.inactive)) + assertFalse(node.allocation().get().membership().retired()); + } + + @Test + public void ensure_retired_groups_time_out() throws InterruptedException { + ManualClock clock = new ManualClock(); + Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); + NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone); + + createReadyNodes(8, nodeRepository, nodeFlavors); + + ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); + + ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty()); + activate(applicationId, cluster, 8, 8, provisioner); + activate(applicationId, cluster, 1, 1, provisioner); + assertEquals(8, nodeRepository.getNodes(applicationId, Node.State.active).size()); + assertEquals(0, nodeRepository.getNodes(applicationId, Node.State.inactive).size()); + + // Cause inactivation of retired nodes + clock.advance(Duration.ofHours(30)); // Retire period spent + MockDeployer deployer = + new MockDeployer(provisioner, + Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, 1, Optional.of("default"), 1))); + new RetiredExpirer(nodeRepository, deployer, clock, Duration.ofHours(12)).run(); + assertEquals(1, nodeRepository.getNodes(applicationId, Node.State.active).size()); + assertEquals(7, nodeRepository.getNodes(applicationId, Node.State.inactive).size()); + assertEquals(1, deployer.redeployments); + + // inactivated nodes are not retired + for (Node node : nodeRepository.getNodes(applicationId, Node.State.inactive)) + assertFalse(node.allocation().get().membership().retired()); + } + + private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodes, int groups, NodeRepositoryProvisioner provisioner) { + List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(nodes), groups, null); + NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); + provisioner.activate(transaction, applicationId, hosts); + transaction.commit(); + } + + private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { + List<Node> nodes = new ArrayList<>(count); + for (int i = 0; i < count; i++) + nodes.add(nodeRepository.createNode("node" + i, "node" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodes = nodeRepository.addNodes(nodes); + nodeRepository.setReady(nodes); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java new file mode 100644 index 00000000000..d2092e5be13 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.monitoring; + +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; + +/** + * @author oyving + */ +public class ProvisionMetricsTest { + + @Test(timeout = 10_000L) + public void test_registered_metric() throws InterruptedException { + final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); + final Curator curator = new MockCurator(); + final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator); + final Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))); + nodeRepository.addNodes(Collections.singletonList(node)); + + final Map<String, Number> expectedMetrics = new HashMap<>(); + expectedMetrics.put("hostedVespa.provisionedHosts", 1); + expectedMetrics.put("hostedVespa.readyHosts", 0); + expectedMetrics.put("hostedVespa.reservedHosts", 0); + expectedMetrics.put("hostedVespa.activeHosts", 0); + expectedMetrics.put("hostedVespa.inactiveHosts", 0); + expectedMetrics.put("hostedVespa.dirtyHosts", 0); + expectedMetrics.put("hostedVespa.failedHosts", 0); + + final TestMetric metric = new TestMetric(expectedMetrics.size()); + final ProvisionMetrics provisionMetrics = new ProvisionMetrics(metric, nodeRepository); + + metric.latch.await(); + assertEquals(expectedMetrics, metric.values); + + provisionMetrics.deconstruct(); + } + + private static class TestMetric implements Metric { + public CountDownLatch latch; + public Map<String, Number> values = new HashMap<>(); + public Map<String, Context> context = new HashMap<>(); + + public TestMetric(int latchNumber) { + this.latch = new CountDownLatch(latchNumber); + } + + @Override + public void set(String key, Number val, Context ctx) { + values.put(key, val); + context.put(key, ctx); + countDownAboveZero(); + } + + @Override + public void add(String key, Number val, Context ctx) { + values.put(key, val); + context.put(key, ctx); + countDownAboveZero(); + } + + @Override + public Context createContext(Map<String, ?> properties) { + return null; + } + + private void countDownAboveZero() { + if (latch.getCount() == 0) { + throw new AssertionError("Countdown latch too low - check metric.set metric.add calls"); + } + + latch.countDown(); + } + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/NodeFlavorsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/NodeFlavorsTest.java new file mode 100644 index 00000000000..d2934187a44 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/NodeFlavorsTest.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node; + +import com.yahoo.vespa.config.nodes.NodeRepositoryConfig; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + + +public class NodeFlavorsTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Test + public void testReplacesWithBadValue() { + NodeRepositoryConfig.Builder builder = new NodeRepositoryConfig.Builder(); + List<NodeRepositoryConfig.Flavor.Builder> flavorBuilderList = new ArrayList<>(); + NodeRepositoryConfig.Flavor.Builder flavorBuilder = new NodeRepositoryConfig.Flavor.Builder(); + NodeRepositoryConfig.Flavor.Replaces.Builder flavorReplacesBuilder = new NodeRepositoryConfig.Flavor.Replaces.Builder(); + flavorReplacesBuilder.name("non-existing-config"); + flavorBuilder.name("strawberry").cost(2).replaces.add(flavorReplacesBuilder); + flavorBuilderList.add(flavorBuilder); + builder.flavor(flavorBuilderList); + NodeRepositoryConfig config = new NodeRepositoryConfig(builder); + exception.expect(IllegalStateException.class); + exception.expectMessage("Replaces for strawberry pointing to a non existing flavor: non-existing-config"); + new NodeFlavors(config); + } + + @Test + public void testConfigParsing() { + NodeRepositoryConfig.Builder builder = new NodeRepositoryConfig.Builder(); + List<NodeRepositoryConfig.Flavor.Builder> flavorBuilderList = new ArrayList<>(); + { + NodeRepositoryConfig.Flavor.Builder flavorBuilder = new NodeRepositoryConfig.Flavor.Builder(); + NodeRepositoryConfig.Flavor.Replaces.Builder flavorReplacesBuilder = new NodeRepositoryConfig.Flavor.Replaces.Builder(); + flavorReplacesBuilder.name("banana"); + flavorBuilder.name("strawberry").cost(2).replaces.add(flavorReplacesBuilder); + flavorBuilderList.add(flavorBuilder); + } + { + NodeRepositoryConfig.Flavor.Builder flavorBuilder = new NodeRepositoryConfig.Flavor.Builder(); + flavorBuilder.name("banana").cost(3); + flavorBuilderList.add(flavorBuilder); + } + builder.flavor(flavorBuilderList); + NodeRepositoryConfig config = new NodeRepositoryConfig(builder); + NodeFlavors nodeFlavors = new NodeFlavors(config); + assertThat(nodeFlavors.getFlavor("banana").get().cost(), is(3)); + } +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java new file mode 100644 index 00000000000..50d0e56d999 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import java.time.Clock; +import java.util.List; +import static org.junit.Assert.assertEquals; + +/** + * @author Oyvind Gronnesby + */ +public class CuratorDatabaseClientTest { + + private Curator curator = new MockCurator(); + private CuratorDatabaseClient zkClient = new CuratorDatabaseClient(FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC()); + + @Test + public void ensure_can_read_stored_host_with_instance_information() throws Exception { + String zkline = "{\"hostname\":\"oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com\",\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"configuration\":{\"flavor\":\"default\"},\"created\":1421054425159,\"allocated\":1421057746687,\"instance\":{\"tenantId\":\"by_mortent\",\"applicationId\":\"music\",\"instanceId\":\"default\",\"serviceId\":\"container/default/0/0\"}}"; + + curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/allocated/oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com", zkline.getBytes()); + + List<Node> allocatedNodes = zkClient.getNodes(Node.State.active); + assertEquals(1, allocatedNodes.size()); + assertEquals("container/default/0/0", allocatedNodes.get(0).allocation().get().membership().stringValue()); + } + + @Test + public void ensure_can_read_stored_host_information() throws Exception { + String zkline = "{\"hostname\":\"oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com\",\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"configuration\":{\"flavor\":\"default\"},\"created\":1421054425159}"; + curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/ready/oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com", zkline.getBytes()); + + List<Node> allocatedNodes = zkClient.getNodes(Node.State.ready); + assertEquals(1, allocatedNodes.size()); + } + + /** Test that locks can be acquired and released */ + @Test + public void testLocking() { + ApplicationId app = ApplicationId.from(TenantName.from("testTenant"), ApplicationName.from("testApp"), InstanceName.from("testInstance")); + + try (CuratorMutex mutex1 = zkClient.lock(app)) { + mutex1.toString(); // reference to avoid warning + throw new RuntimeException(); + } + catch (RuntimeException expected) { + } + + try (CuratorMutex mutex2 = zkClient.lock(app)) { + mutex2.toString(); // reference to avoid warning + } + + try (CuratorMutex mutex3 = zkClient.lock(app)) { + mutex3.toString(); // reference to avoid warning + } + + } + + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java new file mode 100644 index 00000000000..f675a857bab --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.path.Path; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.curator.transaction.CuratorOperations; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import org.junit.Test; + +import java.util.List; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +/** + * Tests the curator db directly. + * This verifies the details of the current implementation of the database, not just its API; + * breaking this does not necessarily mean that a change is wrong. + * + * @author bratseth + */ +public class CuratorDatabaseTest { + + @Test + public void testTransactionsIncreaseTimer() throws Exception { + MockCurator curator = new MockCurator(); + CuratorDatabase database = new CuratorDatabase(curator, Path.fromString("/"), true); + + assertEquals(0L, (long)curator.counter("/changeCounter").get().get().postValue()); + + commitCreate("/1", database); + commitCreate("/2", database); + commitCreate("/1/1", database); + commitCreate("/2/1", database); + assertEquals(4L, (long)curator.counter("/changeCounter").get().get().postValue()); + + List<String> children1Call0 = database.getChildren(Path.fromString("/1")); // prime the db; this call returns a different instance + List<String> children1Call1 = database.getChildren(Path.fromString("/1")); + List<String> children1Call2 = database.getChildren(Path.fromString("/1")); + assertTrue("We reuse cached data when there are no commits", children1Call1 == children1Call2); + assertEquals(1, database.getChildren(Path.fromString("/2")).size()); + commitCreate("/2/2", database); + List<String> children1Call3 = database.getChildren(Path.fromString("/1")); + assertEquals(2, database.getChildren(Path.fromString("/2")).size()); + assertFalse("We do not reuse cached data in different parts of the tree when there are commits", + children1Call3 == children1Call2); + } + + @Test + public void testTransactionsWithDeactivatedCache() throws Exception { + MockCurator curator = new MockCurator(); + CuratorDatabase database = new CuratorDatabase(curator, Path.fromString("/"), false); + + assertEquals(0L, (long)curator.counter("/changeCounter").get().get().postValue()); + + commitCreate("/1", database); + commitCreate("/2", database); + commitCreate("/1/1", database); + commitCreate("/2/1", database); + assertEquals(4L, (long)curator.counter("/changeCounter").get().get().postValue()); + + List<String> children1Call0 = database.getChildren(Path.fromString("/1")); // prime the db; this call returns a different instance + List<String> children1Call1 = database.getChildren(Path.fromString("/1")); + List<String> children1Call2 = database.getChildren(Path.fromString("/1")); + assertTrue("No cache, no reused data", children1Call1 != children1Call2); + } + + @Test + public void testThatCounterIncreasesAlsoOnCommitFailure() throws Exception { + MockCurator curator = new MockCurator(); + CuratorDatabase database = new CuratorDatabase(curator, Path.fromString("/"), true); + + assertEquals(0L, (long)curator.counter("/changeCounter").get().get().postValue()); + + try { + commitCreate("/1/2", database); // fail as parent does not exist + fail("Expected exception"); + } + catch (Exception expected) { + // expected because the parent does not exist + } + assertEquals(1L, (long)curator.counter("/changeCounter").get().get().postValue()); + } + + @Test + public void testThatCounterIncreasesAlsoOnCommitFailureFromExistingTransaction() throws Exception { + MockCurator curator = new MockCurator(); + CuratorDatabase database = new CuratorDatabase(curator, Path.fromString("/"), true); + + assertEquals(0L, (long)curator.counter("/changeCounter").get().get().postValue()); + + try { + NestedTransaction t = new NestedTransaction(); + CuratorTransaction separateC = new CuratorTransaction(curator); + separateC.add(CuratorOperations.create("/1/2")); // fail as parent does not exist + t.add(separateC); + + CuratorTransaction c = database.newCuratorTransactionIn(t); + c.add(CuratorOperations.create("/1")); // does not fail + + t.commit(); + fail("Expected exception"); + } + catch (Exception expected) { + // expected because the parent does not exist + } + assertEquals(1L, (long)curator.counter("/changeCounter").get().get().postValue()); + } + + private void commitCreate(String path, CuratorDatabase database) { + NestedTransaction t = new NestedTransaction(); + CuratorTransaction c = database.newCuratorTransactionIn(t); + c.add(CuratorOperations.create(path)); + t.commit(); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java new file mode 100644 index 00000000000..7dc52148e8b --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -0,0 +1,240 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.test.ManualClock; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.Node.State; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.Generation; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.time.Duration; +import java.util.Optional; + +/** + * @author bratseth + */ +public class SerializationTest { + + private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", "large", "ugccloud-container"); + private final NodeSerializer nodeSerializer = new NodeSerializer(nodeFlavors); + private final ManualClock clock = new ManualClock(); + + @Test + public void testProvisionedNodeSerialization() { + Node node = createNode(); + + Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + assertEquals(node.id(), copy.id()); + assertEquals(node.hostname(), copy.hostname()); + assertEquals(node.state(), copy.state()); + assertFalse(copy.allocation().isPresent()); + assertEquals(0, copy.history().events().size()); + } + + @Test + public void testReservedNodeSerialization() { + Node node = createNode(); + + clock.advance(Duration.ofMinutes(3)); + assertEquals(0, node.history().events().size()); + node = node.allocate(ApplicationId.from(TenantName.from("myTenant"), + ApplicationName.from("myApplication"), + InstanceName.from("myInstance")), + ClusterMembership.from("content/myId/0/0", Optional.empty()), + clock.instant()); + assertEquals(1, node.history().events().size()); + node = node.setRestart(new Generation(1, 2)); + node = node.setReboot(new Generation(3, 4)); + node = node.setFlavor(FlavorConfigBuilder.createDummies("large").getFlavorOrThrow("large")); + node = node.setStatus(node.status().setVespaVersion(Version.fromString("1.2.3"))); + node = node.setStatus(node.status().increaseFailCount().increaseFailCount()); + node = node.setStatus(node.status().setHardwareFailure(true)); + Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + + assertEquals(node.id(), copy.id()); + assertEquals(node.hostname(), copy.hostname()); + assertEquals(node.state(), copy.state()); + assertEquals(1, copy.allocation().get().restartGeneration().wanted()); + assertEquals(2, copy.allocation().get().restartGeneration().current()); + assertEquals(3, copy.status().reboot().wanted()); + assertEquals(4, copy.status().reboot().current()); + assertEquals("large", copy.configuration().flavor().name()); + assertEquals("1.2.3", copy.status().vespaVersion().get().toString()); + assertEquals(2, copy.status().failCount()); + assertEquals(true, copy.status().hardwareFailure()); + assertEquals(node.allocation().get().owner(), copy.allocation().get().owner()); + assertEquals(node.allocation().get().membership(), copy.allocation().get().membership()); + assertEquals(node.allocation().get().removable(), copy.allocation().get().removable()); + assertEquals(1, copy.history().events().size()); + assertEquals(clock.instant(), copy.history().event(History.Event.Type.reserved).get().at()); + } + + @Test + public void testRebootAndRestartNoCurrentValuesSerialization() { + String nodeData = "{\n" + + " \"rebootGeneration\" : 0,\n" + + " \"configuration\" : {\n" + + " \"flavor\" : \"default\"\n" + + " },\n" + + " \"history\" : [\n" + + " {\n" + + " \"type\" : \"reserved\",\n" + + " \"at\" : 1444391402611\n" + + " }\n" + + " ],\n" + + " \"instance\" : {\n" + + " \"applicationId\" : \"myApplication\",\n" + + " \"tenantId\" : \"myTenant\",\n" + + " \"instanceId\" : \"myInstance\",\n" + + " \"serviceId\" : \"content/myId/0\",\n" + + " \"restartGeneration\" : 0,\n" + + " \"removable\" : false\n" + + " },\n" + + " \"openStackId\" : \"myId\",\n" + + " \"hostname\" : \"myHostname\"\n" + + "}"; + + Node node = nodeSerializer.fromJson(Node.State.provisioned, Utf8.toBytes(nodeData)); + + assertEquals(0, node.status().reboot().wanted()); + assertEquals(0, node.status().reboot().current()); + assertEquals(0, node.allocation().get().restartGeneration().wanted()); + assertEquals(0, node.allocation().get().restartGeneration().current()); + } + + @Test + public void testRetiredNodeSerialization() { + Node node = createNode(); + + clock.advance(Duration.ofMinutes(3)); + assertEquals(0, node.history().events().size()); + node = node.allocate(ApplicationId.from(TenantName.from("myTenant"), + ApplicationName.from("myApplication"), + InstanceName.from("myInstance")), + ClusterMembership.from("content/myId/0", Optional.empty()), + clock.instant()); + assertEquals(1, node.history().events().size()); + clock.advance(Duration.ofMinutes(2)); + node = node.retireByApplication(clock.instant()); + Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + assertEquals(2, copy.history().events().size()); + assertEquals(clock.instant(), copy.history().event(History.Event.Type.retired).get().at()); + assertEquals(History.RetiredEvent.Agent.application, + ((History.RetiredEvent) copy.history().event(History.Event.Type.retired).get()).agent()); + assertTrue(copy.allocation().get().membership().retired()); + + Node removable = copy.setAllocation(node.allocation().get().makeRemovable()); + Node removableCopy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(removable)); + assertTrue(removableCopy.allocation().get().removable()); + } + + @Test + public void testAssimilatedDeserialization() { + Node node = nodeSerializer.fromJson(Node.State.active, "{\"hostname\":\"assimilate2.vespahosted.corp.bf1.yahoo.com\",\"openStackId\":\"\",\"configuration\":{\"flavor\":\"ugccloud-container\"},\"instance\":{\"tenantId\":\"by_mortent\",\"applicationId\":\"ugc-assimilate\",\"instanceId\":\"default\",\"serviceId\":\"container/ugccloud-container/0/0\",\"restartGeneration\":0}}\n".getBytes()); + assertEquals(0, node.history().events().size()); + assertTrue(node.allocation().isPresent()); + assertEquals("ugccloud-container", node.allocation().get().membership().cluster().id().value()); + assertEquals("container", node.allocation().get().membership().cluster().type().name()); + assertEquals("0", node.allocation().get().membership().cluster().group().get().value()); + Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + assertEquals(0, copy.history().events().size()); + } + + @Test + public void testSetFailCount() { + Node node = createNode(); + node = node.allocate(ApplicationId.from(TenantName.from("myTenant"), + ApplicationName.from("myApplication"), + InstanceName.from("myInstance")), + ClusterMembership.from("content/myId/0/0", Optional.empty()), + clock.instant()); + + node = node.setStatus(node.status().setFailCount(0)); + Node copy2 = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + + assertEquals(0, copy2.status().failCount()); + } + + @Test + public void serialize_docker_image() { + Node node = createNode(); + + Optional<String> dockerImage = Optional.of("my-docker-image"); + ClusterMembership clusterMembership = ClusterMembership.from("content/myId/0", dockerImage); + + Node nodeWithAllocation = node.setAllocation( + new Allocation( + ApplicationId.from(TenantName.from("myTenant"), + ApplicationName.from("myApplication"), + InstanceName.from("myInstance")), + clusterMembership, + new Generation(0, 0), + false)); + + Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(nodeWithAllocation)); + assertEquals(dockerImage, deserializedNode.allocation().get().membership().cluster().dockerImage()); + } + + @Test + public void serialize_parentHostname() { + final String parentHostname = "parent.yahoo.com"; + Node node = Node.create("myId", "myHostname", Optional.of(parentHostname), new Configuration(nodeFlavors.getFlavorOrThrow("default"))); + + Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node)); + assertEquals(parentHostname, deserializedNode.parentHostname().get()); + } + + // TODO: Remove when 5.120 is released everywhere + @Test + public void serialize_parentHostname_from_dockerHostHostName() { + final String parentHostname = "parent.yahoo.com"; + String nodeData = "{\n" + + " \"rebootGeneration\" : 0,\n" + + " \"configuration\" : {\n" + + " \"flavor\" : \"default\"\n" + + " },\n" + + " \"history\" : [\n" + + " {\n" + + " \"type\" : \"reserved\",\n" + + " \"at\" : 1444391402611\n" + + " }\n" + + " ],\n" + + " \"instance\" : {\n" + + " \"applicationId\" : \"myApplication\",\n" + + " \"tenantId\" : \"myTenant\",\n" + + " \"instanceId\" : \"myInstance\",\n" + + " \"serviceId\" : \"content/myId/0\",\n" + + " \"restartGeneration\" : 0,\n" + + " \"removable\" : false\n" + + " },\n" + + " \"openStackId\" : \"fooId\",\n" + + " \"hostname\" : \"fooHost\",\n" + + " \"dockerHostHostName\" : \"" + parentHostname + "\"\n" + + "}"; + // No parent hostname, but dockerHostHostName is set, so parent hostname should be set after deserialization + + Node deserializedNode2 = nodeSerializer.fromJson(State.provisioned, Utf8.toBytes(nodeData)); + assertEquals(parentHostname, deserializedNode2.parentHostname().get()); + } + + private Node createNode() { + return Node.create("myId", "myHostname", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java new file mode 100644 index 00000000000..4d7a70fc915 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import org.junit.Test; + +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Tests deployment to docker images which share the same physical host. + * + * @author bratseth + */ +public class DockerProvisioningTest { + private static final String dockerFlavor = "docker1"; + + @Test + public void docker_application_deployment() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + ApplicationId application1 = tester.makeApplicationId(); + + for (int i = 1; i < 10; i++) { + tester.makeReadyDockerNodes(1, dockerFlavor, "dockerHost" + i); + } + + List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")), 7, 1, dockerFlavor); + tester.activate(application1, new HashSet<>(hosts)); + + final NodeList nodes = tester.getNodes(application1, Node.State.active); + assertEquals(7, nodes.size()); + assertEquals(dockerFlavor, nodes.asList().get(0).configuration().flavor().canonicalName()); + } + + // In dev, test and staging you get nodes with default flavor, but we should get specified flavor for docker nodes + @Test + public void get_specified_flavor_not_default_flavor_for_docker() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.test, RegionName.from("corp-us-east-1"))); + ApplicationId application1 = tester.makeApplicationId(); + tester.makeReadyDockerNodes(1, dockerFlavor, "dockerHost"); + + List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")), 1, 1, dockerFlavor); + tester.activate(application1, new HashSet<>(hosts)); + + final NodeList nodes = tester.getNodes(application1, Node.State.active); + assertEquals(1, nodes.size()); + assertEquals(dockerFlavor, nodes.asList().get(0).configuration().flavor().canonicalName()); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java new file mode 100644 index 00000000000..7d9fb1e42d6 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -0,0 +1,196 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.maintenance.MockDeployer; +import com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer; +import org.junit.Ignore; +import org.junit.Test; + +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +/** + * @author bratseth + */ +public class MultigroupProvisioningTest { + + @Test + public void test_provisioning_of_multiple_groups() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application1 = tester.makeApplicationId(); + + tester.makeReadyNodes(21, "default"); + + deploy(application1, 6, 1, tester); + deploy(application1, 6, 2, tester); + deploy(application1, 6, 3, tester); + deploy(application1, 6, 6, tester); + deploy(application1, 6, 1, tester); + deploy(application1, 6, 6, tester); + deploy(application1, 6, 6, tester); + deploy(application1, 6, 2, tester); + deploy(application1, 8, 2, tester); + deploy(application1, 9, 3, tester); + deploy(application1, 9, 3, tester); + deploy(application1, 9, 3, tester); + deploy(application1,12, 4, tester); + deploy(application1, 8, 4, tester); + deploy(application1,12, 4, tester); + deploy(application1, 8, 2, tester); + deploy(application1, 6, 3, tester); + } + + /** + * This demonstrates a case where we end up provisioning new nodes rather than reusing retired nodes + * due to asymmetric group sizes after step 2 (second group has 3 additional retired nodes). + * We probably need to switch to a multipass group allocation procedure to fix this case. + */ + @Test @Ignore + public void test_provisioning_of_groups_with_asymmetry() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application1 = tester.makeApplicationId(); + + tester.makeReadyNodes(21, "default"); + + deploy(application1, 12, 2, tester); + deploy(application1, 9, 3, tester); + deploy(application1,12, 3, tester); + } + + @Test + public void test_provisioning_of_multiple_groups_after_flavor_migration() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application1 = tester.makeApplicationId(); + + tester.makeReadyNodes(10, "small"); + tester.makeReadyNodes(10, "large"); + + deploy(application1, 8, 1, "small", tester); + deploy(application1, 8, 1, "large", tester); + deploy(application1, 8, 8, "large", tester); + } + + @Test + public void test_one_node_and_group_to_two() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application1 = tester.makeApplicationId(); + + tester.makeReadyNodes(10, "small"); + + deploy(application1, Capacity.fromRequiredNodeCount(1, "small"), 1, tester); + deploy(application1, Capacity.fromRequiredNodeCount(2, "small"), 2, tester); + } + + @Test + public void test_one_node_and_group_to_two_with_flavor_migration() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application1 = tester.makeApplicationId(); + + tester.makeReadyNodes(10, "small"); + tester.makeReadyNodes(10, "large"); + + deploy(application1, Capacity.fromRequiredNodeCount(1, "small"), 1, tester); + deploy(application1, Capacity.fromRequiredNodeCount(2, "large"), 2, tester); + } + + @Test + public void test_provisioning_of_multiple_groups_after_flavor_migration_and_exiration() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application1 = tester.makeApplicationId(); + + tester.makeReadyNodes(10, "small"); + tester.makeReadyNodes(10, "large"); + + deploy(application1, 8, 1, "small", tester); + deploy(application1, 8, 1, "large", tester); + + // Expire small nodes + tester.advanceTime(Duration.ofDays(7)); + MockDeployer deployer = + new MockDeployer(tester.provisioner(), + Collections.singletonMap(application1, new MockDeployer.ApplicationContext(application1, cluster(), 8, Optional.of("large"), 1))); + new RetiredExpirer(tester.nodeRepository(), deployer, tester.clock(), Duration.ofHours(12)).run(); + + assertEquals(8, tester.getNodes(application1, Node.State.inactive).flavor("small").size()); + deploy(application1, 8, 8, "large", tester); + } + + private void deploy(ApplicationId application, int nodeCount, int groupCount, String flavor, ProvisioningTester tester) { + deploy(application, Capacity.fromNodeCount(nodeCount, Optional.of(flavor)), groupCount, tester); + } + private void deploy(ApplicationId application, int nodeCount, int groupCount, ProvisioningTester tester) { + deploy(application, Capacity.fromNodeCount(nodeCount, "default"), groupCount, tester); + } + private void deploy(ApplicationId application, Capacity capacity, int groupCount, ProvisioningTester tester) { + int nodeCount = capacity.nodeCount(); + String flavor = capacity.flavor().get(); + + int previousActiveNodeCount = tester.getNodes(application, Node.State.active).flavor(flavor).size(); + + tester.activate(application, prepare(application, capacity, groupCount, tester)); + + assertEquals("Superfluous nodes are retired, but no others - went from " + previousActiveNodeCount + " to " + nodeCount + " nodes", + Math.max(0, previousActiveNodeCount - capacity.nodeCount()), + tester.getNodes(application, Node.State.active).retired().flavor(flavor).size()); + assertEquals("Other flavors are retired", + 0, tester.getNodes(application, Node.State.active).nonretired().notFlavor(capacity.flavor().get()).size()); + + // Check invariants for all nodes + Set<Integer> allIndexes = new HashSet<>(); + for (Node node : tester.getNodes(application, Node.State.active).asList()) { + // Node indexes must be unique + int index = node.allocation().get().membership().index(); + assertFalse("Node indexes are unique", allIndexes.contains(index)); + allIndexes.add(index); + + assertTrue(node.allocation().get().membership().cluster().group().isPresent()); + } + + // Count unretired nodes and groups of the requested flavor + Set<Integer> indexes = new HashSet<>(); + Map<ClusterSpec.Group, Integer> groups = new HashMap<>(); + for (Node node : tester.getNodes(application, Node.State.active).nonretired().flavor(flavor).asList()) { + indexes.add(node.allocation().get().membership().index()); + + ClusterSpec.Group group = node.allocation().get().membership().cluster().group().get(); + groups.put(group, groups.getOrDefault(group, 0) + 1); + + if (groupCount > 1) + assertTrue(Integer.parseInt(group.value()) < groupCount); + } + assertEquals("Total nonretired nodes", nodeCount, indexes.size()); + assertEquals("Total nonretired groups", groupCount, groups.size()); + for (Integer groupSize : groups.values()) + assertEquals("Group size", (long)nodeCount / groupCount, (long)groupSize); + } + + private ClusterSpec cluster() { return ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test")); } + + private Set<HostSpec> prepare(ApplicationId application, Capacity capacity, int groupCount, ProvisioningTester tester) { + return new HashSet<>(tester.prepare(application, cluster(), capacity, groupCount)); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionTest.java new file mode 100644 index 00000000000..0d6a2cd6b9e --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionTest.java @@ -0,0 +1,525 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.Node; +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Various allocation sequence scenarios + * + * @author bratseth + */ +public class ProvisionTest { + + @Test + public void application_deployment_constant_application_size() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application1 = tester.makeApplicationId(); + ApplicationId application2 = tester.makeApplicationId(); + + tester.makeReadyNodes(21, "default"); + + // deploy + SystemState state1 = prepare(application1, 2, 2, 3, 3, "default", tester); + tester.activate(application1, state1.allHosts); + + // redeploy + SystemState state2 = prepare(application1, 2, 2, 3, 3, "default", tester); + state2.assertEquals(state1); + tester.activate(application1, state2.allHosts); + + // deploy another application + SystemState state1App2 = prepare(application2, 2, 2, 3, 3, "default", tester); + assertFalse("Hosts to different apps are disjunct", state1App2.allHosts.removeAll(state1.allHosts)); + tester.activate(application2, state1App2.allHosts); + + // prepare twice + SystemState state3 = prepare(application1, 2, 2, 3, 3, "default", tester); + SystemState state4 = prepare(application1, 2, 2, 3, 3, "default", tester); + state3.assertEquals(state2); + state4.assertEquals(state3); + tester.activate(application1, state4.allHosts); + + // remove nodes before deploying + SystemState state5 = prepare(application1, 2, 2, 3, 3, "default", tester); + HostSpec removed = tester.removeOne(state5.allHosts); + tester.activate(application1, state5.allHosts); + assertEquals(removed.hostname(), tester.nodeRepository().getNodes(application1, Node.State.inactive).get(0).hostname()); + + // remove some of the clusters + SystemState state6 = prepare(application1, 0, 2, 0, 3, "default", tester); + tester.activate(application1, state6.allHosts); + assertEquals(5, tester.getNodes(application1, Node.State.active).size()); + assertEquals(5, tester.getNodes(application1, Node.State.inactive).size()); + + // delete app + tester.provisioner().removed(application1); + assertEquals(tester.toHostNames(state1.allHosts), tester.toHostNames(tester.nodeRepository().getNodes(application1, Node.State.inactive))); + assertEquals(0, tester.getNodes(application1, Node.State.active).size()); + + // other application is unaffected + assertEquals(state1App2.hostNames(), tester.toHostNames(tester.nodeRepository().getNodes(application2, Node.State.active))); + + // fail a node from app2 and make sure it does not get inactive nodes from first + HostSpec failed = tester.removeOne(state1App2.allHosts); + tester.fail(failed); + assertEquals(9, tester.getNodes(application2, Node.State.active).size()); + SystemState state2App2 = prepare(application2, 2, 2, 3, 3, "default", tester); + assertFalse("Hosts to different apps are disjunct", state2App2.allHosts.removeAll(state1.allHosts)); + assertEquals("A new node was reserved to replace the failed one", 10, state2App2.allHosts.size()); + assertFalse("The new host is not the failed one", state2App2.allHosts.contains(failed)); + tester.activate(application2, state2App2.allHosts); + + // deploy first app again + SystemState state7 = prepare(application1, 2, 2, 3, 3, "default", tester); + state7.assertEquals(state1); + tester.activate(application1, state7.allHosts); + assertEquals(0, tester.getNodes(application1, Node.State.inactive).size()); + + // restart + HostFilter allFilter = HostFilter.all(); + HostFilter hostFilter = HostFilter.hostname(state6.allHosts.iterator().next().hostname()); + HostFilter clusterTypeFilter = HostFilter.clusterType(ClusterSpec.Type.container); + HostFilter clusterIdFilter = HostFilter.clusterId(ClusterSpec.Id.from("container1")); + + tester.provisioner().restart(application1, allFilter); + tester.provisioner().restart(application1, hostFilter); + tester.provisioner().restart(application1, clusterTypeFilter); + tester.provisioner().restart(application1, clusterIdFilter); + tester.assertRestartCount(application1, allFilter, hostFilter, clusterTypeFilter, clusterIdFilter); + } + + @Test + public void application_deployment_variable_application_size() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application1 = tester.makeApplicationId(); + + tester.makeReadyNodes(24, "default"); + + // deploy + SystemState state1 = prepare(application1, 2, 2, 3, 3, "default", tester); + tester.activate(application1, state1.allHosts); + + // redeploy with increased sizes + SystemState state2 = prepare(application1, 3, 4, 4, 5, "default", tester); + state2.assertExtends(state1); + assertEquals("New nodes are reserved", 6, tester.getNodes(application1, Node.State.reserved).size()); + tester.activate(application1, state2.allHosts); + + // decrease again + SystemState state3 = prepare(application1, 2, 2, 3, 3, "default", tester); + tester.activate(application1, state3.allHosts); + assertEquals("Superfluous container nodes are deactivated", + 3-2 + 4-2, tester.getNodes(application1, Node.State.inactive).size()); + assertEquals("Superfluous content nodes are retired", + 4-3 + 5-3, tester.getNodes(application1, Node.State.active).retired().size()); + + // increase even more, and remove one node before deploying + SystemState state4 = prepare(application1, 4, 5, 5, 6, "default", tester); + assertEquals("Inactive nodes are reused", 0, tester.getNodes(application1, Node.State.inactive).size()); + assertEquals("Earlier retired nodes are not unretired before activate", + 4-3 + 5-3, tester.getNodes(application1, Node.State.active).retired().size()); + state4.assertExtends(state2); + assertEquals("New and inactive nodes are reserved", 4 + 3, tester.getNodes(application1, Node.State.reserved).size()); + HostSpec removed = tester.removeOne(state4.allHosts); + tester.activate(application1, state4.allHosts); + assertEquals(removed.hostname(), tester.getNodes(application1, Node.State.inactive).asList().get(0).hostname()); + assertEquals("Earlier retired nodes are unretired on activate", + 0, tester.getNodes(application1, Node.State.active).retired().size()); + + // decrease again + SystemState state5 = prepare(application1, 2, 2, 3, 3, "default", tester); + tester.activate(application1, state5.allHosts); + assertEquals("Superfluous container nodes are deactivated", + 4-2 + 5-2, tester.getNodes(application1, Node.State.inactive).size()); + assertEquals("Superfluous content nodes are retired", + 5-3 + 6-3, tester.getNodes(application1, Node.State.active).retired().size()); + + // increase content slightly + SystemState state6 = prepare(application1, 2, 2, 4, 3, "default", tester); + tester.activate(application1, state6.allHosts); + assertEquals("One content node is unretired", + 5-4 + 6-3, tester.getNodes(application1, Node.State.active).retired().size()); + + // Then reserve more + SystemState state7 = prepare(application1, 8, 2, 2, 2, "default", tester); + + // delete app + tester.provisioner().removed(application1); + assertEquals(0, tester.getNodes(application1, Node.State.active).size()); + assertEquals(0, tester.getNodes(application1, Node.State.reserved).size()); + } + + @Test + public void application_deployment_multiple_flavors() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application1 = tester.makeApplicationId(); + + tester.makeReadyNodes(12, "small"); + tester.makeReadyNodes(16, "large"); + + // deploy + SystemState state1 = prepare(application1, 2, 2, 4, 4, "small", tester); + tester.activate(application1, state1.allHosts); + + // redeploy with reduced size (to cause us to have retired nodes before switching flavor) + SystemState state2 = prepare(application1, 2, 2, 3, 3, "small", tester); + tester.activate(application1, state2.allHosts); + + // redeploy with increased sizes and new flavor + SystemState state3 = prepare(application1, 3, 4, 4, 5, "large", tester); + assertEquals("New nodes are reserved", 16, tester.nodeRepository().getNodes(application1, Node.State.reserved).size()); + tester.activate(application1, state3.allHosts); + assertEquals("'small' container nodes are retired because we are swapping the entire cluster", + 2 + 2, tester.getNodes(application1, Node.State.active).retired().type(ClusterSpec.Type.container).flavor("small").size()); + assertEquals("'small' content nodes are retired", + 4 + 4, tester.getNodes(application1, Node.State.active).retired().type(ClusterSpec.Type.content).flavor("small").size()); + assertEquals("No 'large' content nodes are retired", + 0, tester.getNodes(application1, Node.State.active).retired().flavor("large").size()); + } + + @Test + public void application_deployment_multiple_flavors_default_per_type() { + ConfigserverConfig.Builder config = new ConfigserverConfig.Builder(); + config.environment("prod"); + config.region("us-east"); + config.defaultFlavor("not-used"); + config.defaultContainerFlavor("small"); + config.defaultContentFlavor("large"); + ProvisioningTester tester = new ProvisioningTester(new Zone(new ConfigserverConfig(config))); + + ApplicationId application1 = tester.makeApplicationId(); + + tester.makeReadyNodes(10, "small"); + tester.makeReadyNodes(9, "large"); + + // deploy + SystemState state1 = prepare(application1, 2, 3, 4, 5, null, tester); + tester.activate(application1, state1.allHosts); + assertEquals("'small' nodes are used for containers", + 2 + 3, tester.getNodes(application1, Node.State.active).flavor("small").size()); + assertEquals("'large' nodes are used for content", + 4 + 5, tester.getNodes(application1, Node.State.active).flavor("large").size()); + } + + @Test + public void application_deployment_multiple_flavors_with_replacement() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application1 = tester.makeApplicationId(); + + tester.makeReadyNodes(8, "large"); + tester.makeReadyNodes(8, "large-variant"); + + // deploy with flavor which will be fulfilled by some old and new nodes + SystemState state1 = prepare(application1, 2, 2, 4, 4, "old-large1", tester); + tester.activate(application1, state1.allHosts); + + // redeploy with increased sizes, this will map to the remaining old/new nodes + SystemState state2 = prepare(application1, 3, 4, 4, 5, "old-large2", tester); + assertEquals("New nodes are reserved", 4, tester.getNodes(application1, Node.State.reserved).size()); + tester.activate(application1, state2.allHosts); + assertEquals("All nodes are used", + 16, tester.getNodes(application1, Node.State.active).size()); + assertEquals("No nodes are retired", + 0, tester.getNodes(application1, Node.State.active).retired().size()); + + // This is a noop as we are already using large nodes and nodes which replace large + SystemState state3 = prepare(application1, 3, 4, 4, 5, "large", tester); + assertEquals("Noop", 0, tester.getNodes(application1, Node.State.reserved).size()); + tester.activate(application1, state3.allHosts); + + try { + SystemState state4 = prepare(application1, 3, 4, 4, 5, "large-variant", tester); + org.junit.Assert.fail("Should fail as we don't have that many large-variant nodes"); + } + catch (OutOfCapacityException expected) { + } + + // make enough nodes to complete the switch to large-variant + tester.makeReadyNodes(8, "large-variant"); + SystemState state4 = prepare(application1, 3, 4, 4, 5, "large-variant", tester); + assertEquals("New 'large-variant' nodes are reserved", 8, tester.getNodes(application1, Node.State.reserved).size()); + tester.activate(application1, state4.allHosts); + // (we can not check for the precise state here without carrying over from earlier as the distribution of + // old and new on different clusters is unknown) + } + + @Test + public void application_deployment_above_then_at_capacity_limit() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application1 = tester.makeApplicationId(); + + tester.makeReadyNodes(5, "default"); + + // deploy + SystemState state1 = prepare(application1, 2, 0, 3, 0, "default", tester); + tester.activate(application1, state1.allHosts); + + // redeploy a too large application + try { + SystemState state2 = prepare(application1, 3, 0, 3, 0, "default", tester); + org.junit.Assert.fail("Expected out of capacity exception"); + } + catch (OutOfCapacityException expected) { + } + + // deploy first state again + SystemState state3 = prepare(application1, 2, 0, 3, 0, "default", tester); + tester.activate(application1, state3.allHosts); + } + + @Test + public void dev_deployment_size() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.dev, RegionName.from("us-east"))); + + ApplicationId application = tester.makeApplicationId(); + tester.makeReadyNodes(4, "default"); + SystemState state = prepare(application, 2, 2, 3, 3, "default", tester); + assertEquals(4, state.allHosts.size()); + tester.activate(application, state.allHosts); + } + + @Test + public void test_deployment_size() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.test, RegionName.from("us-east"))); + + ApplicationId application = tester.makeApplicationId(); + tester.makeReadyNodes(4, "default"); + SystemState state = prepare(application, 2, 2, 3, 3, "default", tester); + assertEquals(4, state.allHosts.size()); + tester.activate(application, state.allHosts); + } + + @Ignore // TODO: Re-activate when the check is reactivate in CapacityPolicies + @Test(expected = IllegalArgumentException.class) + public void prod_deployment_requires_redundancy() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application = tester.makeApplicationId(); + tester.makeReadyNodes(10, "default"); + prepare(application, 1, 2, 3, 3, "default", tester); + } + + /** Dev always uses the zone default flavor */ + @Test + public void dev_deployment_flavor() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.dev, RegionName.from("us-east"))); + + ApplicationId application = tester.makeApplicationId(); + tester.makeReadyNodes(4, "default"); + SystemState state = prepare(application, 2, 2, 3, 3, "large", tester); + assertEquals(4, state.allHosts.size()); + tester.activate(application, state.allHosts); + } + + /** Test always uses the zone default flavor */ + @Test + public void test_deployment_flavor() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.test, RegionName.from("us-east"))); + + ApplicationId application = tester.makeApplicationId(); + tester.makeReadyNodes(4, "default"); + SystemState state = prepare(application, 2, 2, 3, 3, "large", tester); + assertEquals(4, state.allHosts.size()); + tester.activate(application, state.allHosts); + } + + @Test + public void staging_deployment_size() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.staging, RegionName.from("us-east"))); + + ApplicationId application = tester.makeApplicationId(); + tester.makeReadyNodes(14, "default"); + SystemState state = prepare(application, 1, 1, 1, 64, "default", tester); // becomes 1, 1, 1, 6 + assertEquals(9, state.allHosts.size()); + tester.activate(application, state.allHosts); + } + + @Test + public void activate_after_reservation_timeout() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + tester.makeReadyNodes(10, "default"); + ApplicationId application = tester.makeApplicationId(); + SystemState state = prepare(application, 2, 2, 3, 3, "default", tester); + + // Simulate expiry + tester.nodeRepository().deactivate(application); + + try { + tester.activate(application, state.allHosts); + org.junit.Assert.fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("Activation of " + application + " failed")); + } + } + + @Test + public void out_of_capacity() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + tester.makeReadyNodes(9, "default"); // need 2+2+3+3=10 + ApplicationId application = tester.makeApplicationId(); + try { + prepare(application, 2, 2, 3, 3, "default", tester); + org.junit.Assert.fail("Expected exception"); + } + catch (OutOfCapacityException e) { + assertTrue(e.getMessage().startsWith("Could not satisfy request")); + } + } + + @Test + public void out_of_desired_flavor() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + tester.makeReadyNodes(10, "small"); // need 2+2+3+3=10 + tester.makeReadyNodes( 9, "large"); // need 2+2+3+3=10 + ApplicationId application = tester.makeApplicationId(); + try { + prepare(application, 2, 2, 3, 3, "large", tester); + org.junit.Assert.fail("Expected exception"); + } + catch (OutOfCapacityException e) { + assertTrue(e.getMessage().startsWith("Could not satisfy request for 3 nodes of flavor 'large'")); + } + } + + @Test + public void nonexisting_flavor() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application = tester.makeApplicationId(); + try { + prepare(application, 2, 2, 3, 3, "nonexisting", tester); + org.junit.Assert.fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Unknown flavor 'nonexisting' Flavors are [default, docker1, large, old-large1, old-large2, small, v-4-8-100]", e.getMessage()); + } + } + + private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int group0Size, int group1Size, String flavor, ProvisioningTester tester) { + // "deploy prepare" with a two container clusters and a storage cluster having of two groups + ClusterSpec containerCluster0 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("container0"), Optional.empty()); + ClusterSpec containerCluster1 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), Optional.empty()); + ClusterSpec contentGroup0 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.of(ClusterSpec.Group.from("g0"))); + ClusterSpec contentGroup1 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.of(ClusterSpec.Group.from("g1"))); + + Set<HostSpec> container0 = new HashSet<>(tester.prepare(application, containerCluster0, container0Size, 1, flavor)); + Set<HostSpec> container1 = new HashSet<>(tester.prepare(application, containerCluster1, container1Size, 1, flavor)); + Set<HostSpec> group0 = new HashSet<>(tester.prepare(application, contentGroup0, group0Size, 1, flavor)); + Set<HostSpec> group1 = new HashSet<>(tester.prepare(application, contentGroup1, group1Size, 1, flavor)); + + Set<HostSpec> allHosts = new HashSet<>(); + allHosts.addAll(container0); + allHosts.addAll(container1); + allHosts.addAll(group0); + allHosts.addAll(group1); + + int expectedContainer0Size = tester.capacityPolicies().decideSize(Capacity.fromNodeCount(container0Size)); + int expectedContainer1Size = tester.capacityPolicies().decideSize(Capacity.fromNodeCount(container1Size)); + int expectedGroup0Size = tester.capacityPolicies().decideSize(Capacity.fromNodeCount(group0Size)); + int expectedGroup1Size = tester.capacityPolicies().decideSize(Capacity.fromNodeCount(group1Size)); + + assertEquals("Hosts in each group cluster is disjunct and the total number of unretired nodes is correct", + expectedContainer0Size + expectedContainer1Size + expectedGroup0Size + expectedGroup1Size, + tester.nonretired(allHosts).size()); + // Check cluster/group sizes + assertEquals(expectedContainer0Size, tester.nonretired(container0).size()); + assertEquals(expectedContainer1Size, tester.nonretired(container1).size()); + assertEquals(expectedGroup0Size, tester.nonretired(group0).size()); + assertEquals(expectedGroup1Size, tester.nonretired(group1).size()); + // Check cluster membership + tester.assertMembersOf(containerCluster0, container0); + tester.assertMembersOf(containerCluster1, container1); + tester.assertMembersOf(contentGroup0, group0); + tester.assertMembersOf(contentGroup1, group1); + + return new SystemState(allHosts, container0, container1, group0, group1); + } + + private static class SystemState { + + private Set<HostSpec> allHosts; + private Set<HostSpec> container1; + private Set<HostSpec> container2; + private Set<HostSpec> group1; + private Set<HostSpec> group2; + + public SystemState(Set<HostSpec> allHosts, + Set<HostSpec> container1, + Set<HostSpec> container2, + Set<HostSpec> group1, + Set<HostSpec> group2) { + this.allHosts = allHosts; + this.container1 = container1; + this.container2 = container2; + this.group1 = group1; + this.group2 = group2; + } + + public Set<String> hostNames() { + return allHosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); + } + + public void assertExtends(SystemState other) { + assertTrue(this.allHosts.containsAll(other.allHosts)); + assertExtends(this.container1, other.container1); + assertExtends(this.container2, other.container2); + assertExtends(this.group1, other.group1); + assertExtends(this.group2, other.group2); + } + + private void assertExtends(Set<HostSpec> extension, + Set<HostSpec> original) { + for (HostSpec originalHost : original) { + HostSpec newHost = findHost(originalHost.hostname(), extension); + org.junit.Assert.assertEquals(newHost.membership(), originalHost.membership()); + } + } + + private HostSpec findHost(String hostName, Set<HostSpec> hosts) { + for (HostSpec host : hosts) + if (host.hostname().equals(hostName)) + return host; + return null; + } + + public void assertEquals(SystemState other) { + org.junit.Assert.assertEquals(this.allHosts, other.allHosts); + org.junit.Assert.assertEquals(this.container1, other.container1); + org.junit.Assert.assertEquals(this.container2, other.container2); + org.junit.Assert.assertEquals(this.group1, other.group1); + org.junit.Assert.assertEquals(this.group2, other.group2); + } + + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java new file mode 100644 index 00000000000..01d8a0eeabf --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -0,0 +1,249 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.ProvisionLogger; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.test.ManualClock; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.config.nodes.NodeRepositoryConfig; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; + +import java.io.IOException; +import java.time.temporal.TemporalAmount; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.logging.Level; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * A test utility for provisioning tests. + * + * @author bratseth + */ +public class ProvisioningTester implements AutoCloseable { + + private Curator curator = new MockCurator(); + private NodeFlavors nodeFlavors; + private ManualClock clock; + private NodeRepository nodeRepository; + private NodeRepositoryProvisioner provisioner; + private CapacityPolicies capacityPolicies; + private ProvisionLogger provisionLogger; + + public ProvisioningTester(Zone zone) { + try { + nodeFlavors = new NodeFlavors(createConfig()); + clock = new ManualClock(); + nodeRepository = new NodeRepository(nodeFlavors, curator, clock); + provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, clock); + capacityPolicies = new CapacityPolicies(zone, nodeFlavors); + provisionLogger = new NullProvisionLogger(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + private NodeRepositoryConfig createConfig() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("default", 2., 4., 100, "BARE_METAL"); + b.addFlavor("small", 1., 2., 50, "BARE_METAL"); + b.addFlavor("docker1", 1., 1., 10, "DOCKER_CONTAINER"); + b.addFlavor("v-4-8-100", 4., 8., 100, "VIRTUAL_MACHINE"); + b.addFlavor("old-large1", 2., 4., 100, "BARE_METAL"); + b.addFlavor("old-large2", 2., 5., 100, "BARE_METAL"); + NodeRepositoryConfig.Flavor.Builder large = b.addFlavor("large", 4., 8., 100, "BARE_METAL"); + b.addReplaces("old-large1", large); + b.addReplaces("old-large2", large); + NodeRepositoryConfig.Flavor.Builder largeVariant = b.addFlavor("large-variant", 3., 9., 101, "BARE_METAL"); + b.addReplaces("large", largeVariant); + NodeRepositoryConfig.Flavor.Builder largeVariantVariant = b.addFlavor("large-variant-variant", 4., 9., 101, "BARE_METAL"); + b.addReplaces("large-variant", largeVariantVariant); + return b.build(); + } + + private NodeRepositoryConfig.Flavor.Builder addFlavor(String flavorName, NodeRepositoryConfig.Builder b) { + NodeRepositoryConfig.Flavor.Builder flavor = new NodeRepositoryConfig.Flavor.Builder(); + flavor.name(flavorName); + b.flavor(flavor); + return flavor; + } + + private void addReplaces(String replaces, NodeRepositoryConfig.Flavor.Builder flavor) { + NodeRepositoryConfig.Flavor.Replaces.Builder flavorReplaces = new NodeRepositoryConfig.Flavor.Replaces.Builder(); + flavorReplaces.name(replaces); + flavor.replaces(flavorReplaces); + } + + @Override + public void close() throws IOException { + //testingServer.close(); + } + + public void advanceTime(TemporalAmount duration) { clock.advance(duration); } + public NodeRepository nodeRepository() { return nodeRepository; } + public ManualClock clock() { return clock; } + public NodeRepositoryProvisioner provisioner() { return provisioner; } + public CapacityPolicies capacityPolicies() { return capacityPolicies; } + public NodeList getNodes(ApplicationId id, Node.State ... inState) { return new NodeList(nodeRepository.getNodes(id, inState)); } + + public void patchNode(Node node) { nodeRepository.write(node); } + + public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups, String flavor) { + return prepare(application, cluster, Capacity.fromNodeCount(nodeCount, Optional.ofNullable(flavor)), groups); + } + public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, int groups) { + if (capacity.nodeCount() == 0) return Collections.emptyList(); + Set<String> reservedBefore = toHostNames(nodeRepository.getNodes(application, Node.State.reserved)); + Set<String> inactiveBefore = toHostNames(nodeRepository.getNodes(application, Node.State.inactive)); + // prepare twice to ensure idempotence + List<HostSpec> hosts1 = provisioner.prepare(application, cluster, capacity, groups, provisionLogger); + List<HostSpec> hosts2 = provisioner.prepare(application, cluster, capacity, groups, provisionLogger); + assertEquals(hosts1, hosts2); + Set<String> newlyActivated = toHostNames(nodeRepository.getNodes(application, Node.State.reserved)); + newlyActivated.removeAll(reservedBefore); + newlyActivated.removeAll(inactiveBefore); + return hosts2; + } + + public void activate(ApplicationId application, Set<HostSpec> hosts) { + NestedTransaction transaction = new NestedTransaction(); + transaction.add(new CuratorTransaction(curator)); + provisioner.activate(transaction, application, hosts); + transaction.commit(); + assertEquals(toHostNames(hosts), toHostNames(nodeRepository.getNodes(application, Node.State.active))); + } + + public Set<String> toHostNames(Set<HostSpec> hosts) { + return hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); + } + + public Set<String> toHostNames(List<Node> nodes) { + return nodes.stream().map(Node::hostname).collect(Collectors.toSet()); + } + + /** + * Asserts that each active node in this application has a restart count equaling the + * number of matches to the given filters + */ + public void assertRestartCount(ApplicationId application, HostFilter... filters) { + for (Node node : nodeRepository.getNodes(application, Node.State.active)) { + int expectedRestarts = 0; + for (HostFilter filter : filters) + if (NodeHostFilter.from(filter).matches(node)) + expectedRestarts++; + assertEquals(expectedRestarts, node.allocation().get().restartGeneration().wanted()); + } + } + + public void fail(HostSpec host) { + int beforeFailCount = nodeRepository.getNode(Node.State.active, host.hostname()).get().status().failCount(); + Node failedNode = nodeRepository.fail(host.hostname()); + assertTrue(nodeRepository.getNodes(Node.State.failed).contains(failedNode)); + assertEquals(beforeFailCount + 1, failedNode.status().failCount()); + } + + public void assertMembersOf(ClusterSpec requestedCluster, Collection<HostSpec> hosts) { + Set<Integer> indices = new HashSet<>(); + for (HostSpec host : hosts) { + ClusterSpec nodeCluster = host.membership().get().cluster(); + assertTrue(requestedCluster.equalsIgnoringGroup(nodeCluster)); + if (requestedCluster.group().isPresent()) + assertEquals(requestedCluster.group(), nodeCluster.group()); + else + assertEquals("0", nodeCluster.group().get().value()); + + indices.add(host.membership().get().index()); + } + assertEquals("Indexes in " + requestedCluster + " are disjunct", hosts.size(), indices.size()); + } + + public HostSpec removeOne(Set<HostSpec> hosts) { + Iterator<HostSpec> i = hosts.iterator(); + HostSpec removed = i.next(); + i.remove(); + return removed; + } + + public ApplicationId makeApplicationId() { + return ApplicationId.from( + TenantName.from(UUID.randomUUID().toString()), + ApplicationName.from(UUID.randomUUID().toString()), + InstanceName.from(UUID.randomUUID().toString())); + } + + public List<Node> makeReadyNodes(int n, String flavor) { + List<Node> nodes = new ArrayList<>(n); + for (int i = 0; i < n; i++) + nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + Optional.empty(), + new Configuration(nodeFlavors.getFlavorOrThrow(flavor)))); + nodes = nodeRepository.addNodes(nodes); + nodeRepository.setReady(nodes); + return nodes; + } + + /** Creates a set of virtual docker nodes on a single docker host */ + public List<Node> makeReadyDockerNodes(int n, String flavor, String dockerHostId) { + return makeReadyVirtualNodes(n, flavor, Optional.of(dockerHostId)); + } + + /** Creates a set of virtual nodes on a single parent host */ + public List<Node> makeReadyVirtualNodes(int n, String flavor, Optional<String> parentHostId) { + List<Node> nodes = new ArrayList<>(n); + for (int i = 0; i < n; i++) { + final String hostname = UUID.randomUUID().toString(); + nodes.add(nodeRepository.createNode("openstack-id", hostname, parentHostId, + new Configuration(nodeFlavors.getFlavorOrThrow(flavor)))); + } + nodes = nodeRepository.addNodes(nodes); + nodeRepository.setReady(nodes); + return nodes; + } + + public List<Node> makeReadyVirtualNodes(int n, String flavor, String parentHostId) { + return makeReadyVirtualNodes(n, flavor, Optional.of(parentHostId)); + } + + /** Returns the hosts from the input list which are not retired */ + public List<HostSpec> nonretired(Collection<HostSpec> hosts) { + return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList()); + } + + private static class NullProvisionLogger implements ProvisionLogger { + + @Override + public void log(Level level, String message) { + } + + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java new file mode 100644 index 00000000000..a79123959cc --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -0,0 +1,302 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.Node; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Tests provisioning of virtual nodes + * + * @author musum + * @author mpolden + */ +public class VirtualNodeProvisioningTest { + private static final String flavor = "v-4-8-100"; + private static final ClusterSpec contentClusterSpec = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")); + private static final ClusterSpec containerClusterSpec = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer")); + + + private ProvisioningTester tester; + private ApplicationId applicationId; + + @Before + public void setup() { + tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + applicationId = tester.makeApplicationId(); + } + + @Test + public void optimize_sorts_by_more_vms() { + List<Node> readyList = new ArrayList<Node>(); + readyList.addAll(tester.makeReadyNodes(2, flavor)); + readyList.addAll(tester.makeReadyVirtualNodes(2, flavor, "parentHost1")); + readyList.addAll(tester.makeReadyNodes(2, flavor)); + readyList.addAll(tester.makeReadyVirtualNodes(3, flavor, "parentHost2")); + readyList.addAll(tester.makeReadyVirtualNodes(2, flavor, "parentHost3")); + readyList.addAll(tester.makeReadyVirtualNodes(1, flavor, "parentHost4")); + readyList.addAll(tester.makeReadyNodes(3, flavor)); + assertEquals(15, readyList.size()); + List<Node> optimized = GroupPreparer.optimize(readyList); + assertEquals(15, optimized.size()); + assertEquals("parentHost2", optimized.get(0).parentHostname().get()); + assertEquals("parentHost4", optimized.get(3).parentHostname().get()); + assertEquals("parentHost2", optimized.get(4).parentHostname().get()); + assertEquals("parentHost2", optimized.get(7).parentHostname().get()); + assertEquals(false, optimized.get(8).parentHostname().isPresent()); + assertEquals(false, optimized.get(9).parentHostname().isPresent()); + assertEquals(false, optimized.get(10).parentHostname().isPresent()); + assertEquals(false, optimized.get(11).parentHostname().isPresent()); + assertEquals(false, optimized.get(12).parentHostname().isPresent()); + assertEquals(false, optimized.get(13).parentHostname().isPresent()); + assertEquals(false, optimized.get(14).parentHostname().isPresent()); + } + + @Test + public void distinct_parent_host_for_each_node_in_a_cluster() { + tester.makeReadyVirtualNodes(2, flavor, "parentHost1"); + tester.makeReadyVirtualNodes(2, flavor, "parentHost2"); + tester.makeReadyVirtualNodes(2, flavor, "parentHost3"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost4"); + + final int containerNodeCount = 4; + final int contentNodeCount = 3; + final int groups = 1; + List<HostSpec> containerHosts = prepare(containerClusterSpec, containerNodeCount, groups); + List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); + activate(containerHosts, contentHosts); + + final List<Node> nodes = getNodes(applicationId); + assertEquals(contentNodeCount + containerNodeCount, nodes.size()); + assertDistinctParentHosts(nodes, ClusterSpec.Type.container, containerNodeCount); + assertDistinctParentHosts(nodes, ClusterSpec.Type.content, contentNodeCount); + + // Go down to 3 nodes in container cluster + List<HostSpec> containerHosts2 = prepare(containerClusterSpec, containerNodeCount - 1, groups); + activate(containerHosts2); + final List<Node> nodes2 = getNodes(applicationId); + assertDistinctParentHosts(nodes2, ClusterSpec.Type.container, containerNodeCount - 1); + + // Go up to 4 nodes again in container cluster + List<HostSpec> containerHosts3 = prepare(containerClusterSpec, containerNodeCount, groups); + activate(containerHosts3); + final List<Node> nodes3 = getNodes(applicationId); + assertDistinctParentHosts(nodes3, ClusterSpec.Type.container, containerNodeCount); + } + + @Test + public void will_retire_clashing_active() { + tester.makeReadyVirtualNodes(1, flavor, "parentHost1"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost2"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost3"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost4"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost5"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost6"); + + final int containerNodeCount = 2; + final int contentNodeCount = 2; + final int groups = 1; + List<HostSpec> containerHosts = prepare(containerClusterSpec, containerNodeCount, groups); + List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); + activate(containerHosts, contentHosts); + + List<Node> nodes = getNodes(applicationId); + assertEquals(4, nodes.size()); + assertDistinctParentHosts(nodes, ClusterSpec.Type.container, containerNodeCount); + assertDistinctParentHosts(nodes, ClusterSpec.Type.content, contentNodeCount); + + for (Node n : nodes) { + tester.patchNode(n.setParentHostname("clashing")); + } + containerHosts = prepare(containerClusterSpec, containerNodeCount, groups); + contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); + activate(containerHosts, contentHosts); + + nodes = getNodes(applicationId); + assertEquals(6, nodes.size()); + assertEquals(2, nodes.stream().filter(n -> n.allocation().get().membership().retired()).count()); + } + + @Test + public void fail_when_all_hosts_become_clashing() { + tester.makeReadyVirtualNodes(1, flavor, "parentHost1"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost2"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost3"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost4"); + + final int containerNodeCount = 2; + final int contentNodeCount = 2; + final int groups = 1; + List<HostSpec> containerHosts = prepare(containerClusterSpec, containerNodeCount, groups); + List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); + activate(containerHosts, contentHosts); + + List<Node> nodes = getNodes(applicationId); + assertEquals(4, nodes.size()); + assertDistinctParentHosts(nodes, ClusterSpec.Type.container, containerNodeCount); + assertDistinctParentHosts(nodes, ClusterSpec.Type.content, contentNodeCount); + + for (Node n : nodes) { + tester.patchNode(n.setParentHostname("clashing")); + } + OutOfCapacityException expected = null; + try { + containerHosts = prepare(containerClusterSpec, containerNodeCount, groups); + } catch (OutOfCapacityException e) { + expected = e; + } + assertNotNull(expected); + } + + @Test(expected = OutOfCapacityException.class) + // TODO Should fail with something else than OutOfCapacityException + public void fail_when_too_few_distinct_parent_hosts() { + tester.makeReadyVirtualNodes(2, flavor, "parentHost1"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost2"); + + final int contentNodeCount = 3; + List<HostSpec> hosts = prepare(contentClusterSpec, contentNodeCount, 1); + activate(hosts); + + final List<Node> nodes = getNodes(applicationId); + assertDistinctParentHosts(nodes, ClusterSpec.Type.content, contentNodeCount); + } + + @Test + public void incomplete_parent_hosts_has_distinct_distribution() { + tester.makeReadyVirtualNodes(1, flavor, "parentHost1"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost2"); + tester.makeReadyVirtualNodes(1, flavor, Optional.empty()); + + final int contentNodeCount = 3; + final int groups = 1; + final List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); + activate(contentHosts); + assertEquals(3, getNodes(applicationId).size()); + + tester.makeReadyVirtualNodes(1, flavor, "parentHost1"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost2"); + + assertEquals(contentHosts, prepare(contentClusterSpec, contentNodeCount, groups)); + } + + @Test + public void indistinct_distribution_with_known_ready_nodes() { + tester.makeReadyVirtualNodes(3, flavor, Optional.empty()); + + final int contentNodeCount = 3; + final int groups = 1; + final List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); + activate(contentHosts); + + List<Node> nodes = getNodes(applicationId); + assertEquals(3, nodes.size()); + + // Set indistinct parents + tester.patchNode(nodes.get(0).setParentHostname("parentHost1")); + tester.patchNode(nodes.get(1).setParentHostname("parentHost1")); + tester.patchNode(nodes.get(2).setParentHostname("parentHost2")); + nodes = getNodes(applicationId); + assertEquals(3, nodes.stream().filter(n -> n.parentHostname().isPresent()).count()); + + tester.makeReadyVirtualNodes(1, flavor, "parentHost1"); + tester.makeReadyVirtualNodes(2, flavor, "parentHost2"); + + OutOfCapacityException expectedException = null; + try { + prepare(contentClusterSpec, contentNodeCount, groups); + } catch (OutOfCapacityException e) { + expectedException = e; + } + assertNotNull(expectedException); + } + + @Test + public void unknown_distribution_with_known_ready_nodes() { + tester.makeReadyVirtualNodes(3, flavor, Optional.empty()); + + final int contentNodeCount = 3; + final int groups = 1; + final List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); + activate(contentHosts); + assertEquals(3, getNodes(applicationId).size()); + + tester.makeReadyVirtualNodes(1, flavor, "parentHost1"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost2"); + tester.makeReadyVirtualNodes(1, flavor, "parentHost3"); + assertEquals(contentHosts, prepare(contentClusterSpec, contentNodeCount, groups)); + } + + @Test + public void unknown_distribution_with_known_and_unknown_ready_nodes() { + tester.makeReadyVirtualNodes(3, flavor, Optional.empty()); + + final int contentNodeCount = 3; + final int groups = 1; + final List<HostSpec> contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); + activate(contentHosts); + assertEquals(3, getNodes(applicationId).size()); + + tester.makeReadyVirtualNodes(1, flavor, "parentHost1"); + tester.makeReadyVirtualNodes(1, flavor, Optional.empty()); + assertEquals(contentHosts, prepare(contentClusterSpec, contentNodeCount, groups)); + } + + private void assertDistinctParentHosts(List<Node> nodes, ClusterSpec.Type clusterType, int expectedCount) { + List<String> parentHosts = getParentHostsFromNodes(nodes, Optional.of(clusterType)); + + assertEquals(expectedCount, parentHosts.size()); + assertEquals(expectedCount, getDistinctParentHosts(parentHosts).size()); + } + + private List<String> getParentHostsFromNodes(List<Node> nodes, Optional<ClusterSpec.Type> clusterType) { + List<String> parentHosts = new ArrayList<>(); + for (Node node : nodes) { + if (node.parentHostname().isPresent() && (clusterType.isPresent() && clusterType.get() == node.allocation().get().membership().cluster().type())) { + parentHosts.add(node.parentHostname().get()); + } + } + return parentHosts; + } + + private Set<String> getDistinctParentHosts(List<String> hostnames) { + return hostnames.stream() + .distinct() + .collect(Collectors.<String>toSet()); + } + + private List<Node> getNodes(ApplicationId applicationId) { + return tester.getNodes(applicationId, Node.State.active).asList(); + } + + private List<HostSpec> prepare(ClusterSpec clusterSpec, int nodeCount, int groups) { + return tester.prepare(applicationId, clusterSpec, nodeCount, groups, flavor); + } + + @SafeVarargs + private final void activate(List<HostSpec>... hostLists) { + HashSet<HostSpec> hosts = new HashSet<>(); + for (List<HostSpec> h : hostLists) { + hosts.addAll(h); + } + tester.activate(applicationId, hosts); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializerTest.java new file mode 100644 index 00000000000..5987b295dbe --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializerTest.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.vespa.hosted.provision.Node; +import org.junit.Test; + +import java.util.HashSet; +import java.util.Set; + +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; + +/** + * @author bakksjo + */ +public class NodeStateSerializerTest { + @Test + public void allStatesHaveASerializedForm() { + for (Node.State nodeState : Node.State.values()) { + assertThat(NodeStateSerializer.wireNameOf(nodeState), is(notNullValue())); + } + } + + @Test + public void wireNamesDoNotOverlap() { + final Set<String> wireNames = new HashSet<>(); + for (Node.State nodeState : Node.State.values()) { + wireNames.add(NodeStateSerializer.wireNameOf(nodeState)); + } + assertThat(wireNames.size(), is(Node.State.values().length)); + } + + @Test + public void serializationAndDeserializationIsSymmetric() { + for (Node.State nodeState : Node.State.values()) { + final String serialized = NodeStateSerializer.wireNameOf(nodeState); + final Node.State deserialized = NodeStateSerializer.fromWireName(serialized) + .orElseThrow(() -> new RuntimeException( + "Cannot deserialize '" + serialized + "', serialized form of " + nodeState.name())); + assertThat(deserialized, is(nodeState)); + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResourceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResourceTest.java new file mode 100644 index 00000000000..56c2e755c21 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResourceTest.java @@ -0,0 +1,148 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.legacy; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; + +/** + * @author mortent + */ +public class ProvisionResourceTest { + + NodeRepository nodeRepository; + NodeFlavors nodeFlavors; + ProvisionResource provisionResource; + Curator curator; + int capacity = 2; + ApplicationId application; + private NodeRepositoryProvisioner provisioner; + + @Before + public void setUpTest() throws Exception { + curator = new MockCurator(); + nodeFlavors = FlavorConfigBuilder.createDummies("default"); + nodeRepository = new NodeRepository(nodeFlavors, curator); + provisionResource = new ProvisionResource(nodeRepository, nodeFlavors); + application = ApplicationId.from(TenantName.from("myTenant"), ApplicationName.from("myApplication"), InstanceName.from("default")); + provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone()); + } + + private void createNodesInRepository(int readyCount, int provisionedCount) { + List<Node> readyNodes = new ArrayList<>(); + for (HostInfo hostInfo : createHostInfos(readyCount, 0)) + readyNodes.add(nodeRepository.createNode(hostInfo.openStackId, hostInfo.hostname, + Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + readyNodes = nodeRepository.addNodes(readyNodes); + nodeRepository.setReady(readyNodes); + + List<Node> provisionedNodes = new ArrayList<>(); + for (HostInfo hostInfo : createHostInfos(provisionedCount, readyCount)) + provisionedNodes.add(nodeRepository.createNode(hostInfo.openStackId, hostInfo.hostname, + Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodeRepository.addNodes(provisionedNodes); + } + + @Test + public void test_node_allocation() { + createNodesInRepository(10, 0); + List<Node> assignments = assignNode(application, capacity); + assertEquals(2, assignments.size()); + } + + @Test + public void test_node_reallocation() { + createNodesInRepository(10, 0); + List<Node> assignments1 = assignNode(application, capacity); + List<Node> assignments2 = assignNode(application, capacity); + + assertEquals(assignments2.size(), assignments1.size()); + } + + @Test + public void test_node_reallocation_add_hostalias() { + createNodesInRepository(5, 0); + + List<Node> assignments1 = assignNode(application, 2); + List<Node> assignments2 = assignNode(application, 3); + + assertEquals(assignments2.size(), assignments1.size() + 1); + } + + @Test + public void test_node_allocation_remove_hostalias() { + createNodesInRepository(10, 0); + + List<Node> assignments1 = assignNode(application, 3, ClusterSpec.Type.container); + List<Node> assignments2 = assignNode(application, 2, ClusterSpec.Type.container); + + assertEquals(assignments2.size(), assignments1.size() - 1); + ProvisionStatus provisionStatus = provisionResource.getStatus(); + assertEquals(1, provisionStatus.decomissionNodes.size()); + } + + @Test + public void test_recycle_deallocated() { + createNodesInRepository(2, 0); + assignNode(application, 2); + nodeRepository.deactivate(application); + List<Node> nodes = nodeRepository.deallocate(nodeRepository.getNodes(application, Node.State.inactive)); + assertEquals(0, nodeRepository.getNodes(Node.State.ready).size()); + assertEquals(2, nodeRepository.getNodes(Node.State.dirty).size()); + provisionResource.setReady(nodes.get(0).hostname()); + provisionResource.setReady(nodes.get(1).hostname()); + assertEquals(2, nodeRepository.getNodes(Node.State.ready).size()); + assertEquals(0, nodeRepository.getNodes(Node.State.dirty).size()); + } + + @Test(expected = IllegalArgumentException.class) + public void test_ready_node_unknown() { + provisionResource.setReady("does.not.exist"); + } + + private List<HostInfo> createHostInfos(int count, int startIndex) { + String format = "node%d"; + List<HostInfo> hostInfos = new ArrayList<>(); + for (int i = 0; i < count; ++i) + hostInfos.add(HostInfo.createHostInfo(String.format(format, i + startIndex), UUID.randomUUID().toString(), "medium")); + return hostInfos; + } + + private List<Node> assignNode(ApplicationId applicationId, int capacity) { + return assignNode(applicationId, capacity, ClusterSpec.Type.content); + } + + private List<Node> assignNode(ApplicationId applicationId, int capacity, ClusterSpec.Type type) { + ClusterSpec cluster = ClusterSpec.from(type, ClusterSpec.Id.from("test"), Optional.empty()); + List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(capacity), 1, null); + NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); + provisioner.activate(transaction, applicationId, hosts); + transaction.commit(); + return nodeRepository.getNodes(applicationId, Node.State.active); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java new file mode 100644 index 00000000000..9afb14f632a --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java @@ -0,0 +1,113 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v1; + +import com.yahoo.application.container.JDisc; +import com.yahoo.application.Networking; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; +import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class RestApiTest { + + private static final String servicesXml = + "<jdisc version=\"1.0\">" + + " <component id=\"com.yahoo.vespa.hosted.provision.restapi.v1.RestApiTest$MockNodeRepository\"/>" + + " <handler id=\"com.yahoo.vespa.hosted.provision.restapi.v1.NodesApiHandler\">" + + " <binding>http://*/nodes/v1/</binding>" + + " </handler>" + + "</jdisc>"; + + @Test + public void testTopLevelRequest() throws Exception { + try (JDisc container = JDisc.fromServicesXml(servicesXml, Networking.disable)) { + Response response = container.handleRequest(new Request("http://localhost:8080/nodes/v1/")); + assertEquals("{\"provisioned\":[{\"id\":\"node6\",\"hostname\":\"host6.yahoo.com\",\"flavor\":\"default\"}],\"reserved\":[{\"id\":\"node3\",\"hostname\":\"host3.yahoo.com\",\"flavor\":\"default\",\"owner\":{\"tenant\":\"tenant1\",\"application\":\"application1\",\"instance\":\"instance1\"},\"membership\":{\"clustertype\":\"container\",\"clusterid\":\"id1\",\"index\":0,\"retired\":false},\"restartGeneration\":0},{\"id\":\"node2\",\"hostname\":\"host2.yahoo.com\",\"flavor\":\"default\",\"owner\":{\"tenant\":\"tenant1\",\"application\":\"application1\",\"instance\":\"instance1\"},\"membership\":{\"clustertype\":\"container\",\"clusterid\":\"id1\",\"index\":1,\"retired\":false},\"restartGeneration\":0}],\"active\":[{\"id\":\"node1\",\"hostname\":\"host1.yahoo.com\",\"flavor\":\"default\",\"owner\":{\"tenant\":\"tenant2\",\"application\":\"application2\",\"instance\":\"instance2\"},\"membership\":{\"clustertype\":\"content\",\"clusterid\":\"id2\",\"index\":0,\"retired\":false},\"restartGeneration\":0},{\"id\":\"node4\",\"hostname\":\"host4.yahoo.com\",\"flavor\":\"default\",\"owner\":{\"tenant\":\"tenant2\",\"application\":\"application2\",\"instance\":\"instance2\"},\"membership\":{\"clustertype\":\"content\",\"clusterid\":\"id2\",\"index\":1,\"retired\":false},\"restartGeneration\":0}],\"failed\":[{\"id\":\"node5\",\"hostname\":\"host5.yahoo.com\",\"flavor\":\"default\"}]}", + response.getBodyAsString()); + } + } + + @Test + public void testSingleNodeRequest() throws Exception { + try (JDisc container = JDisc.fromServicesXml(servicesXml, Networking.disable)) { + Response response1 = container.handleRequest(new Request("http://localhost:8080/nodes/v1/?hostname=host3.yahoo.com")); + assertEquals("{\"reserved\":[{\"id\":\"node3\",\"hostname\":\"host3.yahoo.com\",\"flavor\":\"default\",\"owner\":{\"tenant\":\"tenant1\",\"application\":\"application1\",\"instance\":\"instance1\"},\"membership\":{\"clustertype\":\"container\",\"clusterid\":\"id1\",\"index\":0,\"retired\":false},\"restartGeneration\":0}]}", + response1.getBodyAsString()); + + Response response2 = container.handleRequest(new Request("http://localhost:8080/nodes/v1/?hostname=host6.yahoo.com")); + assertEquals("{\"provisioned\":[{\"id\":\"node6\",\"hostname\":\"host6.yahoo.com\",\"flavor\":\"default\"}]}", + response2.getBodyAsString()); + + Response response3 = container.handleRequest(new Request("http://localhost:8080/nodes/v1/?hostname=nonexisting-host.yahoo.com")); + assertEquals("{}", + response3.getBodyAsString()); + } + } + + // Instantiated by DI from application package above + public static class MockNodeRepository extends NodeRepository { + + private static final NodeFlavors flavors = FlavorConfigBuilder.createDummies("default"); + + public MockNodeRepository() throws Exception { + super(flavors, new MockCurator(), Clock.systemUTC()); + populate(); + } + + private void populate() { + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, flavors, Zone.defaultZone()); + + NodeFlavors flavors = FlavorConfigBuilder.createDummies("default"); + List<Node> nodes = new ArrayList<>(); + nodes.add(createNode("node1", "host1.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes.add(createNode("node2", "host2.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes.add(createNode("node3", "host3.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes.add(createNode("node4", "host4.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes.add(createNode("node5", "host5.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes.add(createNode("node6", "host6.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes = addNodes(nodes); + nodes.remove(5); + setReady(nodes); + fail("host5.yahoo.com"); + + ApplicationId app1 = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1")); + ClusterSpec cluster1 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Optional.empty()); + provisioner.prepare(app1, cluster1, Capacity.fromNodeCount(2), 1, null); + + ApplicationId app2 = ApplicationId.from(TenantName.from("tenant2"), ApplicationName.from("application2"), InstanceName.from("instance2")); + ClusterSpec cluster2 = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("id2"), Optional.empty()); + List<HostSpec> hosts = provisioner.prepare(app2, cluster2, Capacity.fromNodeCount(2), 1, null); + NestedTransaction transaction = new NestedTransaction(); + provisioner.activate(transaction, app2, hosts); + transaction.commit(); + } + + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java new file mode 100644 index 00000000000..922e0038ea5 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -0,0 +1,268 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.application.Networking; +import com.yahoo.application.container.JDisc; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.io.IOUtils; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class RestApiTest { + + private final static String responsesPath = "src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/"; + + /** This test gives examples of all the requests that can be made to nodes/v2 */ + @Test + public void testRequests() throws Exception { + // GET + assertFile(new Request("http://localhost:8080/nodes/v2/"), "root.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/state/"), "states.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/state/?recursive=true"), "states-recursive.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/state/active?recursive=true"), "active-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/"), "nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true"), "nodes-recursive.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "node2.json"); + + // GET with filters + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&hostname=host3.yahoo.com%20host6.yahoo.com"), "application2-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterType=content"), "active-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterId=id2"), "application2-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&application=tenant2.application2.instance2"), "application2-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&parentHost=parent.yahoo.com,parent.host.yahoo.com"), "parent-nodes.json"); + + // POST restart command + assertRestart(1, new Request("http://localhost:8080/nodes/v2/command/restart?hostname=host3.yahoo.com", + new byte[0], Request.Method.POST)); + assertRestart(2, new Request("http://localhost:8080/nodes/v2/command/restart?application=tenant2.application2.instance2", + new byte[0], Request.Method.POST)); + assertRestart(4, new Request("http://localhost:8080/nodes/v2/command/restart", + new byte[0], Request.Method.POST)); + assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host3.yahoo.com"), + "\"restartGeneration\":3"); + + // POST reboot command + assertReboot(5, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", + new byte[0], Request.Method.POST)); + assertReboot(2, new Request("http://localhost:8080/nodes/v2/command/reboot?application=tenant2.application2.instance2", + new byte[0], Request.Method.POST)); + assertReboot(8, new Request("http://localhost:8080/nodes/v2/command/reboot", + new byte[0], Request.Method.POST)); + assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host3.yahoo.com"), + "\"rebootGeneration\":3"); + + // POST new nodes + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("host8.yahoo.com", "default") + "," + + asNodeJson("host9.yahoo.com", "large-variant") + "," + + asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com") + "]"). + getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 3 nodes to the provisioned state\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), "node8.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com"), "node9.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host11.yahoo.com"), "node11.json"); + + // PUT nodes ready + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host8.yahoo.com to ready\"}"); + assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), + "\"state\":\"ready\""); + // calling ready again is a noop: + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Nothing done; host8.yahoo.com is already ready\"}"); + + // PUT a node in failed ... + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host8.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host8.yahoo.com to failed\"}"); + assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), + "\"state\":\"failed\""); + // ... and put it back in active (after fixing). This is useful to restore data when multiple nodes fail. + assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host8.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host8.yahoo.com to active\"}"); + + // PUT a node in failed ... + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host8.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host8.yahoo.com to failed\"}"); + assertResponseContains(new Request("http://localhost:8080()/nodes/v2/node/host8.yahoo.com"), + "\"state\":\"failed\""); + // ... and delete it + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Removed host8.yahoo.com\"}"); + + // or, PUT a node in failed ... + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host6.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host6.yahoo.com to failed\"}"); + assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), + "\"state\":\"failed\""); + // ... and deallocate it such that it moves to dirty and is recycled + assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host6.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host6.yahoo.com to dirty\"}"); + + // Update (PATCH) a node (multiple fields can also be sent in one request body) + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com/currentRebootGeneration", + Utf8.toBytes("{\"currentRebootGeneration\": 1}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"flavor\": \"medium-disk\"}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentVespaVersion\": \"5.104.142\"}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentHostedVersion\": \"2.1.2408\"}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"convergedStateVersion\": \"5.104.142-2.1.2408\"}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"hardwareFailure\": true}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"parentHostname\": \"parent.yahoo.com\"}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + + assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json"); + } + + @Test + public void testInvalidRequests() throws IOException { + // Attempt to DELETE a node which is not put in failed first + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com", + new byte[0], Request.Method.DELETE), + 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node in the failed state with hostname host8.yahoo.com\"}"); + + // PUT current restart generation with string instead of long + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": \"1\"}"), Request.Method.PATCH), + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Expected a LONG value, got a STRING\"}"); + + // PUT flavor with long instead of string + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"flavor\": 1}"), Request.Method.PATCH), + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'flavor': Expected a STRING value, got a LONG\"}"); + } + + + @Test + public void testNodePatching() throws IOException { + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{" + + "\"currentRestartGeneration\": 1," + + "\"currentRebootGeneration\": 3," + + "\"flavor\": \"medium-disk\"," + + "\"currentVespaVersion\": \"5.104.142\"," + + "\"currentHostedVersion\": \"2.1.2408\"," + + "\"hardwareFailure\": true," + + "\"failCount\": 0," + + "\"parentHostname\": \"parent.yahoo.com\"" + + "}" + ), + Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), + Request.Method.PATCH), + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Node is not allocated\"}"); + } + + /** Tests the rendering of each node separately to make it easier to find errors */ + @Test + public void testSingleNodeRendering() throws IOException { + for (int i = 1; i <= 10; i++) { + if (i == 8 || i == 9) continue; // these nodes are added later + assertFile(new Request("http://localhost:8080/nodes/v2/node/host" + i + ".yahoo.com"), "node" + i + ".json"); + } + } + + private JDisc container; + @Before + public void startContainer() { container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); } + @After + public void stopContainer() { container.close(); } + + private String asDockerNodeJson(String hostname, String parentHostname) { + return "{\"hostname\":\"" + hostname + "\", \"parentHostname\":\"" + parentHostname + + "\", \"openStackId\":\"" + hostname + "\",\"flavor\":\"docker\"}"; + } + + private String asNodeJson(String hostname, String flavor) { + return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\",\"flavor\":\"" + flavor + "\"}"; + } + + /** Asserts a particular response and 200 as response status */ + private void assertResponse(Request request, String responseMessage) throws IOException { + assertResponse(request, 200, responseMessage); + } + + private void assertResponse(Request request, int responseStatus, String responseMessage) throws IOException { + Response response = container.handleRequest(request); + // Compare both status and message at once for easier diagnosis + assertEquals("status: " + responseStatus + "\nmessage: " + responseMessage, + "status: " + response.getStatus() + "\nmessage: " + response.getBodyAsString()); + } + + private void assertResponseContains(Request request, String responseSnippet) throws IOException { + assertTrue("Response contains " + responseSnippet, + container.handleRequest(request).getBodyAsString().contains(responseSnippet)); + } + + private void assertFile(Request request, String responseFile) throws IOException { + String expectedResponse = IOUtils.readFile(new File(responsesPath + responseFile)); + expectedResponse = include(expectedResponse); + expectedResponse = expectedResponse.replaceAll("\\s", ""); + String responseString = container.handleRequest(request).getBodyAsString(); + assertEquals(responseFile, expectedResponse, responseString); + } + + private void assertRestart(int restartCount, Request request) throws IOException { + assertResponse(request, 200, "{\"message\":\"Scheduled restart of " + restartCount + " matching nodes\"}"); + } + + private void assertReboot(int rebootCount, Request request) throws IOException { + assertResponse(request, 200, "{\"message\":\"Scheduled reboot of " + rebootCount + " matching nodes\"}"); + } + + /** Replace @include(localFile) with the content of the file */ + private String include(String response) throws IOException { + // Please don't look at this code + int includeIndex = response.indexOf("@include("); + if (includeIndex < 0) return response; + String prefix = response.substring(0, includeIndex); + String rest = response.substring(includeIndex + "@include(".length()); + int filenameEnd = rest.indexOf(")"); + String includeFileName = rest.substring(0, filenameEnd); + String includedContent = IOUtils.readFile(new File(responsesPath + includeFileName)); + includedContent = include(includedContent); + String postFix = rest.substring(filenameEnd + 1); + postFix = include(postFix); + return prefix + includedContent + postFix; + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json new file mode 100644 index 00000000000..d1df5b83f24 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json @@ -0,0 +1,8 @@ +{ + "nodes": [ + @include(node6.json), + @include(node3.json), + @include(node2.json), + @include(node1.json) + ] +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json new file mode 100644 index 00000000000..f1285766c1b --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json @@ -0,0 +1,6 @@ +{ + "nodes": [ + @include(node6.json), + @include(node3.json) + ] +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json new file mode 100644 index 00000000000..734b6702c1e --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json @@ -0,0 +1,33 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com", + "id": "host1.yahoo.com", + "state": "active", + "hostname": "host1.yahoo.com", + "openStackId": "node1", + "flavor": "default", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "canonicalFlavor": "default", + "environment":"env", + "owner": { + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" + }, + "membership": { + "clustertype": "content", + "clusterid": "id3", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure" : false, + "history":[{"event":"readied","at":123},{"event":"reserved","at":123},{"event":"activated","at":123}] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json new file mode 100644 index 00000000000..d412e803bf5 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json @@ -0,0 +1,38 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com", + "id": "host10.yahoo.com", + "state": "reserved", + "hostname": "host10.yahoo.com", + "parentHostname": "parent.yahoo.com", + "openStackId": "node10", + "flavor": "default", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "canonicalFlavor": "default", + "environment":"env", + "owner": { + "tenant": "tenant1", + "application": "application1", + "instance": "instance1" + }, + "membership": { + "clustertype": "container", + "clusterid": "id1", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage":"image-123", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "vespaVersion": "5.104.142", + "hostedVersion": "2.1.2408", + "convergedStateVersion": "5.104.142-2.1.2408", + "failCount": 0, + "hardwareFailure" : false, + "history":[{"event":"readied","at":123},{"event":"reserved","at":123}] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json new file mode 100644 index 00000000000..6d1922e7fc0 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json @@ -0,0 +1,20 @@ +{ + "url":"http://localhost:8080/nodes/v2/node/host11.yahoo.com", + "id":"host11.yahoo.com", + "state":"provisioned", + "hostname":"host11.yahoo.com", + "parentHostname":"parent.host.yahoo.com", + "openStackId":"host11.yahoo.com", + "flavor":"docker", + "minDiskAvailableGb":100.0, + "minMainMemoryAvailableGb":0.5, + "description":"Flavor-name-is-docker", + "minCpuCores":0.2, + "canonicalFlavor":"docker", + "environment":"docker", + "rebootGeneration":0, + "currentRebootGeneration":0, + "failCount":0, + "hardwareFailure":false, + "history":[] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json new file mode 100644 index 00000000000..830c866ae81 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json @@ -0,0 +1,33 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com", + "id": "host2.yahoo.com", + "state": "active", + "hostname": "host2.yahoo.com", + "openStackId": "node2", + "flavor": "default", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "canonicalFlavor": "default", + "environment":"env", + "owner": { + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" + }, + "membership": { + "clustertype": "content", + "clusterid": "id3", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure" : false, + "history":[{"event":"readied","at":123},{"event":"reserved","at":123},{"event":"activated","at":123}] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json new file mode 100644 index 00000000000..5bf8631797a --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json @@ -0,0 +1,30 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com", + "id": "host3.yahoo.com", + "state": "active", + "hostname": "host3.yahoo.com", + "openStackId": "node3", + "flavor":"expensive", + "description":"Flavor-name-is-expensive", + "canonicalFlavor":"default", + "cost":200, + "owner": { + "tenant": "tenant2", + "application": "application2", + "instance": "instance2" + }, + "membership": { + "clustertype": "content", + "clusterid": "id2", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure" : false, + "history":[{"event":"readied","at":123},{"event":"reserved","at":123},{"event":"activated","at":123}] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json new file mode 100644 index 00000000000..ef88154fc5c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -0,0 +1,48 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", + "id": "host4.yahoo.com", + "state": "reserved", + "hostname": "host4.yahoo.com", + "parentHostname": "parent.yahoo.com", + "openStackId": "node4", + "flavor": "medium-disk", + "minDiskAvailableGb": 56.0, + "minMainMemoryAvailableGb": 12.0, + "description": "Flavor-name-is-medium-disk", + "minCpuCores": 6.0, + "canonicalFlavor": "medium-disk", + "environment": "foo", + "owner": { + "tenant": "tenant1", + "application": "application1", + "instance": "instance1" + }, + "membership": { + "clustertype": "container", + "clusterid": "id1", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 1, + "wantedDockerImage": "image-123", + "rebootGeneration": 1, + "currentRebootGeneration": 1, + "vespaVersion": "5.104.142", + "hostedVersion": "2.1.2408", + "currentDockerImage": "image-12", + "convergedStateVersion": "5.104.142-2.1.2408", + "failCount": 0, + "hardwareFailure": true, + "history": [ + { + "event": "readied", + "at": 123 + }, + { + "event": "reserved", + "at": 123 + } + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json new file mode 100644 index 00000000000..a1bc67705ce --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json @@ -0,0 +1,36 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", + "id": "host4.yahoo.com", + "state": "reserved", + "hostname": "host4.yahoo.com", + "parentHostname":"dockerhost4", + "openStackId": "node4", + "flavor": "default", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "canonicalFlavor": "default", + "environment":"env", + "owner": { + "tenant": "tenant1", + "application": "application1", + "instance": "instance1" + }, + "membership": { + "clustertype": "container", + "clusterid": "id1", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage":"image-123", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "currentDockerImage":"image-12", + "failCount": 0, + "hardwareFailure" : false, + "history":[{"event":"readied","at":123},{"event":"reserved","at":123}] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json new file mode 100644 index 00000000000..da4b49280c7 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json @@ -0,0 +1,21 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com", + "id": "host5.yahoo.com", + "state": "failed", + "hostname": "host5.yahoo.com", + "parentHostname":"dockerhost", + "openStackId": "node5", + "flavor": "default", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "canonicalFlavor": "default", + "environment":"env", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "currentDockerImage":"image-123", + "failCount": 1, + "hardwareFailure" : false, + "history":[{"event":"readied","at":123},{"event":"failed","at":123}] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json new file mode 100644 index 00000000000..9ef8adb1f07 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json @@ -0,0 +1,33 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", + "id": "host6.yahoo.com", + "state": "active", + "hostname": "host6.yahoo.com", + "openStackId": "node6", + "flavor": "default", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "canonicalFlavor": "default", + "environment":"env", + "owner": { + "tenant": "tenant2", + "application": "application2", + "instance": "instance2" + }, + "membership": { + "clustertype": "content", + "clusterid": "id2", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure" : false, + "history":[{"event":"readied","at":123},{"event":"reserved","at":123},{"event":"activated","at":123}] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json new file mode 100644 index 00000000000..52f01407b2b --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json @@ -0,0 +1,19 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com", + "id": "host7.yahoo.com", + "state": "provisioned", + "hostname": "host7.yahoo.com", + "openStackId": "node7", + "flavor": "default", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "canonicalFlavor": "default", + "environment":"env", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure" : false, + "history":[] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json new file mode 100644 index 00000000000..c00b6ed797c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json @@ -0,0 +1,19 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host8.yahoo.com", + "id": "host8.yahoo.com", + "state": "provisioned", + "hostname": "host8.yahoo.com", + "openStackId": "host8.yahoo.com", + "flavor": "default", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Flavor-name-is-default", + "minCpuCores":2.0, + "canonicalFlavor": "default", + "environment":"env", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure" : false, + "history":[] +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json new file mode 100644 index 00000000000..73a0eb8a266 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json @@ -0,0 +1,19 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host9.yahoo.com", + "id": "host9.yahoo.com", + "state": "provisioned", + "hostname": "host9.yahoo.com", + "openStackId": "host9.yahoo.com", + "flavor": "large-variant", + "minDiskAvailableGb":2000.0, + "minMainMemoryAvailableGb":128.0, + "description":"Flavor-name-is-large-variant", + "minCpuCores":64.0, + "canonicalFlavor": "large", + "environment":"env", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure" : false, + "history":[] +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json new file mode 100644 index 00000000000..8ea48599f0e --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json @@ -0,0 +1,12 @@ +{ + "nodes": [ + @include(node7.json), + @include(node10.json), + @include(node4.json), + @include(node6.json), + @include(node3.json), + @include(node2.json), + @include(node1.json), + @include(node5.json) + ] +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json new file mode 100644 index 00000000000..73947ded547 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json @@ -0,0 +1,28 @@ +{ + "nodes": [ + { + "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com" + }, + { + "url":"http://localhost:8080/nodes/v2/node/host10.yahoo.com" + }, + { + "url":"http://localhost:8080/nodes/v2/node/host4.yahoo.com" + }, + { + "url":"http://localhost:8080/nodes/v2/node/host6.yahoo.com" + }, + { + "url":"http://localhost:8080/nodes/v2/node/host3.yahoo.com" + }, + { + "url":"http://localhost:8080/nodes/v2/node/host2.yahoo.com" + }, + { + "url":"http://localhost:8080/nodes/v2/node/host1.yahoo.com" + }, + { + "url":"http://localhost:8080/nodes/v2/node/host5.yahoo.com" + } + ] +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json new file mode 100644 index 00000000000..28a17b03cc6 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json @@ -0,0 +1,5 @@ +{ + "nodes": [ + @include(node10.json) +] +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json new file mode 100644 index 00000000000..9648c059af6 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json @@ -0,0 +1,13 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/nodes/v2/state/" + }, + { + "url": "http://localhost:8080/nodes/v2/node/" + }, + { + "url": "http://localhost:8080/nodes/v2/command/" + } + ] +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json new file mode 100644 index 00000000000..b83616d2e0b --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json @@ -0,0 +1,47 @@ +{ + "states": { + "provisioned": { + "url": "http://localhost:8080/nodes/v2/state/provisioned", + "nodes": [ + @include(node7.json) + ] + }, + "ready": { + "url": "http://localhost:8080/nodes/v2/state/ready", + "nodes": [ + ] + }, + "reserved": { + "url": "http://localhost:8080/nodes/v2/state/reserved", + "nodes": [ + @include(node10.json), + @include(node4.json) + ] + }, + "active": { + "url": "http://localhost:8080/nodes/v2/state/active", + "nodes": [ + @include(node6.json), + @include(node3.json), + @include(node2.json), + @include(node1.json) + ] + }, + "inactive": { + "url": "http://localhost:8080/nodes/v2/state/inactive", + "nodes": [ + ] + }, + "dirty": { + "url": "http://localhost:8080/nodes/v2/state/dirty", + "nodes": [ + ] + }, + "failed": { + "url": "http://localhost:8080/nodes/v2/state/failed", + "nodes": [ + @include(node5.json) + ] + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json new file mode 100644 index 00000000000..b2d7354a6c9 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json @@ -0,0 +1,25 @@ +{ + "states": { + "provisioned": { + "url": "http://localhost:8080/nodes/v2/state/provisioned" + }, + "ready": { + "url": "http://localhost:8080/nodes/v2/state/ready" + }, + "reserved": { + "url": "http://localhost:8080/nodes/v2/state/reserved" + }, + "active": { + "url": "http://localhost:8080/nodes/v2/state/active" + }, + "inactive": { + "url": "http://localhost:8080/nodes/v2/state/inactive" + }, + "dirty": { + "url": "http://localhost:8080/nodes/v2/state/dirty" + }, + "failed": { + "url": "http://localhost:8080/nodes/v2/state/failed" + } + } +}
\ No newline at end of file diff --git a/node-repository/src/test/resources/hosts.xml b/node-repository/src/test/resources/hosts.xml new file mode 100644 index 00000000000..8f833840c5c --- /dev/null +++ b/node-repository/src/test/resources/hosts.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="hostname1"> + <alias>myhostalias1</alias> + </host> + + <host name="hostname2"> + <alias>myhostalias2</alias> + </host> + + <host name="hostname3"> + <alias>myhostalias3</alias> + </host> + + <host name="hostname4"> + <alias>myhostalias4</alias> + </host> + + <host name="hostname5"> + <alias>myhostalias5</alias> + </host> + + <host name="hostname6"> + <alias>myhostalias6</alias> + </host> +</hosts> diff --git a/node-repository/src/test/resources/services.xml b/node-repository/src/test/resources/services.xml new file mode 100644 index 00000000000..ac1ecfb02de --- /dev/null +++ b/node-repository/src/test/resources/services.xml @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + <jdisc version="1.0" id="default"> + <nodes> + <node hostalias="myhostalias1"/> + <node hostalias="myhostalias2"/> + </nodes> + </jdisc> + <content version="1.0"> <!-- id="default" --> + <nodes> + <node hostalias="myhostalias3" distribution-key="99"/> <!-- arbitrary distribution keys --> + <node hostalias="myhostalias4" distribution-key="42"/> + </nodes> + </content> + <content version="1.0" id="mycontent"> <!-- second content cluster --> + <group> <!-- element name is group instead of nodes --> + <node hostalias="myhostalias5" distribution-key="0"/> + <node hostalias="myhostalias6" distribution-key="1"/> + </group> + </content> +</services> |