summaryrefslogtreecommitdiffstats
path: root/node-admin
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2019-08-27 13:01:44 +0200
committerHarald Musum <musum@verizonmedia.com>2019-08-27 13:01:44 +0200
commitdef172b42a3a676183b5c81aa27f206c663d7e85 (patch)
treecb1bb0b24877ddd7f95144130c177ff924bcb4e2 /node-admin
parent8ec7cb0116fa57973a15e69f9955c179db394a54 (diff)
parent89183f028fe4e980f20d92c2958ce1520cb39acb (diff)
Merge branch 'master' into revert-10388-revert-10370-revert-10346-revert-10336-hmusum/upgrade-to-zookeeper-3.4.14
Diffstat (limited to 'node-admin')
-rwxr-xr-xnode-admin/scripts/etc-hosts.sh106
-rwxr-xr-xnode-admin/scripts/node-repo.sh360
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java27
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java64
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java155
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java30
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java3
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java6
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java2
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java31
10 files changed, 100 insertions, 684 deletions
diff --git a/node-admin/scripts/etc-hosts.sh b/node-admin/scripts/etc-hosts.sh
deleted file mode 100755
index 68d882da22a..00000000000
--- a/node-admin/scripts/etc-hosts.sh
+++ /dev/null
@@ -1,106 +0,0 @@
-#!/bin/bash
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-set -e
-
-declare -r NETWORK_PREFIX=172.18
-declare -r CONFIG_SERVER_HOSTNAME=config-server
-declare -r CONFIG_SERVER_IP="$NETWORK_PREFIX.1.1"
-
-declare -r APP_HOSTNAME_PREFIX=cnode-
-declare -r APP_NETWORK_PREFIX="$NETWORK_PREFIX.2"
-declare -r NUM_APP_CONTAINERS=20 # Statically allocated number of nodes.
-
-declare -r SYSTEM_TEST_HOSTNAME_PREFIX=stest-
-declare -r SYSTEM_TEST_NETWORK_PREFIX="$NETWORK_PREFIX.3"
-declare -r NUM_SYSTEM_TEST_CONTAINERS=5 # Statically allocated number of nodes.
-
-declare -r HOSTS_FILE=/etc/hosts
-declare -r HOSTS_LINE_SUFFIX=" # Managed by etc-hosts.sh"
-
-function IsInHostsAlready {
- local ip="$1"
- local hostname="$2"
- local file="$3"
-
- # TODO: Escape $ip to make sure it's matched as a literal in the regex.
- local matching_ip_line
- matching_ip_line=$(grep -E "^$ip[ \\t]" "$file")
-
- local -i num_ip_lines=0
- # This 'if' is needed because wc -l <<< "" is 1.
- if [ -n "$matching_ip_line" ]
- then
- num_ip_lines=$(wc -l <<< "$matching_ip_line")
- fi
-
- local matching_hostname_line
- matching_hostname_line=$(grep -E "^[^#]*[ \\t]$hostname(\$|[ \\t])" "$file")
-
- local -i num_hostname_lines=0
- # This 'if' is needed because wc -l <<< "" is 1.
- if [ -n "$matching_hostname_line" ]
- then
- num_hostname_lines=$(wc -l <<< "$matching_hostname_line")
- fi
-
- if ((num_ip_lines == 1)) && ((num_hostname_lines == 1)) &&
- [ "$matching_ip_line" == "$matching_hostname_line" ]
- then
- return 0
- elif ((num_ip_lines == 0)) && ((num_hostname_lines == 0))
- then
- return 1
- else
- printf "$file contains a conflicting host specification for $hostname/$ip"
- exit 1
- fi
-}
-
-function AddHost {
- local ip="$1"
- local hostname="$2"
- local file="$3"
-
- if IsInHostsAlready "$ip" "$hostname" "$file"
- then
- return
- fi
-
- echo -n "Adding host $hostname ($ip) to $file... "
- printf "%-11s %s%s\n" "$ip" "$hostname" "$HOSTS_LINE_SUFFIX" >> "$file"
- echo done
-}
-
-function Stop {
- # TODO: Remove entries.
- :
-}
-
-function StartAsRoot {
- # May need sudo
- if [ ! -w "$HOSTS_FILE" ]
- then
- Fail "$HOSTS_FILE is not writeable (run script with sudo)"
- fi
-
- AddHost "$CONFIG_SERVER_IP" "$CONFIG_SERVER_HOSTNAME" "$HOSTS_FILE"
-
- local -i index=1
- for ((; index <= NUM_APP_CONTAINERS; ++index))
- do
- local ip="$APP_NETWORK_PREFIX.$index"
- local container_name="$APP_HOSTNAME_PREFIX$index"
- AddHost "$ip" "$container_name" "$HOSTS_FILE"
- done
-
- local -i index=1
- for ((; index <= NUM_SYSTEM_TEST_CONTAINERS; ++index))
- do
- local ip="$SYSTEM_TEST_NETWORK_PREFIX.$index"
- local container_name="$SYSTEM_TEST_HOSTNAME_PREFIX$index"
- AddHost "$ip" "$container_name" "$HOSTS_FILE"
- done
-}
-
-StartAsRoot "$@"
diff --git a/node-admin/scripts/node-repo.sh b/node-admin/scripts/node-repo.sh
deleted file mode 100755
index 82f99f28e72..00000000000
--- a/node-admin/scripts/node-repo.sh
+++ /dev/null
@@ -1,360 +0,0 @@
-#!/bin/bash
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-set -e
-
-declare -r VESPA_WEB_SERVICE_PORT=4080
-
-# Output from InnerCurlNodeRepo, see there for details.
-declare CURL_RESPONSE
-
-function Usage {
- cat <<EOF
-Usage: ${0##*/} <command> [<args>...]
-Script for manipulating the Node Repository.
-
-Commands
- add [-c <configserverhost>] -p <parent-hostname> [-f <parent-flavor>]
- [-n <flavor> <hostname>...]
- With -f, provision "host" node <parent-hostname> with flavor
- <parent-flavor>. With -n, provision "tenant" nodes <hostname...> with
- flavor <flavor> and parent host <parenthostname>.
- reprovision [-c <configserverhost>] -p <parenthostname> <hostname>...
- Fail node <hostname>, then rm and add.
- rm [-c <configserverhost>] <hostname>...
- Remove nodes from node repo.
- set-state [-c <configserverhost>] <state> <hostname>...
- Set the node repo state.
-
-By default, <configserverhost> is config-server.
-
-Example
- To remove docker-1--1 through docker-1--5 from the node repo at configserver.com,
-
- ${0##*/} rm -c configserver.com \
- docker-1--{1,2,3,4,5}.dockerhosts.com
-EOF
-
- exit 1
-}
-
-function Fail {
- printf "%s\nRun '${0##*/} help' for usage\n" "$*"
- exit 1
-}
-
-# Invoke curl such that:
-#
-# - Arguments to this function are passed to curl.
-#
-# - Additional arguments are passed to curl to filter noise (--silent
-# --show-error).
-#
-# - The curl stdout (i.e. the response) is stored in the global CURL_RESPONSE
-# variable on return of the function.
-#
-# - If curl returns 0 (i.e. a successful HTTP response) with a JSON response
-# that contains an "error-code" key, this function will instead return 22. 22
-# does not conflict with a curl return code, because curl only ever returns
-# 22 when --fail is specified.
-#
-# Except, if the JSON response contains a "message" of the form "Cannot add
-# tmp-cnode-0: A node with this name already exists", InnerCurlNodeRepo
-# returns 0 instead of 22, even if the HTTP response status code is an error.
-#
-# Note: Why not use --fail with curl? Because as of 2015-11-24, if the node
-# exists when provisioning a node, the node repo returns a 400 Bad Request
-# with a JSON body containing a "message" field as described above. With
-# --fail, this would result in curl exiting with code 22, which is
-# indistinguishable from other HTTP errors. Can the output from --show-error
-# be used in combination with --fail? No, because that ends up saying "curl:
-# (22) The requested URL returned error: 400 Bad Request" when the node
-# exists, making it indistinguishable from other malformed request error
-# messages.
-#
-# TODO: Make node repo return a unique HTTP error code when node already
-# exists. It's also fragile to test for the error message in the response.
-function InnerCurlNodeRepo {
- # --show-error causes error message to be printed on error, even with
- # --silent, which is useful when we print the error message to Fail.
- local -a command=(curl --silent --show-error "$@")
-
- # We need the 'if' here, because a non-zero exit code of a command will
- # exit the process, with 'set -e'.
- if CURL_RESPONSE=$("${command[@]}" 2>&1)
- then
- # Match a JSON of the form:
- # {
- # "error-code": "BAD_REQUEST",
- # "message": "Cannot add cnode-0: A node with this name already exists"
- # }
- if [[ "$CURL_RESPONSE" =~ '"error-code"' ]]
- then
- if [[ "$CURL_RESPONSE" =~ '"message"'[^\"]*\"(.*)\" ]]
- then
- local message="${BASH_REMATCH[1]}"
- if [[ "$message" =~ 'already exists' ]]
- then
- return 0
- fi
- fi
-
- return 22
- fi
-
- return 0
- else
- # Do not move this statement outside of this else: $? gets cleared when
- # the execution passes out of the else-fi block.
- return $?
- fi
-}
-
-function CurlOrFail {
- if InnerCurlNodeRepo "$@"
- then
- : # This form of if-else is used to preserve $?.
- else
- local error_code=$?
-
- # Terminate the current progress-bar-like line
- printf ' failed\n'
-
- Fail "Error ($error_code) from the node repo at '$url': '$CURL_RESPONSE'"
- fi
-}
-
-function ProvisionDockerNode {
- local config_server_hostname="$1"
- local container_hostname="$2"
- local parent_hostname="$3"
- local flavor="$4"
-
- local json="[
- {
- \"hostname\":\"$container_hostname\",
- \"parentHostname\":\"$parent_hostname\",
- \"openStackId\":\"fake-$container_hostname\",
- \"flavor\":\"$flavor\",
- \"type\":\"tenant\"
- }
- ]"
-
- ProvisionNode $config_server_hostname "$json"
-}
-
-
-# Docker host, the docker nodes points to this host in parentHostname in their node config
-function ProvisionDockerHost {
- local config_server_hostname="$1"
- local docker_host_hostname="$2"
- local flavor="$3"
-
- local json="[
- {
- \"hostname\":\"$docker_host_hostname\",
- \"openStackId\":\"$docker_host_hostname\",
- \"flavor\":\"$flavor\",
- \"type\":\"host\"
- }
- ]"
-
- ProvisionNode $config_server_hostname "$json"
-}
-
-# Docker node in node repo (both docker hosts and docker nodes)
-function ProvisionNode {
- local config_server_hostname="$1"
- local json="$2"
-
- local url="http://$config_server_hostname:$VESPA_WEB_SERVICE_PORT/nodes/v2/node"
-
- CurlOrFail -H "Content-Type: application/json" -X POST -d "$json" "$url"
-}
-
-function SetNodeState {
- local config_server_hostname="$1"
- local hostname="$2"
- local state="$3"
-
- local url="http://$config_server_hostname:$VESPA_WEB_SERVICE_PORT/nodes/v2/state/$state/$hostname"
- CurlOrFail -X PUT "$url"
-}
-
-function AddCommand {
- local config_server_hostname=config-server
- local parent_hostname=
-
- OPTIND=1
- local option
- while getopts "c:p:f:n:" option
- do
- case "$option" in
- c) config_server_hostname="$OPTARG" ;;
- p) parent_hostname="$OPTARG" ;;
- f) parent_host_flavor="$OPTARG" ;;
- n) node_flavor="$OPTARG" ;;
- ?) exit 1 ;; # E.g. option lacks argument, in case error has been
- # already been printed
- *) Fail "Unknown option '$option' with value '$OPTARG'"
- esac
- done
-
- if [ -z "$parent_hostname" ]
- then
- Fail "Parent hostname not specified (-p)"
- fi
-
- shift $((OPTIND - 1))
-
- if [ -n "$parent_host_flavor" ]
- then
- echo "Provisioning Docker host $parent_hostname with flavor $parent_host_flavor"
- ProvisionDockerHost "$config_server_hostname" \
- "$parent_hostname" \
- "$parent_host_flavor"
- fi
-
- if [ -n "$node_flavor" ]
- then
- echo -n "Provisioning $# nodes with parent host $parent_hostname"
- local container_hostname
- for container_hostname in "$@"
- do
- ProvisionDockerNode "$config_server_hostname" \
- "$container_hostname" \
- "$parent_hostname" \
- "$node_flavor"
- echo -n .
- done
-
- echo " done"
- fi
-}
-
-function ReprovisionCommand {
- local config_server_hostname=config-server
- local parent_hostname=
-
- OPTIND=1
- local option
- while getopts "c:p:" option
- do
- case "$option" in
- c) config_server_hostname="$OPTARG" ;;
- p) parent_hostname="$OPTARG" ;;
- ?) exit 1 ;; # E.g. option lacks argument, in case error has been
- # already been printed
- *) Fail "Unknown option '$option' with value '$OPTARG'"
- esac
- done
-
- if [ -z "$parent_hostname" ]
- then
- Fail "Parent hostname not specified (-p)"
- fi
-
- shift $((OPTIND - 1))
-
- if (($# == 0))
- then
- Fail "No node hostnames were specified"
- fi
-
- # Simulate calls to the following commands.
- SetStateCommand -c "$config_server_hostname" failed "$@"
- RemoveCommand -c "$config_server_hostname" "$@"
- AddCommand -c "$config_server_hostname" -p "$parent_hostname" "$@"
-}
-
-function RemoveCommand {
- local config_server_hostname=config-server
-
- OPTIND=1
- local option
- while getopts "c:" option
- do
- case "$option" in
- c) config_server_hostname="$OPTARG" ;;
- ?) exit 1 ;; # E.g. option lacks argument, in case error has been
- # already been printed
- *) Fail "Unknown option '$option' with value '$OPTARG'"
- esac
- done
-
- shift $((OPTIND - 1))
-
- if (($# == 0))
- then
- Fail "No nodes were specified"
- fi
-
- echo -n "Removing $# nodes"
-
- local hostname
- for hostname in "$@"
- do
- local url="http://$config_server_hostname:$VESPA_WEB_SERVICE_PORT/nodes/v2/node/$hostname"
- CurlOrFail -X DELETE "$url"
- echo -n .
- done
-
- echo " done"
-}
-
-function SetStateCommand {
- local config_server_hostname=config-server
-
- OPTIND=1
- local option
- while getopts "c:" option
- do
- case "$option" in
- c) config_server_hostname="$OPTARG" ;;
- ?) exit 1 ;; # E.g. option lacks argument, in case error has been
- # already been printed
- *) Fail "Unknown option '$option' with value '$OPTARG'"
- esac
- done
-
- shift $((OPTIND - 1))
-
- if (($# <= 1))
- then
- Fail "Too few arguments"
- fi
-
- local state="$1"
- shift
-
- echo -n "Setting $# nodes to $state"
-
- local hostname
- for hostname in "$@"
- do
- SetNodeState "$config_server_hostname" "$hostname" "$state"
- echo -n .
- done
-
- echo " done"
-}
-
-function Main {
- if (($# == 0))
- then
- Usage
- fi
- local command="$1"
- shift
-
- case "$command" in
- add) AddCommand "$@" ;;
- reprovision) ReprovisionCommand "$@" ;;
- rm) RemoveCommand "$@" ;;
- set-state) SetStateCommand "$@" ;;
- help) Usage "$@" ;;
- *) Usage ;;
- esac
-}
-
-Main "$@"
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java
index b83bd9895fc..233fe8318ae 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java
@@ -1,9 +1,10 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
+import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.host.FlavorOverrides;
-import java.util.Collections;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
@@ -15,29 +16,29 @@ public class AddNode {
public final String hostname;
public final Optional<String> parentHostname;
- public final String nodeFlavor;
+ public final Optional<String> nodeFlavor;
+ public final Optional<FlavorOverrides> flavorOverrides;
+ public final Optional<NodeResources> nodeResources;
public final NodeType nodeType;
public final Set<String> ipAddresses;
public final Set<String> additionalIpAddresses;
- /**
- * Constructor for a host node (has no parent)
- */
- public AddNode(String hostname, String nodeFlavor, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) {
- this(hostname, Optional.empty(), nodeFlavor, nodeType, ipAddresses, additionalIpAddresses);
+ public static AddNode forHost(String hostname, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) {
+ return new AddNode(hostname, Optional.empty(), Optional.of(nodeFlavor), flavorOverrides, Optional.empty(), nodeType, ipAddresses, additionalIpAddresses);
}
- /**
- * Constructor for a child node (Must set parentHostname, no additionalIpAddresses)
- */
- public AddNode(String hostname, String parentHostname, String nodeFlavor, NodeType nodeType, Set<String> ipAddresses) {
- this(hostname, Optional.of(parentHostname), nodeFlavor, nodeType, ipAddresses, Collections.emptySet());
+ public static AddNode forNode(String hostname, String parentHostname, NodeResources nodeResources, NodeType nodeType, Set<String> ipAddresses) {
+ return new AddNode(hostname, Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), nodeType, ipAddresses, Set.of());
}
- public AddNode(String hostname, Optional<String> parentHostname, String nodeFlavor, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) {
+ private AddNode(String hostname, Optional<String> parentHostname,
+ Optional<String> nodeFlavor, Optional<FlavorOverrides> flavorOverrides, Optional<NodeResources> nodeResources,
+ NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) {
this.hostname = hostname;
this.parentHostname = parentHostname;
this.nodeFlavor = nodeFlavor;
+ this.flavorOverrides = flavorOverrides;
+ this.nodeResources = nodeResources;
this.nodeType = nodeType;
this.ipAddresses = ipAddresses;
this.additionalIpAddresses = additionalIpAddresses;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java
deleted file mode 100644
index c41e050d534..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
-
-import com.yahoo.config.provision.ApplicationId;
-
-/**
- * @author freva
- */
-public class NodeOwner {
- private final String tenant;
- private final String application;
- private final String instance;
-
- public NodeOwner(String tenant, String application, String instance) {
- this.tenant = tenant;
- this.application = application;
- this.instance = instance;
- }
-
- public String tenant() {
- return tenant;
- }
-
- public String application() {
- return application;
- }
-
- public String instance() {
- return instance;
- }
-
- public ApplicationId asApplicationId() {
- return ApplicationId.from(tenant, application, instance);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- NodeOwner owner = (NodeOwner) o;
-
- if (!tenant.equals(owner.tenant)) return false;
- if (!application.equals(owner.application)) return false;
- return instance.equals(owner.instance);
-
- }
-
- @Override
- public int hashCode() {
- int result = tenant.hashCode();
- result = 31 * result + application.hashCode();
- result = 31 * result + instance.hashCode();
- return result;
- }
-
- public String toString() {
- return "Owner {" +
- " tenant = " + tenant +
- " application = " + application +
- " instance = " + instance +
- " }";
- }
-}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
index 6fb6d44bd6f..c2e68bdb329 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
@@ -3,7 +3,9 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
import com.fasterxml.jackson.databind.JsonNode;
import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.NodeType;
import java.time.Instant;
@@ -11,6 +13,9 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast;
+import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow;
+
/**
* @author stiankri
*/
@@ -19,7 +24,6 @@ public class NodeSpec {
private final NodeState state;
private final NodeType type;
private final String flavor;
- private final String canonicalFlavor;
private final Optional<DockerImage> wantedDockerImage;
private final Optional<DockerImage> currentDockerImage;
@@ -43,15 +47,10 @@ public class NodeSpec {
private final Optional<Boolean> allowedToBeDown;
private final Optional<Boolean> wantToDeprovision;
- private final Optional<NodeOwner> owner;
+ private final Optional<ApplicationId> owner;
private final Optional<NodeMembership> membership;
- private final double vcpus;
- private final double memoryGb;
- private final double diskGb;
-
- private final boolean fastDisk;
- private final double bandwidth;
+ private final NodeResources resources;
private final Set<String> ipAddresses;
private final Set<String> additionalIpAddresses;
@@ -66,14 +65,13 @@ public class NodeSpec {
NodeState state,
NodeType type,
String flavor,
- String canonicalFlavor,
Optional<Version> wantedVespaVersion,
Optional<Version> currentVespaVersion,
Optional<Version> wantedOsVersion,
Optional<Version> currentOsVersion,
Optional<Boolean> allowedToBeDown,
Optional<Boolean> wantToDeprovision,
- Optional<NodeOwner> owner,
+ Optional<ApplicationId> owner,
Optional<NodeMembership> membership,
Optional<Long> wantedRestartGeneration,
Optional<Long> currentRestartGeneration,
@@ -82,11 +80,7 @@ public class NodeSpec {
Optional<Instant> wantedFirmwareCheck,
Optional<Instant> currentFirmwareCheck,
Optional<String> modelName,
- double vcpus,
- double memoryGb,
- double diskGb,
- boolean fastDisk,
- double bandwidth,
+ NodeResources resources,
Set<String> ipAddresses,
Set<String> additionalIpAddresses,
NodeReports reports,
@@ -104,7 +98,6 @@ public class NodeSpec {
this.state = Objects.requireNonNull(state);
this.type = Objects.requireNonNull(type);
this.flavor = Objects.requireNonNull(flavor);
- this.canonicalFlavor = canonicalFlavor;
this.modelName = modelName;
this.wantedVespaVersion = Objects.requireNonNull(wantedVespaVersion);
this.currentVespaVersion = Objects.requireNonNull(currentVespaVersion);
@@ -120,11 +113,7 @@ public class NodeSpec {
this.currentRebootGeneration = currentRebootGeneration;
this.wantedFirmwareCheck = Objects.requireNonNull(wantedFirmwareCheck);
this.currentFirmwareCheck = Objects.requireNonNull(currentFirmwareCheck);
- this.vcpus = vcpus;
- this.memoryGb = memoryGb;
- this.diskGb = diskGb;
- this.fastDisk = fastDisk;
- this.bandwidth = bandwidth;
+ this.resources = Objects.requireNonNull(resources);
this.ipAddresses = Objects.requireNonNull(ipAddresses);
this.additionalIpAddresses = Objects.requireNonNull(additionalIpAddresses);
this.reports = Objects.requireNonNull(reports);
@@ -147,10 +136,6 @@ public class NodeSpec {
return flavor;
}
- public String canonicalFlavor() {
- return canonicalFlavor;
- }
-
public Optional<DockerImage> wantedDockerImage() {
return wantedDockerImage;
}
@@ -211,7 +196,7 @@ public class NodeSpec {
return wantToDeprovision;
}
- public Optional<NodeOwner> owner() {
+ public Optional<ApplicationId> owner() {
return owner;
}
@@ -219,24 +204,28 @@ public class NodeSpec {
return membership;
}
+ public NodeResources resources() {
+ return resources;
+ }
+
public double vcpus() {
- return vcpus;
+ return resources.vcpu();
}
public double memoryGb() {
- return memoryGb;
+ return resources.memoryGb();
}
public double diskGb() {
- return diskGb;
+ return resources.diskGb();
}
public boolean isFastDisk() {
- return fastDisk;
+ return resources.diskSpeed() == fast;
}
- public double bandwidth() {
- return bandwidth;
+ public double bandwidthGbps() {
+ return resources.bandwidthGbps();
}
public Set<String> ipAddresses() {
@@ -266,7 +255,6 @@ public class NodeSpec {
Objects.equals(state, that.state) &&
Objects.equals(type, that.type) &&
Objects.equals(flavor, that.flavor) &&
- Objects.equals(canonicalFlavor, that.canonicalFlavor) &&
Objects.equals(wantedVespaVersion, that.wantedVespaVersion) &&
Objects.equals(currentVespaVersion, that.currentVespaVersion) &&
Objects.equals(wantedOsVersion, that.wantedOsVersion) &&
@@ -281,11 +269,7 @@ public class NodeSpec {
Objects.equals(currentRebootGeneration, that.currentRebootGeneration) &&
Objects.equals(wantedFirmwareCheck, that.wantedFirmwareCheck) &&
Objects.equals(currentFirmwareCheck, that.currentFirmwareCheck) &&
- Objects.equals(vcpus, that.vcpus) &&
- Objects.equals(memoryGb, that.memoryGb) &&
- Objects.equals(diskGb, that.diskGb) &&
- Objects.equals(fastDisk, that.fastDisk) &&
- Objects.equals(bandwidth, that.bandwidth) &&
+ Objects.equals(resources, that.resources) &&
Objects.equals(ipAddresses, that.ipAddresses) &&
Objects.equals(additionalIpAddresses, that.additionalIpAddresses) &&
Objects.equals(reports, that.reports) &&
@@ -301,7 +285,6 @@ public class NodeSpec {
state,
type,
flavor,
- canonicalFlavor,
wantedVespaVersion,
currentVespaVersion,
wantedOsVersion,
@@ -316,11 +299,7 @@ public class NodeSpec {
currentRebootGeneration,
wantedFirmwareCheck,
currentFirmwareCheck,
- vcpus,
- memoryGb,
- diskGb,
- fastDisk,
- bandwidth,
+ resources,
ipAddresses,
additionalIpAddresses,
reports,
@@ -336,7 +315,6 @@ public class NodeSpec {
+ " state=" + state
+ " type=" + type
+ " flavor=" + flavor
- + " canonicalFlavor=" + canonicalFlavor
+ " wantedVespaVersion=" + wantedVespaVersion
+ " currentVespaVersion=" + currentVespaVersion
+ " wantedOsVersion=" + wantedOsVersion
@@ -345,17 +323,13 @@ public class NodeSpec {
+ " wantToDeprovision=" + wantToDeprovision
+ " owner=" + owner
+ " membership=" + membership
- + " vcpus=" + vcpus
+ " wantedRestartGeneration=" + wantedRestartGeneration
+ " currentRestartGeneration=" + currentRestartGeneration
+ " wantedRebootGeneration=" + wantedRebootGeneration
+ " currentRebootGeneration=" + currentRebootGeneration
+ " wantedFirmwareCheck=" + wantedFirmwareCheck
+ " currentFirmwareCheck=" + currentFirmwareCheck
- + " memoryGb=" + memoryGb
- + " diskGb=" + diskGb
- + " fastDisk=" + fastDisk
- + " bandwidth=" + bandwidth
+ + " resources=" + resources
+ " ipAddresses=" + ipAddresses
+ " additionalIpAddresses=" + additionalIpAddresses
+ " reports=" + reports
@@ -368,7 +342,6 @@ public class NodeSpec {
private NodeState state;
private NodeType type;
private String flavor;
- private String canonicalFlavor;
private Optional<DockerImage> wantedDockerImage = Optional.empty();
private Optional<DockerImage> currentDockerImage = Optional.empty();
private Optional<Version> wantedVespaVersion = Optional.empty();
@@ -377,7 +350,7 @@ public class NodeSpec {
private Optional<Version> currentOsVersion = Optional.empty();
private Optional<Boolean> allowedToBeDown = Optional.empty();
private Optional<Boolean> wantToDeprovision = Optional.empty();
- private Optional<NodeOwner> owner = Optional.empty();
+ private Optional<ApplicationId> owner = Optional.empty();
private Optional<NodeMembership> membership = Optional.empty();
private Optional<Long> wantedRestartGeneration = Optional.empty();
private Optional<Long> currentRestartGeneration = Optional.empty();
@@ -386,11 +359,7 @@ public class NodeSpec {
private Optional<Instant> wantedFirmwareCheck = Optional.empty();
private Optional<Instant> currentFirmwareCheck = Optional.empty();
private Optional<String> modelName = Optional.empty();
- private double vcpus;
- private double memoryGb;
- private double diskGb;
- private boolean fastDisk;
- private double bandwidth;
+ private NodeResources resources = new NodeResources(0, 0, 0, 0, slow);
private Set<String> ipAddresses = Set.of();
private Set<String> additionalIpAddresses = Set.of();
private NodeReports reports = new NodeReports();
@@ -403,12 +372,7 @@ public class NodeSpec {
state(node.state);
type(node.type);
flavor(node.flavor);
- canonicalFlavor(node.canonicalFlavor);
- vcpus(node.vcpus);
- memoryGb(node.memoryGb);
- diskGb(node.diskGb);
- fastDisk(node.fastDisk);
- bandwidth(node.bandwidth);
+ resources(node.resources);
ipAddresses(node.ipAddresses);
additionalIpAddresses(node.additionalIpAddresses);
wantedRebootGeneration(node.wantedRebootGeneration);
@@ -462,11 +426,6 @@ public class NodeSpec {
return this;
}
- public Builder canonicalFlavor(String canonicalFlavor) {
- this.canonicalFlavor = canonicalFlavor;
- return this;
- }
-
public Builder wantedVespaVersion(Version wantedVespaVersion) {
this.wantedVespaVersion = Optional.of(wantedVespaVersion);
return this;
@@ -497,7 +456,7 @@ public class NodeSpec {
return this;
}
- public Builder owner(NodeOwner owner) {
+ public Builder owner(ApplicationId owner) {
this.owner = Optional.of(owner);
return this;
}
@@ -537,29 +496,29 @@ public class NodeSpec {
return this;
}
- public Builder vcpus(double minCpuCores) {
- this.vcpus = minCpuCores;
+ public Builder resources(NodeResources resources) {
+ this.resources = resources;
return this;
}
- public Builder memoryGb(double minMainMemoryAvailableGb) {
- this.memoryGb = minMainMemoryAvailableGb;
- return this;
+ public Builder vcpus(double vcpus) {
+ return resources(resources.withVcpu(vcpus));
}
- public Builder diskGb(double minDiskAvailableGb) {
- this.diskGb = minDiskAvailableGb;
- return this;
+ public Builder memoryGb(double memoryGb) {
+ return resources(resources.withMemoryGb(memoryGb));
+ }
+
+ public Builder diskGb(double diskGb) {
+ return resources(resources.withDiskGb(diskGb));
}
public Builder fastDisk(boolean fastDisk) {
- this.fastDisk = fastDisk;
- return this;
+ return resources(resources.withDiskSpeed(fastDisk ? fast : slow));
}
- public Builder bandwidth(double bandwidth) {
- this.bandwidth = bandwidth;
- return this;
+ public Builder bandwidthGbps(double bandwidthGbps) {
+ return resources(resources.withBandwidthGbps(bandwidthGbps));
}
public Builder ipAddresses(Set<String> ipAddresses) {
@@ -627,10 +586,6 @@ public class NodeSpec {
return flavor;
}
- public String canonicalFlavor() {
- return canonicalFlavor;
- }
-
public Optional<Version> wantedVespaVersion() {
return wantedVespaVersion;
}
@@ -655,7 +610,7 @@ public class NodeSpec {
return wantToDeprovision;
}
- public Optional<NodeOwner> owner() {
+ public Optional<ApplicationId> owner() {
return owner;
}
@@ -679,24 +634,8 @@ public class NodeSpec {
return currentRebootGeneration;
}
- public double vcpus() {
- return vcpus;
- }
-
- public double memoryGb() {
- return memoryGb;
- }
-
- public double diskGb() {
- return diskGb;
- }
-
- public boolean isFastDisk() {
- return fastDisk;
- }
-
- public double bandwidth() {
- return bandwidth;
+ public NodeResources resources() {
+ return resources;
}
public Set<String> ipAddresses() {
@@ -716,15 +655,13 @@ public class NodeSpec {
}
public NodeSpec build() {
- return new NodeSpec(hostname, wantedDockerImage, currentDockerImage, state, type,
- flavor, canonicalFlavor,
+ return new NodeSpec(hostname, wantedDockerImage, currentDockerImage, state, type, flavor,
wantedVespaVersion, currentVespaVersion, wantedOsVersion, currentOsVersion, allowedToBeDown, wantToDeprovision,
owner, membership,
wantedRestartGeneration, currentRestartGeneration,
wantedRebootGeneration, currentRebootGeneration,
wantedFirmwareCheck, currentFirmwareCheck, modelName,
- vcpus, memoryGb, diskGb,
- fastDisk, bandwidth, ipAddresses, additionalIpAddresses,
+ resources, ipAddresses, additionalIpAddresses,
reports, parentHostname);
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java
index fe19b81614d..582ba3dd09c 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java
@@ -4,8 +4,11 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.base.Strings;
import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.host.FlavorOverrides;
import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
import com.yahoo.vespa.hosted.node.admin.configserver.HttpException;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetAclResponse;
@@ -25,6 +28,9 @@ import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast;
+import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow;
+
/**
* @author stiankri, dybis
*/
@@ -154,14 +160,13 @@ public class RealNodeRepository implements NodeRepository {
nodeState,
nodeType,
node.flavor,
- node.canonicalFlavor,
Optional.ofNullable(node.wantedVespaVersion).map(Version::fromString),
Optional.ofNullable(node.vespaVersion).map(Version::fromString),
Optional.ofNullable(node.wantedOsVersion).map(Version::fromString),
Optional.ofNullable(node.currentOsVersion).map(Version::fromString),
Optional.ofNullable(node.allowedToBeDown),
Optional.ofNullable(node.wantToDeprovision),
- Optional.ofNullable(node.owner).map(o -> new NodeOwner(o.tenant, o.application, o.instance)),
+ Optional.ofNullable(node.owner).map(o -> ApplicationId.from(o.tenant, o.application, o.instance)),
membership,
Optional.ofNullable(node.restartGeneration),
Optional.ofNullable(node.currentRestartGeneration),
@@ -170,11 +175,12 @@ public class RealNodeRepository implements NodeRepository {
Optional.ofNullable(node.wantedFirmwareCheck).map(Instant::ofEpochMilli),
Optional.ofNullable(node.currentFirmwareCheck).map(Instant::ofEpochMilli),
Optional.ofNullable(node.modelName),
- node.minCpuCores,
- node.minMainMemoryAvailableGb,
- node.minDiskAvailableGb,
- node.fastDisk,
- node.bandwidth,
+ new NodeResources(
+ node.minCpuCores,
+ node.minMainMemoryAvailableGb,
+ node.minDiskAvailableGb,
+ node.bandwidth / 1000,
+ node.fastDisk ? fast : slow),
node.ipAddresses,
node.additionalIpAddresses,
reports,
@@ -186,7 +192,15 @@ public class RealNodeRepository implements NodeRepository {
node.openStackId = "fake-" + addNode.hostname;
node.hostname = addNode.hostname;
node.parentHostname = addNode.parentHostname.orElse(null);
- node.flavor = addNode.nodeFlavor;
+ addNode.nodeFlavor.ifPresent(f -> node.flavor = f);
+ addNode.flavorOverrides.flatMap(FlavorOverrides::diskGb).ifPresent(d -> node.minDiskAvailableGb = d);
+ addNode.nodeResources.ifPresent(resources -> {
+ node.minCpuCores = resources.vcpu();
+ node.minMainMemoryAvailableGb = resources.memoryGb();
+ node.minDiskAvailableGb = resources.diskGb();
+ node.bandwidth = resources.bandwidthGbps() * 1000;
+ node.fastDisk = resources.diskSpeed() == NodeResources.DiskSpeed.fast;
+ });
node.type = addNode.nodeType.name();
node.ipAddresses = addNode.ipAddresses;
node.additionalIpAddresses = addNode.additionalIpAddresses;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java
index 2b4f9277fc7..90d70a94144 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java
@@ -28,8 +28,6 @@ public class NodeRepositoryNode {
public String openStackId;
@JsonProperty("flavor")
public String flavor;
- @JsonProperty("canonicalFlavor")
- public String canonicalFlavor;
@JsonProperty("membership")
public Membership membership;
@JsonProperty("owner")
@@ -98,7 +96,6 @@ public class NodeRepositoryNode {
", openStackId='" + openStackId + '\'' +
", modelName='" + modelName + '\'' +
", flavor='" + flavor + '\'' +
- ", canonicalFlavor='" + canonicalFlavor + '\'' +
", membership=" + membership +
", owner=" + owner +
", restartGeneration=" + restartGeneration +
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index f4355ed3afa..91f4924cefa 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -141,9 +141,9 @@ public class StorageMaintainer {
context.node().parentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent));
context.node().currentVespaVersion().ifPresent(version -> attributes.put("vespa_version", version.toFullString()));
context.node().owner().ifPresent(owner -> {
- attributes.put("tenant", owner.tenant());
- attributes.put("application", owner.application());
- attributes.put("instance", owner.instance());
+ attributes.put("tenant", owner.tenant().value());
+ attributes.put("application", owner.application().value());
+ attributes.put("instance", owner.instance().value());
});
return Collections.unmodifiableMap(attributes);
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index b7e7b97cdd8..161775b0702 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
import com.yahoo.vespa.hosted.dockerapi.exception.ContainerNotFoundException;
import com.yahoo.vespa.hosted.dockerapi.exception.DockerException;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
@@ -345,7 +344,6 @@ public class NodeAgentImpl implements NodeAgent {
double cpuCap = noCpuCap(context.zone()) ?
0 :
context.node().owner()
- .map(NodeOwner::asApplicationId)
.map(appId -> containerCpuCap.with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm()))
.orElse(containerCpuCap)
.with(FetchVector.Dimension.HOSTNAME, context.node().hostname())
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java
index 0938eb23b49..e30572fc63e 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java
@@ -1,11 +1,12 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
import com.yahoo.application.Networking;
import com.yahoo.application.container.JDisc;
import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.host.FlavorOverrides;
import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl;
import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig;
@@ -17,11 +18,10 @@ import java.io.IOException;
import java.net.ServerSocket;
import java.net.URI;
import java.time.Instant;
-import java.util.Arrays;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
import java.util.Optional;
+import java.util.Set;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertEquals;
@@ -162,23 +162,22 @@ public class RealNodeRepositoryTest {
@Test
public void testAddNodes() {
- AddNode host = new AddNode("host123.domain.tld", "default", NodeType.confighost,
- Collections.singleton("::1"), new HashSet<>(Arrays.asList("::2", "::3")));
-
- AddNode node = new AddNode("host123-1.domain.tld", "host123.domain.tld", "docker", NodeType.config,
- new HashSet<>(Arrays.asList("::2", "::3")));
+ AddNode host = AddNode.forHost("host123.domain.tld", "default", Optional.of(FlavorOverrides.ofDisk(123)), NodeType.confighost,
+ Set.of("::1"), Set.of("::2", "::3"));
- List<AddNode> nodesToAdd = Arrays.asList(host, node);
+ NodeResources nodeResources = new NodeResources(1, 2, 3, 4, NodeResources.DiskSpeed.slow);
+ AddNode node = AddNode.forNode("host123-1.domain.tld", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3"));
assertFalse(nodeRepositoryApi.getOptionalNode("host123.domain.tld").isPresent());
- nodeRepositoryApi.addNodes(nodesToAdd);
-
- NodeSpec hostSpecInNodeRepo = nodeRepositoryApi.getOptionalNode("host123.domain.tld")
- .orElseThrow(RuntimeException::new);
+ nodeRepositoryApi.addNodes(List.of(host, node));
- assertEquals(host.nodeFlavor, hostSpecInNodeRepo.flavor());
- assertEquals(host.nodeType, hostSpecInNodeRepo.type());
+ NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode("host123.domain.tld").orElseThrow();
+ assertEquals("default", hostSpec.flavor());
+ assertEquals(123, hostSpec.diskGb(), 0);
+ assertEquals(NodeType.confighost, hostSpec.type());
- assertTrue(nodeRepositoryApi.getOptionalNode("host123-1.domain.tld").isPresent());
+ NodeSpec nodeSpec = nodeRepositoryApi.getOptionalNode("host123-1.domain.tld").orElseThrow();
+ assertEquals(nodeResources, nodeSpec.resources());
+ assertEquals(NodeType.config, nodeSpec.type());
}
}