diff options
10 files changed, 100 insertions, 684 deletions
diff --git a/node-admin/scripts/etc-hosts.sh b/node-admin/scripts/etc-hosts.sh deleted file mode 100755 index 68d882da22a..00000000000 --- a/node-admin/scripts/etc-hosts.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -declare -r NETWORK_PREFIX=172.18 -declare -r CONFIG_SERVER_HOSTNAME=config-server -declare -r CONFIG_SERVER_IP="$NETWORK_PREFIX.1.1" - -declare -r APP_HOSTNAME_PREFIX=cnode- -declare -r APP_NETWORK_PREFIX="$NETWORK_PREFIX.2" -declare -r NUM_APP_CONTAINERS=20 # Statically allocated number of nodes. - -declare -r SYSTEM_TEST_HOSTNAME_PREFIX=stest- -declare -r SYSTEM_TEST_NETWORK_PREFIX="$NETWORK_PREFIX.3" -declare -r NUM_SYSTEM_TEST_CONTAINERS=5 # Statically allocated number of nodes. - -declare -r HOSTS_FILE=/etc/hosts -declare -r HOSTS_LINE_SUFFIX=" # Managed by etc-hosts.sh" - -function IsInHostsAlready { - local ip="$1" - local hostname="$2" - local file="$3" - - # TODO: Escape $ip to make sure it's matched as a literal in the regex. - local matching_ip_line - matching_ip_line=$(grep -E "^$ip[ \\t]" "$file") - - local -i num_ip_lines=0 - # This 'if' is needed because wc -l <<< "" is 1. - if [ -n "$matching_ip_line" ] - then - num_ip_lines=$(wc -l <<< "$matching_ip_line") - fi - - local matching_hostname_line - matching_hostname_line=$(grep -E "^[^#]*[ \\t]$hostname(\$|[ \\t])" "$file") - - local -i num_hostname_lines=0 - # This 'if' is needed because wc -l <<< "" is 1. - if [ -n "$matching_hostname_line" ] - then - num_hostname_lines=$(wc -l <<< "$matching_hostname_line") - fi - - if ((num_ip_lines == 1)) && ((num_hostname_lines == 1)) && - [ "$matching_ip_line" == "$matching_hostname_line" ] - then - return 0 - elif ((num_ip_lines == 0)) && ((num_hostname_lines == 0)) - then - return 1 - else - printf "$file contains a conflicting host specification for $hostname/$ip" - exit 1 - fi -} - -function AddHost { - local ip="$1" - local hostname="$2" - local file="$3" - - if IsInHostsAlready "$ip" "$hostname" "$file" - then - return - fi - - echo -n "Adding host $hostname ($ip) to $file... " - printf "%-11s %s%s\n" "$ip" "$hostname" "$HOSTS_LINE_SUFFIX" >> "$file" - echo done -} - -function Stop { - # TODO: Remove entries. - : -} - -function StartAsRoot { - # May need sudo - if [ ! -w "$HOSTS_FILE" ] - then - Fail "$HOSTS_FILE is not writeable (run script with sudo)" - fi - - AddHost "$CONFIG_SERVER_IP" "$CONFIG_SERVER_HOSTNAME" "$HOSTS_FILE" - - local -i index=1 - for ((; index <= NUM_APP_CONTAINERS; ++index)) - do - local ip="$APP_NETWORK_PREFIX.$index" - local container_name="$APP_HOSTNAME_PREFIX$index" - AddHost "$ip" "$container_name" "$HOSTS_FILE" - done - - local -i index=1 - for ((; index <= NUM_SYSTEM_TEST_CONTAINERS; ++index)) - do - local ip="$SYSTEM_TEST_NETWORK_PREFIX.$index" - local container_name="$SYSTEM_TEST_HOSTNAME_PREFIX$index" - AddHost "$ip" "$container_name" "$HOSTS_FILE" - done -} - -StartAsRoot "$@" diff --git a/node-admin/scripts/node-repo.sh b/node-admin/scripts/node-repo.sh deleted file mode 100755 index 82f99f28e72..00000000000 --- a/node-admin/scripts/node-repo.sh +++ /dev/null @@ -1,360 +0,0 @@ -#!/bin/bash -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -declare -r VESPA_WEB_SERVICE_PORT=4080 - -# Output from InnerCurlNodeRepo, see there for details. -declare CURL_RESPONSE - -function Usage { - cat <<EOF -Usage: ${0##*/} <command> [<args>...] -Script for manipulating the Node Repository. - -Commands - add [-c <configserverhost>] -p <parent-hostname> [-f <parent-flavor>] - [-n <flavor> <hostname>...] - With -f, provision "host" node <parent-hostname> with flavor - <parent-flavor>. With -n, provision "tenant" nodes <hostname...> with - flavor <flavor> and parent host <parenthostname>. - reprovision [-c <configserverhost>] -p <parenthostname> <hostname>... - Fail node <hostname>, then rm and add. - rm [-c <configserverhost>] <hostname>... - Remove nodes from node repo. - set-state [-c <configserverhost>] <state> <hostname>... - Set the node repo state. - -By default, <configserverhost> is config-server. - -Example - To remove docker-1--1 through docker-1--5 from the node repo at configserver.com, - - ${0##*/} rm -c configserver.com \ - docker-1--{1,2,3,4,5}.dockerhosts.com -EOF - - exit 1 -} - -function Fail { - printf "%s\nRun '${0##*/} help' for usage\n" "$*" - exit 1 -} - -# Invoke curl such that: -# -# - Arguments to this function are passed to curl. -# -# - Additional arguments are passed to curl to filter noise (--silent -# --show-error). -# -# - The curl stdout (i.e. the response) is stored in the global CURL_RESPONSE -# variable on return of the function. -# -# - If curl returns 0 (i.e. a successful HTTP response) with a JSON response -# that contains an "error-code" key, this function will instead return 22. 22 -# does not conflict with a curl return code, because curl only ever returns -# 22 when --fail is specified. -# -# Except, if the JSON response contains a "message" of the form "Cannot add -# tmp-cnode-0: A node with this name already exists", InnerCurlNodeRepo -# returns 0 instead of 22, even if the HTTP response status code is an error. -# -# Note: Why not use --fail with curl? Because as of 2015-11-24, if the node -# exists when provisioning a node, the node repo returns a 400 Bad Request -# with a JSON body containing a "message" field as described above. With -# --fail, this would result in curl exiting with code 22, which is -# indistinguishable from other HTTP errors. Can the output from --show-error -# be used in combination with --fail? No, because that ends up saying "curl: -# (22) The requested URL returned error: 400 Bad Request" when the node -# exists, making it indistinguishable from other malformed request error -# messages. -# -# TODO: Make node repo return a unique HTTP error code when node already -# exists. It's also fragile to test for the error message in the response. -function InnerCurlNodeRepo { - # --show-error causes error message to be printed on error, even with - # --silent, which is useful when we print the error message to Fail. - local -a command=(curl --silent --show-error "$@") - - # We need the 'if' here, because a non-zero exit code of a command will - # exit the process, with 'set -e'. - if CURL_RESPONSE=$("${command[@]}" 2>&1) - then - # Match a JSON of the form: - # { - # "error-code": "BAD_REQUEST", - # "message": "Cannot add cnode-0: A node with this name already exists" - # } - if [[ "$CURL_RESPONSE" =~ '"error-code"' ]] - then - if [[ "$CURL_RESPONSE" =~ '"message"'[^\"]*\"(.*)\" ]] - then - local message="${BASH_REMATCH[1]}" - if [[ "$message" =~ 'already exists' ]] - then - return 0 - fi - fi - - return 22 - fi - - return 0 - else - # Do not move this statement outside of this else: $? gets cleared when - # the execution passes out of the else-fi block. - return $? - fi -} - -function CurlOrFail { - if InnerCurlNodeRepo "$@" - then - : # This form of if-else is used to preserve $?. - else - local error_code=$? - - # Terminate the current progress-bar-like line - printf ' failed\n' - - Fail "Error ($error_code) from the node repo at '$url': '$CURL_RESPONSE'" - fi -} - -function ProvisionDockerNode { - local config_server_hostname="$1" - local container_hostname="$2" - local parent_hostname="$3" - local flavor="$4" - - local json="[ - { - \"hostname\":\"$container_hostname\", - \"parentHostname\":\"$parent_hostname\", - \"openStackId\":\"fake-$container_hostname\", - \"flavor\":\"$flavor\", - \"type\":\"tenant\" - } - ]" - - ProvisionNode $config_server_hostname "$json" -} - - -# Docker host, the docker nodes points to this host in parentHostname in their node config -function ProvisionDockerHost { - local config_server_hostname="$1" - local docker_host_hostname="$2" - local flavor="$3" - - local json="[ - { - \"hostname\":\"$docker_host_hostname\", - \"openStackId\":\"$docker_host_hostname\", - \"flavor\":\"$flavor\", - \"type\":\"host\" - } - ]" - - ProvisionNode $config_server_hostname "$json" -} - -# Docker node in node repo (both docker hosts and docker nodes) -function ProvisionNode { - local config_server_hostname="$1" - local json="$2" - - local url="http://$config_server_hostname:$VESPA_WEB_SERVICE_PORT/nodes/v2/node" - - CurlOrFail -H "Content-Type: application/json" -X POST -d "$json" "$url" -} - -function SetNodeState { - local config_server_hostname="$1" - local hostname="$2" - local state="$3" - - local url="http://$config_server_hostname:$VESPA_WEB_SERVICE_PORT/nodes/v2/state/$state/$hostname" - CurlOrFail -X PUT "$url" -} - -function AddCommand { - local config_server_hostname=config-server - local parent_hostname= - - OPTIND=1 - local option - while getopts "c:p:f:n:" option - do - case "$option" in - c) config_server_hostname="$OPTARG" ;; - p) parent_hostname="$OPTARG" ;; - f) parent_host_flavor="$OPTARG" ;; - n) node_flavor="$OPTARG" ;; - ?) exit 1 ;; # E.g. option lacks argument, in case error has been - # already been printed - *) Fail "Unknown option '$option' with value '$OPTARG'" - esac - done - - if [ -z "$parent_hostname" ] - then - Fail "Parent hostname not specified (-p)" - fi - - shift $((OPTIND - 1)) - - if [ -n "$parent_host_flavor" ] - then - echo "Provisioning Docker host $parent_hostname with flavor $parent_host_flavor" - ProvisionDockerHost "$config_server_hostname" \ - "$parent_hostname" \ - "$parent_host_flavor" - fi - - if [ -n "$node_flavor" ] - then - echo -n "Provisioning $# nodes with parent host $parent_hostname" - local container_hostname - for container_hostname in "$@" - do - ProvisionDockerNode "$config_server_hostname" \ - "$container_hostname" \ - "$parent_hostname" \ - "$node_flavor" - echo -n . - done - - echo " done" - fi -} - -function ReprovisionCommand { - local config_server_hostname=config-server - local parent_hostname= - - OPTIND=1 - local option - while getopts "c:p:" option - do - case "$option" in - c) config_server_hostname="$OPTARG" ;; - p) parent_hostname="$OPTARG" ;; - ?) exit 1 ;; # E.g. option lacks argument, in case error has been - # already been printed - *) Fail "Unknown option '$option' with value '$OPTARG'" - esac - done - - if [ -z "$parent_hostname" ] - then - Fail "Parent hostname not specified (-p)" - fi - - shift $((OPTIND - 1)) - - if (($# == 0)) - then - Fail "No node hostnames were specified" - fi - - # Simulate calls to the following commands. - SetStateCommand -c "$config_server_hostname" failed "$@" - RemoveCommand -c "$config_server_hostname" "$@" - AddCommand -c "$config_server_hostname" -p "$parent_hostname" "$@" -} - -function RemoveCommand { - local config_server_hostname=config-server - - OPTIND=1 - local option - while getopts "c:" option - do - case "$option" in - c) config_server_hostname="$OPTARG" ;; - ?) exit 1 ;; # E.g. option lacks argument, in case error has been - # already been printed - *) Fail "Unknown option '$option' with value '$OPTARG'" - esac - done - - shift $((OPTIND - 1)) - - if (($# == 0)) - then - Fail "No nodes were specified" - fi - - echo -n "Removing $# nodes" - - local hostname - for hostname in "$@" - do - local url="http://$config_server_hostname:$VESPA_WEB_SERVICE_PORT/nodes/v2/node/$hostname" - CurlOrFail -X DELETE "$url" - echo -n . - done - - echo " done" -} - -function SetStateCommand { - local config_server_hostname=config-server - - OPTIND=1 - local option - while getopts "c:" option - do - case "$option" in - c) config_server_hostname="$OPTARG" ;; - ?) exit 1 ;; # E.g. option lacks argument, in case error has been - # already been printed - *) Fail "Unknown option '$option' with value '$OPTARG'" - esac - done - - shift $((OPTIND - 1)) - - if (($# <= 1)) - then - Fail "Too few arguments" - fi - - local state="$1" - shift - - echo -n "Setting $# nodes to $state" - - local hostname - for hostname in "$@" - do - SetNodeState "$config_server_hostname" "$hostname" "$state" - echo -n . - done - - echo " done" -} - -function Main { - if (($# == 0)) - then - Usage - fi - local command="$1" - shift - - case "$command" in - add) AddCommand "$@" ;; - reprovision) ReprovisionCommand "$@" ;; - rm) RemoveCommand "$@" ;; - set-state) SetStateCommand "$@" ;; - help) Usage "$@" ;; - *) Usage ;; - esac -} - -Main "$@" diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java index b83bd9895fc..233fe8318ae 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java @@ -1,9 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.host.FlavorOverrides; -import java.util.Collections; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -15,29 +16,29 @@ public class AddNode { public final String hostname; public final Optional<String> parentHostname; - public final String nodeFlavor; + public final Optional<String> nodeFlavor; + public final Optional<FlavorOverrides> flavorOverrides; + public final Optional<NodeResources> nodeResources; public final NodeType nodeType; public final Set<String> ipAddresses; public final Set<String> additionalIpAddresses; - /** - * Constructor for a host node (has no parent) - */ - public AddNode(String hostname, String nodeFlavor, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { - this(hostname, Optional.empty(), nodeFlavor, nodeType, ipAddresses, additionalIpAddresses); + public static AddNode forHost(String hostname, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { + return new AddNode(hostname, Optional.empty(), Optional.of(nodeFlavor), flavorOverrides, Optional.empty(), nodeType, ipAddresses, additionalIpAddresses); } - /** - * Constructor for a child node (Must set parentHostname, no additionalIpAddresses) - */ - public AddNode(String hostname, String parentHostname, String nodeFlavor, NodeType nodeType, Set<String> ipAddresses) { - this(hostname, Optional.of(parentHostname), nodeFlavor, nodeType, ipAddresses, Collections.emptySet()); + public static AddNode forNode(String hostname, String parentHostname, NodeResources nodeResources, NodeType nodeType, Set<String> ipAddresses) { + return new AddNode(hostname, Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), nodeType, ipAddresses, Set.of()); } - public AddNode(String hostname, Optional<String> parentHostname, String nodeFlavor, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { + private AddNode(String hostname, Optional<String> parentHostname, + Optional<String> nodeFlavor, Optional<FlavorOverrides> flavorOverrides, Optional<NodeResources> nodeResources, + NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { this.hostname = hostname; this.parentHostname = parentHostname; this.nodeFlavor = nodeFlavor; + this.flavorOverrides = flavorOverrides; + this.nodeResources = nodeResources; this.nodeType = nodeType; this.ipAddresses = ipAddresses; this.additionalIpAddresses = additionalIpAddresses; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java deleted file mode 100644 index c41e050d534..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeOwner.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.yahoo.config.provision.ApplicationId; - -/** - * @author freva - */ -public class NodeOwner { - private final String tenant; - private final String application; - private final String instance; - - public NodeOwner(String tenant, String application, String instance) { - this.tenant = tenant; - this.application = application; - this.instance = instance; - } - - public String tenant() { - return tenant; - } - - public String application() { - return application; - } - - public String instance() { - return instance; - } - - public ApplicationId asApplicationId() { - return ApplicationId.from(tenant, application, instance); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - NodeOwner owner = (NodeOwner) o; - - if (!tenant.equals(owner.tenant)) return false; - if (!application.equals(owner.application)) return false; - return instance.equals(owner.instance); - - } - - @Override - public int hashCode() { - int result = tenant.hashCode(); - result = 31 * result + application.hashCode(); - result = 31 * result + instance.hashCode(); - return result; - } - - public String toString() { - return "Owner {" + - " tenant = " + tenant + - " application = " + application + - " instance = " + instance + - " }"; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index 6fb6d44bd6f..c2e68bdb329 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import java.time.Instant; @@ -11,6 +13,9 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; +import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; + /** * @author stiankri */ @@ -19,7 +24,6 @@ public class NodeSpec { private final NodeState state; private final NodeType type; private final String flavor; - private final String canonicalFlavor; private final Optional<DockerImage> wantedDockerImage; private final Optional<DockerImage> currentDockerImage; @@ -43,15 +47,10 @@ public class NodeSpec { private final Optional<Boolean> allowedToBeDown; private final Optional<Boolean> wantToDeprovision; - private final Optional<NodeOwner> owner; + private final Optional<ApplicationId> owner; private final Optional<NodeMembership> membership; - private final double vcpus; - private final double memoryGb; - private final double diskGb; - - private final boolean fastDisk; - private final double bandwidth; + private final NodeResources resources; private final Set<String> ipAddresses; private final Set<String> additionalIpAddresses; @@ -66,14 +65,13 @@ public class NodeSpec { NodeState state, NodeType type, String flavor, - String canonicalFlavor, Optional<Version> wantedVespaVersion, Optional<Version> currentVespaVersion, Optional<Version> wantedOsVersion, Optional<Version> currentOsVersion, Optional<Boolean> allowedToBeDown, Optional<Boolean> wantToDeprovision, - Optional<NodeOwner> owner, + Optional<ApplicationId> owner, Optional<NodeMembership> membership, Optional<Long> wantedRestartGeneration, Optional<Long> currentRestartGeneration, @@ -82,11 +80,7 @@ public class NodeSpec { Optional<Instant> wantedFirmwareCheck, Optional<Instant> currentFirmwareCheck, Optional<String> modelName, - double vcpus, - double memoryGb, - double diskGb, - boolean fastDisk, - double bandwidth, + NodeResources resources, Set<String> ipAddresses, Set<String> additionalIpAddresses, NodeReports reports, @@ -104,7 +98,6 @@ public class NodeSpec { this.state = Objects.requireNonNull(state); this.type = Objects.requireNonNull(type); this.flavor = Objects.requireNonNull(flavor); - this.canonicalFlavor = canonicalFlavor; this.modelName = modelName; this.wantedVespaVersion = Objects.requireNonNull(wantedVespaVersion); this.currentVespaVersion = Objects.requireNonNull(currentVespaVersion); @@ -120,11 +113,7 @@ public class NodeSpec { this.currentRebootGeneration = currentRebootGeneration; this.wantedFirmwareCheck = Objects.requireNonNull(wantedFirmwareCheck); this.currentFirmwareCheck = Objects.requireNonNull(currentFirmwareCheck); - this.vcpus = vcpus; - this.memoryGb = memoryGb; - this.diskGb = diskGb; - this.fastDisk = fastDisk; - this.bandwidth = bandwidth; + this.resources = Objects.requireNonNull(resources); this.ipAddresses = Objects.requireNonNull(ipAddresses); this.additionalIpAddresses = Objects.requireNonNull(additionalIpAddresses); this.reports = Objects.requireNonNull(reports); @@ -147,10 +136,6 @@ public class NodeSpec { return flavor; } - public String canonicalFlavor() { - return canonicalFlavor; - } - public Optional<DockerImage> wantedDockerImage() { return wantedDockerImage; } @@ -211,7 +196,7 @@ public class NodeSpec { return wantToDeprovision; } - public Optional<NodeOwner> owner() { + public Optional<ApplicationId> owner() { return owner; } @@ -219,24 +204,28 @@ public class NodeSpec { return membership; } + public NodeResources resources() { + return resources; + } + public double vcpus() { - return vcpus; + return resources.vcpu(); } public double memoryGb() { - return memoryGb; + return resources.memoryGb(); } public double diskGb() { - return diskGb; + return resources.diskGb(); } public boolean isFastDisk() { - return fastDisk; + return resources.diskSpeed() == fast; } - public double bandwidth() { - return bandwidth; + public double bandwidthGbps() { + return resources.bandwidthGbps(); } public Set<String> ipAddresses() { @@ -266,7 +255,6 @@ public class NodeSpec { Objects.equals(state, that.state) && Objects.equals(type, that.type) && Objects.equals(flavor, that.flavor) && - Objects.equals(canonicalFlavor, that.canonicalFlavor) && Objects.equals(wantedVespaVersion, that.wantedVespaVersion) && Objects.equals(currentVespaVersion, that.currentVespaVersion) && Objects.equals(wantedOsVersion, that.wantedOsVersion) && @@ -281,11 +269,7 @@ public class NodeSpec { Objects.equals(currentRebootGeneration, that.currentRebootGeneration) && Objects.equals(wantedFirmwareCheck, that.wantedFirmwareCheck) && Objects.equals(currentFirmwareCheck, that.currentFirmwareCheck) && - Objects.equals(vcpus, that.vcpus) && - Objects.equals(memoryGb, that.memoryGb) && - Objects.equals(diskGb, that.diskGb) && - Objects.equals(fastDisk, that.fastDisk) && - Objects.equals(bandwidth, that.bandwidth) && + Objects.equals(resources, that.resources) && Objects.equals(ipAddresses, that.ipAddresses) && Objects.equals(additionalIpAddresses, that.additionalIpAddresses) && Objects.equals(reports, that.reports) && @@ -301,7 +285,6 @@ public class NodeSpec { state, type, flavor, - canonicalFlavor, wantedVespaVersion, currentVespaVersion, wantedOsVersion, @@ -316,11 +299,7 @@ public class NodeSpec { currentRebootGeneration, wantedFirmwareCheck, currentFirmwareCheck, - vcpus, - memoryGb, - diskGb, - fastDisk, - bandwidth, + resources, ipAddresses, additionalIpAddresses, reports, @@ -336,7 +315,6 @@ public class NodeSpec { + " state=" + state + " type=" + type + " flavor=" + flavor - + " canonicalFlavor=" + canonicalFlavor + " wantedVespaVersion=" + wantedVespaVersion + " currentVespaVersion=" + currentVespaVersion + " wantedOsVersion=" + wantedOsVersion @@ -345,17 +323,13 @@ public class NodeSpec { + " wantToDeprovision=" + wantToDeprovision + " owner=" + owner + " membership=" + membership - + " vcpus=" + vcpus + " wantedRestartGeneration=" + wantedRestartGeneration + " currentRestartGeneration=" + currentRestartGeneration + " wantedRebootGeneration=" + wantedRebootGeneration + " currentRebootGeneration=" + currentRebootGeneration + " wantedFirmwareCheck=" + wantedFirmwareCheck + " currentFirmwareCheck=" + currentFirmwareCheck - + " memoryGb=" + memoryGb - + " diskGb=" + diskGb - + " fastDisk=" + fastDisk - + " bandwidth=" + bandwidth + + " resources=" + resources + " ipAddresses=" + ipAddresses + " additionalIpAddresses=" + additionalIpAddresses + " reports=" + reports @@ -368,7 +342,6 @@ public class NodeSpec { private NodeState state; private NodeType type; private String flavor; - private String canonicalFlavor; private Optional<DockerImage> wantedDockerImage = Optional.empty(); private Optional<DockerImage> currentDockerImage = Optional.empty(); private Optional<Version> wantedVespaVersion = Optional.empty(); @@ -377,7 +350,7 @@ public class NodeSpec { private Optional<Version> currentOsVersion = Optional.empty(); private Optional<Boolean> allowedToBeDown = Optional.empty(); private Optional<Boolean> wantToDeprovision = Optional.empty(); - private Optional<NodeOwner> owner = Optional.empty(); + private Optional<ApplicationId> owner = Optional.empty(); private Optional<NodeMembership> membership = Optional.empty(); private Optional<Long> wantedRestartGeneration = Optional.empty(); private Optional<Long> currentRestartGeneration = Optional.empty(); @@ -386,11 +359,7 @@ public class NodeSpec { private Optional<Instant> wantedFirmwareCheck = Optional.empty(); private Optional<Instant> currentFirmwareCheck = Optional.empty(); private Optional<String> modelName = Optional.empty(); - private double vcpus; - private double memoryGb; - private double diskGb; - private boolean fastDisk; - private double bandwidth; + private NodeResources resources = new NodeResources(0, 0, 0, 0, slow); private Set<String> ipAddresses = Set.of(); private Set<String> additionalIpAddresses = Set.of(); private NodeReports reports = new NodeReports(); @@ -403,12 +372,7 @@ public class NodeSpec { state(node.state); type(node.type); flavor(node.flavor); - canonicalFlavor(node.canonicalFlavor); - vcpus(node.vcpus); - memoryGb(node.memoryGb); - diskGb(node.diskGb); - fastDisk(node.fastDisk); - bandwidth(node.bandwidth); + resources(node.resources); ipAddresses(node.ipAddresses); additionalIpAddresses(node.additionalIpAddresses); wantedRebootGeneration(node.wantedRebootGeneration); @@ -462,11 +426,6 @@ public class NodeSpec { return this; } - public Builder canonicalFlavor(String canonicalFlavor) { - this.canonicalFlavor = canonicalFlavor; - return this; - } - public Builder wantedVespaVersion(Version wantedVespaVersion) { this.wantedVespaVersion = Optional.of(wantedVespaVersion); return this; @@ -497,7 +456,7 @@ public class NodeSpec { return this; } - public Builder owner(NodeOwner owner) { + public Builder owner(ApplicationId owner) { this.owner = Optional.of(owner); return this; } @@ -537,29 +496,29 @@ public class NodeSpec { return this; } - public Builder vcpus(double minCpuCores) { - this.vcpus = minCpuCores; + public Builder resources(NodeResources resources) { + this.resources = resources; return this; } - public Builder memoryGb(double minMainMemoryAvailableGb) { - this.memoryGb = minMainMemoryAvailableGb; - return this; + public Builder vcpus(double vcpus) { + return resources(resources.withVcpu(vcpus)); } - public Builder diskGb(double minDiskAvailableGb) { - this.diskGb = minDiskAvailableGb; - return this; + public Builder memoryGb(double memoryGb) { + return resources(resources.withMemoryGb(memoryGb)); + } + + public Builder diskGb(double diskGb) { + return resources(resources.withDiskGb(diskGb)); } public Builder fastDisk(boolean fastDisk) { - this.fastDisk = fastDisk; - return this; + return resources(resources.withDiskSpeed(fastDisk ? fast : slow)); } - public Builder bandwidth(double bandwidth) { - this.bandwidth = bandwidth; - return this; + public Builder bandwidthGbps(double bandwidthGbps) { + return resources(resources.withBandwidthGbps(bandwidthGbps)); } public Builder ipAddresses(Set<String> ipAddresses) { @@ -627,10 +586,6 @@ public class NodeSpec { return flavor; } - public String canonicalFlavor() { - return canonicalFlavor; - } - public Optional<Version> wantedVespaVersion() { return wantedVespaVersion; } @@ -655,7 +610,7 @@ public class NodeSpec { return wantToDeprovision; } - public Optional<NodeOwner> owner() { + public Optional<ApplicationId> owner() { return owner; } @@ -679,24 +634,8 @@ public class NodeSpec { return currentRebootGeneration; } - public double vcpus() { - return vcpus; - } - - public double memoryGb() { - return memoryGb; - } - - public double diskGb() { - return diskGb; - } - - public boolean isFastDisk() { - return fastDisk; - } - - public double bandwidth() { - return bandwidth; + public NodeResources resources() { + return resources; } public Set<String> ipAddresses() { @@ -716,15 +655,13 @@ public class NodeSpec { } public NodeSpec build() { - return new NodeSpec(hostname, wantedDockerImage, currentDockerImage, state, type, - flavor, canonicalFlavor, + return new NodeSpec(hostname, wantedDockerImage, currentDockerImage, state, type, flavor, wantedVespaVersion, currentVespaVersion, wantedOsVersion, currentOsVersion, allowedToBeDown, wantToDeprovision, owner, membership, wantedRestartGeneration, currentRestartGeneration, wantedRebootGeneration, currentRebootGeneration, wantedFirmwareCheck, currentFirmwareCheck, modelName, - vcpus, memoryGb, diskGb, - fastDisk, bandwidth, ipAddresses, additionalIpAddresses, + resources, ipAddresses, additionalIpAddresses, reports, parentHostname); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index fe19b81614d..582ba3dd09c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -4,8 +4,11 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Strings; import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetAclResponse; @@ -25,6 +28,9 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; +import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; + /** * @author stiankri, dybis */ @@ -154,14 +160,13 @@ public class RealNodeRepository implements NodeRepository { nodeState, nodeType, node.flavor, - node.canonicalFlavor, Optional.ofNullable(node.wantedVespaVersion).map(Version::fromString), Optional.ofNullable(node.vespaVersion).map(Version::fromString), Optional.ofNullable(node.wantedOsVersion).map(Version::fromString), Optional.ofNullable(node.currentOsVersion).map(Version::fromString), Optional.ofNullable(node.allowedToBeDown), Optional.ofNullable(node.wantToDeprovision), - Optional.ofNullable(node.owner).map(o -> new NodeOwner(o.tenant, o.application, o.instance)), + Optional.ofNullable(node.owner).map(o -> ApplicationId.from(o.tenant, o.application, o.instance)), membership, Optional.ofNullable(node.restartGeneration), Optional.ofNullable(node.currentRestartGeneration), @@ -170,11 +175,12 @@ public class RealNodeRepository implements NodeRepository { Optional.ofNullable(node.wantedFirmwareCheck).map(Instant::ofEpochMilli), Optional.ofNullable(node.currentFirmwareCheck).map(Instant::ofEpochMilli), Optional.ofNullable(node.modelName), - node.minCpuCores, - node.minMainMemoryAvailableGb, - node.minDiskAvailableGb, - node.fastDisk, - node.bandwidth, + new NodeResources( + node.minCpuCores, + node.minMainMemoryAvailableGb, + node.minDiskAvailableGb, + node.bandwidth / 1000, + node.fastDisk ? fast : slow), node.ipAddresses, node.additionalIpAddresses, reports, @@ -186,7 +192,15 @@ public class RealNodeRepository implements NodeRepository { node.openStackId = "fake-" + addNode.hostname; node.hostname = addNode.hostname; node.parentHostname = addNode.parentHostname.orElse(null); - node.flavor = addNode.nodeFlavor; + addNode.nodeFlavor.ifPresent(f -> node.flavor = f); + addNode.flavorOverrides.flatMap(FlavorOverrides::diskGb).ifPresent(d -> node.minDiskAvailableGb = d); + addNode.nodeResources.ifPresent(resources -> { + node.minCpuCores = resources.vcpu(); + node.minMainMemoryAvailableGb = resources.memoryGb(); + node.minDiskAvailableGb = resources.diskGb(); + node.bandwidth = resources.bandwidthGbps() * 1000; + node.fastDisk = resources.diskSpeed() == NodeResources.DiskSpeed.fast; + }); node.type = addNode.nodeType.name(); node.ipAddresses = addNode.ipAddresses; node.additionalIpAddresses = addNode.additionalIpAddresses; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java index 2b4f9277fc7..90d70a94144 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java @@ -28,8 +28,6 @@ public class NodeRepositoryNode { public String openStackId; @JsonProperty("flavor") public String flavor; - @JsonProperty("canonicalFlavor") - public String canonicalFlavor; @JsonProperty("membership") public Membership membership; @JsonProperty("owner") @@ -98,7 +96,6 @@ public class NodeRepositoryNode { ", openStackId='" + openStackId + '\'' + ", modelName='" + modelName + '\'' + ", flavor='" + flavor + '\'' + - ", canonicalFlavor='" + canonicalFlavor + '\'' + ", membership=" + membership + ", owner=" + owner + ", restartGeneration=" + restartGeneration + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index f4355ed3afa..91f4924cefa 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -141,9 +141,9 @@ public class StorageMaintainer { context.node().parentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent)); context.node().currentVespaVersion().ifPresent(version -> attributes.put("vespa_version", version.toFullString())); context.node().owner().ifPresent(owner -> { - attributes.put("tenant", owner.tenant()); - attributes.put("application", owner.application()); - attributes.put("instance", owner.instance()); + attributes.put("tenant", owner.tenant().value()); + attributes.put("application", owner.application().value()); + attributes.put("instance", owner.instance().value()); }); return Collections.unmodifiableMap(attributes); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index b7e7b97cdd8..161775b0702 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.exception.ContainerNotFoundException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; @@ -345,7 +344,6 @@ public class NodeAgentImpl implements NodeAgent { double cpuCap = noCpuCap(context.zone()) ? 0 : context.node().owner() - .map(NodeOwner::asApplicationId) .map(appId -> containerCpuCap.with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm())) .orElse(containerCpuCap) .with(FetchVector.Dimension.HOSTNAME, context.node().hostname()) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 0938eb23b49..e30572fc63e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -1,11 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.yahoo.application.Networking; import com.yahoo.application.container.JDisc; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; @@ -17,11 +18,10 @@ import java.io.IOException; import java.net.ServerSocket; import java.net.URI; import java.time.Instant; -import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; @@ -162,23 +162,22 @@ public class RealNodeRepositoryTest { @Test public void testAddNodes() { - AddNode host = new AddNode("host123.domain.tld", "default", NodeType.confighost, - Collections.singleton("::1"), new HashSet<>(Arrays.asList("::2", "::3"))); - - AddNode node = new AddNode("host123-1.domain.tld", "host123.domain.tld", "docker", NodeType.config, - new HashSet<>(Arrays.asList("::2", "::3"))); + AddNode host = AddNode.forHost("host123.domain.tld", "default", Optional.of(FlavorOverrides.ofDisk(123)), NodeType.confighost, + Set.of("::1"), Set.of("::2", "::3")); - List<AddNode> nodesToAdd = Arrays.asList(host, node); + NodeResources nodeResources = new NodeResources(1, 2, 3, 4, NodeResources.DiskSpeed.slow); + AddNode node = AddNode.forNode("host123-1.domain.tld", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3")); assertFalse(nodeRepositoryApi.getOptionalNode("host123.domain.tld").isPresent()); - nodeRepositoryApi.addNodes(nodesToAdd); - - NodeSpec hostSpecInNodeRepo = nodeRepositoryApi.getOptionalNode("host123.domain.tld") - .orElseThrow(RuntimeException::new); + nodeRepositoryApi.addNodes(List.of(host, node)); - assertEquals(host.nodeFlavor, hostSpecInNodeRepo.flavor()); - assertEquals(host.nodeType, hostSpecInNodeRepo.type()); + NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode("host123.domain.tld").orElseThrow(); + assertEquals("default", hostSpec.flavor()); + assertEquals(123, hostSpec.diskGb(), 0); + assertEquals(NodeType.confighost, hostSpec.type()); - assertTrue(nodeRepositoryApi.getOptionalNode("host123-1.domain.tld").isPresent()); + NodeSpec nodeSpec = nodeRepositoryApi.getOptionalNode("host123-1.domain.tld").orElseThrow(); + assertEquals(nodeResources, nodeSpec.resources()); + assertEquals(NodeType.config, nodeSpec.type()); } } |