diff options
author | Harald Musum <musum@verizonmedia.com> | 2021-07-14 13:01:17 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-07-14 13:01:17 +0200 |
commit | b55147670eba8a1b8c7a0c75077d2f865bc50499 (patch) | |
tree | 95606419071e63cf9223e6b3c5d34bf8b859746f | |
parent | 3534840687517be5d8f90af0a7bde4ebb43bc06a (diff) |
Revert "Do not use serializable NodeRepositoryNode internally in controller"
24 files changed, 633 insertions, 592 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index 25357ad7f98..5f46b949844 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; +import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; @@ -8,29 +9,26 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeHistory; import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.UUID; /** * A node in hosted Vespa. * - * This is immutable and all fields are guaranteed to be non-null. This should never leak any wire format types or - * types from third-party libraries. - * - * Use {@link Node#builder()} or {@link Node#builder(Node)} to create instances of this. - * * @author mpolden * @author jonmv */ public class Node { - private final String id; private final HostName hostname; private final Optional<HostName> parentHostname; private final State state; @@ -52,278 +50,204 @@ public class Node { private final long rebootGeneration; private final long wantedRebootGeneration; private final int cost; - private final int failCount; private final String flavor; private final String clusterId; private final ClusterType clusterType; - private final String group; private final boolean retired; private final boolean wantToRetire; private final boolean wantToDeprovision; private final boolean wantToRebuild; private final Optional<TenantName> reservedTo; private final Optional<ApplicationId> exclusiveTo; - private final Map<String, String> reports; - private final List<Event> history; - private final Set<String> ipAddresses; + private final Map<String, JsonNode> reports; + private final List<NodeHistory> history; private final Set<String> additionalIpAddresses; - private final Set<String> additionalHostnames; + private final String openStackId; private final Optional<String> switchHostname; private final Optional<String> modelName; - private final Environment environment; - - private Node(String id, HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, - NodeResources resources, Optional<ApplicationId> owner, Version currentVersion, Version wantedVersion, - Version currentOsVersion, Version wantedOsVersion, Optional<Instant> currentFirmwareCheck, - Optional<Instant> wantedFirmwareCheck, ServiceState serviceState, Optional<Instant> suspendedSince, - long restartGeneration, long wantedRestartGeneration, long rebootGeneration, - long wantedRebootGeneration, int cost, int failCount, String flavor, String clusterId, - ClusterType clusterType, String group, boolean retired, boolean wantToRetire, boolean wantToDeprovision, - boolean wantToRebuild, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveTo, - DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, String> reports, - List<Event> history, Set<String> ipAddresses, Set<String> additionalIpAddresses, - Set<String> additionalHostnames, Optional<String> switchHostname, - Optional<String> modelName, Environment environment) { - this.id = Objects.requireNonNull(id, "id must be non-null"); - this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); - this.parentHostname = Objects.requireNonNull(parentHostname, "parentHostname must be non-null"); - this.state = Objects.requireNonNull(state, "state must be non-null"); - this.type = Objects.requireNonNull(type, "type must be non-null"); - this.resources = Objects.requireNonNull(resources, "resources must be non-null"); - this.owner = Objects.requireNonNull(owner, "owner must be non-null"); - this.currentVersion = Objects.requireNonNull(currentVersion, "currentVersion must be non-null"); - this.wantedVersion = Objects.requireNonNull(wantedVersion, "wantedVersion must be non-null"); - this.currentOsVersion = Objects.requireNonNull(currentOsVersion, "currentOsVersion must be non-null"); - this.wantedOsVersion = Objects.requireNonNull(wantedOsVersion, "wantedOsVersion must be non-null"); - this.currentFirmwareCheck = Objects.requireNonNull(currentFirmwareCheck, "currentFirmwareCheck must be non-null"); - this.wantedFirmwareCheck = Objects.requireNonNull(wantedFirmwareCheck, "wantedFirmwareCheck must be non-null"); - this.serviceState = Objects.requireNonNull(serviceState, "serviceState must be non-null"); - this.suspendedSince = Objects.requireNonNull(suspendedSince, "suspendedSince must be non-null"); + + public Node(HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, NodeResources resources, Optional<ApplicationId> owner, + Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, + Optional<Instant> currentFirmwareCheck, Optional<Instant> wantedFirmwareCheck, ServiceState serviceState, + Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, + int cost, String flavor, String clusterId, ClusterType clusterType, boolean retired, boolean wantToRetire, boolean wantToDeprovision, + boolean wantToRebuild, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveTo, + DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, JsonNode> reports, List<NodeHistory> history, + Set<String> additionalIpAddresses, String openStackId, Optional<String> switchHostname, Optional<String> modelName) { + this.hostname = hostname; + this.parentHostname = parentHostname; + this.state = state; + this.type = type; + this.resources = resources; + this.owner = owner; + this.currentVersion = currentVersion; + this.wantedVersion = wantedVersion; + this.currentOsVersion = currentOsVersion; + this.wantedOsVersion = wantedOsVersion; + this.currentFirmwareCheck = currentFirmwareCheck; + this.wantedFirmwareCheck = wantedFirmwareCheck; + this.serviceState = serviceState; + this.suspendedSince = suspendedSince; this.restartGeneration = restartGeneration; this.wantedRestartGeneration = wantedRestartGeneration; this.rebootGeneration = rebootGeneration; this.wantedRebootGeneration = wantedRebootGeneration; this.cost = cost; - this.failCount = failCount; - this.flavor = Objects.requireNonNull(flavor, "flavor must be non-null"); - this.clusterId = Objects.requireNonNull(clusterId, "clusterId must be non-null"); - this.clusterType = Objects.requireNonNull(clusterType, "clusterType must be non-null"); + this.flavor = flavor; + this.clusterId = clusterId; + this.clusterType = clusterType; this.retired = retired; - this.group = Objects.requireNonNull(group, "group must be non-null"); this.wantToRetire = wantToRetire; this.wantToDeprovision = wantToDeprovision; - this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo must be non-null"); - this.exclusiveTo = Objects.requireNonNull(exclusiveTo, "exclusiveTo must be non-null"); - this.wantedDockerImage = Objects.requireNonNull(wantedDockerImage, "wantedDockerImage must be non-null"); - this.currentDockerImage = Objects.requireNonNull(currentDockerImage, "currentDockerImage must be non-null"); + this.reservedTo = reservedTo; + this.exclusiveTo = exclusiveTo; + this.wantedDockerImage = wantedDockerImage; + this.currentDockerImage = currentDockerImage; this.wantToRebuild = wantToRebuild; - this.reports = Map.copyOf(Objects.requireNonNull(reports, "reports must be non-null")); - this.history = List.copyOf(Objects.requireNonNull(history, "history must be non-null")); - this.ipAddresses = Set.copyOf(Objects.requireNonNull(ipAddresses, "ipAddresses must be non-null")); - this.additionalIpAddresses = Set.copyOf(Objects.requireNonNull(additionalIpAddresses, "additionalIpAddresses must be non-null")); - this.additionalHostnames = Set.copyOf(Objects.requireNonNull(additionalHostnames, "additionalHostnames must be non-null")); - this.switchHostname = Objects.requireNonNull(switchHostname, "switchHostname must be non-null"); - this.modelName = Objects.requireNonNull(modelName, "modelName must be non-null"); - this.environment = Objects.requireNonNull(environment, "environment must be non-ull"); + this.reports = reports; + this.history = history; + this.openStackId = openStackId; + this.additionalIpAddresses = additionalIpAddresses; + this.switchHostname = switchHostname; + this.modelName = modelName; } - /** The cloud provider's unique ID for this */ - public String id() { - return id; - } - - /** The hostname of this */ public HostName hostname() { return hostname; } - /** The parent hostname of this, if any */ public Optional<HostName> parentHostname() { return parentHostname; } - /** Current state of this */ public State state() { return state; } - /** The node type of this */ public NodeType type() { return type; } - /** Resources, such as CPU and memory, of this */ public NodeResources resources() { return resources; } - /** The application owning this, if any */ public Optional<ApplicationId> owner() { return owner; } - /** The Vespa version this is currently running */ public Version currentVersion() { return currentVersion; } - /** The wanted Vespa version */ public Version wantedVersion() { return wantedVersion; } - /** The OS version this is currently running */ public Version currentOsVersion() { return currentOsVersion; } - /** The wanted OS version */ public Version wantedOsVersion() { return wantedOsVersion; } - /** The container image of this is currently running */ public DockerImage currentDockerImage() { return currentDockerImage; } - /** The wanted Docker image */ public DockerImage wantedDockerImage() { return wantedDockerImage; } - /** The last time this checked for a firmware update */ public Optional<Instant> currentFirmwareCheck() { return currentFirmwareCheck; } - /** The wanted time this should check for a firmware update */ public Optional<Instant> wantedFirmwareCheck() { return wantedFirmwareCheck; } - /** The current service state of this */ public ServiceState serviceState() { return serviceState; } - /** The most recent time this suspended, if any */ public Optional<Instant> suspendedSince() { return suspendedSince; } - /** The current restart generation */ public long restartGeneration() { return restartGeneration; } - /** The wanted restart generation */ public long wantedRestartGeneration() { return wantedRestartGeneration; } - /** The current reboot generation */ public long rebootGeneration() { return rebootGeneration; } - /** The wanted reboot generation */ public long wantedRebootGeneration() { return wantedRebootGeneration; } - /** A number representing the cost of this */ public int cost() { return cost; } - /** How many times this has failed */ - public int failCount() { - return failCount; - } - - /** The flavor of this */ public String flavor() { return flavor; } - /** The cluster ID of this, empty string if unallocated */ public String clusterId() { return clusterId; } - /** The cluster type of this */ public ClusterType clusterType() { return clusterType; } - /** Whether this is retired */ public boolean retired() { return retired; } - /** The group of this node, empty string if unallocated */ - public String group() { - return group; - } - - /** Whether this node has been requested to retire */ public boolean wantToRetire() { return wantToRetire; } - /** Whether this node has been requested to deprovision */ public boolean wantToDeprovision() { return wantToDeprovision; } - /** Whether this node has been requested to rebuild */ public boolean wantToRebuild() { return wantToRebuild; } - /** The tenant this has been reserved to, if any */ public Optional<TenantName> reservedTo() { return reservedTo; } - /** The application this has been provisioned exclusively for, if any */ public Optional<ApplicationId> exclusiveTo() { return exclusiveTo; } - /** Returns the reports of this node. Key is the report ID. Value is untyped, but is typically a JSON string */ - public Map<String, String> reports() { + public Map<String, JsonNode> reports() { return reports; } - /** History of events affecting this */ - public List<Event> history() { + public List<NodeHistory> history() { return history; } - /** IP addresses of this */ - public Set<String> ipAddresses() { - return ipAddresses; - } - - /** Additional IP addresses available on this, usable by child nodes */ public Set<String> additionalIpAddresses() { return additionalIpAddresses; } - /** Additional hostnames available on this, usable by child nodes */ - public Set<String> additionalHostnames() { - return additionalHostnames; + public String openStackId() { + return openStackId; } - /** Hostname of the switch this is connected to, if any */ public Optional<String> switchHostname() { return switchHostname; } - /** The server model of this, if any */ public Optional<String> modelName() { return modelName; } - /** The environment this runs in */ - public Environment environment() { - return environment; - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -369,106 +293,48 @@ public class Node { combined, unknown } - - /** Known nope environments */ - public enum Environment { - bareMetal, - virtualMachine, - dockerContainer, - } - - /** A node event */ - public static class Event { - - private final Instant at; - private final String agent; - private final String name; - - public Event(Instant at, String agent, String name) { - this.at = Objects.requireNonNull(at); - this.agent = Objects.requireNonNull(agent); - this.name = Objects.requireNonNull(name); - } - - /** The time this occurred */ - public Instant at() { - return at; - } - - /** The agent responsible for this */ - public String agent() { - return agent; - } - - /** Name of the event */ - public String name() { - return name; - } - - } - - public static Builder builder() { - return new Builder(); - } - public static Builder builder(Node node) { - return new Builder(node); - } - - /** - * Builder for a {@link Node}. - * - * The appropriate builder method must be called for any field that does not have a default value. - */ public static class Builder { - private HostName hostname; - - private String id = UUID.randomUUID().toString(); private Optional<HostName> parentHostname = Optional.empty(); - private State state = State.active; - private NodeType type = NodeType.host; - private NodeResources resources = NodeResources.unspecified(); + private State state; + private NodeType type; + private NodeResources resources; private Optional<ApplicationId> owner = Optional.empty(); - private Version currentVersion = Version.emptyVersion; - private Version wantedVersion = Version.emptyVersion; - private Version currentOsVersion = Version.emptyVersion; - private Version wantedOsVersion = Version.emptyVersion; - private DockerImage currentDockerImage = DockerImage.EMPTY; - private DockerImage wantedDockerImage = DockerImage.EMPTY; + private Version currentVersion; + private Version wantedVersion; + private Version currentOsVersion; + private Version wantedOsVersion; + private DockerImage currentDockerImage; + private DockerImage wantedDockerImage; private Optional<Instant> currentFirmwareCheck = Optional.empty(); private Optional<Instant> wantedFirmwareCheck = Optional.empty(); - private ServiceState serviceState = ServiceState.unknown; + private ServiceState serviceState; private Optional<Instant> suspendedSince = Optional.empty(); - private long restartGeneration = 0; - private long wantedRestartGeneration = 0; - private long rebootGeneration = 0; - private long wantedRebootGeneration = 0; - private int cost = 0; - private int failCount = 0; - private String flavor = "default"; - private String clusterId = ""; - private ClusterType clusterType = ClusterType.unknown; - private String group = ""; - private boolean retired = false; - private boolean wantToRetire = false; - private boolean wantToDeprovision = false; - private boolean wantToRebuild = false; + private long restartGeneration; + private long wantedRestartGeneration; + private long rebootGeneration; + private long wantedRebootGeneration; + private int cost; + private String flavor; + private String clusterId; + private ClusterType clusterType; + private boolean retired; + private boolean wantToRetire; + private boolean wantToDeprovision; + private boolean wantToRebuild; private Optional<TenantName> reservedTo = Optional.empty(); private Optional<ApplicationId> exclusiveTo = Optional.empty(); - private Map<String, String> reports = Map.of(); - private List<Event> history = List.of(); - private Set<String> ipAddresses = Set.of(); - private Set<String> additionalIpAddresses = Set.of(); - private Set<String> additionalHostnames = Set.of(); + private Map<String, JsonNode> reports = new HashMap<>(); + private List<NodeHistory> history = new ArrayList<>(); + private Set<String> additionalIpAddresses = new HashSet<>(); + private String openStackId; private Optional<String> switchHostname = Optional.empty(); private Optional<String> modelName = Optional.empty(); - private Environment environment = Environment.bareMetal; - private Builder() {} + public Builder() { } - private Builder(Node node) { - this.id = node.id; + public Builder(Node node) { this.hostname = node.hostname; this.parentHostname = node.parentHostname; this.state = node.state; @@ -481,20 +347,18 @@ public class Node { this.wantedOsVersion = node.wantedOsVersion; this.currentDockerImage = node.currentDockerImage; this.wantedDockerImage = node.wantedDockerImage; - this.serviceState = node.serviceState; - this.suspendedSince = node.suspendedSince; this.currentFirmwareCheck = node.currentFirmwareCheck; this.wantedFirmwareCheck = node.wantedFirmwareCheck; + this.serviceState = node.serviceState; + this.suspendedSince = node.suspendedSince; this.restartGeneration = node.restartGeneration; this.wantedRestartGeneration = node.wantedRestartGeneration; this.rebootGeneration = node.rebootGeneration; this.wantedRebootGeneration = node.wantedRebootGeneration; this.cost = node.cost; - this.failCount = node.failCount; this.flavor = node.flavor; this.clusterId = node.clusterId; this.clusterType = node.clusterType; - this.group = node.group; this.retired = node.retired; this.wantToRetire = node.wantToRetire; this.wantToDeprovision = node.wantToDeprovision; @@ -503,21 +367,10 @@ public class Node { this.exclusiveTo = node.exclusiveTo; this.reports = node.reports; this.history = node.history; - this.ipAddresses = node.ipAddresses; this.additionalIpAddresses = node.additionalIpAddresses; - this.additionalHostnames = node.additionalHostnames; + this.openStackId = node.openStackId; this.switchHostname = node.switchHostname; this.modelName = node.modelName; - this.environment = node.environment; - } - - public Builder id(String id) { - this.id = id; - return this; - } - - public Builder hostname(String hostname) { - return hostname(HostName.from(hostname)); } public Builder hostname(HostName hostname) { @@ -525,10 +378,6 @@ public class Node { return this; } - public Builder parentHostname(String parentHostname) { - return parentHostname(HostName.from(parentHostname)); - } - public Builder parentHostname(HostName parentHostname) { this.parentHostname = Optional.ofNullable(parentHostname); return this; @@ -629,11 +478,6 @@ public class Node { return this; } - public Builder failCount(int failCount) { - this.failCount = failCount; - return this; - } - public Builder flavor(String flavor) { this.flavor = flavor; return this; @@ -649,11 +493,6 @@ public class Node { return this; } - public Builder group(String group) { - this.group = group; - return this; - } - public Builder retired(boolean retired) { this.retired = retired; return this; @@ -684,23 +523,18 @@ public class Node { return this; } - public Builder history(List<Event> history) { + public Builder history(List<NodeHistory> history) { this.history = history; return this; } - public Builder ipAddresses(Set<String> ipAdresses) { - this.ipAddresses = ipAdresses; - return this; - } - public Builder additionalIpAddresses(Set<String> additionalIpAddresses) { this.additionalIpAddresses = additionalIpAddresses; return this; } - public Builder additionalHostnames(Set<String> additionalHostnames) { - this.additionalHostnames = additionalHostnames; + public Builder openStackId(String openStackId) { + this.openStackId = openStackId; return this; } @@ -714,26 +548,18 @@ public class Node { return this; } - public Builder reports(Map<String, String> reports) { + public Builder reports(Map<String, JsonNode> reports) { this.reports = reports; return this; } - public Builder environment(Environment environment) { - this.environment = environment; - return this; - } - public Node build() { - return new Node(id, hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, + return new Node(hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState, - suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, - wantedRebootGeneration, cost, failCount, flavor, clusterId, clusterType, group, retired, - wantToRetire, wantToDeprovision, wantToRebuild, reservedTo, exclusiveTo, wantedDockerImage, - currentDockerImage, reports, history, ipAddresses, additionalIpAddresses, - additionalHostnames, switchHostname, modelName, environment); + suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration, + cost, flavor, clusterId, clusterType, retired, wantToRetire, wantToDeprovision, wantToRebuild, reservedTo, exclusiveTo, + wantedDockerImage, currentDockerImage, reports, history, additionalIpAddresses, openStackId, switchHostname, modelName); } } - } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 3c16eac06c7..ac4ff0a80a0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -3,15 +3,21 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.OrchestratorStatus; import java.net.URI; import java.time.Duration; +import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Map; @@ -20,7 +26,7 @@ import java.util.Set; import java.util.stream.Collectors; /** - * Node repository interface intended for use by the controller. + * A minimal interface to the node repository, providing only the operations used by the controller. * * @author mpolden */ @@ -32,7 +38,10 @@ public interface NodeRepository { void setState(ZoneId zone, NodeState nodeState, String hostname); - Node getNode(ZoneId zone, String hostname); + NodeRepositoryNode getNode(ZoneId zone, String hostname); + + // TODO: Migrate any callers to list() and remove this method + NodeList listNodes(ZoneId zone); /** List all nodes in given zone */ List<Node> list(ZoneId zone, boolean includeDeprovisioned); @@ -87,4 +96,145 @@ public interface NodeRepository { /** Checks whether the zone has the spare capacity to remove the given hosts */ boolean isReplaceable(ZoneId zoneId, List<HostName> hostNames); + static Node toNode(NodeRepositoryNode node) { + var application = Optional.ofNullable(node.getOwner()) + .map(owner -> ApplicationId.from(owner.getTenant(), owner.getApplication(), + owner.getInstance())); + var parentHostname = Optional.ofNullable(node.getParentHostname()).map(HostName::from); + var resources = new NodeResources( + toDouble(node.getResources().getVcpu()), + toDouble(node.getResources().getMemoryGb()), + toDouble(node.getResources().getDiskGb()), + toDouble(node.getResources().getBandwidthGbps()), + diskSpeedFromString(node.getResources().getDiskSpeed()), + storageTypeFromString(node.getResources().getStorageType())); + return new Node(HostName.from(node.getHostname()), + parentHostname, + fromJacksonState(node.getState()), + fromJacksonType(node.getType()), + resources, + application, + versionFrom(node.getVespaVersion()), + versionFrom(node.getWantedVespaVersion()), + versionFrom(node.getCurrentOsVersion()), + versionFrom(node.getWantedOsVersion()), + Optional.ofNullable(node.getCurrentFirmwareCheck()).map(Instant::ofEpochMilli), + Optional.ofNullable(node.getWantedFirmwareCheck()).map(Instant::ofEpochMilli), + toServiceState(node.getOrchestratorStatus()), + Optional.ofNullable(node.suspendedSinceMillis()).map(Instant::ofEpochMilli), + toInt(node.getCurrentRestartGeneration()), + toInt(node.getRestartGeneration()), + toInt(node.getCurrentRebootGeneration()), + toInt(node.getRebootGeneration()), + toInt(node.getCost()), + node.getFlavor(), + clusterIdOf(node.getMembership()), + clusterTypeOf(node.getMembership()), + Optional.ofNullable(node.getMembership()).map(NodeMembership::getRetired).orElse(false), + node.getWantToRetire(), + node.getWantToDeprovision(), + node.getWantToRebuild(), Optional.ofNullable(node.getReservedTo()).map(TenantName::from), + Optional.ofNullable(node.getExclusiveTo()).map(ApplicationId::fromSerializedForm), + dockerImageFrom(node.getWantedDockerImage()), + dockerImageFrom(node.getCurrentDockerImage()), + node.getReports(), + node.getHistory(), + node.getAdditionalIpAddresses(), + node.getOpenStackId(), + Optional.ofNullable(node.getSwitchHostname()), + Optional.ofNullable(node.getModelName())); + } + + private static String clusterIdOf(NodeMembership nodeMembership) { + return nodeMembership == null ? "" : nodeMembership.clusterid; + } + + private static Node.ClusterType clusterTypeOf(NodeMembership nodeMembership) { + if (nodeMembership == null) return Node.ClusterType.unknown; + switch (nodeMembership.clustertype) { + case "admin": return Node.ClusterType.admin; + case "content": return Node.ClusterType.content; + case "container": return Node.ClusterType.container; + case "combined": return Node.ClusterType.combined; + } + return Node.ClusterType.unknown; + } + + // Convert Jackson type to config.provision type + private static NodeType fromJacksonType(com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType nodeType) { + switch (nodeType) { + case tenant: return NodeType.tenant; + case host: return NodeType.host; + case proxy: return NodeType.proxy; + case proxyhost: return NodeType.proxyhost; + case config: return NodeType.config; + case confighost: return NodeType.confighost; + case controller: return NodeType.controller; + case controllerhost: return NodeType.controllerhost; + default: throw new IllegalArgumentException("Unknown type: " + nodeType); + } + } + + private static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State fromJacksonState(NodeState state) { + switch (state) { + case provisioned: return Node.State.provisioned; + case ready: return Node.State.ready; + case reserved: return Node.State.reserved; + case active: return Node.State.active; + case inactive: return Node.State.inactive; + case dirty: return Node.State.dirty; + case failed: return Node.State.failed; + case parked: return Node.State.parked; + case breakfixed: return Node.State.breakfixed; + case deprovisioned: return Node.State.deprovisioned; + } + return Node.State.unknown; + } + + private static NodeResources.DiskSpeed diskSpeedFromString(String diskSpeed) { + if (diskSpeed == null) return NodeResources.DiskSpeed.getDefault(); + switch (diskSpeed) { + case "fast": return NodeResources.DiskSpeed.fast; + case "slow": return NodeResources.DiskSpeed.slow; + case "any": return NodeResources.DiskSpeed.any; + default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed + "'"); + } + } + + private static NodeResources.StorageType storageTypeFromString(String storageType) { + if (storageType == null) return NodeResources.StorageType.getDefault(); + switch (storageType) { + case "remote": return NodeResources.StorageType.remote; + case "local": return NodeResources.StorageType.local; + case "any": return NodeResources.StorageType.any; + default: throw new IllegalArgumentException("Unknown storage type '" + storageType + "'"); + } + } + + private static Node.ServiceState toServiceState(OrchestratorStatus orchestratorStatus) { + switch (orchestratorStatus) { + case ALLOWED_TO_BE_DOWN: return Node.ServiceState.allowedDown; + case PERMANENTLY_DOWN: return Node.ServiceState.permanentlyDown; + case NO_REMARKS: return Node.ServiceState.expectedUp; + } + + return Node.ServiceState.unknown; + } + + private static double toDouble(Double d) { + return d == null ? 0 : d; + } + + private static int toInt(Integer i) { + return i == null ? 0 : i; + } + + private static Version versionFrom(String s) { + return s == null ? Version.emptyVersion : Version.fromString(s); + } + + private static DockerImage dockerImageFrom(String s) { + return s == null ? DockerImage.EMPTY : DockerImage.fromString(s); + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java index 97c6222e77d..c2425fe0f72 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.repair; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; @@ -52,8 +53,8 @@ public class RepairTicketReport { return REPORT_ID; } - public static RepairTicketReport fromJsonNode(String jsonReport) { - return uncheck(() -> objectMapper.readValue(jsonReport, RepairTicketReport.class)); + public static RepairTicketReport fromJsonNode(JsonNode node) { + return uncheck(() -> objectMapper.treeToValue(node, RepairTicketReport.class)); } public JsonNode toJsonNode() { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/VCMRReport.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/VCMRReport.java index 8ebfde9f475..a3c0af95053 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/VCMRReport.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/VCMRReport.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.vcmr; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; @@ -68,12 +69,12 @@ public class VCMRReport { /** * Serialization functions - mapped to {@link Node#reports()} */ - public static VCMRReport fromReports(Map<String, String> reports) { + public static VCMRReport fromReports(Map<String, JsonNode> reports) { var serialized = reports.get(REPORT_ID); if (serialized == null) return new VCMRReport(); - return uncheck(() -> objectMapper.readValue(serialized, VCMRReport.class)); + return uncheck(() -> objectMapper.treeToValue(serialized, VCMRReport.class)); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java index fc7b256a644..d74578d9adc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java @@ -2,10 +2,13 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; import java.util.Collection; import java.util.List; @@ -13,9 +16,6 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -/** - * @author smorgrav - */ public class ChangeManagementAssessor { private final NodeRepository nodeRepository; @@ -25,31 +25,31 @@ public class ChangeManagementAssessor { } public Assessment assessment(List<String> impactedHostnames, ZoneId zone) { - return assessmentInner(impactedHostnames, nodeRepository.list(zone, false), zone); + return assessmentInner(impactedHostnames, nodeRepository.listNodes(zone).nodes(), zone); } - Assessment assessmentInner(List<String> impactedHostnames, List<Node> allNodes, ZoneId zone) { + Assessment assessmentInner(List<String> impactedHostnames, List<NodeRepositoryNode> allNodes, ZoneId zone) { List<String> impactedParentHosts = toParentHosts(impactedHostnames, allNodes); // Group impacted application nodes by parent host - Map<Node, List<Node>> prParentHost = allNodes.stream() - .filter(node -> node.state() == Node.State.active) //TODO look at more states? - .filter(node -> impactedParentHosts.contains(node.parentHostname().map(HostName::value).orElse(""))) + Map<NodeRepositoryNode, List<NodeRepositoryNode>> prParentHost = allNodes.stream() + .filter(nodeRepositoryNode -> nodeRepositoryNode.getState() == NodeState.active) //TODO look at more states? + .filter(node -> impactedParentHosts.contains(node.getParentHostname() == null ? "" : node.getParentHostname())) .collect(Collectors.groupingBy(node -> allNodes.stream() - .filter(parent -> parent.hostname().equals(node.parentHostname().get())) + .filter(parent -> parent.getHostname().equals(node.getParentHostname())) .findFirst().orElseThrow() )); // Group nodes pr cluster - Map<Cluster, List<Node>> prCluster = prParentHost.values() + Map<Cluster, List<NodeRepositoryNode>> prCluster = prParentHost.values() .stream() .flatMap(Collection::stream) .collect(Collectors.groupingBy(ChangeManagementAssessor::clusterKey)); var tenantHosts = prParentHost.keySet().stream() - .filter(node -> node.type() == NodeType.host) - .map(node -> node.hostname()) + .filter(node -> node.getType() == NodeType.host) + .map(node -> HostName.from(node.getHostname())) .collect(Collectors.toList()); boolean allHostsReplacable = tenantHosts.isEmpty() || nodeRepository.isReplaceable( @@ -60,7 +60,7 @@ public class ChangeManagementAssessor { // Report assessment pr cluster var clusterAssessments = prCluster.entrySet().stream().map((entry) -> { Cluster cluster = entry.getKey(); - List<Node> nodes = entry.getValue(); + List<NodeRepositoryNode> nodes = entry.getValue(); long[] totalStats = clusterStats(cluster, allNodes); long[] impactedStats = clusterStats(cluster, nodes); @@ -87,8 +87,8 @@ public class ChangeManagementAssessor { var hostAssessments = prParentHost.entrySet().stream().map((entry) -> { HostAssessment hostAssessment = new HostAssessment(); - hostAssessment.hostName = entry.getKey().hostname().value(); - hostAssessment.switchName = entry.getKey().switchHostname().orElse(null); + hostAssessment.hostName = entry.getKey().getHostname(); + hostAssessment.switchName = entry.getKey().getSwitchHostname(); hostAssessment.numberOfChildren = entry.getValue().size(); //TODO: Some better heuristic for what's considered problematic @@ -103,31 +103,31 @@ public class ChangeManagementAssessor { return new Assessment(clusterAssessments, hostAssessments); } - private List<String> toParentHosts(List<String> impactedHostnames, List<Node> allNodes) { + private List<String> toParentHosts(List<String> impactedHostnames, List<NodeRepositoryNode> allNodes) { return impactedHostnames.stream() .flatMap(hostname -> allNodes.stream() - .filter(node -> List.of(NodeType.config, NodeType.proxy, NodeType.host).contains(node.type())) - .filter(node -> hostname.equals(node.hostname().value()) || hostname.equals(node.parentHostname().map(HostName::value).orElse(""))) + .filter(node -> List.of(NodeType.config, NodeType.proxy, NodeType.host).contains(node.getType())) + .filter(node -> hostname.equals(node.getHostname()) || hostname.equals(node.getParentHostname())) .map(node -> { - if (node.type() == NodeType.host) - return node.hostname().value(); - return node.parentHostname().get().value(); + if (node.getType() == NodeType.host) + return node.getHostname(); + return node.getParentHostname(); }).findFirst().stream() ) .collect(Collectors.toList()); } - private static Cluster clusterKey(Node node) { - if (node.owner().isEmpty()) + private static Cluster clusterKey(NodeRepositoryNode node) { + if (node.getOwner() == null) return Cluster.EMPTY; - String appId = node.owner().get().serializedForm(); - return new Cluster(node.clusterType(), node.clusterId(), appId, node.type()); + String appId = Text.format("%s:%s:%s", node.getOwner().tenant, node.getOwner().application, node.getOwner().instance); + return new Cluster(Node.ClusterType.valueOf(node.getMembership().clustertype), node.getMembership().clusterid, appId, node.getType()); } - private static long[] clusterStats(Cluster cluster, List<Node> containerNodes) { - List<Node> clusterNodes = containerNodes.stream().filter(node -> cluster.equals(clusterKey(node))).collect(Collectors.toList()); - long groups = clusterNodes.stream().map(Node::group).distinct().count(); + private static long[] clusterStats(Cluster cluster, List<NodeRepositoryNode> containerNodes) { + List<NodeRepositoryNode> clusterNodes = containerNodes.stream().filter(nodeRepositoryNode -> cluster.equals(clusterKey(nodeRepositoryNode))).collect(Collectors.toList()); + long groups = clusterNodes.stream().map(nodeRepositoryNode -> nodeRepositoryNode.getMembership() != null ? nodeRepositoryNode.getMembership().group : "").distinct().count(); return new long[] { clusterNodes.size(), groups}; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java index 19617a1f293..38e10aa6750 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; @@ -175,7 +174,7 @@ public class VCMRMaintainer extends ControllerMaintainer { } catch (Exception e) { logger.warning("Failed to retire host " + node.hostname() + ": " + Exceptions.toMessageString(e)); // Check if retirement actually failed - if (!nodeRepository.getNode(changeRequest.getZoneId(), node.hostname().value()).wantToRetire()) { + if (!nodeRepository.getNode(changeRequest.getZoneId(), node.hostname().value()).getWantToRetire()) { return hostAction; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 1a4fa3fcd79..c1e6c362c83 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -880,7 +880,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { toSlime(node.resources(), nodeObject); nodeObject.setString("clusterId", node.clusterId()); nodeObject.setString("clusterType", valueOf(node.clusterType())); - nodeObject.setBool("down", node.history().stream().anyMatch(event -> "down".equals(event.name()))); + nodeObject.setBool("down", node.history().stream().anyMatch(event -> "down".equals(event.getEvent()))); nodeObject.setBool("retired", node.retired() || node.wantToRetire()); nodeObject.setBool("restarting", node.wantedRestartGeneration() > node.restartGeneration()); nodeObject.setBool("rebooting", node.wantedRebootGeneration() > node.rebootGeneration()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 5e79f1e4d12..d8bddac4187 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -197,10 +197,10 @@ public class InternalStepRunnerTest { app.instanceId()).iterator().next(); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().minus(Duration.ofSeconds(1))); tester.configServer().nodeRepository().putNodes(JobType.systemTest.zone(system()), - Node.builder(systemTestNode) - .serviceState(Node.ServiceState.allowedDown) - .suspendedSince(tester.clock().instant()) - .build()); + new Node.Builder(systemTestNode) + .serviceState(Node.ServiceState.allowedDown) + .suspendedSince(tester.clock().instant()) + .build()); tester.runner().run(); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index c66cc3710c9..92c8cbc4889 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -63,7 +63,10 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.Supplier; import java.util.logging.Level; import java.util.stream.Collectors; @@ -135,23 +138,23 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer Node parent = nodeRepository().list(zone, SystemApplication.tenantHost.id()).stream().findAny() .orElseThrow(() -> new IllegalStateException("No parent hosts in " + zone)); - nodeRepository().putNodes(zone, Node.builder().hostname(hostFor(application, zone)) - .state(Node.State.reserved) - .type(NodeType.tenant) - .owner(application) - .parentHostname(parent.hostname()) - .currentVersion(initialVersion) - .wantedVersion(initialVersion) - .currentDockerImage(initialDockerImage) - .wantedDockerImage(initialDockerImage) - .currentOsVersion(Version.emptyVersion) - .wantedOsVersion(Version.emptyVersion) - .resources(new NodeResources(2, 8, 50, 1, slow, remote)) - .serviceState(Node.ServiceState.unorchestrated) - .flavor("d-2-8-50") - .clusterId(clusterId.value()) - .clusterType(Node.ClusterType.container) - .build()); + nodeRepository().putNodes(zone, new Node.Builder().hostname(hostFor(application, zone)) + .state(Node.State.reserved) + .type(NodeType.tenant) + .owner(application) + .parentHostname(parent.hostname()) + .currentVersion(initialVersion) + .wantedVersion(initialVersion) + .currentDockerImage(initialDockerImage) + .wantedDockerImage(initialDockerImage) + .currentOsVersion(Version.emptyVersion) + .wantedOsVersion(Version.emptyVersion) + .resources(new NodeResources(2, 8, 50, 1, slow, remote)) + .serviceState(Node.ServiceState.unorchestrated) + .flavor("d-2-8-50") + .clusterId(clusterId.value()) + .clusterType(Node.ClusterType.container) + .build()); } public HostName hostFor(ApplicationId application, ZoneId zone) { @@ -171,16 +174,16 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer for (ZoneId zone : zones) { for (SystemApplication application : applications) { for (int i = 1; i <= 3; i++) { - Node node = Node.builder() - .hostname(HostName.from("node-" + i + "-" + application.id().application() + Node node = new Node.Builder() + .hostname(HostName.from("node-" + i + "-" + application.id().application() .value() + "-" + zone.value())) - .state(Node.State.active) - .type(application.nodeType()) - .owner(application.id()) - .currentVersion(initialVersion).wantedVersion(initialVersion) - .currentOsVersion(Version.emptyVersion).wantedOsVersion(Version.emptyVersion) - .build(); - nodeRepository().putNodes(zone, node); + .state(Node.State.active) + .type(application.nodeType()) + .owner(application.id()) + .currentVersion(initialVersion).wantedVersion(initialVersion) + .currentOsVersion(Version.emptyVersion).wantedOsVersion(Version.emptyVersion) + .build(); + nodeRepository().putNode(zone, node); } convergeServices(application.id(), zone); } @@ -241,9 +244,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer for (var node : nodes) { Node newNode; if (osVersion) { - newNode = Node.builder(node).currentOsVersion(version).wantedOsVersion(version).build(); + newNode = new Node.Builder(node).currentOsVersion(version).wantedOsVersion(version).build(); } else { - newNode = Node.builder(node).currentVersion(version).wantedVersion(version).build(); + newNode = new Node.Builder(node).currentVersion(version).wantedVersion(version).build(); } nodeRepository().putNodes(zone, newNode); } @@ -407,10 +410,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer application.activate(); List<Node> nodes = nodeRepository.list(id.zoneId(), id.applicationId()); for (Node node : nodes) { - nodeRepository.putNodes(id.zoneId(), Node.builder(node) - .state(Node.State.active) - .wantedVersion(application.version().get()) - .build()); + nodeRepository.putNodes(id.zoneId(), new Node.Builder(node) + .state(Node.State.active) + .wantedVersion(application.version().get()) + .build()); } serviceStatus.put(id, new ServiceConvergence(id.applicationId(), id.zoneId(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index b16817c0f3d..4079591730d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; +import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; @@ -17,11 +18,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepoStats; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.configserver.TargetVersions; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; import java.net.URI; import java.time.Duration; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -30,6 +33,7 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; +import java.util.function.Function; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -46,12 +50,115 @@ public class NodeRepositoryMock implements NodeRepository { private final Map<DeploymentId, Pair<Double, Double>> trafficFractions = new HashMap<>(); private final Map<ZoneId, Map<TenantName, URI>> archiveUris = new HashMap<>(); + // A separate/alternative list of NodeRepositoryNode nodes. + // Methods operating with Node and NodeRepositoryNode lives separate lives. + private final Map<ZoneId, List<NodeRepositoryNode>> nodeRepoNodes = new HashMap<>(); + private boolean allowPatching = false; private boolean hasSpareCapacity = false; + /** Add or update given nodes in zone */ + public void putNodes(ZoneId zone, List<Node> nodes) { + Map<HostName, Node> zoneNodes = nodeRepository.computeIfAbsent(zone, (k) -> new HashMap<>()); + for (var node : nodes) { + zoneNodes.put(node.hostname(), node); + } + } + + public void putNode(ZoneId zone, Node node) { + nodeRepository.computeIfAbsent(zone, (k) -> new HashMap<>()).put(node.hostname(), node); + } + + public void putApplication(ZoneId zone, Application application) { + applications.putIfAbsent(zone, new HashMap<>()); + applications.get(zone).put(application.id(), application); + } + + @Override + public NodeRepoStats getStats(ZoneId zone) { + List<ApplicationStats> applicationStats = + applications.containsKey(zone) + ? applications.get(zone).keySet().stream() + .map(id -> new ApplicationStats(id, Load.zero(), 0, 0)) + .collect(Collectors.toList()) + : List.of(); + + return new NodeRepoStats(Load.zero(), Load.zero(), applicationStats); + } + + public Pair<Double, Double> getTrafficFraction(ApplicationId application, ZoneId zone) { + return trafficFractions.get(new DeploymentId(application, zone)); + } + + /** Add or update given node in zone */ + public void putNodes(ZoneId zone, Node node) { + putNodes(zone, Collections.singletonList(node)); + } + + /** Remove given nodes from zone */ + public void removeNodes(ZoneId zone, List<Node> nodes) { + nodes.forEach(node -> nodeRepository.get(zone).remove(node.hostname())); + } + + /** Remove all nodes in all zones */ + public void clear() { + nodeRepository.clear(); + nodeRepoNodes.clear(); + } + + /** Replace nodes in zone with given nodes */ + public void setNodes(ZoneId zone, List<Node> nodes) { + nodeRepository.put(zone, nodes.stream().collect(Collectors.toMap(Node::hostname, Function.identity()))); + } + + public Node require(HostName hostName) { + return nodeRepository.values().stream() + .map(zoneNodes -> zoneNodes.get(hostName)) + .filter(Objects::nonNull) + .findFirst() + .orElseThrow(() -> new NoSuchElementException("No node with the hostname " + hostName + " is known.")); + } + + /** Replace nodes in zone with a fixed set of nodes */ + public void setFixedNodes(ZoneId zone) { + var nodeA = new Node.Builder() + .hostname(HostName.from("hostA")) + .parentHostname(HostName.from("parentHostA")) + .state(Node.State.active) + .type(NodeType.tenant) + .owner(ApplicationId.from("tenant1", "app1", "default")) + .currentVersion(Version.fromString("7.42")) + .wantedVersion(Version.fromString("7.42")) + .currentOsVersion(Version.fromString("7.6")) + .wantedOsVersion(Version.fromString("7.6")) + .serviceState(Node.ServiceState.expectedUp) + .resources(new NodeResources(24, 24, 500, 1)) + .clusterId("clusterA") + .clusterType(Node.ClusterType.container) + .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) + .build(); + var nodeB = new Node.Builder() + .hostname(HostName.from("hostB")) + .parentHostname(HostName.from("parentHostB")) + .state(Node.State.active) + .type(NodeType.tenant) + .owner(ApplicationId.from("tenant2", "app2", "default")) + .currentVersion(Version.fromString("7.42")) + .wantedVersion(Version.fromString("7.42")) + .currentOsVersion(Version.fromString("7.6")) + .wantedOsVersion(Version.fromString("7.6")) + .serviceState(Node.ServiceState.expectedUp) + .resources(new NodeResources(40, 24, 500, 1)) + .cost(20) + .clusterId("clusterB") + .clusterType(Node.ClusterType.container) + .build(); + setNodes(zone, List.of(nodeA, nodeB)); + } + @Override public void addNodes(ZoneId zone, Collection<NodeRepositoryNode> nodes) { - throw new UnsupportedOperationException(); + nodeRepoNodes.put(zone, new ArrayList<>(nodes)); } @Override @@ -64,18 +171,23 @@ public class NodeRepositoryMock implements NodeRepository { var existing = list(zone, List.of(HostName.from(hostName))); if (existing.size() != 1) throw new IllegalArgumentException("Node " + hostName + " not found in " + zone); - var node = Node.builder(existing.get(0)) - .state(Node.State.valueOf(nodeState.name())) - .build(); + var node = new Node.Builder(existing.get(0)) + .state(Node.State.valueOf(nodeState.name())) + .build(); putNodes(zone, node); } @Override - public Node getNode(ZoneId zone, String hostname) { + public NodeRepositoryNode getNode(ZoneId zone, String hostname) { throw new UnsupportedOperationException(); } @Override + public NodeList listNodes(ZoneId zone) { + return new NodeList(nodeRepoNodes.get(zone)); + } + + @Override public List<Node> list(ZoneId zone, boolean includeDeprovisioned) { return List.copyOf(nodeRepository.getOrDefault(zone, Map.of()).values()); } @@ -106,18 +218,6 @@ public class NodeRepositoryMock implements NodeRepository { } @Override - public NodeRepoStats getStats(ZoneId zone) { - List<ApplicationStats> applicationStats = - applications.containsKey(zone) - ? applications.get(zone).keySet().stream() - .map(id -> new ApplicationStats(id, Load.zero(), 0, 0)) - .collect(Collectors.toList()) - : List.of(); - - return new NodeRepoStats(Load.zero(), Load.zero(), applicationStats); - } - - @Override public Map<TenantName, URI> getArchiveUris(ZoneId zone) { return Map.copyOf(archiveUris.getOrDefault(zone, Map.of())); } @@ -144,7 +244,7 @@ public class NodeRepositoryMock implements NodeRepository { nodeRepository.getOrDefault(zone, Map.of()).values() .stream() .filter(node -> node.type() == type) - .map(node -> Node.builder(node).wantedVersion(version).build()) + .map(node -> new Node.Builder(node).wantedVersion(version).build()) .forEach(node -> putNodes(zone, node)); } @@ -161,7 +261,7 @@ public class NodeRepositoryMock implements NodeRepository { nodeRepository.getOrDefault(zone, Map.of()).values() .stream() .filter(node -> node.type() == type) - .map(node -> Node.builder(node).wantedOsVersion(version).build()) + .map(node -> new Node.Builder(node).wantedOsVersion(version).build()) .forEach(node -> putNodes(zone, node)); } @@ -190,20 +290,15 @@ public class NodeRepositoryMock implements NodeRepository { if (existing.size() != 1) throw new IllegalArgumentException("Node " + hostName + " not found in " + zoneId); // Note: Only supports switchHostname, modelName and wantToRetire - Node.Builder newNode = Node.builder(existing.get(0)); + Node.Builder newNode = new Node.Builder(existing.get(0)); if (node.getSwitchHostname() != null) newNode.switchHostname(node.getSwitchHostname()); if (node.getModelName() != null) newNode.modelName(node.getModelName()); if (node.getWantToRetire() != null) newNode.wantToRetire(node.getWantToRetire()); - - Map<String, String> reports = new HashMap<>(); - for (var kv : node.getReports().entrySet()) { - if (kv.getValue() == null) continue; // Null value clears a report - reports.put(kv.getKey(), kv.getValue().toString()); - } - newNode.reports(reports); + if (!node.getReports().isEmpty()) + newNode.reports(node.getReports()); putNodes(zoneId, newNode.build()); } @@ -218,83 +313,6 @@ public class NodeRepositoryMock implements NodeRepository { return hasSpareCapacity; } - /** Add or update given nodes in zone */ - public void putNodes(ZoneId zone, List<Node> nodes) { - Map<HostName, Node> zoneNodes = nodeRepository.computeIfAbsent(zone, (k) -> new HashMap<>()); - for (var node : nodes) { - zoneNodes.put(node.hostname(), node); - } - } - - /** Add or update given node in zone */ - public void putNodes(ZoneId zone, Node node) { - putNodes(zone, List.of(node)); - } - - public void putApplication(ZoneId zone, Application application) { - applications.computeIfAbsent(zone, (k) -> new HashMap<>()) - .put(application.id(), application); - } - - public Pair<Double, Double> getTrafficFraction(ApplicationId application, ZoneId zone) { - return trafficFractions.get(new DeploymentId(application, zone)); - } - - /** Remove given nodes from zone */ - public void removeNodes(ZoneId zone, List<Node> nodes) { - nodes.forEach(node -> nodeRepository.get(zone).remove(node.hostname())); - } - - /** Remove all nodes in all zones */ - public void clear() { - nodeRepository.clear(); - } - - public Node require(HostName hostName) { - return nodeRepository.values().stream() - .map(zoneNodes -> zoneNodes.get(hostName)) - .filter(Objects::nonNull) - .findFirst() - .orElseThrow(() -> new NoSuchElementException("No node with the hostname " + hostName + " is known.")); - } - - /** Add a fixed set of nodes to given zone */ - public void addFixedNodes(ZoneId zone) { - var nodeA = Node.builder() - .hostname(HostName.from("hostA")) - .parentHostname(HostName.from("parentHostA")) - .state(Node.State.active) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(24, 24, 500, 1)) - .clusterId("clusterA") - .clusterType(Node.ClusterType.container) - .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) - .build(); - var nodeB = Node.builder() - .hostname(HostName.from("hostB")) - .parentHostname(HostName.from("parentHostB")) - .state(Node.State.active) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant2", "app2", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(40, 24, 500, 1)) - .cost(20) - .clusterId("clusterB") - .clusterType(Node.ClusterType.container) - .build(); - putNodes(zone, List.of(nodeA, nodeB)); - } - public Optional<Duration> osUpgradeBudget(ZoneId zone, NodeType type, Version version) { return Optional.ofNullable(osUpgradeBudgets.get(Objects.hash(zone, type, version))); } @@ -302,10 +320,10 @@ public class NodeRepositoryMock implements NodeRepository { public void doUpgrade(DeploymentId deployment, Optional<HostName> hostName, Version version) { modifyNodes(deployment, hostName, node -> { assert node.wantedVersion().equals(version); - return Node.builder(node) - .currentVersion(version) - .currentDockerImage(node.wantedDockerImage()) - .build(); + return new Node.Builder(node) + .currentVersion(version) + .currentDockerImage(node.wantedDockerImage()) + .build(); }); } @@ -318,29 +336,23 @@ public class NodeRepositoryMock implements NodeRepository { } public void requestRestart(DeploymentId deployment, Optional<HostName> hostname) { - modifyNodes(deployment, hostname, node -> Node.builder(node).wantedRestartGeneration(node.wantedRestartGeneration() + 1).build()); + modifyNodes(deployment, hostname, node -> new Node.Builder(node).wantedRestartGeneration(node.wantedRestartGeneration() + 1).build()); } public void doRestart(DeploymentId deployment, Optional<HostName> hostname) { - modifyNodes(deployment, hostname, node -> Node.builder(node).restartGeneration(node.restartGeneration() + 1).build()); + modifyNodes(deployment, hostname, node -> new Node.Builder(node).restartGeneration(node.restartGeneration() + 1).build()); } public void requestReboot(DeploymentId deployment, Optional<HostName> hostname) { - modifyNodes(deployment, hostname, node -> Node.builder(node).wantedRebootGeneration(node.wantedRebootGeneration() + 1).build()); + modifyNodes(deployment, hostname, node -> new Node.Builder(node).wantedRebootGeneration(node.wantedRebootGeneration() + 1).build()); } public void doReboot(DeploymentId deployment, Optional<HostName> hostname) { - modifyNodes(deployment, hostname, node -> Node.builder(node).rebootGeneration(node.rebootGeneration() + 1).build()); + modifyNodes(deployment, hostname, node -> new Node.Builder(node).rebootGeneration(node.rebootGeneration() + 1).build()); } - public void addReport(ZoneId zoneId, HostName hostName, String reportId, String report) { - Node node = nodeRepository.getOrDefault(zoneId, Map.of()).get(hostName); - if (node == null) throw new IllegalArgumentException("No node named " + hostName + " in " + zoneId); - - Map<String, String> reports = new HashMap<>(node.reports()); - reports.put(reportId, report); - Node newNode = Node.builder(node).reports(reports).build(); - putNodes(zoneId, newNode); + public void addReport(ZoneId zoneId, HostName hostName, String reportId, JsonNode report) { + nodeRepository.get(zoneId).get(hostName).reports().put(reportId, report); } public NodeRepositoryMock allowPatching(boolean allowPatching) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java index 01c16139667..476d2465202 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java @@ -2,32 +2,32 @@ package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import org.junit.Test; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -/** - * @author smorgrav - */ public class ChangeManagementAssessorTest { - private final ChangeManagementAssessor changeManagementAssessor = new ChangeManagementAssessor(new NodeRepositoryMock()); + private ChangeManagementAssessor changeManagementAssessor = new ChangeManagementAssessor(new NodeRepositoryMock()); @Test public void empty_input_variations() { ZoneId zone = ZoneId.from("prod", "eu-trd"); List<String> hostNames = new ArrayList<>(); - List<Node> allNodesInZone = new ArrayList<>(); + List<NodeRepositoryNode> allNodesInZone = new ArrayList<>(); // Both zone and hostnames are empty ChangeManagementAssessor.Assessment assessment @@ -39,7 +39,7 @@ public class ChangeManagementAssessorTest { public void one_host_one_cluster_no_groups() { ZoneId zone = ZoneId.from("prod", "eu-trd"); List<String> hostNames = Collections.singletonList("host1"); - List<Node> allNodesInZone = new ArrayList<>(); + List<NodeRepositoryNode> allNodesInZone = new ArrayList<>(); allNodesInZone.add(createNode("node1", "host1", "default", 0 )); allNodesInZone.add(createNode("node2", "host1", "default", 0 )); allNodesInZone.add(createNode("node3", "host1", "default", 0 )); @@ -69,8 +69,8 @@ public class ChangeManagementAssessorTest { @Test public void one_of_two_groups_in_one_of_two_clusters() { ZoneId zone = ZoneId.from("prod", "eu-trd"); - List<String> hostNames = List.of("host1", "host2", "host5"); - List<Node> allNodesInZone = new ArrayList<>(); + List<String> hostNames = Arrays.asList("host1", "host2", "host5"); + List<NodeRepositoryNode> allNodesInZone = new ArrayList<>(); // Two impacted nodes on host1 allNodesInZone.add(createNode("node1", "host1", "default", 0 )); @@ -123,8 +123,8 @@ public class ChangeManagementAssessorTest { @Test public void two_config_nodes() { var zone = ZoneId.from("prod", "eu-trd"); - var hostNames = List.of("config1", "config2"); - var allNodesInZone = new ArrayList<Node>(); + var hostNames = Arrays.asList("config1", "config2"); + var allNodesInZone = new ArrayList<NodeRepositoryNode>(); // Add config nodes and parents allNodesInZone.add(createNode("config1", "confighost1", "config", 0, NodeType.config)); @@ -141,8 +141,8 @@ public class ChangeManagementAssessorTest { @Test public void one_of_three_proxy_nodes() { var zone = ZoneId.from("prod", "eu-trd"); - var hostNames = List.of("routing1"); - var allNodesInZone = new ArrayList<Node>(); + var hostNames = Arrays.asList("routing1"); + var allNodesInZone = new ArrayList<NodeRepositoryNode>(); // Add routing nodes and parents allNodesInZone.add(createNode("routing1", "parentrouting1", "routing", 0, NodeType.proxy)); @@ -156,33 +156,47 @@ public class ChangeManagementAssessorTest { assertEquals("33% of routing nodes impacted. Consider reprovisioning if too many", assessment.get(0).impact); } - private Node createNode(String nodename, String hostname, String clusterId, int group) { - return createNode(nodename, hostname, clusterId, group, NodeType.tenant); + private NodeOwner createOwner() { + NodeOwner owner = new NodeOwner(); + owner.tenant = "mytenant"; + owner.application = "myapp"; + owner.instance = "default"; + return owner; + } + + private NodeMembership createMembership(String clusterId, int group) { + NodeMembership membership = new NodeMembership(); + membership.group = "" + group; + membership.clusterid = clusterId; + membership.clustertype = "content"; + membership.index = 2; + membership.retired = false; + return membership; } - private Node createNode(String nodename, String hostname, String clusterId, int group, NodeType nodeType) { - return Node.builder().hostname(nodename) - .parentHostname(hostname) - .state(Node.State.active) - .owner(ApplicationId.from("mytenant", "myapp", "default")) - .group(String.valueOf(group)) - .clusterId(clusterId) - .clusterType(Node.ClusterType.content) - .type(nodeType) - .build(); + private NodeRepositoryNode createNode(String nodename, String hostname, String clusterId, int group) { + return createNode(nodename, hostname, clusterId, group, NodeType.tenant); } - private Node createHost(String hostname, NodeType nodeType) { - return Node.builder() - .hostname(hostname) - .switchHostname("switch1") - .state(Node.State.active) - .owner(ApplicationId.from("mytenant", "myapp", "default")) - .group(String.valueOf(0)) - .clusterId(nodeType.name()) - .clusterType(Node.ClusterType.content) - .type(nodeType) - .build(); + private NodeRepositoryNode createNode(String nodename, String hostname, String clusterId, int group, NodeType nodeType) { + NodeRepositoryNode node = new NodeRepositoryNode(); + node.setHostname(nodename); + node.setParentHostname(hostname); + node.setState(NodeState.active); + node.setOwner(createOwner()); + node.setMembership(createMembership(clusterId, group)); + node.setType(nodeType); + + return node; } + private NodeRepositoryNode createHost(String hostname, NodeType nodeType) { + NodeRepositoryNode node = new NodeRepositoryNode(); + node.setHostname(hostname); + node.setSwitchHostname("switch1"); + node.setType(nodeType); + node.setOwner(createOwner()); + node.setMembership(createMembership(nodeType.name(), 0)); + return node; + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java index e838a693be1..680743055c9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java @@ -121,10 +121,10 @@ public class CloudEventReporterTest { } private Node createNode(String hostname, NodeType nodeType) { - return Node.builder() - .hostname(HostName.from(hostname)) - .type(nodeType) - .build(); + return new Node.Builder() + .hostname(HostName.from(hostname)) + .type(nodeType) + .build(); } private Set<String> getHostnames(ZoneId zoneId) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java index d78b48c362a..cbffd6d610f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java @@ -54,7 +54,7 @@ public class CostReportMaintainerTest { private void addNodes() { for (var zone : tester.zoneRegistry().zones().all().zones()) { - tester.configServer().nodeRepository().addFixedNodes(zone.getId()); + tester.configServer().nodeRepository().setFixedNodes(zone.getId()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java index 56f8de494f0..0baee28143c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java @@ -83,12 +83,12 @@ public class HostInfoUpdaterTest { // Updates node registered under a different hostname ZoneId zone = tester.zoneRegistry().zones().controllerUpgraded().all().ids().get(0); String hostnameSuffix = ".prod." + zone.value(); - Node configNode = Node.builder().hostname(HostName.from("cfg3" + hostnameSuffix)) - .type(NodeType.config) - .build(); - Node configHost = Node.builder().hostname(HostName.from("cfghost3" + hostnameSuffix)) - .type(NodeType.confighost) - .build(); + Node configNode = new Node.Builder().hostname(HostName.from("cfg3" + hostnameSuffix)) + .type(NodeType.config) + .build(); + Node configHost = new Node.Builder().hostname(HostName.from("cfghost3" + hostnameSuffix)) + .type(NodeType.confighost) + .build(); tester.serviceRegistry().configServer().nodeRepository().putNodes(zone, List.of(configNode, configHost)); String switchHostname = switchHostname(configHost); NodeEntity configNodeEntity = new NodeEntity("cfg3" + hostnameSuffix, "RD350G", "Lenovo", switchHostname); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 3789777a8fc..29c5573a1f5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -28,6 +28,7 @@ import org.junit.Test; import java.time.Duration; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -545,7 +546,7 @@ public class MetricsReporterTest { ControllerTester tester) { var currentNodes = getNodes(zone, nodes, tester); var updatedNodes = currentNodes.stream() - .map(node -> builderOps.apply(Node.builder(node)).build()) + .map(node -> builderOps.apply(new Node.Builder(node)).build()) .collect(Collectors.toList()); tester.configServer().nodeRepository().putNodes(zone, updatedNodes); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index eca000ff969..c29e10ab643 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -273,7 +273,7 @@ public class OsUpgraderTest { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); - nodeRepository().putNodes(zone.getVirtualId(), Node.builder(node).state(Node.State.failed).build()); + nodeRepository().putNodes(zone.getVirtualId(), new Node.Builder(node).state(Node.State.failed).build()); } /** Simulate OS upgrade of nodes allocated to application. In a real system this is done by the node itself */ @@ -285,9 +285,9 @@ public class OsUpgraderTest { assertWanted(wantedVersion, application, zones); for (ZoneApi zone : zones) { for (Node node : nodesRequiredToUpgrade(zone, application)) { - nodeRepository().putNodes(zone.getVirtualId(), Node.builder(node).wantedOsVersion(version) - .currentOsVersion(version) - .build()); + nodeRepository().putNodes(zone.getVirtualId(), new Node.Builder(node).wantedOsVersion(version) + .currentOsVersion(version) + .build()); } assertCurrent(version, application, zone); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index 5ef64b460b9..e61516cbb1a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -114,8 +114,8 @@ public class ResourceMeterMaintainerTest { ZoneApiMock zone1 = ZoneApiMock.newBuilder().withId("prod.region-2").build(); ZoneApiMock zone2 = ZoneApiMock.newBuilder().withId("test.region-3").build(); tester.zoneRegistry().setZones(zone1, zone2); - tester.configServer().nodeRepository().addFixedNodes(zone1.getId()); - tester.configServer().nodeRepository().addFixedNodes(zone2.getId()); + tester.configServer().nodeRepository().setFixedNodes(zone1.getId()); + tester.configServer().nodeRepository().setFixedNodes(zone2.getId()); tester.configServer().nodeRepository().putNodes(zone1.getId(), createNodes()); } @@ -126,21 +126,21 @@ public class ResourceMeterMaintainerTest { Node.State.failed, Node.State.parked, Node.State.active) - .map(state -> Node.builder() - .hostname(HostName.from("host" + state)) - .parentHostname(HostName.from("parenthost" + state)) - .state(state) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(24, 24, 500, 1)) - .clusterId("clusterA") - .clusterType(state == Node.State.active ? Node.ClusterType.admin : Node.ClusterType.container) - .build()) + .map(state -> new Node.Builder() + .hostname(HostName.from("host" + state)) + .parentHostname(HostName.from("parenthost" + state)) + .state(state) + .type(NodeType.tenant) + .owner(ApplicationId.from("tenant1", "app1", "default")) + .currentVersion(Version.fromString("7.42")) + .wantedVersion(Version.fromString("7.42")) + .currentOsVersion(Version.fromString("7.6")) + .wantedOsVersion(Version.fromString("7.6")) + .serviceState(Node.ServiceState.expectedUp) + .resources(new NodeResources(24, 24, 500, 1)) + .clusterId("clusterA") + .clusterType(state == Node.State.active ? Node.ClusterType.admin : Node.ClusterType.container) + .build()) .collect(Collectors.toUnmodifiableList()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java index a90c8a9593b..516c28ab5cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -15,6 +15,7 @@ import org.junit.Test; import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Optional; import static com.yahoo.vespa.hosted.controller.maintenance.ResourceTagMaintainer.SHARED_HOST_APPLICATION; import static org.junit.Assert.assertEquals; @@ -24,7 +25,7 @@ import static org.junit.Assert.assertEquals; */ public class ResourceTagMaintainerTest { - private final ControllerTester tester = new ControllerTester(); + final ControllerTester tester = new ControllerTester(); @Test public void maintain() { @@ -50,24 +51,24 @@ public class ResourceTagMaintainerTest { } public void setNodes(ZoneId zone) { - var hostA = Node.builder() - .hostname(HostName.from("parentHostA." + zone.value())) - .type(NodeType.host) - .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) - .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) - .build(); - var nodeA = Node.builder() - .hostname(HostName.from("hostA." + zone.value())) - .type(NodeType.tenant) - .parentHostname(HostName.from("parentHostA." + zone.value())) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .build(); - var hostB = Node.builder() - .hostname(HostName.from("parentHostB." + zone.value())) - .type(NodeType.host) - .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) - .build(); - tester.configServer().nodeRepository().putNodes(zone, List.of(hostA, nodeA, hostB)); + var hostA = new Node.Builder() + .hostname(HostName.from("parentHostA." + zone.value())) + .type(NodeType.host) + .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) + .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) + .build(); + var nodeA = new Node.Builder() + .hostname(HostName.from("hostA." + zone.value())) + .type(NodeType.tenant) + .parentHostname(HostName.from("parentHostA." + zone.value())) + .owner(ApplicationId.from("tenant1", "app1", "default")) + .build(); + var hostB = new Node.Builder() + .hostname(HostName.from("parentHostB." + zone.value())) + .type(NodeType.host) + .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) + .build(); + tester.configServer().nodeRepository().setNodes(zone, List.of(hostA, nodeA, hostB)); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 8090765b5f9..db2353860ae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -305,7 +305,7 @@ public class SystemUpgraderTest { for (Node node : listNodes(zone, application)) { nodeRepository().putNodes( zone.getId(), - Node.builder(node).currentVersion(node.wantedVersion()).build()); + new Node.Builder(node).currentVersion(node.wantedVersion()).build()); } assertCurrentVersion(application, version, zone); }); @@ -329,7 +329,7 @@ public class SystemUpgraderTest { Node node = nodes.get(0); nodeRepository().putNodes( zone.getId(), - Node.builder(node).state(Node.State.failed).build()); + new Node.Builder(node).state(Node.State.failed).build()); } private void assertSystemVersion(Version version) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainerTest.java index f957b14ef95..16ed6b7ef98 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainerTest.java @@ -20,12 +20,8 @@ import java.time.Duration; import java.time.Instant; import java.time.ZonedDateTime; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; /** * @author olaa @@ -53,12 +49,9 @@ public class VCMRMaintainerTest { vcmrReport.addVcmr("id123", ZonedDateTime.now(), ZonedDateTime.now()); var parkedNode = createNode(host1, NodeType.host, Node.State.parked, true); var failedNode = createNode(host2, NodeType.host, Node.State.failed, false); - Map<String, String> reports = vcmrReport.toNodeReports().entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, - kv -> kv.getValue().toString())); - parkedNode = Node.builder(parkedNode) - .reports(reports) - .build(); + parkedNode = new Node.Builder(parkedNode) + .reports(vcmrReport.toNodeReports()) + .build(); nodeRepo.putNodes(zoneId, List.of(parkedNode, failedNode)); @@ -70,7 +63,8 @@ public class VCMRMaintainerTest { assertEquals(Node.State.dirty, nodeList.get(0).state()); assertEquals(Node.State.failed, nodeList.get(1).state()); - assertTrue(nodeList.get(0).reports().isEmpty()); + var report = nodeList.get(0).reports(); + assertNull(report.get(VCMRReport.getReportId())); var writtenChangeRequest = tester.curator().readChangeRequest(changeRequestId).get(); assertEquals(Status.COMPLETED, writtenChangeRequest.getStatus()); @@ -241,11 +235,11 @@ public class VCMRMaintainerTest { } private Node createNode(HostName hostname, NodeType nodeType, Node.State state, boolean wantToRetire) { - return Node.builder() - .hostname(hostname) - .type(nodeType) - .state(state) - .wantToRetire(wantToRetire) - .build(); + return new Node.Builder() + .hostname(hostname) + .type(nodeType) + .state(state) + .wantToRetire(wantToRetire) + .build(); } -} +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java index 5846ab5f2a4..80cee3af58b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java @@ -2,17 +2,23 @@ package com.yahoo.vespa.hosted.controller.restapi.changemanagement; import com.yahoo.application.container.handler.Request; -import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestSource; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import org.intellij.lang.annotations.Language; import org.junit.Before; import org.junit.Test; @@ -36,7 +42,8 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { public void before() { tester = new ContainerTester(container, responses); addUserToHostedOperatorRole(operator); - tester.serviceRegistry().configServer().nodeRepository().putNodes(ZoneId.from("prod.us-east-3"), createNodes()); + tester.serviceRegistry().configServer().nodeRepository().addNodes(ZoneId.from("prod.us-east-3"), createNodes()); + tester.serviceRegistry().configServer().nodeRepository().putNodes(ZoneId.from("prod.us-east-3"), createNode()); tester.controller().curator().writeChangeRequest(createChangeRequest()); } @@ -78,11 +85,23 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { assertEquals(VespaChangeRequest.Status.COMPLETED, changeRequest.getStatus()); } + private void assertResponse(Request request, @Language("JSON") String body, int statusCode) { + addIdentityToRequest(request, operator); + tester.assertResponse(request, body, statusCode); + } + private void assertFile(Request request, String filename) { addIdentityToRequest(request, operator); tester.assertResponse(request, new File(filename)); } + private Node createNode() { + return new Node.Builder() + .hostname(HostName.from("host1")) + .switchHostname("switch1") + .build(); + } + private VespaChangeRequest createChangeRequest() { var instant = Instant.ofEpochMilli(9001); var date = ZonedDateTime.ofInstant(instant, java.time.ZoneId.of("UTC")); @@ -105,8 +124,8 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { ); } - private List<Node> createNodes() { - List<Node> nodes = new ArrayList<>(); + private List<NodeRepositoryNode> createNodes() { + List<NodeRepositoryNode> nodes = new ArrayList<>(); nodes.add(createNode("node1", "host1", "default", 0 )); nodes.add(createNode("node2", "host1", "default", 0 )); nodes.add(createNode("node3", "host1", "default", 0 )); @@ -116,27 +135,44 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { return nodes; } - private Node createNode(String nodename, String hostname, String clusterId, int group) { - return Node.builder() - .hostname(nodename) - .parentHostname(hostname).state(Node.State.active) - .owner(ApplicationId.from("mytenant", "myapp", "default")) - .type(com.yahoo.config.provision.NodeType.tenant) - .clusterId(clusterId) - .group(String.valueOf(group)) - .clusterType(Node.ClusterType.content) - .build(); + private NodeOwner createOwner() { + NodeOwner owner = new NodeOwner(); + owner.tenant = "mytenant"; + owner.application = "myapp"; + owner.instance = "default"; + return owner; + } + + private NodeMembership createMembership(String clusterId, int group) { + NodeMembership membership = new NodeMembership(); + membership.group = "" + group; + membership.clusterid = clusterId; + membership.clustertype = "content"; + membership.index = 2; + membership.retired = false; + return membership; + } + + private NodeRepositoryNode createNode(String nodename, String hostname, String clusterId, int group) { + NodeRepositoryNode node = new NodeRepositoryNode(); + node.setHostname(nodename); + node.setParentHostname(hostname); + node.setState(NodeState.active); + node.setOwner(createOwner()); + node.setMembership(createMembership(clusterId, group)); + node.setType(NodeType.tenant); + + return node; } - private Node createHost(String hostname, String switchName) { - return Node.builder() - .hostname(hostname) - .switchHostname(switchName) - .owner(ApplicationId.from("mytenant", "myapp", "default")) - .type(com.yahoo.config.provision.NodeType.host) - .clusterId("host") - .group("0") - .build(); + private NodeRepositoryNode createHost(String hostname, String switchName) { + NodeRepositoryNode node = new NodeRepositoryNode(); + node.setHostname(hostname); + node.setSwitchHostname(switchName); + node.setOwner(createOwner()); + node.setType(NodeType.host); + node.setMembership(createMembership("host", 0)); + return node; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index c1358decf19..7364723f5f0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -142,7 +142,7 @@ public class OsApiTest extends ControllerContainerTest { var targetVersion = nodeRepository().targetVersionsOf(zone).osVersion(application.nodeType()); for (Node node : nodeRepository().list(zone, application.id())) { var version = targetVersion.orElse(node.wantedOsVersion()); - nodeRepository().putNodes(zone, Node.builder(node).currentOsVersion(version).wantedOsVersion(version).build()); + nodeRepository().putNodes(zone, new Node.Builder(node).currentOsVersion(version).wantedOsVersion(version).build()); } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 6e70fb8c3cb..4dd283cf5d7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -70,7 +70,7 @@ public class VersionStatusTest { // Upgrade some config servers for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) { for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { - Node upgradedNode = Node.builder(node).currentVersion(version1).build(); + Node upgradedNode = new Node.Builder(node).currentVersion(version1).build(); tester.configServer().nodeRepository().putNodes(zone.getId(), upgradedNode); break; } @@ -114,7 +114,7 @@ public class VersionStatusTest { Version ancientVersion = Version.fromString("5.1"); for (ZoneApi zone : tester.controller().zoneRegistry().zones().all().zones()) { for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { - Node downgradedNode = Node.builder(node).currentVersion(ancientVersion).build(); + Node downgradedNode = new Node.Builder(node).currentVersion(ancientVersion).build(); tester.configServer().nodeRepository().putNodes(zone.getId(), downgradedNode); break; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java index 7f5b6ef453e..a0b78461b34 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java @@ -33,6 +33,9 @@ public class OsVersions { /** The maximum number of concurrent upgrades per node type triggered by {@link DelegatingOsUpgrader} */ private static final int MAX_DELEGATED_UPGRADES = 30; + /** The maximum number of concurrent upgrades (rebuilds) per node type triggered by {@link RebuildingOsUpgrader} */ + private static final int MAX_REBUILDS = 10; + private final NodeRepository nodeRepository; private final CuratorDatabaseClient db; private final boolean reprovisionToUpgradeOs; |