diff options
Diffstat (limited to 'config-provisioning/src/main/java/com/yahoo')
5 files changed, 193 insertions, 47 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index f723575c342..1a1cd243f89 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -90,7 +90,7 @@ public final class Capacity { /** Creates this from a node type */ public static Capacity fromRequiredNodeType(NodeType type) { - return from(new ClusterResources(0, 0, NodeResources.unspecified), true, false, type); + return from(new ClusterResources(0, 0, NodeResources.unspecified()), true, false, type); } private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index 8d767e9f4ad..c8ec45b0ec8 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -109,15 +109,25 @@ public class Flavor { public Optional<FlavorOverrides> flavorOverrides() { return flavorOverrides; } + // TODO: Remove after June 2020 + @Deprecated public double getMinMainMemoryAvailableGb() { return resources.memoryGb(); } + // TODO: Remove after June 2020 + @Deprecated public double getMinDiskAvailableGb() { return resources.diskGb(); } + // TODO: Remove after June 2020 + @Deprecated public boolean hasFastDisk() { return resources.diskSpeed() == NodeResources.DiskSpeed.fast; } + // TODO: Remove after June 2020 + @Deprecated public double getBandwidthGbps() { return resources.bandwidthGbps(); } /** Returns the number of cores available in this flavor, not scaled for speed. */ + // TODO: Remove after June 2020 + @Deprecated public double getMinCpuCores() { return minCpuCores; } public Type getType() { return type; } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index 2a5d27a0fe7..52a2008b658 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -22,73 +22,167 @@ public class HostSpec implements Comparable<HostSpec> { /** Aliases of this host */ private final List<String> aliases; + private final NodeResources realResources; + private final NodeResources advertisedResources; + private final NodeResources requestedResources; + /** The current membership role of this host in the cluster it belongs to */ private final Optional<ClusterMembership> membership; - private final Optional<Flavor> flavor; - private final Optional<Version> version; private final Optional<DockerImage> dockerImageRepo; private final Optional<NetworkPorts> networkPorts; - private final Optional<NodeResources> requestedResources; + /** Create a host in a non-cloud system, where hosts are specified in config */ + public HostSpec(String hostname, List<String> aliases, Optional<NetworkPorts> networkPorts) { + this(hostname, aliases, + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + Optional.empty(), Optional.empty(), networkPorts, Optional.empty()); + } + + /** Create a host in a hosted system */ + public HostSpec(String hostname, + NodeResources realResources, + NodeResources advertisedResurces, + NodeResources requestedResources, + ClusterMembership membership, + Optional<Version> version, + Optional<NetworkPorts> networkPorts, + Optional<DockerImage> dockerImageRepo) { + this(hostname, List.of(), + realResources, + advertisedResurces, + requestedResources, + Optional.of(membership), + version, + networkPorts, + dockerImageRepo); + } + + private HostSpec(String hostname, + List<String> aliases, + NodeResources realResources, + NodeResources advertisedResurces, + NodeResources requestedResources, + Optional<ClusterMembership> membership, + Optional<Version> version, + Optional<NetworkPorts> networkPorts, + Optional<DockerImage> dockerImageRepo) { + if (hostname == null || hostname.isEmpty()) throw new IllegalArgumentException("Hostname must be specified"); + this.hostname = hostname; + this.aliases = List.copyOf(aliases); + this.realResources = Objects.requireNonNull(realResources); + this.advertisedResources = Objects.requireNonNull(advertisedResurces); + this.requestedResources = Objects.requireNonNull(requestedResources, "RequestedResources cannot be null"); + this.membership = Objects.requireNonNull(membership); + this.version = Objects.requireNonNull(version, "Version cannot be null but can be empty"); + this.networkPorts = Objects.requireNonNull(networkPorts, "Network ports cannot be null but can be empty"); + this.dockerImageRepo = Objects.requireNonNull(dockerImageRepo, "Docker image repo cannot be null but can be empty"); + } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, Optional<ClusterMembership> membership) { - this(hostname, new ArrayList<>(), Optional.empty(), membership); + this(hostname, new ArrayList<>(), + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + membership, + Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, ClusterMembership membership, Flavor flavor, Optional<Version> version) { - this(hostname, new ArrayList<>(), Optional.of(flavor), Optional.of(membership), version); + this(hostname, new ArrayList<>(), + flavor.resources(), flavor.resources(), NodeResources.unspecified(), + Optional.of(membership), version, Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List<String> aliases) { - this(hostname, aliases, Optional.empty(), Optional.empty()); + this(hostname, aliases, + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List<String> aliases, Flavor flavor) { - this(hostname, aliases, Optional.of(flavor), Optional.empty()); + this(hostname, aliases, + flavor.resources(), flavor.resources(), NodeResources.unspecified(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List<String> aliases, ClusterMembership membership) { - this(hostname, aliases, Optional.empty(), Optional.of(membership)); + this(hostname, aliases, + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + Optional.of(membership), + Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership) { - this(hostname, aliases, flavor, membership, Optional.empty()); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + NodeResources.unspecified(), + membership, Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<Version> version) { - this(hostname, aliases, flavor, membership, version, Optional.empty()); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + NodeResources.unspecified(), + membership, version, + Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<Version> version, Optional<NetworkPorts> networkPorts) { - this(hostname, aliases, flavor, membership, version, networkPorts, Optional.empty()); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + NodeResources.unspecified(), + membership, version, networkPorts, + Optional.empty()); } - public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, + // TODO: Remove after June 2020 + @Deprecated + public HostSpec(String hostname, List<String> aliases, + Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<Version> version, Optional<NetworkPorts> networkPorts, Optional<NodeResources> requestedResources) { - this(hostname, aliases, flavor, membership, version, networkPorts, requestedResources, Optional.empty()); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + requestedResources.orElse(NodeResources.unspecified()), + membership, version, networkPorts, Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<Version> version, Optional<NetworkPorts> networkPorts, Optional<NodeResources> requestedResources, Optional<DockerImage> dockerImageRepo) { - if (hostname == null || hostname.isEmpty()) throw new IllegalArgumentException("Hostname must be specified"); - this.hostname = hostname; - this.aliases = List.copyOf(aliases); - this.flavor = flavor; - this.membership = membership; - this.version = Objects.requireNonNull(version, "Version cannot be null but can be empty"); - this.networkPorts = Objects.requireNonNull(networkPorts, "Network ports cannot be null but can be empty"); - this.requestedResources = Objects.requireNonNull(requestedResources, "RequestedResources cannot be null"); - this.dockerImageRepo = Objects.requireNonNull(dockerImageRepo, "Docker image repo cannot be null but can be empty"); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + requestedResources.orElse(NodeResources.unspecified()), + membership, version, networkPorts, dockerImageRepo); } /** Returns the name identifying this host */ @@ -97,7 +191,17 @@ public class HostSpec implements Comparable<HostSpec> { /** Returns the aliases of this host as an immutable list. This may be empty but never null. */ public List<String> aliases() { return aliases; } - public Optional<Flavor> flavor() { return flavor; } + /** The real resources available for Vespa processes on this node, after subtracting infrastructure overhead. */ + public NodeResources realResources() { return realResources; } + + /** The total advertised resources of this node, typically matching what's requested. */ + public NodeResources advertisedResources() { return advertisedResources; } + + /** A flavor contained the advertised resources of this host */ + // TODO: Remove after June 2020 + public Optional<Flavor> flavor() { + return advertisedResources.asOptional().map(resources -> new Flavor(resources)); + } /** Returns the current version of Vespa running on this node, or empty if not known */ public Optional<com.yahoo.component.Version> version() { return version; } @@ -108,13 +212,13 @@ public class HostSpec implements Comparable<HostSpec> { /** Returns the network port allocations on this host, or empty if not present */ public Optional<NetworkPorts> networkPorts() { return networkPorts; } - /** Returns the requested resources leading to this host being provisioned, or empty if not known */ - public Optional<NodeResources> requestedResources() { return requestedResources; } + /** Returns the requested resources leading to this host being provisioned, or empty if unspecified */ + public Optional<NodeResources> requestedResources() { return requestedResources.asOptional(); } public Optional<DockerImage> dockerImageRepo() { return dockerImageRepo; } public HostSpec withPorts(Optional<NetworkPorts> ports) { - return new HostSpec(hostname, aliases, flavor, membership, version, ports, requestedResources, dockerImageRepo); + return new HostSpec(hostname, aliases, realResources, advertisedResources, requestedResources, membership, version, ports, dockerImageRepo); } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 91604cd667d..e02aa7a5f22 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -3,6 +3,7 @@ package com.yahoo.config.provision; import java.util.Locale; import java.util.Objects; +import java.util.Optional; /** * The node resources required by an application cluster @@ -16,6 +17,7 @@ public class NodeResources { private static final double memoryUnitCost = 0.012; private static final double diskUnitCost = 0.0004; + // TODO: Make private after June 2020 public static final NodeResources unspecified = new NodeResources(0, 0, 0, 0); public enum DiskSpeed { @@ -252,7 +254,14 @@ public class NodeResources { return true; } - public boolean isUnspecified() { return this == unspecified; } + public static NodeResources unspecified() { return unspecified; } + + public boolean isUnspecified() { return this.equals(unspecified); } + + /** Returns this.isUnspecified() ? Optional.empty() : Optional.of(this) */ + public Optional<NodeResources> asOptional() { + return this.isUnspecified() ? Optional.empty() : Optional.of(this); + } private boolean equal(double a, double b) { return Math.abs(a - b) < 0.00000001; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index 9ba26be072c..96b189cefe6 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -45,6 +45,9 @@ public class AllocatedHostsSerializer { private static final String aliasesKey = "aliases"; private static final String hostSpecMembershipKey = "membership"; + private static final String realResourcesKey = "realResources"; + private static final String advertisedResourcesKey = "advertisedResources"; + // Flavor can be removed when all allocated nodes are docker nodes private static final String flavorKey = "flavor"; @@ -90,7 +93,9 @@ public class AllocatedHostsSerializer { object.setString(hostSpecDockerImageRepoKey, repo.repository()); }); }); - host.flavor().ifPresent(flavor -> toSlime(flavor, object)); + host.flavor().ifPresent(flavor -> toSlime(flavor, object)); // TODO: Remove this line after June 2020 + toSlime(host.realResources(), object.setObject(realResourcesKey)); + toSlime(host.advertisedResources(), object.setObject(advertisedResourcesKey)); host.requestedResources().ifPresent(resources -> toSlime(resources, object.setObject(requestedResourcesKey))); host.version().ifPresent(version -> object.setString(hostSpecCurrentVespaVersionKey, version.toFullString())); host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray(hostSpecNetworkPortsKey))); @@ -133,18 +138,25 @@ public class AllocatedHostsSerializer { } private static HostSpec hostFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { - return new HostSpec(object.field(hostSpecHostNameKey).asString(), - aliasesFromSlime(object), - flavorFromSlime(object, nodeFlavors), - object.field(hostSpecMembershipKey).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(), - optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new), - NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)), - nodeResourcesFromSlime(object.field(requestedResourcesKey)), - optionalDockerImage(object.field(hostSpecDockerImageRepoKey))); + if (object.field(hostSpecMembershipKey).valid()) { // Hosted + return new HostSpec(object.field(hostSpecHostNameKey).asString(), + nodeResourcesFromSlime(object.field(realResourcesKey), object, nodeFlavors), + nodeResourcesFromSlime(object.field(advertisedResourcesKey), object, nodeFlavors), + optionalNodeResourcesFromSlime(object.field(requestedResourcesKey)), // TODO: Make non-optional after June 2020 + membershipFromSlime(object), + optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new), + NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)), + optionalDockerImage(object.field(hostSpecDockerImageRepoKey))); + } + else { + return new HostSpec(object.field(hostSpecHostNameKey).asString(), + aliasesFromSlime(object), + NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey))); + } } private static List<String> aliasesFromSlime(Inspector object) { - if ( ! object.field(aliasesKey).valid()) return Collections.emptyList(); + if ( ! object.field(aliasesKey).valid()) return List.of(); List<String> aliases = new ArrayList<>(); object.field(aliasesKey).traverse((ArrayTraverser)(index, alias) -> aliases.add(alias.asString())); return aliases; @@ -154,17 +166,28 @@ public class AllocatedHostsSerializer { if (object.field(flavorKey).valid() && nodeFlavors.isPresent() && nodeFlavors.get().exists(object.field(flavorKey).asString())) return nodeFlavors.get().getFlavor(object.field(flavorKey).asString()); else - return nodeResourcesFromSlime(object.field(resourcesKey)).map(resources -> new Flavor(resources)); + return Optional.empty(); + } + + private static NodeResources nodeResourcesFromSlime(Inspector resources) { + return new NodeResources(resources.field(vcpuKey).asDouble(), + resources.field(memoryKey).asDouble(), + resources.field(diskKey).asDouble(), + resources.field(bandwidthKey).asDouble(), + diskSpeedFromSlime(resources.field(diskSpeedKey)), + storageTypeFromSlime(resources.field(storageTypeKey))); + } + + private static NodeResources optionalNodeResourcesFromSlime(Inspector resources) { + if ( ! resources.valid()) return NodeResources.unspecified(); + return nodeResourcesFromSlime(resources); } - private static Optional<NodeResources> nodeResourcesFromSlime(Inspector resources) { - if ( ! resources.valid()) return Optional.empty(); - return Optional.of(new NodeResources(resources.field(vcpuKey).asDouble(), - resources.field(memoryKey).asDouble(), - resources.field(diskKey).asDouble(), - resources.field(bandwidthKey).asDouble(), - diskSpeedFromSlime(resources.field(diskSpeedKey)), - storageTypeFromSlime(resources.field(storageTypeKey)))); + private static NodeResources nodeResourcesFromSlime(Inspector resources, Inspector parent, + Optional<NodeFlavors> nodeFlavors) { + if ( ! resources.valid()) // TODO: Remove the fallback using nodeFlavors after June 2020 + return flavorFromSlime(parent, nodeFlavors).map(f -> f.resources()).orElse(NodeResources.unspecified); + return nodeResourcesFromSlime(resources); } private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { |