From 2a7af074f571fb557f3bdc756be131a15288bff6 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 22 May 2020 14:08:22 +0200 Subject: Clean up HostSpec constructors --- .../java/com/yahoo/config/provision/Capacity.java | 2 +- .../java/com/yahoo/config/provision/HostSpec.java | 119 ++++++++++++++++++--- .../com/yahoo/config/provision/NodeResources.java | 11 +- .../serialization/AllocatedHostsSerializer.java | 24 ++++- .../yahoo/config/provision/NodeFlavorsTest.java | 2 +- .../AllocatedHostsSerializerTest.java | 46 ++++---- 6 files changed, 164 insertions(+), 40 deletions(-) (limited to 'config-provisioning') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index f723575c342..1a1cd243f89 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -90,7 +90,7 @@ public final class Capacity { /** Creates this from a node type */ public static Capacity fromRequiredNodeType(NodeType type) { - return from(new ClusterResources(0, 0, NodeResources.unspecified), true, false, type); + return from(new ClusterResources(0, 0, NodeResources.unspecified()), true, false, type); } private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index 2a5d27a0fe7..c8c8a349fee 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -22,11 +22,13 @@ public class HostSpec implements Comparable { /** Aliases of this host */ private final List aliases; + private final NodeResources realResources; + + private final NodeResources advertisedResources; + /** The current membership role of this host in the cluster it belongs to */ private final Optional membership; - private final Optional flavor; - private final Optional version; private final Optional dockerImageRepo; @@ -35,55 +37,130 @@ public class HostSpec implements Comparable { private final Optional requestedResources; + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, Optional membership) { - this(hostname, new ArrayList<>(), Optional.empty(), membership); + this(hostname, new ArrayList<>(), + NodeResources.unspecified(), NodeResources.unspecified(), + membership, + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, ClusterMembership membership, Flavor flavor, Optional version) { - this(hostname, new ArrayList<>(), Optional.of(flavor), Optional.of(membership), version); + this(hostname, new ArrayList<>(), + flavor.resources(), flavor.resources(), + Optional.of(membership), version, Optional.empty(), Optional.empty(), Optional.empty()); } + /** Create a host in a non-cloud system, where hosts are specified in config */ public HostSpec(String hostname, List aliases) { - this(hostname, aliases, Optional.empty(), Optional.empty()); + this(hostname, aliases, + NodeResources.unspecified(), NodeResources.unspecified(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List aliases, Flavor flavor) { - this(hostname, aliases, Optional.of(flavor), Optional.empty()); + this(hostname, aliases, + flavor.resources(), flavor.resources(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List aliases, ClusterMembership membership) { - this(hostname, aliases, Optional.empty(), Optional.of(membership)); + this(hostname, aliases, + NodeResources.unspecified(), NodeResources.unspecified(), + Optional.of(membership), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List aliases, Optional flavor, Optional membership) { - this(hostname, aliases, flavor, membership, Optional.empty()); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + membership, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List aliases, Optional flavor, Optional membership, Optional version) { - this(hostname, aliases, flavor, membership, version, Optional.empty()); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + membership, version, + Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List aliases, Optional flavor, Optional membership, Optional version, Optional networkPorts) { - this(hostname, aliases, flavor, membership, version, networkPorts, Optional.empty()); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + membership, version, networkPorts, + Optional.empty(), + Optional.empty()); } - public HostSpec(String hostname, List aliases, Optional flavor, + // TODO: Remove after June 2020 + @Deprecated + public HostSpec(String hostname, List aliases, + Optional flavor, Optional membership, Optional version, Optional networkPorts, Optional requestedResources) { - this(hostname, aliases, flavor, membership, version, networkPorts, requestedResources, Optional.empty()); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + membership, version, networkPorts, requestedResources, Optional.empty()); } + // TODO: Remove after June 2020 + @Deprecated public HostSpec(String hostname, List aliases, Optional flavor, Optional membership, Optional version, Optional networkPorts, Optional requestedResources, Optional dockerImageRepo) { + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + membership, version, networkPorts, requestedResources, dockerImageRepo); + } + + /** Create a host in a hosted system */ + public HostSpec(String hostname, + NodeResources realResources, + NodeResources advertisedResurces, + NodeResources requestedResources, + ClusterMembership membership, + Optional version, + Optional networkPorts, + Optional dockerImageRepo) { + this(hostname, List.of(), + realResources, advertisedResurces, + Optional.of(membership), + version, networkPorts, requestedResources.asOptional(), dockerImageRepo); + } + + /** Create a fully specified host for any system */ + public HostSpec(String hostname, List aliases, + NodeResources realResources, NodeResources advertisedResurces, + Optional membership, Optional version, + Optional networkPorts, Optional requestedResources, + Optional dockerImageRepo) { if (hostname == null || hostname.isEmpty()) throw new IllegalArgumentException("Hostname must be specified"); this.hostname = hostname; this.aliases = List.copyOf(aliases); - this.flavor = flavor; + this.realResources = realResources; + this.advertisedResources = advertisedResurces; this.membership = membership; this.version = Objects.requireNonNull(version, "Version cannot be null but can be empty"); this.networkPorts = Objects.requireNonNull(networkPorts, "Network ports cannot be null but can be empty"); @@ -97,7 +174,17 @@ public class HostSpec implements Comparable { /** Returns the aliases of this host as an immutable list. This may be empty but never null. */ public List aliases() { return aliases; } - public Optional flavor() { return flavor; } + /** The real resources available for Vespa processes on this node, after subtracting infrastructure overhead. */ + public NodeResources realResources() { return realResources; } + + /** The total advertised resources of this node, typically matching what's requested. */ + public NodeResources advertisedResources() { return advertisedResources; } + + /** A flavor contained the advertised resources of this host */ + // TODO: Remove after June 2020 + public Optional flavor() { + return advertisedResources.asOptional().map(resources -> new Flavor(resources)); + } /** Returns the current version of Vespa running on this node, or empty if not known */ public Optional version() { return version; } @@ -108,13 +195,13 @@ public class HostSpec implements Comparable { /** Returns the network port allocations on this host, or empty if not present */ public Optional networkPorts() { return networkPorts; } - /** Returns the requested resources leading to this host being provisioned, or empty if not known */ + /** Returns the requested resources leading to this host being provisioned, or empty if unspecified */ public Optional requestedResources() { return requestedResources; } public Optional dockerImageRepo() { return dockerImageRepo; } public HostSpec withPorts(Optional ports) { - return new HostSpec(hostname, aliases, flavor, membership, version, ports, requestedResources, dockerImageRepo); + return new HostSpec(hostname, aliases, realResources, advertisedResources, membership, version, ports, requestedResources, dockerImageRepo); } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 91604cd667d..e02aa7a5f22 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -3,6 +3,7 @@ package com.yahoo.config.provision; import java.util.Locale; import java.util.Objects; +import java.util.Optional; /** * The node resources required by an application cluster @@ -16,6 +17,7 @@ public class NodeResources { private static final double memoryUnitCost = 0.012; private static final double diskUnitCost = 0.0004; + // TODO: Make private after June 2020 public static final NodeResources unspecified = new NodeResources(0, 0, 0, 0); public enum DiskSpeed { @@ -252,7 +254,14 @@ public class NodeResources { return true; } - public boolean isUnspecified() { return this == unspecified; } + public static NodeResources unspecified() { return unspecified; } + + public boolean isUnspecified() { return this.equals(unspecified); } + + /** Returns this.isUnspecified() ? Optional.empty() : Optional.of(this) */ + public Optional asOptional() { + return this.isUnspecified() ? Optional.empty() : Optional.of(this); + } private boolean equal(double a, double b) { return Math.abs(a - b) < 0.00000001; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index 9ba26be072c..4ce9aa2d599 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -45,6 +45,9 @@ public class AllocatedHostsSerializer { private static final String aliasesKey = "aliases"; private static final String hostSpecMembershipKey = "membership"; + private static final String realResourcesKey = "realResources"; + private static final String advertisedResourcesKey = "advertisedResources"; + // Flavor can be removed when all allocated nodes are docker nodes private static final String flavorKey = "flavor"; @@ -90,7 +93,9 @@ public class AllocatedHostsSerializer { object.setString(hostSpecDockerImageRepoKey, repo.repository()); }); }); - host.flavor().ifPresent(flavor -> toSlime(flavor, object)); + host.flavor().ifPresent(flavor -> toSlime(flavor, object)); // TODO: Remove this line after June 2020 + toSlime(host.realResources(), object.setObject(realResourcesKey)); + toSlime(host.advertisedResources(), object.setObject(advertisedResourcesKey)); host.requestedResources().ifPresent(resources -> toSlime(resources, object.setObject(requestedResourcesKey))); host.version().ifPresent(version -> object.setString(hostSpecCurrentVespaVersionKey, version.toFullString())); host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray(hostSpecNetworkPortsKey))); @@ -135,7 +140,8 @@ public class AllocatedHostsSerializer { private static HostSpec hostFromSlime(Inspector object, Optional nodeFlavors) { return new HostSpec(object.field(hostSpecHostNameKey).asString(), aliasesFromSlime(object), - flavorFromSlime(object, nodeFlavors), + nodeResourcesFromSlime(object.field(realResourcesKey), object, nodeFlavors), + nodeResourcesFromSlime(object.field(advertisedResourcesKey), object, nodeFlavors), object.field(hostSpecMembershipKey).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(), optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new), NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)), @@ -144,7 +150,7 @@ public class AllocatedHostsSerializer { } private static List aliasesFromSlime(Inspector object) { - if ( ! object.field(aliasesKey).valid()) return Collections.emptyList(); + if ( ! object.field(aliasesKey).valid()) return List.of(); List aliases = new ArrayList<>(); object.field(aliasesKey).traverse((ArrayTraverser)(index, alias) -> aliases.add(alias.asString())); return aliases; @@ -167,6 +173,18 @@ public class AllocatedHostsSerializer { storageTypeFromSlime(resources.field(storageTypeKey)))); } + private static NodeResources nodeResourcesFromSlime(Inspector resources, Inspector parent, + Optional nodeFlavors) { + if ( ! resources.valid()) // TODO: Remove the fallback using nodeFlavors after June 2020 + return flavorFromSlime(parent, nodeFlavors).map(f -> f.resources()).orElse(NodeResources.unspecified); + return new NodeResources(resources.field(vcpuKey).asDouble(), + resources.field(memoryKey).asDouble(), + resources.field(diskKey).asDouble(), + resources.field(bandwidthKey).asDouble(), + diskSpeedFromSlime(resources.field(diskSpeedKey)), + storageTypeFromSlime(resources.field(storageTypeKey))); + } + private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { switch (diskSpeed.asString()) { case "fast" : return NodeResources.DiskSpeed.fast; diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java index c259f31df16..e184e85c399 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java @@ -34,7 +34,7 @@ public class NodeFlavorsTest { NodeFlavors nodeFlavors = new NodeFlavors(config); Flavor banana = nodeFlavors.getFlavor("banana").get(); assertEquals(3, banana.cost()); - assertEquals(10, banana.resources().vcpu(), delta); + assertEquals(13, banana.resources().vcpu(), delta); assertEquals("10 * 1.3", 13, banana.resources().vcpu(), delta); } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java index 33157a01c07..a7e62f762e7 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java @@ -14,7 +14,6 @@ import com.yahoo.config.provisioning.FlavorsConfig; import org.junit.Test; import java.io.IOException; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; @@ -34,33 +33,44 @@ public class AllocatedHostsSerializerTest { NodeFlavors configuredFlavors = configuredFlavorsFrom("C/12/45/100", 12, 45, 100, 50, Flavor.Type.BARE_METAL); Set hosts = new LinkedHashSet<>(); hosts.add(new HostSpec("empty", - Optional.empty())); + List.of())); hosts.add(new HostSpec("with-aliases", List.of("alias1", "alias2"))); hosts.add(new HostSpec("allocated", - List.of(), - Optional.empty(), - Optional.of(ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), - Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar")))), - Optional.empty(), Optional.empty(), Optional.empty(), + NodeResources.unspecified(), + NodeResources.unspecified(), + NodeResources.unspecified(), + ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), + Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))), + Optional.empty(), Optional.empty(), Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar")))); - hosts.add(new HostSpec("flavor-from-resources-1", - Collections.emptyList(), new Flavor(new NodeResources(0.5, 3.1, 4, 1)))); hosts.add(new HostSpec("flavor-from-resources-2", - Collections.emptyList(), - Optional.of(new Flavor(new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.slow))), - Optional.empty(), + new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.slow), + new NodeResources(1.0, 6.2, 8, 2, NodeResources.DiskSpeed.slow), + new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.any), + ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), + Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))), Optional.empty(), Optional.empty(), - Optional.of(new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.any)))); - hosts.add(new HostSpec("configured-flavor", - Collections.emptyList(), configuredFlavors.getFlavorOrThrow("C/12/45/100"))); + Optional.empty())); hosts.add(new HostSpec("with-version", - Collections.emptyList(), Optional.empty(), Optional.empty(), Optional.of(Version.fromString("3.4.5")))); + NodeResources.unspecified(), + NodeResources.unspecified(), + NodeResources.unspecified(), + ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), + Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))), + Optional.of(Version.fromString("3.4.5")), + Optional.empty(), Optional.empty())); hosts.add(new HostSpec("with-ports", - Collections.emptyList(), Optional.empty(), Optional.empty(), Optional.empty(), + NodeResources.unspecified(), + NodeResources.unspecified(), + NodeResources.unspecified(), + ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), + Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))), + Optional.empty(), Optional.of(new NetworkPorts(List.of(new NetworkPorts.Allocation(1234, "service1", "configId1", "suffix1"), - new NetworkPorts.Allocation(4567, "service2", "configId2", "suffix2")))))); + new NetworkPorts.Allocation(4567, "service2", "configId2", "suffix2")))), + Optional.empty())); assertAllocatedHosts(AllocatedHosts.withHosts(hosts), configuredFlavors); } -- cgit v1.2.3