diff options
author | Harald Musum <musum@verizonmedia.com> | 2019-08-14 20:25:52 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-08-14 20:25:52 +0200 |
commit | 4e52564e8d01d3af68d611d9bd88e497bbd08e25 (patch) | |
tree | ef0e2e43d1189ad03565e9d911a991474c0f9f6e /config-provisioning/src/main | |
parent | fc6ebf45c0ef126043eb9db4cf613958ce665411 (diff) |
Revert "Bratseth/remove allocation by flavor"
Diffstat (limited to 'config-provisioning/src/main')
6 files changed, 193 insertions, 54 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index dfa9ab7f6b8..f8535eda44f 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -41,7 +41,7 @@ public final class Capacity { @Deprecated public Optional<String> flavor() { if (nodeResources().isEmpty()) return Optional.empty(); - return nodeResources.map(n -> n.toString()); + return nodeResources.get().legacyName(); } /** Returns the resources requested for each node, or empty to leave this decision to provisioning */ diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index 2bc70efbc15..48c84b8ecb7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -3,14 +3,15 @@ package com.yahoo.config.provision; import com.yahoo.config.provisioning.FlavorsConfig; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; import java.util.Objects; /** * A host or node flavor. * *Host* flavors come from a configured set which corresponds to the actual flavors available in a zone. - * *Node* flavors are simply a wrapper of a NodeResources object. + * *Node* flavors are simply a wrapper of a NodeResources object (for now (May 2019) with the exception of some + * legacy behavior where nodes are allocated by specifying a physical host flavor directly). * * @author bratseth */ @@ -19,8 +20,11 @@ public class Flavor { private boolean configured; private final String name; private final int cost; + private final boolean isStock; private final Type type; private final double bandwidth; + private final boolean retired; + private List<Flavor> replacesFlavors; /** The hardware resources of this flavor */ private NodeResources resources; @@ -30,22 +34,31 @@ public class Flavor { this.configured = true; this.name = flavorConfig.name(); this.cost = flavorConfig.cost(); + this.isStock = flavorConfig.stock(); this.type = Type.valueOf(flavorConfig.environment()); this.resources = new NodeResources(flavorConfig.minCpuCores(), flavorConfig.minMainMemoryAvailableGb(), flavorConfig.minDiskAvailableGb(), flavorConfig.fastDisk() ? NodeResources.DiskSpeed.fast : NodeResources.DiskSpeed.slow); this.bandwidth = flavorConfig.bandwidth(); + this.retired = flavorConfig.retired(); + this.replacesFlavors = new ArrayList<>(); } /** Creates a *node* flavor from a node resources spec */ public Flavor(NodeResources resources) { Objects.requireNonNull(resources, "Resources cannot be null"); + if (resources.allocateByLegacyName()) + throw new IllegalArgumentException("Can not create flavor '" + resources.legacyName() + "' from a flavor: " + + "Non-docker flavors must be of a configured flavor"); this.configured = false; - this.name = resources.toString(); + this.name = resources.legacyName().orElse(resources.toString()); this.cost = 0; + this.isStock = true; this.type = Type.DOCKER_CONTAINER; this.bandwidth = 1; + this.retired = false; + this.replacesFlavors = List.of(); this.resources = resources; } @@ -60,6 +73,8 @@ public class Flavor { */ public int cost() { return cost; } + public boolean isStock() { return isStock; } + /** * True if this is a configured flavor used for hosts, * false if it is a virtual flavor created on the fly from node resources @@ -78,31 +93,65 @@ public class Flavor { public double getMinCpuCores() { return resources.vcpu(); } + /** Returns whether the flavor is retired */ + public boolean isRetired() { + return retired; + } + public Type getType() { return type; } /** Convenience, returns getType() == Type.DOCKER_CONTAINER */ public boolean isDocker() { return type == Type.DOCKER_CONTAINER; } - // TODO: Remove after August 2019 - public String canonicalName() { return name; } - - // TODO: Remove after August 2019 - public boolean satisfies(Flavor flavor) { return this.equals(flavor); } - - // TODO: Remove after August 2019 - public boolean isStock() { return false; } - - // TODO: Remove after August 2019 - public boolean isRetired() { return false; } + /** + * Returns the canonical name of this flavor - which is the name which should be used as an interface to users. + * The canonical name of this flavor is: + * <ul> + * <li>If it replaces one flavor, the canonical name of the flavor it replaces + * <li>If it replaces multiple or no flavors - itself + * </ul> + * + * The logic is that we can use this to capture the gritty details of configurations in exact flavor names + * but also encourage users to refer to them by a common name by letting such flavor variants declare that they + * replace the canonical name we want. However, if a node replaces multiple names, we have no basis for choosing one + * of them as the canonical, so we return the current as canonical. + */ + public String canonicalName() { + return isCanonical() ? name : replacesFlavors.get(0).canonicalName(); + } + + /** Returns whether this is a canonical flavor */ + public boolean isCanonical() { + return replacesFlavors.size() != 1; + } - // TODO: Remove after August 2019 - public boolean isCanonical() { return false; } + /** + * The flavors this (directly) replaces. + * This is immutable if this is frozen, and a mutable list otherwise. + */ + public List<Flavor> replaces() { return replacesFlavors; } - // TODO: Remove after August 2019 - public List<Flavor> replaces() { return Collections.emptyList(); } + /** + * Returns whether this flavor satisfies the requested flavor, either directly + * (by being the same), or by directly or indirectly replacing it + */ + public boolean satisfies(Flavor flavor) { + if (this.equals(flavor)) { + return true; + } + if (this.retired) { + return false; + } + for (Flavor replaces : replacesFlavors) + if (replaces.satisfies(flavor)) + return true; + return false; + } - // TODO: Remove after August 2019 - public void freeze() {} + /** Irreversibly freezes the content of this */ + public void freeze() { + replacesFlavors = List.copyOf(replacesFlavors); + } @Override public int hashCode() { return name.hashCode(); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java index a9f031cae70..4d4d3c8cf86 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java @@ -41,7 +41,10 @@ public class NodeFlavors { return Optional.of(configuredFlavors.get(name)); NodeResources nodeResources = NodeResources.fromLegacyName(name); - return Optional.of(new Flavor(nodeResources)); + if (nodeResources.allocateByLegacyName()) + return Optional.empty(); + else + return Optional.of(new Flavor(nodeResources)); } /** @@ -49,7 +52,8 @@ public class NodeFlavors { * and cannot be created on the fly. */ public Flavor getFlavorOrThrow(String flavorName) { - return getFlavor(flavorName).orElseThrow(() -> new IllegalArgumentException("Unknown flavor '" + flavorName + "'")); + return getFlavor(flavorName).orElseThrow(() -> new IllegalArgumentException("Unknown flavor '" + flavorName + + "'. Flavors are " + canonicalFlavorNames())); } /** Returns true if this flavor is configured or can be created on the fly */ @@ -57,8 +61,43 @@ public class NodeFlavors { return getFlavor(flavorName).isPresent(); } + private List<String> canonicalFlavorNames() { + return configuredFlavors.values().stream().map(Flavor::canonicalName).distinct().sorted().collect(Collectors.toList()); + } + private static Collection<Flavor> toFlavors(FlavorsConfig config) { - return config.flavor().stream().map(Flavor::new).collect(Collectors.toList()); + Map<String, Flavor> flavors = new HashMap<>(); + // First pass, create all flavors, but do not include flavorReplacesConfig. + for (FlavorsConfig.Flavor flavorConfig : config.flavor()) { + flavors.put(flavorConfig.name(), new Flavor(flavorConfig)); + } + // Second pass, set flavorReplacesConfig to point to correct flavor. + for (FlavorsConfig.Flavor flavorConfig : config.flavor()) { + Flavor flavor = flavors.get(flavorConfig.name()); + for (FlavorsConfig.Flavor.Replaces flavorReplacesConfig : flavorConfig.replaces()) { + if (! flavors.containsKey(flavorReplacesConfig.name())) { + throw new IllegalStateException("Replaces for " + flavor.name() + + " pointing to a non existing flavor: " + flavorReplacesConfig.name()); + } + flavor.replaces().add(flavors.get(flavorReplacesConfig.name())); + } + flavor.freeze(); + } + // Third pass, ensure that retired flavors have a replacement + for (Flavor flavor : flavors.values()) { + if (flavor.isRetired() && !hasReplacement(flavors.values(), flavor)) { + throw new IllegalStateException( + String.format("Flavor '%s' is retired, but has no replacement", flavor.name()) + ); + } + } + return flavors.values(); + } + + private static boolean hasReplacement(Collection<Flavor> flavors, Flavor flavor) { + return flavors.stream() + .filter(f -> !f.equals(flavor)) + .anyMatch(f -> f.satisfies(flavor)); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 8ef48f7048f..7e90767c9c5 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import java.util.Objects; import java.util.Optional; /** @@ -21,6 +22,11 @@ public class NodeResources { private final double diskGb; private final DiskSpeed diskSpeed; + private final boolean allocateByLegacyName; + + /** The legacy (flavor) name of this, or null if none */ + private final String legacyName; + /** Create node resources requiring fast disk */ public NodeResources(double vcpu, double memoryGb, double diskGb) { this(vcpu, memoryGb, diskGb, DiskSpeed.fast); @@ -31,6 +37,18 @@ public class NodeResources { this.memoryGb = memoryGb; this.diskGb = diskGb; this.diskSpeed = diskSpeed; + this.allocateByLegacyName = false; + this.legacyName = null; + } + + private NodeResources(double vcpu, double memoryGb, double diskGb, DiskSpeed diskSpeed, + boolean allocateByLegacyName, String legacyName) { + this.vcpu = vcpu; + this.memoryGb = memoryGb; + this.diskGb = diskGb; + this.diskSpeed = diskSpeed; + this.allocateByLegacyName = allocateByLegacyName; + this.legacyName = legacyName; } public double vcpu() { return vcpu; } @@ -64,17 +82,24 @@ public class NodeResources { combine(this.diskSpeed, other.diskSpeed)); } - // TODO: Remove after August 2019 + /** + * If this is true, a non-docker legacy name was used to specify this and we'll respect that by mapping directly. + * The other getters of this will return 0. + */ + public boolean allocateByLegacyName() { return allocateByLegacyName; } + + /** Returns the legacy name of this, or empty if none. */ public Optional<String> legacyName() { - return Optional.of(toString()); + return Optional.ofNullable(legacyName); } - // TODO: Remove after August 2019 - public boolean allocateByLegacyName() { return false; } - private boolean isInterchangeableWith(NodeResources other) { + if (this.allocateByLegacyName != other.allocateByLegacyName) return false; + if (this.allocateByLegacyName) return legacyName.equals(other.legacyName); + if (this.diskSpeed != DiskSpeed.any && other.diskSpeed != DiskSpeed.any && this.diskSpeed != other.diskSpeed) return false; + return true; } @@ -90,26 +115,40 @@ public class NodeResources { if (o == this) return true; if ( ! (o instanceof NodeResources)) return false; NodeResources other = (NodeResources)o; - if (this.vcpu != other.vcpu) return false; - if (this.memoryGb != other.memoryGb) return false; - if (this.diskGb != other.diskGb) return false; - if (this.diskSpeed != other.diskSpeed) return false; - return true; + if (allocateByLegacyName) { + return this.legacyName.equals(other.legacyName); + } + else { + if (this.vcpu != other.vcpu) return false; + if (this.memoryGb != other.memoryGb) return false; + if (this.diskGb != other.diskGb) return false; + if (this.diskSpeed != other.diskSpeed) return false; + return true; + } } @Override public int hashCode() { - return (int)(2503 * vcpu + 22123 * memoryGb + 26987 * diskGb + diskSpeed.hashCode()); + if (allocateByLegacyName) + return legacyName.hashCode(); + else + return (int)(2503 * vcpu + 22123 * memoryGb + 26987 * diskGb + diskSpeed.hashCode()); } @Override public String toString() { - return "[vcpu: " + vcpu + ", memory: " + memoryGb + " Gb, disk " + diskGb + " Gb" + - (diskSpeed != DiskSpeed.fast ? ", disk speed: " + diskSpeed : "") + "]"; + if (allocateByLegacyName) + return "flavor '" + legacyName + "'"; + else + return "[vcpu: " + vcpu + ", memory: " + memoryGb + " Gb, disk " + diskGb + " Gb" + + (diskSpeed != DiskSpeed.fast ? ", disk speed: " + diskSpeed : "") + "]"; } /** Returns true if all the resources of this are the same or larger than the given resources */ public boolean satisfies(NodeResources other) { + if (this.allocateByLegacyName || other.allocateByLegacyName) // resources are not available + return Objects.equals(this.legacyName, other.legacyName); + if (this.vcpu < other.vcpu) return false; if (this.memoryGb < other.memoryGb) return false; if (this.diskGb < other.diskGb) return false; @@ -124,6 +163,9 @@ public class NodeResources { /** Returns true if all the resources of this are the same as or compatible with the given resources */ public boolean compatibleWith(NodeResources other) { + if (this.allocateByLegacyName || other.allocateByLegacyName) // resources are not available + return Objects.equals(this.legacyName, other.legacyName); + if (this.vcpu != other.vcpu) return false; if (this.memoryGb != other.memoryGb) return false; if (this.diskGb != other.diskGb) return false; @@ -137,20 +179,20 @@ public class NodeResources { * * @throws IllegalArgumentException if the given string cannot be parsed as a serial form of this */ - public static NodeResources fromLegacyName(String name) { - if ( ! name.startsWith("d-")) - throw new IllegalArgumentException("A node specification string must start by 'd-' but was '" + name + "'"); - String[] parts = name.split("-"); - if (parts.length != 4) - throw new IllegalArgumentException("A node specification string must contain three numbers separated by '-' but was '" + name + "'"); - - double cpu = Integer.parseInt(parts[1]); - double mem = Integer.parseInt(parts[2]); - double dsk = Integer.parseInt(parts[3]); - if (cpu == 0) cpu = 0.5; - if (cpu == 2 && mem == 8 ) cpu = 1.5; - if (cpu == 2 && mem == 12 ) cpu = 2.3; - return new NodeResources(cpu, mem, dsk, DiskSpeed.fast); + public static NodeResources fromLegacyName(String flavorString) { + if (flavorString.startsWith("d-")) { // A legacy docker flavor: We still allocate by numbers + String[] parts = flavorString.split("-"); + double cpu = Integer.parseInt(parts[1]); + double mem = Integer.parseInt(parts[2]); + double dsk = Integer.parseInt(parts[3]); + if (cpu == 0) cpu = 0.5; + if (cpu == 2 && mem == 8 ) cpu = 1.5; + if (cpu == 2 && mem == 12 ) cpu = 2.3; + return new NodeResources(cpu, mem, dsk, DiskSpeed.fast, false, flavorString); + } + else { // Another legacy flavor: Allocate by direct matching + return new NodeResources(0, 0, 0, DiskSpeed.fast, true, flavorString); + } } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index 9fcee6b60ed..9e01718bfc6 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -154,6 +154,7 @@ public class AllocatedHostsSerializer { } private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { + if ( ! diskSpeed.valid()) return NodeResources.DiskSpeed.fast; // TODO: Remove this line after June 2019 switch (diskSpeed.asString()) { case "fast" : return NodeResources.DiskSpeed.fast; case "slow" : return NodeResources.DiskSpeed.slow; diff --git a/config-provisioning/src/main/resources/configdefinitions/flavors.def b/config-provisioning/src/main/resources/configdefinitions/flavors.def index 131c23054a2..1cfb18d2cd2 100644 --- a/config-provisioning/src/main/resources/configdefinitions/flavors.def +++ b/config-provisioning/src/main/resources/configdefinitions/flavors.def @@ -7,14 +7,22 @@ namespace=config.provisioning # If a certain flavor has no config it is not necessary to list it here to use it. flavor[].name string -# NOT USED: TODO: Remove after August 2019 +# Names of other flavors (whether mentioned in this config or not) which this flavor +# is a replacement for: If one of these flavor names are requested, this flavor may +# be assigned instead. +# Replacements are transitive: If flavor a replaces b replaces c, then a request for flavor +# c may be satisfied by assigning nodes of flavor a. flavor[].replaces[].name string # The monthly Total Cost of Ownership (TCO) in USD. Typically calculated as TCO divided by # the expected lifetime of the node (usually three years). flavor[].cost int default=0 -# NOT USED: TODO: Remove after August 2019 +# A stock flavor is any flavor which we expect to buy more of in the future. +# Stock flavors are assigned to applications by cost priority. +# +# Non-stock flavors are used for nodes for which a fixed amount has already been purchased +# for some historical reason. These nodes are assigned to applications by exact match and ignoring cost. flavor[].stock bool default=true # The type of node: BARE_METAL, VIRTUAL_MACHINE or DOCKER_CONTAINER @@ -35,6 +43,6 @@ flavor[].fastDisk bool default=true # Expected network interface bandwidth available for this flavor, in Mbit/s. flavor[].bandwidth double default=0.0 -# NOT USED: TODO: Remove after August 2019 +# The flavor is retired and should no longer be used. flavor[].retired bool default=false |