From 1add32ea899b62a38008cc460a42437e15f31b15 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 6 May 2019 10:02:49 +0200 Subject: Allow node allocation by resource spec --- config-provisioning/abi-spec.json | 49 +++++----- .../java/com/yahoo/config/provision/Capacity.java | 30 +++--- .../java/com/yahoo/config/provision/Flavor.java | 39 +++++--- .../com/yahoo/config/provision/FlavorSpec.java | 107 --------------------- .../com/yahoo/config/provision/NodeFlavors.java | 6 +- .../com/yahoo/config/provision/NodeResources.java | 106 ++++++++++++++++++++ 6 files changed, 177 insertions(+), 160 deletions(-) delete mode 100644 config-provisioning/src/main/java/com/yahoo/config/provision/FlavorSpec.java create mode 100644 config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java (limited to 'config-provisioning') diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 684b260c98c..b71aa9976a7 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -135,13 +135,13 @@ "methods": [ "public int nodeCount()", "public java.util.Optional flavor()", - "public java.util.Optional flavorSpec()", + "public java.util.Optional nodeResources()", "public boolean isRequired()", "public boolean canFail()", "public com.yahoo.config.provision.NodeType type()", "public java.lang.String toString()", "public static com.yahoo.config.provision.Capacity fromNodeCount(int)", - "public static com.yahoo.config.provision.Capacity fromCount(int, com.yahoo.config.provision.FlavorSpec, boolean, boolean)", + "public static com.yahoo.config.provision.Capacity fromCount(int, com.yahoo.config.provision.NodeResources, boolean, boolean)", "public static com.yahoo.config.provision.Capacity fromCount(int, java.util.Optional, boolean, boolean)", "public static com.yahoo.config.provision.Capacity fromNodeCount(int, java.util.Optional, boolean, boolean)", "public static com.yahoo.config.provision.Capacity fromRequiredNodeType(com.yahoo.config.provision.NodeType)" @@ -380,7 +380,7 @@ ], "methods": [ "public void (com.yahoo.config.provisioning.FlavorsConfig$Flavor)", - "public void (com.yahoo.config.provision.FlavorSpec)", + "public void (com.yahoo.config.provision.NodeResources)", "public java.lang.String name()", "public int cost()", "public boolean isStock()", @@ -400,33 +400,14 @@ "public boolean satisfies(com.yahoo.config.provision.Flavor)", "public void freeze()", "public boolean isLargerThan(com.yahoo.config.provision.Flavor)", - "public com.yahoo.config.provision.FlavorSpec asSpec()", + "public boolean isConfigured()", + "public com.yahoo.config.provision.NodeResources resources()", "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()" ], "fields": [] }, - "com.yahoo.config.provision.FlavorSpec": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void (double, double, double)", - "public double cpuCores()", - "public double memoryGb()", - "public double diskGb()", - "public boolean allocateByLegacyName()", - "public java.lang.String legacyFlavorName()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()", - "public static com.yahoo.config.provision.FlavorSpec fromLegacyFlavorName(java.lang.String)" - ], - "fields": [] - }, "com.yahoo.config.provision.HostFilter": { "superClass": "java.lang.Object", "interfaces": [], @@ -590,6 +571,26 @@ ], "fields": [] }, + "com.yahoo.config.provision.NodeResources": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void (double, double, double)", + "public double vcpu()", + "public double memoryGb()", + "public double diskGb()", + "public boolean allocateByLegacyName()", + "public java.util.Optional legacyName()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()", + "public static com.yahoo.config.provision.NodeResources fromLegacyName(java.lang.String)" + ], + "fields": [] + }, "com.yahoo.config.provision.NodeType": { "superClass": "java.lang.Enum", "interfaces": [], diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index f635b986558..60ce73be234 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -17,15 +17,15 @@ public final class Capacity { private final boolean canFail; - private final Optional flavor; + private final Optional nodeResources; private final NodeType type; - private Capacity(int nodeCount, Optional flavor, boolean required, boolean canFail, NodeType type) { + private Capacity(int nodeCount, Optional nodeResources, boolean required, boolean canFail, NodeType type) { this.nodeCount = nodeCount; this.required = required; this.canFail = canFail; - this.flavor = flavor; + this.nodeResources = nodeResources; this.type = type; } @@ -33,13 +33,19 @@ public final class Capacity { public int nodeCount() { return nodeCount; } /** - * The node flavor requested, or empty if no particular flavor is specified. - * This may be satisfied by the requested flavor or a suitable replacement + * The node flavor requested, or empty if no legacy flavor name has been used. + * This may be satisfied by the requested flavor or a suitable replacement. + * + * @deprecated use nodeResources instead */ - public Optional flavor() { return flavor.map(FlavorSpec::legacyFlavorName); } + @Deprecated + public Optional flavor() { + if (nodeResources().isEmpty()) return Optional.empty(); + return nodeResources.get().legacyName(); + } - /** Returns the capacity specified for each node, or empty to leave this decision to provisioning */ - public Optional flavorSpec() { return flavor; } + /** Returns the resources requested for each node, or empty to leave this decision to provisioning */ + public Optional nodeResources() { return nodeResources; } /** Returns whether the requested number of nodes must be met exactly for a request for this to succeed */ public boolean isRequired() { return required; } @@ -60,7 +66,7 @@ public final class Capacity { @Override public String toString() { - return nodeCount + " nodes " + ( flavor.isPresent() ? "of flavor " + flavor.get() : "(default flavor)" ); + return nodeCount + " nodes " + (nodeResources.isPresent() ? "of flavor " + nodeResources.get() : "(default flavor)" ); } /** Creates this from a desired node count: The request may be satisfied with a smaller number of nodes. */ @@ -68,16 +74,16 @@ public final class Capacity { return fromNodeCount(capacity, Optional.empty(), false, true); } - public static Capacity fromCount(int nodeCount, FlavorSpec flavor, boolean required, boolean canFail) { + public static Capacity fromCount(int nodeCount, NodeResources flavor, boolean required, boolean canFail) { return new Capacity(nodeCount, Optional.of(flavor), required, canFail, NodeType.tenant); } - public static Capacity fromCount(int nodeCount, Optional flavor, boolean required, boolean canFail) { + public static Capacity fromCount(int nodeCount, Optional flavor, boolean required, boolean canFail) { return new Capacity(nodeCount, flavor, required, canFail, NodeType.tenant); } public static Capacity fromNodeCount(int nodeCount, Optional flavor, boolean required, boolean canFail) { - return new Capacity(nodeCount, flavor.map(FlavorSpec::fromLegacyFlavorName), required, canFail, NodeType.tenant); + return new Capacity(nodeCount, flavor.map(NodeResources::fromLegacyName), required, canFail, NodeType.tenant); } /** Creates this from a node type */ diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index 8b6fa863af6..189d49e5c80 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -7,6 +7,7 @@ import com.yahoo.config.provisioning.FlavorsConfig; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; /** * A host flavor (type). This is a value object where the identity is the name. @@ -16,6 +17,11 @@ import java.util.List; */ public class Flavor { + private boolean configured; + + /** The hardware resources of this flavor */ + private NodeResources resources; + private final String name; private final int cost; private final boolean isStock; @@ -36,6 +42,7 @@ public class Flavor { * @param flavorConfig config to be used for Flavor. */ public Flavor(FlavorsConfig.Flavor flavorConfig) { + this.configured = true; this.name = flavorConfig.name(); this.replacesFlavors = new ArrayList<>(); this.cost = flavorConfig.cost(); @@ -49,25 +56,28 @@ public class Flavor { this.description = flavorConfig.description(); this.retired = flavorConfig.retired(); this.idealHeadroom = flavorConfig.idealHeadroom(); + this.resources = new NodeResources(minCpuCores, minMainMemoryAvailableGb, minDiskAvailableGb); } /** Create a Flavor from a Flavor spec and all other fields set to Docker defaults */ - public Flavor(FlavorSpec spec) { - if (spec.allocateByLegacyName()) - throw new IllegalArgumentException("Can not create flavor '" + spec.legacyFlavorName() + "' from a spec: " + + public Flavor(NodeResources resources) { + if (resources.allocateByLegacyName()) + throw new IllegalArgumentException("Can not create flavor '" + resources.legacyName() + "' from a flavor: " + "Non-docker flavors must be of a configured flavor"); - this.name = spec.legacyFlavorName(); + this.configured = false; + this.name = resources.legacyName().orElse(resources.toString()); this.cost = 0; this.isStock = true; this.type = Type.DOCKER_CONTAINER; - this.minCpuCores = spec.cpuCores(); - this.minMainMemoryAvailableGb = spec.memoryGb(); - this.minDiskAvailableGb = spec.diskGb(); + this.minCpuCores = resources.vcpu(); + this.minMainMemoryAvailableGb = resources.memoryGb(); + this.minDiskAvailableGb = resources.diskGb(); this.fastDisk = true; this.bandwidth = 1; this.description = ""; this.retired = false; this.replacesFlavors = Collections.emptyList(); + this.resources = resources; } /** Returns the unique identity of this flavor */ @@ -160,7 +170,7 @@ public class Flavor { replacesFlavors = ImmutableList.copyOf(replacesFlavors); } - /** Returns whether this flavor has at least as much as each hardware resource as the given flavor */ + /** Returns whether this flavor has at least as much of each hardware resource as the given flavor */ public boolean isLargerThan(Flavor other) { return this.minCpuCores >= other.minCpuCores && this.minDiskAvailableGb >= other.minDiskAvailableGb && @@ -168,12 +178,13 @@ public class Flavor { this.fastDisk || ! other.fastDisk; } - public FlavorSpec asSpec() { - if (isDocker()) - return new FlavorSpec(minCpuCores, minMainMemoryAvailableGb, minDiskAvailableGb); - else - return FlavorSpec.fromLegacyFlavorName(name); - } + /** + * True if this is a configured flavor used for hosts, + * false if it is a virtual flavor created on the fly from node resources + */ + public boolean isConfigured() { return configured; } + + public NodeResources resources() { return resources; } @Override public int hashCode() { return name.hashCode(); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/FlavorSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/FlavorSpec.java deleted file mode 100644 index 62cfb59c51c..00000000000 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/FlavorSpec.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision; - -import java.util.Objects; - -/** - * The node capacity specified by an application, which is matched to an actual flavor during provisioning. - * - * @author bratseth - */ -public class FlavorSpec { - - private final double cpuCores; - private final double memoryGb; - private final double diskGb; - - private final boolean allocateByLegacyName; - private final String legacyFlavorName; - - public FlavorSpec(double cpuCores, double memoryGb, double diskGb) { - this.cpuCores = cpuCores; - this.memoryGb = memoryGb; - this.diskGb = diskGb; - this.allocateByLegacyName = false; - this.legacyFlavorName = null; - } - - private FlavorSpec(double cpuCores, double memoryGb, double diskGb, boolean allocateByLegacyName, String legacyFlavorName) { - this.cpuCores = cpuCores; - this.memoryGb = memoryGb; - this.diskGb = diskGb; - this.allocateByLegacyName = allocateByLegacyName; - this.legacyFlavorName = legacyFlavorName; - } - - public double cpuCores() { return cpuCores; } - public double memoryGb() { return memoryGb; } - public double diskGb() { return diskGb; } - - /** - * If this is true, a non-docker legacy name was used to specify this and we'll respect that by mapping directly. - * The other getters of this will return 0. - */ - public boolean allocateByLegacyName() { return allocateByLegacyName; } - - /** Returns the legacy flavor string of this. This is never null. */ - public String legacyFlavorName() { - if (legacyFlavorName != null) - return legacyFlavorName; - else - return "d-" + (int)Math.ceil(cpuCores) + "-" + (int)Math.ceil(memoryGb) + "-" + (int)Math.ceil(diskGb); - } - - @Override - public boolean equals(Object o) { - if (o == this) return true; - if ( ! (o instanceof FlavorSpec)) return false; - FlavorSpec other = (FlavorSpec)o; - if (allocateByLegacyName) { - return this.legacyFlavorName.equals(other.legacyFlavorName); - } - else { - if (this.cpuCores != other.cpuCores) return false; - if (this.memoryGb != other.memoryGb) return false; - if (this.diskGb != other.diskGb) return false; - return true; - } - } - - @Override - public int hashCode() { - if (allocateByLegacyName) - return legacyFlavorName.hashCode(); - else - return (int)(2503 * cpuCores + 22123 * memoryGb + 26987 * diskGb); - } - - @Override - public String toString() { - if (allocateByLegacyName) - return "flavor '" + legacyFlavorName + "'"; - else - return "cpu cores: " + cpuCores + ", memory: " + memoryGb + " Gb, disk " + diskGb + " Gb"; - } - - /** - * Create this from a legacy flavor string. - * - * @throws IllegalArgumentException if the given string does not map to a legacy flavor - */ - public static FlavorSpec fromLegacyFlavorName(String flavorString) { - if (flavorString.startsWith("d-")) { // A docker flavor - String[] parts = flavorString.split("-"); - double cpu = Integer.parseInt(parts[1]); - double mem = Integer.parseInt(parts[2]); - double dsk = Integer.parseInt(parts[3]); - if (cpu == 0) cpu = 0.5; - if (cpu == 2 && mem == 8 ) cpu = 1.5; - if (cpu == 2 && mem == 12 ) cpu = 2.3; - return new FlavorSpec(cpu, mem, dsk, false, flavorString); - } - else { - return new FlavorSpec(0, 0, 0, true, flavorString); - } - } - -} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java index b87f6eeec31..1d29ed85c08 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java @@ -40,11 +40,11 @@ public class NodeFlavors { if (configuredFlavors.containsKey(name)) return Optional.of(configuredFlavors.get(name)); - FlavorSpec flavorSpec = FlavorSpec.fromLegacyFlavorName(name); - if (flavorSpec.allocateByLegacyName()) + NodeResources nodeResources = NodeResources.fromLegacyName(name); + if (nodeResources.allocateByLegacyName()) return Optional.empty(); else - return Optional.of(new Flavor(flavorSpec)); + return Optional.of(new Flavor(nodeResources)); } /** diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java new file mode 100644 index 00000000000..005bfac6b5c --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -0,0 +1,106 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import java.util.Optional; + +/** + * The node resources required by an application cluster + * + * @author bratseth + */ +public class NodeResources { + + private final double vcpu; + private final double memoryGb; + private final double diskGb; + + private final boolean allocateByLegacyName; + + /** The legacy (flavor) name of this, or null if none */ + private final String legacyName; + + public NodeResources(double vcpu, double memoryGb, double diskGb) { + this.vcpu = vcpu; + this.memoryGb = memoryGb; + this.diskGb = diskGb; + this.allocateByLegacyName = false; + this.legacyName = null; + } + + private NodeResources(double vcpu, double memoryGb, double diskGb, boolean allocateByLegacyName, String legacyName) { + this.vcpu = vcpu; + this.memoryGb = memoryGb; + this.diskGb = diskGb; + this.allocateByLegacyName = allocateByLegacyName; + this.legacyName = legacyName; + } + + public double vcpu() { return vcpu; } + public double memoryGb() { return memoryGb; } + public double diskGb() { return diskGb; } + + /** + * If this is true, a non-docker legacy name was used to specify this and we'll respect that by mapping directly. + * The other getters of this will return 0. + */ + public boolean allocateByLegacyName() { return allocateByLegacyName; } + + /** Returns the legacy name of this, or empty if none. */ + public Optional legacyName() { + return Optional.ofNullable(legacyName); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof NodeResources)) return false; + NodeResources other = (NodeResources)o; + if (allocateByLegacyName) { + return this.legacyName.equals(other.legacyName); + } + else { + if (this.vcpu != other.vcpu) return false; + if (this.memoryGb != other.memoryGb) return false; + if (this.diskGb != other.diskGb) return false; + return true; + } + } + + @Override + public int hashCode() { + if (allocateByLegacyName) + return legacyName.hashCode(); + else + return (int)(2503 * vcpu + 22123 * memoryGb + 26987 * diskGb); + } + + @Override + public String toString() { + if (allocateByLegacyName) + return "flavor '" + legacyName + "'"; + else + return "[vcpu: " + vcpu + ", memory: " + memoryGb + " Gb, disk " + diskGb + " Gb]"; + } + + /** + * Create this from serial form. + * + * @throws IllegalArgumentException if the given string cannot be parsed as a serial form of this + */ + public static NodeResources fromLegacyName(String flavorString) { + if (flavorString.startsWith("d-")) { // A legacy docker flavor: We still allocate by numbers + String[] parts = flavorString.split("-"); + double cpu = Integer.parseInt(parts[1]); + double mem = Integer.parseInt(parts[2]); + double dsk = Integer.parseInt(parts[3]); + if (cpu == 0) cpu = 0.5; + if (cpu == 2 && mem == 8 ) cpu = 1.5; + if (cpu == 2 && mem == 12 ) cpu = 2.3; + return new NodeResources(cpu, mem, dsk, false, flavorString); + } + else { // Another legacy flavor: Allocate by direct matching + return new NodeResources(0, 0, 0, true, flavorString); + } + } + +} -- cgit v1.2.3