diff options
author | Jon Bratseth <bratseth@verizonmedia.com> | 2019-11-13 14:26:34 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@verizonmedia.com> | 2019-11-13 14:26:34 +0100 |
commit | 276c047bfdc581bbbef42012b24ffd4424873430 (patch) | |
tree | 10492056209697f8404e8f7844b27c2a904089a2 /config-provisioning | |
parent | 25d00feb031fab5a34fbea78f9eeb9b6ed7cf7f8 (diff) |
Add NodeResources.storageType
Diffstat (limited to 'config-provisioning')
5 files changed, 129 insertions, 45 deletions
diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 64dba165d36..5038cc8742a 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -599,6 +599,25 @@ "public static final enum com.yahoo.config.provision.NodeResources$DiskSpeed any" ] }, + "com.yahoo.config.provision.NodeResources$StorageType": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.config.provision.NodeResources$StorageType[] values()", + "public static com.yahoo.config.provision.NodeResources$StorageType valueOf(java.lang.String)", + "public static int compare(com.yahoo.config.provision.NodeResources$StorageType, com.yahoo.config.provision.NodeResources$StorageType)" + ], + "fields": [ + "public static final enum com.yahoo.config.provision.NodeResources$StorageType remote", + "public static final enum com.yahoo.config.provision.NodeResources$StorageType local", + "public static final enum com.yahoo.config.provision.NodeResources$StorageType any" + ] + }, "com.yahoo.config.provision.NodeResources": { "superClass": "java.lang.Object", "interfaces": [], @@ -606,21 +625,23 @@ "public" ], "methods": [ - "public void <init>(double, double, double)", - "public void <init>(double, double, double, com.yahoo.config.provision.NodeResources$DiskSpeed)", "public void <init>(double, double, double, double)", "public void <init>(double, double, double, double, com.yahoo.config.provision.NodeResources$DiskSpeed)", + "public void <init>(double, double, double, double, com.yahoo.config.provision.NodeResources$DiskSpeed, com.yahoo.config.provision.NodeResources$StorageType)", "public double vcpu()", "public double memoryGb()", "public double diskGb()", "public double bandwidthGbps()", "public com.yahoo.config.provision.NodeResources$DiskSpeed diskSpeed()", + "public com.yahoo.config.provision.NodeResources$StorageType storageType()", "public com.yahoo.config.provision.NodeResources withVcpu(double)", "public com.yahoo.config.provision.NodeResources withMemoryGb(double)", "public com.yahoo.config.provision.NodeResources withDiskGb(double)", "public com.yahoo.config.provision.NodeResources withBandwidthGbps(double)", "public com.yahoo.config.provision.NodeResources withDiskSpeed(com.yahoo.config.provision.NodeResources$DiskSpeed)", - "public com.yahoo.config.provision.NodeResources anySpeed()", + "public com.yahoo.config.provision.NodeResources with(com.yahoo.config.provision.NodeResources$DiskSpeed)", + "public com.yahoo.config.provision.NodeResources with(com.yahoo.config.provision.NodeResources$StorageType)", + "public com.yahoo.config.provision.NodeResources numbersOnly()", "public com.yahoo.config.provision.NodeResources subtract(com.yahoo.config.provision.NodeResources)", "public com.yahoo.config.provision.NodeResources add(com.yahoo.config.provision.NodeResources)", "public boolean equals(java.lang.Object)", diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index bb4aca0e34b..7a4df8febcf 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -38,7 +38,8 @@ public class Flavor { flavorConfig.minMainMemoryAvailableGb(), flavorConfig.minDiskAvailableGb(), flavorConfig.bandwidth() / 1000, - flavorConfig.fastDisk() ? NodeResources.DiskSpeed.fast : NodeResources.DiskSpeed.slow), + flavorConfig.fastDisk() ? NodeResources.DiskSpeed.fast : NodeResources.DiskSpeed.slow, + flavorConfig.remoteStorage() ? NodeResources.StorageType.remote : NodeResources.StorageType.local), Optional.empty(), Type.valueOf(flavorConfig.environment()), true, @@ -71,11 +72,7 @@ public class Flavor { if (!configured) throw new IllegalArgumentException("Cannot override non-configured flavor"); - NodeResources newResources = new NodeResources(resources.vcpu(), - resources.memoryGb(), - flavorOverrides.diskGb().orElseGet(resources::diskGb), - resources.bandwidthGbps(), - resources.diskSpeed()); + NodeResources newResources = resources.withDiskGb(flavorOverrides.diskGb().orElseGet(resources::diskGb)); return new Flavor(name, newResources, Optional.of(flavorOverrides), type, true, cost, minCpuCores); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 66baecf6c82..15922aa69a4 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -12,9 +12,9 @@ public class NodeResources { public enum DiskSpeed { - fast, // SSD disk or similar speed is needed - slow, // This is tuned to work with the speed of spinning disks - any; // The performance of the cluster using this does not depend on disk speed + fast, // Has/requires SSD disk or similar speed + slow, // Has spinning disk/Is tuned to work with the speed of spinning disks + any; // In requests only: The performance of the cluster using this does not depend on disk speed /** * Compares disk speeds by cost: Slower is cheaper, and therefore before. @@ -29,6 +29,41 @@ public class NodeResources { return 0; } + private DiskSpeed combineWith(DiskSpeed other) { + if (this == any) return other; + if (other == any) return this; + if (this == other) return this; + throw new IllegalArgumentException(this + " cannot be combined with " + other); + } + + } + + public enum StorageType { + + remote, // Has remote (network) storage/Is tuned to work with network storage + local, // Storage is/must be attached to the local host + any; // In requests only: Can use both local and remote storage + + /** + * Compares storage type by cost: Remote is cheaper, and therefore before. + * Any can be remote and therefore costs the same as slow. + */ + public static int compare(StorageType a, StorageType b) { + if (a == any) a = remote; + if (b == any) b = remote; + + if (a == remote && b == local) return -1; + if (a == local && b == remote) return 1; + return 0; + } + + private StorageType combineWith(StorageType other) { + if (this == any) return other; + if (other == any) return this; + if (this == other) return this; + throw new IllegalArgumentException(this + " cannot be combined with " + other); + } + } private final double vcpu; @@ -36,18 +71,7 @@ public class NodeResources { private final double diskGb; private final double bandwidthGbps; private final DiskSpeed diskSpeed; - - /** Create node resources requiring fast disk and no bandwidth */ - @Deprecated // Remove Oct. 2019 - public NodeResources(double vcpu, double memoryGb, double diskGb) { - this(vcpu, memoryGb, diskGb, DiskSpeed.fast); - } - - /** Create node resources requiring no bandwidth */ - @Deprecated // Remove Oct. 2019 - public NodeResources(double vcpu, double memoryGb, double diskGb, DiskSpeed diskSpeed) { - this(vcpu, memoryGb, diskGb, 0.3, diskSpeed); - } + private final StorageType storageType; /** Create node resources requiring fast disk */ public NodeResources(double vcpu, double memoryGb, double diskGb, double bandwidthGbps) { @@ -55,11 +79,16 @@ public class NodeResources { } public NodeResources(double vcpu, double memoryGb, double diskGb, double bandwidthGbps, DiskSpeed diskSpeed) { + this(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, StorageType.any); + } + + public NodeResources(double vcpu, double memoryGb, double diskGb, double bandwidthGbps, DiskSpeed diskSpeed, StorageType storageType) { this.vcpu = vcpu; this.memoryGb = memoryGb; this.diskGb = diskGb; this.bandwidthGbps = bandwidthGbps; this.diskSpeed = diskSpeed; + this.storageType = storageType; } public double vcpu() { return vcpu; } @@ -67,30 +96,40 @@ public class NodeResources { public double diskGb() { return diskGb; } public double bandwidthGbps() { return bandwidthGbps; } public DiskSpeed diskSpeed() { return diskSpeed; } + public StorageType storageType() { return storageType; } public NodeResources withVcpu(double vcpu) { - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed); + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources withMemoryGb(double memoryGb) { - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed); + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources withDiskGb(double diskGb) { - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed); + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources withBandwidthGbps(double bandwidthGbps) { - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed); + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } + // TODO: Remove after November 2019 public NodeResources withDiskSpeed(DiskSpeed speed) { - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, speed); + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, speed, storageType); + } + + public NodeResources with(DiskSpeed speed) { + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, speed, storageType); + } + + public NodeResources with(StorageType storageType) { + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } - /** A shorthand for withDiskSpeed(NodeResources.DiskSpeed.any) */ - public NodeResources anySpeed() { - return withDiskSpeed(NodeResources.DiskSpeed.any); + /** Returns this with disk speed and storage type set to any */ + public NodeResources numbersOnly() { + return with(NodeResources.DiskSpeed.any).with(StorageType.any); } public NodeResources subtract(NodeResources other) { @@ -100,7 +139,8 @@ public class NodeResources { memoryGb - other.memoryGb, diskGb - other.diskGb, bandwidthGbps - other.bandwidthGbps, - combine(this.diskSpeed, other.diskSpeed)); + this.diskSpeed.combineWith(other.diskSpeed), + this.storageType.combineWith(other.storageType)); } public NodeResources add(NodeResources other) { @@ -110,22 +150,18 @@ public class NodeResources { memoryGb + other.memoryGb, diskGb + other.diskGb, bandwidthGbps + other.bandwidthGbps, - combine(this.diskSpeed, other.diskSpeed)); + this.diskSpeed.combineWith(other.diskSpeed), + this.storageType.combineWith(other.storageType)); } private boolean isInterchangeableWith(NodeResources other) { if (this.diskSpeed != DiskSpeed.any && other.diskSpeed != DiskSpeed.any && this.diskSpeed != other.diskSpeed) return false; + if (this.storageType != StorageType.any && other.storageType != StorageType.any && this.storageType != other.storageType) + return false; return true; } - private DiskSpeed combine(DiskSpeed a, DiskSpeed b) { - if (a == DiskSpeed.any) return b; - if (b == DiskSpeed.any) return a; - if (a == b) return a; - throw new IllegalArgumentException(a + " cannot be combined with " + b); - } - @Override public boolean equals(Object o) { if (o == this) return true; @@ -136,19 +172,21 @@ public class NodeResources { if (this.diskGb != other.diskGb) return false; if (this.bandwidthGbps != other.bandwidthGbps) return false; if (this.diskSpeed != other.diskSpeed) return false; + if (this.storageType != other.storageType) return false; return true; } @Override public int hashCode() { - return Objects.hash(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed); + return Objects.hash(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } @Override public String toString() { return "[vcpu: " + vcpu + ", memory: " + memoryGb + " Gb, disk " + diskGb + " Gb" + (bandwidthGbps > 0 ? ", bandwidth: " + bandwidthGbps + " Gbps" : "") + - (diskSpeed != DiskSpeed.fast ? ", disk speed: " + diskSpeed : "") + "]"; + (diskSpeed != DiskSpeed.fast ? ", disk speed: " + diskSpeed : "") + + (storageType != StorageType.any ? ", storage type: " + storageType : "") + "]"; } /** Returns true if all the resources of this are the same or larger than the given resources */ @@ -163,6 +201,9 @@ public class NodeResources { // draw conclusions about performance on the basis of better resources than you think you have if (other.diskSpeed != DiskSpeed.any && other.diskSpeed != this.diskSpeed) return false; + // Same reasoning as the above + if (other.storageType != StorageType.any && other.storageType != this.storageType) return false; + return true; } @@ -173,6 +214,7 @@ public class NodeResources { if (this.diskGb != other.diskGb) return false; if (this.bandwidthGbps != other.bandwidthGbps) return false; if (other.diskSpeed != DiskSpeed.any && other.diskSpeed != this.diskSpeed) return false; + if (other.storageType != StorageType.any && other.storageType != this.storageType) return false; return true; } @@ -195,7 +237,7 @@ public class NodeResources { if (cpu == 0) cpu = 0.5; if (cpu == 2 && mem == 8 ) cpu = 1.5; if (cpu == 2 && mem == 12 ) cpu = 2.3; - return new NodeResources(cpu, mem, dsk, 0.3, DiskSpeed.fast); + return new NodeResources(cpu, mem, dsk, 0.3, DiskSpeed.fast, StorageType.any); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index 4a060fb5143..c7630f06552 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -55,6 +55,7 @@ public class AllocatedHostsSerializer { private static final String diskKey = "disk"; private static final String bandwidthKey = "bandwidth"; private static final String diskSpeedKey = "diskSpeed"; + private static final String storageTypeKey = "storageType"; /** Wanted version */ private static final String hostSpecVespaVersionKey = "vespaVersion"; @@ -109,6 +110,7 @@ public class AllocatedHostsSerializer { resourcesObject.setDouble(diskKey, resources.diskGb()); resourcesObject.setDouble(bandwidthKey, resources.bandwidthGbps()); resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); + resourcesObject.setString(storageTypeKey, storageTypeToString(resources.storageType())); } public static AllocatedHosts fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { @@ -152,7 +154,8 @@ public class AllocatedHostsSerializer { resources.field(memoryKey).asDouble(), resources.field(diskKey).asDouble(), resources.field(bandwidthKey).asDouble(), - diskSpeedFromSlime(resources.field(diskSpeedKey)))); + diskSpeedFromSlime(resources.field(diskSpeedKey)), + storageTypeFromSlime(resources.field(storageTypeKey)))); } private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { @@ -171,7 +174,25 @@ public class AllocatedHostsSerializer { case any : return "any"; default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed + "'"); } + } + + private static NodeResources.StorageType storageTypeFromSlime(Inspector storageType) { + if ( ! storageType.valid()) return NodeResources.StorageType.any; // TODO: Remove this line after December 2019 + switch (storageType.asString()) { + case "remote" : return NodeResources.StorageType.remote; + case "local" : return NodeResources.StorageType.local; + case "any" : return NodeResources.StorageType.any; + default: throw new IllegalStateException("Illegal storage-type value '" + storageType.asString() + "'"); + } + } + private static String storageTypeToString(NodeResources.StorageType storageType) { + switch (storageType) { + case remote : return "remote"; + case local : return "local"; + case any : return "any"; + default: throw new IllegalStateException("Illegal storage-type value '" + storageType + "'"); + } } private static ClusterMembership membershipFromSlime(Inspector object) { diff --git a/config-provisioning/src/main/resources/configdefinitions/flavors.def b/config-provisioning/src/main/resources/configdefinitions/flavors.def index dbdca724158..c5040c58b74 100644 --- a/config-provisioning/src/main/resources/configdefinitions/flavors.def +++ b/config-provisioning/src/main/resources/configdefinitions/flavors.def @@ -30,5 +30,8 @@ flavor[].minDiskAvailableGb double default=0.0 # Whether the disk is fast (typically SSD) or slow (typically spinning HDD). flavor[].fastDisk bool default=true +# Whether the storage is remote (network) or local. +flavor[].remoteStorage bool default=true + # Expected network interface bandwidth available for this flavor, in Mbit/s. flavor[].bandwidth double default=0.0 |