diff options
author | HÃ¥kon Hallingstad <hakon@verizonmedia.com> | 2020-10-28 11:54:20 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-10-28 11:54:20 +0100 |
commit | 352801f85a390ae635da9ff6995052b565ddd33f (patch) | |
tree | c8beb76c702cb85ea114988a76d0948ac5478da4 /flags | |
parent | 9e1f5c78bc60c595912c9fcdbdcd481df74406b2 (diff) | |
parent | 0274f45fd9c0108d3c08a1b75b77737ae421eee5 (diff) |
Merge pull request #15050 from vespa-engine/hakonhall/support-provisioning-of-shared-hosts
Support provisioning of shared hosts
Diffstat (limited to 'flags')
5 files changed, 214 insertions, 97 deletions
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 20629c05bf0..1d4cd21975d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.flags; import com.yahoo.component.Vtag; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.flags.custom.HostCapacity; +import com.yahoo.vespa.flags.custom.SharedHost; import java.math.BigDecimal; import java.util.List; @@ -62,13 +63,6 @@ public class Flags { "Takes effect on next tick.", HOSTNAME); - public static final UnboundLongFlag THIN_POOL_GB = defineLongFlag( - "thin-pool-gb", -1, - "The size of the disk reserved for the thin pool with dynamic provisioning in AWS, in base-2 GB. " + - "If <0, the default is used (which may depend on the zone and node type).", - "Takes effect immediately (but used only during provisioning).", - NODE_TYPE); - public static final UnboundDoubleFlag CONTAINER_CPU_CAP = defineDoubleFlag( "container-cpu-cap", 0, "Hard limit on how many CPUs a container may use. This value is multiplied by CPU allocated to node, so " + @@ -96,6 +90,12 @@ public class Flags { "Otherwise it specifies the total (unallocated or not) capacity.", "Takes effect on next iteration of DynamicProvisioningMaintainer."); + public static final UnboundJacksonFlag<SharedHost> SHARED_HOST = defineJacksonFlag( + "shared-host", SharedHost.createDisabled(), SharedHost.class, + "Specifies whether shared hosts can be provisioned, and if so, the advertised " + + "node resources of the host, the maximum number of containers, etc.", + "Takes effect on next iteration of DynamicProvisioningMaintainer."); + public static final UnboundListFlag<String> INACTIVE_MAINTENANCE_JOBS = defineListFlag( "inactive-maintenance-jobs", List.of(), String.class, "The list of maintenance jobs that are inactive.", diff --git a/flags/src/main/java/com/yahoo/vespa/flags/custom/HostResources.java b/flags/src/main/java/com/yahoo/vespa/flags/custom/HostResources.java new file mode 100644 index 00000000000..c0b5d7a523c --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/custom/HostResources.java @@ -0,0 +1,132 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.custom; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * The advertised node resources of a host, similar to config-provision's NodeResources, + * but with additional host-specific resources like the number of containers. + * + * @author freva + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class HostResources { + private static final Set<String> validDiskSpeeds = Set.of("slow", "fast"); + private static final Set<String> validStorageTypes = Set.of("remote", "local"); + + private final double vcpu; + + private final double memoryGb; + + private final double diskGb; + + private final double bandwidthGbps; + + private final String diskSpeed; + + private final String storageType; + + private final int containers; + + @JsonCreator + public HostResources(@JsonProperty("vcpu") Double vcpu, + @JsonProperty("memoryGb") Double memoryGb, + @JsonProperty("diskGb") Double diskGb, + @JsonProperty("bandwidthGbps") Double bandwidthGbps, + @JsonProperty("diskSpeed") String diskSpeed, + @JsonProperty("storageType") String storageType, + @JsonProperty("containers") Integer containers) { + this.vcpu = requirePositive("vcpu", vcpu); + this.memoryGb = requirePositive("memoryGb", memoryGb); + this.diskGb = requirePositive("diskGb", diskGb); + this.bandwidthGbps = requirePositive("bandwidthGbps", Optional.ofNullable(bandwidthGbps).orElse(0.3)); + this.diskSpeed = validateEnum("diskSpeed", validDiskSpeeds, diskSpeed); + this.storageType = validateEnum("storageType", validStorageTypes, storageType); + this.containers = requirePositive("containers", containers); + } + + @JsonProperty("vcpu") + public double vcpu() { return vcpu; } + + @JsonProperty("memoryGb") + public double memoryGb() { return memoryGb; } + + @JsonProperty("diskGb") + public double diskGb() { return diskGb; } + + @JsonProperty("bandwidthGbps") + public double bandwidthGbps() { return bandwidthGbps; } + + @JsonProperty("diskSpeed") + public String diskSpeed() { return diskSpeed; } + + @JsonProperty("storageType") + public String storageType() { return storageType; } + + @JsonProperty("containers") + public int containers() { return containers; } + + private static double requirePositive(String name, Double value) { + requireNonNull(name, value); + if (value <= 0) + throw new IllegalArgumentException("'" + name + "' must be positive, was " + value); + return value; + } + + private static int requirePositive(String name, Integer value) { + requireNonNull(name, value); + if (value <= 0) + throw new IllegalArgumentException("'" + name + "' must be positive, was " + value); + return value; + } + + private static String validateEnum(String name, Set<String> validValues, String value) { + requireNonNull(name, value); + if (!validValues.contains(value)) + throw new IllegalArgumentException("Invalid " + name + ", valid values are: " + + validValues + ", got: " + value); + return value; + } + + private static <T> T requireNonNull(String name, T value) { + return Objects.requireNonNull(value, () -> "'" + name + "' has not been specified"); + } + + @Override + public String toString() { + return "HostResources{" + + "vcpu=" + vcpu + + ", memoryGb=" + memoryGb + + ", diskGb=" + diskGb + + ", bandwidthGbps=" + bandwidthGbps + + ", diskSpeed='" + diskSpeed + '\'' + + ", storageType='" + storageType + '\'' + + ", containers=" + containers + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HostResources resources = (HostResources) o; + return Double.compare(resources.vcpu, vcpu) == 0 && + Double.compare(resources.memoryGb, memoryGb) == 0 && + Double.compare(resources.diskGb, diskGb) == 0 && + Double.compare(resources.bandwidthGbps, bandwidthGbps) == 0 && + diskSpeed.equals(resources.diskSpeed) && + storageType.equals(resources.storageType) && + containers == resources.containers; + } + + @Override + public int hashCode() { + return Objects.hash(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, containers); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/custom/NodeResources.java b/flags/src/main/java/com/yahoo/vespa/flags/custom/NodeResources.java deleted file mode 100644 index 38afc8a1481..00000000000 --- a/flags/src/main/java/com/yahoo/vespa/flags/custom/NodeResources.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.flags.custom; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * @author freva - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class NodeResources { - private static final Set<String> validDiskSpeeds = Set.of("any", "slow", "fast"); - - @JsonProperty("vcpu") - private final double vcpu; - - @JsonProperty("memoryGb") - private final double memoryGb; - - @JsonProperty("diskGb") - private final double diskGb; - - @JsonProperty("bandwidthGbps") - private final double bandwidthGbps; - - @JsonProperty("diskSpeed") - private final String diskSpeed; - - public NodeResources(@JsonProperty("vcpu") double vcpu, - @JsonProperty("memoryGb") double memoryGb, - @JsonProperty("diskGb") double diskGb, - @JsonProperty("bandwidthGbps") Double bandwidthGbps, - @JsonProperty("diskSpeed") String diskSpeed) { - this.vcpu = requirePositive("vcpu", vcpu); - this.memoryGb = requirePositive("memoryGb", memoryGb); - this.diskGb = requirePositive("diskGb", diskGb); - this.bandwidthGbps = requirePositive("bandwidthGbps", Optional.ofNullable(bandwidthGbps).orElse(0.3)); - this.diskSpeed = Optional.ofNullable(diskSpeed).orElse("fast"); - - if (!validDiskSpeeds.contains(this.diskSpeed)) - throw new IllegalArgumentException("Invalid diskSpeed, valid values are: " + validDiskSpeeds + ", got: " + diskSpeed); - } - - public double vcpu() { - return vcpu; - } - - public double memoryGb() { - return memoryGb; - } - - public double diskGb() { - return diskGb; - } - - public double bandwidthGbps() { - return bandwidthGbps; - } - - public String diskSpeed() { - return diskSpeed; - } - - private static double requirePositive(String name, double value) { - if (value <= 0) - throw new IllegalArgumentException("'" + name + "' must be positive, was " + value); - return value; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - NodeResources resources = (NodeResources) o; - return Double.compare(resources.vcpu, vcpu) == 0 && - Double.compare(resources.memoryGb, memoryGb) == 0 && - Double.compare(resources.diskGb, diskGb) == 0 && - Double.compare(resources.bandwidthGbps, bandwidthGbps) == 0 && - diskSpeed.equals(resources.diskSpeed); - } - - @Override - public int hashCode() { - return Objects.hash(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed); - } -} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/custom/SharedHost.java b/flags/src/main/java/com/yahoo/vespa/flags/custom/SharedHost.java new file mode 100644 index 00000000000..e463159eb8f --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/custom/SharedHost.java @@ -0,0 +1,64 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.custom; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.flags.Flags; + +import java.util.List; +import java.util.Objects; + +/** + * Defines properties related to shared hosts, see {@link Flags#SHARED_HOST}. + * + * @author hakon + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(value = JsonInclude.Include.NON_NULL) +public class SharedHost { + private final List<HostResources> resources; + + public static SharedHost createDisabled() { + return new SharedHost(null); + } + + @JsonCreator + public SharedHost(@JsonProperty("resources") List<HostResources> resources) { + this.resources = resources == null ? List.of() : List.copyOf(resources); + } + + @JsonProperty("resources") + public List<HostResources> getResourcesOrNull() { + return resources.isEmpty() ? null : resources; + } + + @JsonIgnore + public List<HostResources> getHostResources() { + return resources; + } + + public boolean isEnabled() { + return resources.size() > 0; + } + + @Override + public String toString() { + return resources.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SharedHost that = (SharedHost) o; + return resources.equals(that.resources); + } + + @Override + public int hashCode() { + return Objects.hash(resources); + } +} diff --git a/flags/src/test/java/com/yahoo/vespa/flags/FlagsTest.java b/flags/src/test/java/com/yahoo/vespa/flags/FlagsTest.java index 0f486794c3a..28e84bcf3e5 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/FlagsTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/FlagsTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.flags; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.BooleanNode; +import com.yahoo.vespa.flags.custom.HostResources; +import com.yahoo.vespa.flags.custom.SharedHost; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -107,6 +109,15 @@ public class FlagsTest { List.of(instance)); } + @Test + public void testSharedHostFlag() { + SharedHost sharedHost = new SharedHost(List.of(new HostResources( + 4.0, 16.0, 50.0, null, + "fast", "local", + 10))); + testGeneric(Flags.SHARED_HOST, sharedHost); + } + private <T> void testGeneric(UnboundFlag<T, ?, ?> unboundFlag, T value) { FlagSource source = mock(FlagSource.class); Flag<T, ?> flag = unboundFlag.bindTo(source); |