diff options
Diffstat (limited to 'config-provisioning/src/main')
6 files changed, 145 insertions, 56 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index f3c214da6ec..735f4afd974 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import java.time.Duration; import java.util.Objects; import java.util.Optional; -import java.util.stream.Stream; /** * A capacity request. @@ -35,6 +35,8 @@ public final class Capacity { if (max.smallerThan(min)) throw new IllegalArgumentException("The max capacity must be larger than the min capacity, but got min " + min + " and max " + max); + if (cloudAccount.isEmpty() && ! clusterInfo.hostTTL().isZero()) + throw new IllegalArgumentException("Cannot set hostTTL without a custom cloud account"); this.min = min; this.max = max; this.groupSize = groupSize; @@ -105,36 +107,40 @@ public final class Capacity { } public static Capacity from(ClusterResources resources, boolean required, boolean canFail) { - return from(resources, required, canFail, NodeType.tenant); + return from(resources, required, canFail, Duration.ZERO); } - // TODO: Remove after March 2023 - public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { - return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount, ClusterInfo.empty()); + public static Capacity from(ClusterResources resources, boolean required, boolean canFail, Duration hostTTL) { + return from(resources, required, canFail, NodeType.tenant, hostTTL); + } + + public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, + Optional<CloudAccount> cloudAccount, ClusterInfo clusterInfo) { + return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount, clusterInfo); } - // TODO: Remove after March 2023 + // TODO: remove at some point, much later than March 2023 ... ? public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail) { return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, Optional.empty(), ClusterInfo.empty()); } - // TODO: Remove after March 2023 + // TODO: remove at some point, much later than March 2023 ... ? public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, cloudAccount, ClusterInfo.empty()); } - public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, - Optional<CloudAccount> cloudAccount, ClusterInfo clusterInfo) { - return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount, clusterInfo); + // TODO: remove at some point, much later than March 2023 ... ? + public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { + return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount, ClusterInfo.empty()); } /** Creates this from a node type */ public static Capacity fromRequiredNodeType(NodeType type) { - return from(new ClusterResources(0, 1, NodeResources.unspecified()), true, false, type); + return from(new ClusterResources(0, 1, NodeResources.unspecified()), true, false, type, Duration.ZERO); } - private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type) { - return new Capacity(resources, resources, IntRange.empty(), required, canFail, type, Optional.empty(), ClusterInfo.empty()); + private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type, Duration hostTTL) { + return new Capacity(resources, resources, IntRange.empty(), required, canFail, type, Optional.empty(), new ClusterInfo.Builder().hostTTL(hostTTL).build()); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java b/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java index 215afbca255..5e14e287a12 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java @@ -1,39 +1,43 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import ai.vespa.validation.PatternedStringWrapper; -import ai.vespa.validation.Validation; - +import java.util.Map; +import java.util.Objects; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Identifies an account in a public cloud, such as {@link CloudName#AWS} or {@link CloudName#GCP}. * * @author mpolden */ -public class CloudAccount extends PatternedStringWrapper<CloudAccount> { +public class CloudAccount implements Comparable<CloudAccount> { - private static final String EMPTY = ""; - private static final String AWS_ACCOUNT_ID = "[0-9]{12}"; - private static final Pattern AWS_ACCOUNT_ID_PATTERN = Pattern.compile(AWS_ACCOUNT_ID); - private static final String GCP_PROJECT_ID = "[a-z][a-z0-9-]{4,28}[a-z0-9]"; - private static final Pattern GCP_PROJECT_ID_PATTERN = Pattern.compile(GCP_PROJECT_ID); + private record CloudMeta(String accountType, Pattern pattern) { + private boolean matches(String account) { return pattern.matcher(account).matches(); } + } + private static final Map<String, CloudMeta> META_BY_CLOUD = Map.of( + "aws", new CloudMeta("Account ID", Pattern.compile("[0-9]{12}")), + "gcp", new CloudMeta("Project ID", Pattern.compile("[a-z][a-z0-9-]{4,28}[a-z0-9]"))); /** Empty value. When this is used, either implicitly or explicitly, the zone will use its default account */ - public static final CloudAccount empty = new CloudAccount("", EMPTY, "cloud account"); + public static final CloudAccount empty = new CloudAccount("", CloudName.DEFAULT); - /** Verifies accountId is a valid AWS account ID, or throw an IllegalArgumentException. */ - public static void requireAwsAccountId(String accountId) { - Validation.requireMatch(accountId, "AWS account ID", AWS_ACCOUNT_ID_PATTERN); - } + private final String account; + private final CloudName cloudName; - /** Verifies accountId is a valid GCP project ID, or throw an IllegalArgumentException. */ - public static void requireGcpProjectId(String projectId) { - Validation.requireMatch(projectId, "GCP project ID", GCP_PROJECT_ID_PATTERN); + private CloudAccount(String account, CloudName cloudName) { + this.account = account; + this.cloudName = cloudName; } - private CloudAccount(String value, String regex, String description) { - super(value, Pattern.compile("^(" + regex + ")$"), description); + public String account() { return account; } + public CloudName cloudName() { return cloudName; } + + /** Returns the serialized value of this account that can be deserialized with {@link CloudAccount#from} */ + public final String value() { + if (isUnspecified()) return account; + return cloudName.value() + ':' + account; } public boolean isUnspecified() { @@ -47,27 +51,56 @@ public class CloudAccount extends PatternedStringWrapper<CloudAccount> { !equals(zone.cloud().account()); } - /** Verifies this account is a valid AWS account ID, or throw an IllegalArgumentException. */ - public void requireAwsAccountId() { - requireAwsAccountId(value()); + @Override + public String toString() { + return isUnspecified() ? "unspecified account" : "account '" + account + "' in " + cloudName; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CloudAccount that = (CloudAccount) o; + return account.equals(that.account) && cloudName.equals(that.cloudName); + } + + @Override + public int hashCode() { + return Objects.hash(account, cloudName); } - /** Verifies this account is a valid GCP project ID, or throw an IllegalArgumentException. */ - public void requireGcpProjectId() { - requireGcpProjectId(value()); + @Override + public int compareTo(CloudAccount o) { + return this.value().compareTo(o.value()); } + public static CloudAccount from(String cloudAccount) { - return switch (cloudAccount) { + int index = cloudAccount.indexOf(':'); + if (index < 0) { // Tenants are allowed to specify "default" in services.xml. - case "", "default" -> empty; - default -> new CloudAccount(cloudAccount, AWS_ACCOUNT_ID + "|" + GCP_PROJECT_ID, "cloud account"); - }; + if (cloudAccount.isEmpty() || cloudAccount.equals("default")) + return empty; + if (META_BY_CLOUD.get("aws").matches(cloudAccount)) + return new CloudAccount(cloudAccount, CloudName.AWS); + if (META_BY_CLOUD.get("gcp").matches(cloudAccount)) // TODO (freva): Remove July 2023 + return new CloudAccount(cloudAccount, CloudName.GCP); + throw illegal(cloudAccount, "Must be on format '<cloud-name>:<account>' or 'default'"); + } + + String cloud = cloudAccount.substring(0, index); + String account = cloudAccount.substring(index + 1); + CloudMeta cloudMeta = META_BY_CLOUD.get(cloud); + if (cloudMeta == null) + throw illegal(cloudAccount, "Cloud name must be one of: " + META_BY_CLOUD.keySet().stream().sorted().collect(Collectors.joining(", "))); + + if (!cloudMeta.matches(account)) + throw illegal(cloudAccount, cloudMeta.accountType + " must match '" + cloudMeta.pattern.pattern() + "'"); + return new CloudAccount(account, CloudName.from(cloud)); } - @Override - public String toString() { - return isUnspecified() ? "unspecified account" : "account '" + value() + "'"; + private static IllegalArgumentException illegal(String cloudAccount, String details) { + return new IllegalArgumentException("Invalid cloud account '" + cloudAccount + "': " + details); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/CloudName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/CloudName.java index ba262136abe..e1d7afdc9f0 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/CloudName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/CloudName.java @@ -3,7 +3,6 @@ package com.yahoo.config.provision; import ai.vespa.validation.PatternedStringWrapper; -import java.util.Objects; import java.util.regex.Pattern; /** @@ -14,17 +13,23 @@ import java.util.regex.Pattern; public class CloudName extends PatternedStringWrapper<CloudName> { private static final Pattern pattern = Pattern.compile("[a-z]([a-z0-9-]*[a-z0-9])*"); - public static final CloudName AWS = from("aws"); - public static final CloudName GCP = from("gcp"); - public static final CloudName DEFAULT = from("default"); - public static final CloudName YAHOO = from("yahoo"); + public static final CloudName AWS = new CloudName("aws"); + public static final CloudName GCP = new CloudName("gcp"); + public static final CloudName DEFAULT = new CloudName("default"); + public static final CloudName YAHOO = new CloudName("yahoo"); private CloudName(String cloud) { super(cloud, pattern, "cloud name"); } public static CloudName from(String cloud) { - return new CloudName(cloud); + return switch (cloud) { + case "aws" -> AWS; + case "gcp" -> GCP; + case "default" -> DEFAULT; + case "yahoo" -> YAHOO; + default -> new CloudName(cloud); + }; } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java index fe8acb0c3c0..d9076557ac7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java @@ -3,6 +3,8 @@ package com.yahoo.config.provision; import java.time.Duration; import java.util.Objects; +import static ai.vespa.validation.Validation.requireAtLeast; + /** * Auxiliary information about a cluster, provided by the config model to the node repo during a * capacity request. @@ -14,13 +16,18 @@ public class ClusterInfo { private static final ClusterInfo empty = new ClusterInfo.Builder().build(); private final Duration bcpDeadline; + private final Duration hostTTL; private ClusterInfo(Builder builder) { this.bcpDeadline = builder.bcpDeadline; + this.hostTTL = builder.hostTTL; + requireAtLeast(hostTTL, "host TTL", Duration.ZERO); } public Duration bcpDeadline() { return bcpDeadline; } + public Duration hostTTL() { return hostTTL; } + public static ClusterInfo empty() { return empty; } public boolean isEmpty() { return this.equals(empty); } @@ -30,28 +37,35 @@ public class ClusterInfo { if (o == this) return true; if ( ! (o instanceof ClusterInfo other)) return false; if ( ! other.bcpDeadline.equals(this.bcpDeadline)) return false; + if ( ! other.hostTTL.equals(this.hostTTL)) return false; return true; } @Override public int hashCode() { - return Objects.hash(bcpDeadline); + return Objects.hash(bcpDeadline, hostTTL); } @Override public String toString() { - return "cluster info: [bcp deadline: " + bcpDeadline + "]"; + return "cluster info: [bcp deadline: " + bcpDeadline + ", host TTL: " + hostTTL + "]"; } public static class Builder { - private Duration bcpDeadline = Duration.ofMinutes(0); + private Duration bcpDeadline = Duration.ZERO; + private Duration hostTTL = Duration.ZERO; public Builder bcpDeadline(Duration duration) { this.bcpDeadline = Objects.requireNonNull(duration); return this; } + public Builder hostTTL(Duration duration) { + this.hostTTL = Objects.requireNonNull(duration); + return this; + } + public ClusterInfo build() { return new ClusterInfo(this); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index d1fd409fc93..a431dd61b0d 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -213,6 +213,7 @@ public class NodeResources { public boolean vcpuIsUnspecified() { return vcpu == 0; } public boolean memoryGbIsUnspecified() { return memoryGb == 0; } public boolean diskGbIsUnspecified() { return diskGb == 0; } + public boolean bandwidthGbpsIsUnspecified() { return bandwidthGbps == 0; } /** Returns the standard cost of these resources, in dollars per hour */ public double cost() { @@ -267,6 +268,19 @@ public class NodeResources { return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); } + public NodeResources withUnspecifiedNumbersFrom(NodeResources fullySpecified) { + var resources = this; + if (resources.vcpuIsUnspecified()) + resources = resources.withVcpu(fullySpecified.vcpu()); + if (resources.memoryGbIsUnspecified()) + resources = resources.withMemoryGb(fullySpecified.memoryGb()); + if (resources.diskGbIsUnspecified()) + resources = resources.withDiskGb(fullySpecified.diskGb()); + if (resources.bandwidthGbpsIsUnspecified()) + resources = resources.withBandwidthGbps(fullySpecified.bandwidthGbps()); + return resources; + } + /** Returns this with disk speed, storage type and architecture set to any */ public NodeResources justNumbers() { if (isUnspecified()) return unspecified(); @@ -362,7 +376,7 @@ public class NodeResources { appendDouble(sb, vcpu); sb.append(", memory: "); appendDouble(sb, memoryGb); - sb.append(" Gb, disk "); + sb.append(" Gb, disk: "); appendDouble(sb, diskGb); sb.append(" Gb"); if (bandwidthGbps > 0) { @@ -461,7 +475,7 @@ public class NodeResources { throw new IllegalStateException("Cannot perform this on unspecified resources"); } - // Returns squared euclidean distance of the relevant numerical values of two node resources + // Returns squared Euclidean distance of the relevant numerical values of two node resources public double distanceTo(NodeResources other) { if ( ! this.diskSpeed().compatibleWith(other.diskSpeed())) return Double.MAX_VALUE; if ( ! this.storageType().compatibleWith(other.storageType())) return Double.MAX_VALUE; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/QuotaExceededException.java b/config-provisioning/src/main/java/com/yahoo/config/provision/QuotaExceededException.java new file mode 100644 index 00000000000..12289f44c6a --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/QuotaExceededException.java @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +/** + * @author hmusum + */ +public class QuotaExceededException extends RuntimeException { + + public QuotaExceededException(Throwable t) { + super(t); + } + + public QuotaExceededException(String message) { + super(message); + } + +} |