diff options
author | Øyvind Grønnesby <oyving@verizonmedia.com> | 2020-09-30 11:03:43 +0200 |
---|---|---|
committer | Øyvind Grønnesby <oyving@verizonmedia.com> | 2020-09-30 11:03:43 +0200 |
commit | 68176288649b5ebc3bbf00ae934875c17b9a75b7 (patch) | |
tree | b262777aba8cde3e3c05dd6a2cc3583db0778a00 | |
parent | 628a9a9dd57d9eadd3ad786c5cde40ccfec1ba63 (diff) |
Use BigDecimal for budget inside the Quota
- To allow budgets below $1/hour we change the internal representation of budget
to a decimal number. Some interfaces that assume integers are kept to keep the
API stable.
- Created a nicer method .unlimited() instead of .empt() to better show semantics.
- Added some serialisation tests to make sure we support integers and decimals.
7 files changed, 98 insertions, 21 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 0c78aafcf20..b1145d49f59 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -112,7 +112,7 @@ public interface ModelContext { default double feedCoreThreadPoolSizeFactor() { return 4.0; } default Quota quota() { - return Quota.empty(); + return Quota.unlimited(); } // TODO(bjorncs): Temporary feature flag diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java index cb600bc0a5e..3a57e5bb66c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java @@ -4,7 +4,9 @@ package com.yahoo.config.model.api; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import com.yahoo.slime.Type; +import java.math.BigDecimal; import java.util.Objects; import java.util.Optional; @@ -14,39 +16,54 @@ import java.util.Optional; * @author ogronnesby */ public class Quota { + private static final Quota UNLIMITED = new Quota(Optional.empty(), Optional.empty()); + private final Optional<Integer> maxClusterSize; - private final Optional<Integer> budget; + private final Optional<BigDecimal> budget; + + // TODO: Remove constructor once Vespa < 7.300 is gone from production + public Quota(Optional<Integer> maxClusterSize, Optional<Integer> budget) { + this(maxClusterSize, budget.map(BigDecimal::new), true); + } - public Quota(Optional<Integer> maybeClusterSize, Optional<Integer> budget) { - this.maxClusterSize = maybeClusterSize; - this.budget = budget; + private Quota(Optional<Integer> maxClusterSize, Optional<BigDecimal> budget, boolean isDecimal) { + this.maxClusterSize = Objects.requireNonNull(maxClusterSize); + this.budget = Objects.requireNonNull(budget); } public static Quota fromSlime(Inspector inspector) { var clusterSize = SlimeUtils.optionalLong(inspector.field("clusterSize")); - var budget = SlimeUtils.optionalLong(inspector.field("budget")); - return new Quota(clusterSize.map(Long::intValue), budget.map(Long::intValue)); + var budget = budgetFromSlime(inspector.field("budget")); + return new Quota(clusterSize.map(Long::intValue), budget, true); + } + + public Quota withBudget(Optional<BigDecimal> budget) { + return new Quota(this.maxClusterSize, budget, true); + } + + public Quota withClusterSize(Optional<Integer> clusterSize) { + return new Quota(clusterSize, this.budget, true); } public Slime toSlime() { var slime = new Slime(); var root = slime.setObject(); maxClusterSize.ifPresent(clusterSize -> root.setLong("clusterSize", clusterSize)); - budget.ifPresent(b -> root.setLong("budget", b)); + budget.ifPresent(b -> root.setString("budget", b.toPlainString())); return slime; } - public static Quota empty() { - return new Quota(Optional.empty(), Optional.empty()); - } + public static Quota unlimited() { return UNLIMITED; } public Optional<Integer> maxClusterSize() { return maxClusterSize; } - public Optional<Integer> budget() { - return budget; - } + public Optional<BigDecimal> budgetAsDecimal() { return budget; } + + // TODO: Remove once Vespa < 7.300 is gone from production + public static Quota empty() { return unlimited(); } + public Optional<Integer> budget() { return budget.map(BigDecimal::intValue); } @Override public boolean equals(Object o) { @@ -69,4 +86,21 @@ public class Quota { ", budget=" + budget + '}'; } + + /** + * Since Slime does not support any decimal numeric value that isn't a floating point of some sort, we need + * to be liberal in what we accept. Since we are dealing with currency, ideally we would have a decimal + * data type all the way through. + * + * There are three ways of communicating the budget to the Quota class: + * 1. A LONG means we are looking at the budget in whole dollars. This is the legacy way. + * 2. A STRING formatted as a number is a full precision decimal number. This is the proper way. + * 3. A DOUBLE gets translated into a decimal type, but loses precision. This is the CYA way. + */ + private static Optional<BigDecimal> budgetFromSlime(Inspector inspector) { + if (inspector.type() == Type.STRING) return Optional.of(inspector.asString()).map(BigDecimal::new); + if (inspector.type() == Type.LONG) return Optional.of(inspector.asLong()).map(BigDecimal::new); + if (inspector.type() == Type.DOUBLE) return Optional.of(inspector.asDouble()).map(BigDecimal::new); + return Optional.empty(); + } } diff --git a/config-model-api/src/test/java/com/yahoo/config/model/api/QuotaTest.java b/config-model-api/src/test/java/com/yahoo/config/model/api/QuotaTest.java new file mode 100644 index 00000000000..83d4ffbbe72 --- /dev/null +++ b/config-model-api/src/test/java/com/yahoo/config/model/api/QuotaTest.java @@ -0,0 +1,41 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +import com.yahoo.slime.SlimeUtils; +import org.junit.Test; + + +import java.math.BigDecimal; + +import static org.junit.Assert.assertEquals; + +public class QuotaTest { + + @Test + public void test_serialization_with_integers() { + var json = "{\"budget\": 123}"; + var slime = SlimeUtils.jsonToSlime(json); + var quota = Quota.fromSlime(slime.get()); + assertEquals((Integer) 123, quota.budget().get()); + assertEquals(123, quota.budgetAsDecimal().get().intValueExact()); + } + + @Test + public void test_serialization_with_floats() { + var json = "{\"budget\": 123.4}"; + var slime = SlimeUtils.jsonToSlime(json); + var quota = Quota.fromSlime(slime.get()); + assertEquals((Integer) 123, quota.budget().get()); + assertEquals(123.4, quota.budgetAsDecimal().get().doubleValue(), 0.01); + } + + @Test + public void test_serialization_with_string() { + var json = "{\"budget\": \"123.4\"}"; + var slime = SlimeUtils.jsonToSlime(json); + var quota = Quota.fromSlime(slime.get()); + assertEquals((Integer) 123, quota.budget().get()); + assertEquals(new BigDecimal("123.4"), quota.budgetAsDecimal().get()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 31c72e1be69..1455b4d8007 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -51,7 +51,7 @@ public class TestProperties implements ModelContext.Properties { private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private AthenzDomain athenzDomain; private ApplicationRoles applicationRoles; - private Quota quota = Quota.empty(); + private Quota quota = Quota.unlimited(); @Override public boolean multitenant() { return multitenant; } @Override public ApplicationId applicationId() { return applicationId; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index c0a2fe4fdf2..d06d616efac 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; +import java.math.BigDecimal; import java.util.Locale; import java.util.stream.Collectors; @@ -19,17 +20,17 @@ public class QuotaValidator extends Validator { public void validate(VespaModel model, DeployState deployState) { var quota = deployState.getProperties().quota(); quota.maxClusterSize().ifPresent(maxClusterSize -> validateMaxClusterSize(maxClusterSize, model)); - quota.budget().ifPresent(budget -> validateBudget(budget, model)); + quota.budgetAsDecimal().ifPresent(budget -> validateBudget(budget, model)); } - private void validateBudget(int budget, VespaModel model) { + private void validateBudget(BigDecimal budget, VespaModel model) { var spend = model.allClusters().stream() .map(clusterId -> model.provisioned().all().get(clusterId)) .map(Capacity::maxResources) .map(clusterCapacity -> clusterCapacity.nodeResources().cost() * clusterCapacity.nodes()) .reduce(0.0, Double::sum); - if (spend > budget) { + if (budget.doubleValue() < spend) { throwBudgetExceeded(spend, budget); } } @@ -51,7 +52,7 @@ public class QuotaValidator extends Validator { } } - private void throwBudgetExceeded(double spend, double budget) { + private void throwBudgetExceeded(double spend, BigDecimal budget) { var message = String.format(Locale.US, "Hourly spend for maximum specified resources ($%.2f) exceeds budget from quota ($%.2f)!", spend, budget); throw new IllegalArgumentException(message); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 10199bfe6b9..70a6bbbf659 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.provision.Environment; import org.junit.Test; +import java.math.BigDecimal; import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -16,7 +17,7 @@ import static org.junit.Assert.fail; */ public class QuotaValidatorTest { - private final Quota quota = new Quota(Optional.of(10), Optional.of(1)); + private final Quota quota = Quota.unlimited().withClusterSize(Optional.of(10)).withBudget(Optional.of(BigDecimal.valueOf(1))); @Test public void test_deploy_under_quota() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 48d3fd6a176..c55eb3dbff2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -234,7 +234,7 @@ public class ModelContextImpl implements ModelContext { this.applicationRoles = applicationRoles; feedCoreThreadPoolSizeFactor = Flags.FEED_CORE_THREAD_POOL_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - this.quota = maybeQuota.orElseGet(Quota::empty); + this.quota = maybeQuota.orElseGet(Quota::unlimited); this.useNewRestapiHandler = Flags.USE_NEW_RESTAPI_HANDLER.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()) .value(); |