diff options
15 files changed, 92 insertions, 29 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 9e867a4c3bc..28ff8dff620 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -25,6 +25,7 @@ import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.QuotaExceededException; import com.yahoo.config.provision.TransientException; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.VespaVersion; @@ -222,7 +223,7 @@ public class VespaModelFactory implements ModelFactory { Exceptions.toMessageString(e)); else rethrowUnlessIgnoreErrors(e, validationParameters.ignoreValidationErrors()); - } catch (IllegalArgumentException | TransientException e) { + } catch (IllegalArgumentException | TransientException | QuotaExceededException e) { rethrowUnlessIgnoreErrors(e, validationParameters.ignoreValidationErrors()); } catch (Exception e) { throw new RuntimeException(e); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index 475a4174f9a..5405a528150 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.QuotaExceededException; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.model.VespaModel; @@ -80,19 +81,19 @@ public class QuotaValidator extends Validator { if (!invalidClusters.isEmpty()) { var clusterNames = String.join(", ", invalidClusters); - throw new IllegalArgumentException("Clusters " + clusterNames + " exceeded max cluster size of " + maxClusterSize); + throw new QuotaExceededException("Clusters " + clusterNames + " exceeded max cluster size of " + maxClusterSize); } } private static void throwIfBudgetNegative(double spend, BigDecimal budget, SystemName systemName) { if (budget.doubleValue() < 0) { - throw new IllegalArgumentException(quotaMessage("Please free up some capacity.", systemName, spend, budget, true)); + throw new QuotaExceededException(quotaMessage("Please free up some capacity.", systemName, spend, budget, true)); } } private static void throwIfBudgetExceeded(double spend, BigDecimal budget, SystemName systemName, boolean actual) { if (budget.doubleValue() < spend) { - throw new IllegalArgumentException(quotaMessage("Contact support to upgrade your plan.", systemName, spend, budget, actual)); + throw new QuotaExceededException(quotaMessage("Contact support to upgrade your plan.", systemName, spend, budget, actual)); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/QuotaExceededException.java b/config-provisioning/src/main/java/com/yahoo/config/provision/QuotaExceededException.java new file mode 100644 index 00000000000..12289f44c6a --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/QuotaExceededException.java @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +/** + * @author hmusum + */ +public class QuotaExceededException extends RuntimeException { + + public QuotaExceededException(Throwable t) { + super(t); + } + + public QuotaExceededException(String message) { + super(message); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java index 3b5269cdf11..c87d77eaf07 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java @@ -53,7 +53,8 @@ public class HttpErrorResponse extends HttpResponse { LOAD_BALANCER_NOT_READY, CONFIG_NOT_CONVERGED, REINDEXING_STATUS_UNAVAILABLE, - PRECONDITION_FAILED + PRECONDITION_FAILED, + QUOTA_EXCEEDED } public static HttpErrorResponse notFoundError(String msg) { @@ -120,6 +121,10 @@ public class HttpErrorResponse extends HttpResponse { return new HttpErrorResponse(PRECONDITION_FAILED, ErrorCode.PRECONDITION_FAILED.name(), msg); } + public static HttpResponse quotaExceeded(String msg) { + return new HttpErrorResponse(BAD_REQUEST, ErrorCode.QUOTA_EXCEEDED.name(), msg); + } + @Override public void render(OutputStream stream) throws IOException { new JsonFormat(true).encode(stream, slime); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index a0e814f32d8..58651af54f3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.CertificateNotReadyException; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.ParentHostUnavailableException; +import com.yahoo.config.provision.QuotaExceededException; import com.yahoo.config.provision.exception.ActivationConflictException; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.container.jdisc.HttpRequest; @@ -73,6 +74,8 @@ public class HttpHandler extends ThreadedHttpRequestHandler { return HttpErrorResponse.reindexingStatusUnavailable(getMessage(e, request)); } catch (PreconditionFailedException e) { return HttpErrorResponse.preconditionFailed(getMessage(e, request)); + } catch (QuotaExceededException e) { + return HttpErrorResponse.quotaExceeded(getMessage(e, request)); } catch (Exception e) { log.log(Level.WARNING, "Unexpected exception handling a config server request", e); return HttpErrorResponse.internalServerError(getMessage(e, request)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index bd1837707d9..4faa475fa08 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -14,6 +14,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeAllocationException; +import com.yahoo.config.provision.QuotaExceededException; import com.yahoo.config.provision.TransientException; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.http.InternalServerException; @@ -122,7 +123,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { buildLatestModelForThisMajor, majorVersion)); buildLatestModelForThisMajor = false; // We have successfully built latest model version, do it only for this major } - catch (NodeAllocationException | ApplicationLockException | TransientException e) { + catch (NodeAllocationException | ApplicationLockException | TransientException | QuotaExceededException e) { // Don't wrap this exception, and don't try to load other model versions as this is (most likely) // caused by the state of the system, not the model version/application combination throw e; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployNodeAllocationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployNodeAllocationTest.java index e546569b255..e9dca44ed81 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployNodeAllocationTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployNodeAllocationTest.java @@ -4,14 +4,17 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.component.Version; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.Quota; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ProvisionLogger; +import com.yahoo.config.provision.QuotaExceededException; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.MockProvisioner; +import com.yahoo.vespa.config.server.session.PrepareParams; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -24,6 +27,7 @@ import java.util.stream.Collectors; import static com.yahoo.vespa.config.server.deploy.DeployTester.createHostedModelFactory; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class HostedDeployNodeAllocationTest { @@ -50,6 +54,26 @@ public class HostedDeployNodeAllocationTest { assertEquals(resources(2), get("host4", hosts).advertisedResources()); } + @Test + public void testExceedsQuota() { + List<ModelFactory> modelFactories = List.of(createHostedModelFactory(Version.fromString("7.2")), + createHostedModelFactory(Version.fromString("7.3"))); + var provisioner = new VersionProvisioner(); + DeployTester tester = new DeployTester.Builder(temporaryFolder).modelFactories(modelFactories) + .provisioner(new MockProvisioner().hostProvisioner(provisioner)) + .hostedConfigserverConfig(Zone.defaultZone()) + .build(); + + try { + tester.deployApp("src/test/apps/hosted/", new PrepareParams.Builder() + .vespaVersion("7.3") + .quota(new Quota(Optional.of(4), Optional.of(0)))); + fail("Expected to get a QuotaExceededException"); + } catch (QuotaExceededException e) { + assertEquals("main: The resources used cost $1.02 but your quota is $0.00: Contact support to upgrade your plan.", e.getMessage()); + } + } + private HostSpec get(String hostname, Set<HostSpec> hosts) { return hosts.stream().filter(host -> host.hostname().equals(hostname)).findAny().orElseThrow(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java index 2b35334e14b..7b5166dc84e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java @@ -1,13 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.SlimeUtils; - -import java.util.stream.Stream; - -import static java.nio.charset.StandardCharsets.UTF_8; - /** * An exception due to server error, a bad request, or similar. * @@ -49,17 +42,8 @@ public class ConfigServerException extends RuntimeException { CERTIFICATE_NOT_READY, LOAD_BALANCER_NOT_READY, INCOMPLETE_RESPONSE, - CONFIG_NOT_CONVERGED - } - - public static ConfigServerException readException(byte[] body, String context) { - Inspector root = SlimeUtils.jsonToSlime(body).get(); - String codeName = root.field("error-code").asString(); - ErrorCode code = Stream.of(ErrorCode.values()) - .filter(value -> value.name().equals(codeName)) - .findAny().orElse(ErrorCode.INCOMPLETE_RESPONSE); - String message = root.field("message").valid() ? root.field("message").asString() : new String(body, UTF_8); - return new ConfigServerException(code, message, context); + CONFIG_NOT_CONVERGED, + QUOTA_EXCEEDED } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 71ab1c4d7da..d992c0a273b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -71,13 +71,13 @@ import static com.yahoo.config.application.api.Notifications.When.failing; import static com.yahoo.config.application.api.Notifications.When.failingCommit; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.cancelled; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.invalidApplication; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.quotaExceeded; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; @@ -268,6 +268,10 @@ public class InternalStepRunner implements StepRunner { logger.log(WARNING, e.getMessage()); return Optional.of(deploymentFailed); } + case QUOTA_EXCEEDED -> { + logger.log(WARNING, e.getMessage()); + return Optional.of(quotaExceeded); + } } throw e; @@ -828,6 +832,9 @@ public class InternalStepRunner implements StepRunner { case error: case endpointCertificateTimeout: break; + case quotaExceeded: + updater.accept("quota exceeded. Contact support to upgrade your plan."); + return; default: logger.log(WARNING, "Don't know what to set console notification to for run status '" + run.status() + "'"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java index e2b231e0946..b9bff5f777e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -25,6 +25,7 @@ public class JobMetrics { public static final String abort = "deployment.abort"; public static final String cancel = "deployment.cancel"; public static final String success = "deployment.success"; + public static final String quotaExceeded = "deployment.quotaExceeded"; private final Metric metric; @@ -61,6 +62,7 @@ public class JobMetrics { case cancelled -> cancel; case aborted -> abort; case success -> success; + case quotaExceeded -> quotaExceeded; default -> throw new IllegalArgumentException("Unexpected run status '" + status + "'"); }; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index b89e89e7002..5d625285a7d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -45,6 +45,9 @@ public enum RunStatus { cancelled, /** Run should be reset to its starting state. Used for production tests. */ - reset + reset, + + /** Deployment of the real application was rejected due to exceeding quota. */ + quotaExceeded } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 4da7aa4b2bd..4547eed24c8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -38,6 +38,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installatio import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.invalidApplication; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.quotaExceeded; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; @@ -345,6 +346,7 @@ class RunSerializer { case aborted -> "aborted"; case cancelled -> "cancelled"; case reset -> "reset"; + case quotaExceeded -> "quotaExceeded"; }; } @@ -363,6 +365,7 @@ class RunSerializer { case "aborted" -> aborted; case "cancelled" -> cancelled; case "reset" -> reset; + case "quotaExceeded" -> quotaExceeded; default -> throw new IllegalArgumentException("No run status defined by '" + status + "'!"); }; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 2f93ce999cd..d1d0fb54eef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.StepType; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; @@ -240,6 +239,7 @@ class JobControllerApiHandlerHelper { case installationFailed -> "installationFailed"; case invalidApplication, deploymentFailed -> "deploymentFailed"; case success -> "success"; + case quotaExceeded -> "quotaExceeded"; }; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 862fa08ab86..feb8a89b057 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -268,6 +268,7 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { case installationFailed -> "installationFailed"; case invalidApplication, deploymentFailed -> "deploymentFailed"; case success -> "success"; + case quotaExceeded -> "quotaExceeded"; }; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 2c2cc333f9c..7783f9af5a4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -31,7 +31,6 @@ import org.junit.jupiter.api.Test; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.Executors; @@ -45,12 +44,12 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.inst import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.quotaExceeded; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static java.time.temporal.ChronoUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -540,6 +539,18 @@ public class InternalStepRunnerTest { assertEquals(RunStatus.error, tester.jobs().run(id).status()); } + + @Test + public void quotaExceededAbortsJob() { + RuntimeException exception = new ConfigServerException(ConfigServerException.ErrorCode.QUOTA_EXCEEDED, + "Quota exceeded", + "deploy failure"); + tester.configServer().throwOnNextPrepare(exception); + tester.jobs().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), applicationPackage()); + assertEquals(failed, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().stepStatuses().get(Step.deployReal)); + assertEquals(quotaExceeded, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().status()); + } + private void assertTestLogEntries(RunId id, Step step, LogEntry... entries) { assertEquals(List.of(entries), tester.jobs().details(id).get().get(step)); } |