diff options
author | Harald Musum <musum@yahooinc.com> | 2023-05-22 13:33:56 +0200 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2023-05-22 13:33:56 +0200 |
commit | f872038482382aef5f329954a9815469c48f4538 (patch) | |
tree | a9d90086d0435bd856f5fa217d5502c3f5f21263 /controller-server | |
parent | beddb01f9c8c3f9c34ef110cd38a0a666834102b (diff) |
Handle quota exceeded from config server
Diffstat (limited to 'controller-server')
7 files changed, 32 insertions, 5 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 71ab1c4d7da..d992c0a273b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -71,13 +71,13 @@ import static com.yahoo.config.application.api.Notifications.When.failing; import static com.yahoo.config.application.api.Notifications.When.failingCommit; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.cancelled; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.invalidApplication; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.quotaExceeded; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; @@ -268,6 +268,10 @@ public class InternalStepRunner implements StepRunner { logger.log(WARNING, e.getMessage()); return Optional.of(deploymentFailed); } + case QUOTA_EXCEEDED -> { + logger.log(WARNING, e.getMessage()); + return Optional.of(quotaExceeded); + } } throw e; @@ -828,6 +832,9 @@ public class InternalStepRunner implements StepRunner { case error: case endpointCertificateTimeout: break; + case quotaExceeded: + updater.accept("quota exceeded. Contact support to upgrade your plan."); + return; default: logger.log(WARNING, "Don't know what to set console notification to for run status '" + run.status() + "'"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java index e2b231e0946..b9bff5f777e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -25,6 +25,7 @@ public class JobMetrics { public static final String abort = "deployment.abort"; public static final String cancel = "deployment.cancel"; public static final String success = "deployment.success"; + public static final String quotaExceeded = "deployment.quotaExceeded"; private final Metric metric; @@ -61,6 +62,7 @@ public class JobMetrics { case cancelled -> cancel; case aborted -> abort; case success -> success; + case quotaExceeded -> quotaExceeded; default -> throw new IllegalArgumentException("Unexpected run status '" + status + "'"); }; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index b89e89e7002..5d625285a7d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -45,6 +45,9 @@ public enum RunStatus { cancelled, /** Run should be reset to its starting state. Used for production tests. */ - reset + reset, + + /** Deployment of the real application was rejected due to exceeding quota. */ + quotaExceeded } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 4da7aa4b2bd..4547eed24c8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -38,6 +38,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installatio import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.invalidApplication; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.quotaExceeded; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; @@ -345,6 +346,7 @@ class RunSerializer { case aborted -> "aborted"; case cancelled -> "cancelled"; case reset -> "reset"; + case quotaExceeded -> "quotaExceeded"; }; } @@ -363,6 +365,7 @@ class RunSerializer { case "aborted" -> aborted; case "cancelled" -> cancelled; case "reset" -> reset; + case "quotaExceeded" -> quotaExceeded; default -> throw new IllegalArgumentException("No run status defined by '" + status + "'!"); }; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 2f93ce999cd..d1d0fb54eef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.StepType; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; @@ -240,6 +239,7 @@ class JobControllerApiHandlerHelper { case installationFailed -> "installationFailed"; case invalidApplication, deploymentFailed -> "deploymentFailed"; case success -> "success"; + case quotaExceeded -> "quotaExceeded"; }; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 862fa08ab86..feb8a89b057 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -268,6 +268,7 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { case installationFailed -> "installationFailed"; case invalidApplication, deploymentFailed -> "deploymentFailed"; case success -> "success"; + case quotaExceeded -> "quotaExceeded"; }; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 2c2cc333f9c..7783f9af5a4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -31,7 +31,6 @@ import org.junit.jupiter.api.Test; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.Executors; @@ -45,12 +44,12 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.inst import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.quotaExceeded; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static java.time.temporal.ChronoUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -540,6 +539,18 @@ public class InternalStepRunnerTest { assertEquals(RunStatus.error, tester.jobs().run(id).status()); } + + @Test + public void quotaExceededAbortsJob() { + RuntimeException exception = new ConfigServerException(ConfigServerException.ErrorCode.QUOTA_EXCEEDED, + "Quota exceeded", + "deploy failure"); + tester.configServer().throwOnNextPrepare(exception); + tester.jobs().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), applicationPackage()); + assertEquals(failed, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().stepStatuses().get(Step.deployReal)); + assertEquals(quotaExceeded, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().status()); + } + private void assertTestLogEntries(RunId id, Step step, LogEntry... entries) { assertEquals(List.of(entries), tester.jobs().details(id).get().get(step)); } |