From f20c6b5e0539a4ec1eb1fcc913656f8a2409aa0e Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 3 Mar 2020 10:36:32 +0100 Subject: Allow unknown packages in manual prod deployment jobs in CD --- .../hosted/controller/ApplicationController.java | 63 ++++++++++++++++- .../controller/deployment/InternalStepRunner.java | 22 ++---- .../controller/deployment/JobController.java | 7 +- .../restapi/application/ApplicationApiHandler.java | 3 + .../deployment/DeploymentTriggerTest.java | 65 ++++++++++++++++- .../deployment/InternalStepRunnerTest.java | 10 +-- .../restapi/application/ApplicationApiTest.java | 16 ++++- .../application/responses/deployment-overview.json | 29 +++++++- .../restapi/application/responses/instance.json | 82 +++++++++++++--------- .../application/responses/instance1-recursive.json | 82 +++++++++++++--------- .../restapi/application/responses/jobs.json | 22 +++++- 11 files changed, 291 insertions(+), 110 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 3fad76b2e07..bc31a0a02fa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -37,6 +37,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareRes import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -286,9 +287,65 @@ public class ApplicationController { return deploy(applicationId, zone, applicationPackageFromDeployer, Optional.empty(), options); } - /** Deploys an application. If the application does not exist it is created. */ - // TODO: Get rid of the options arg - // TODO jonmv: Split this, and choose between deployDirectly and deploy in handler, excluding internally built from the latter. + /** Deploys an application package for an existing application instance. */ + public ActivateResult deploy2(JobId job, boolean deploySourceVersions) { // TODO jonmv: make it number one! + if (job.application().instance().isTester()) + throw new IllegalArgumentException("'" + job.application() + "' is a tester application!"); + + TenantAndApplicationId applicationId = TenantAndApplicationId.from(job.application()); + ZoneId zone = job.type().zone(controller.system()); + + try (Lock deploymentLock = lockForDeployment(job.application(), zone)) { + Set endpoints; + Optional endpointCertificateMetadata; + + Run run = controller.jobController().last(job) + .orElseThrow(() -> new IllegalStateException("No known run of '" + job + "'")); + + if (run.hasEnded()) + throw new IllegalStateException("No deployment expected for " + job + " now, as no job is running"); + + Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform()); + ApplicationVersion revision = run.versions().sourceApplication().filter(__ -> deploySourceVersions).orElse(run.versions().targetApplication()); + ApplicationPackage applicationPackage = getApplicationPackage(job.application(), zone, revision); + + try (Lock lock = lock(applicationId)) { + LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); + Instance instance = application.get().require(job.application().instance()); + + Deployment deployment = instance.deployments().get(zone); + if ( zone.environment().isProduction() && deployment != null + && ( platform.compareTo(deployment.version()) < 0 && ! instance.change().isPinned() + || revision.compareTo(deployment.applicationVersion()) < 0 && ! (revision.isUnknown() && controller.system().isCd()))) + throw new IllegalArgumentException(String.format("Rejecting deployment of application %s to %s, as the requested versions (platform: %s, application: %s)" + + " are older than the currently deployed (platform: %s, application: %s).", + job.application(), zone, platform, revision, deployment.version(), deployment.applicationVersion())); + + if ( ! applicationPackage.trustedCertificates().isEmpty() + && run.testerCertificate().isPresent()) + applicationPackage = applicationPackage.withTrustedCertificate(run.testerCertificate().get()); + + endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(instance, zone); + + endpoints = controller.routing().registerEndpointsInDns(applicationPackage.deploymentSpec(), instance, zone); + } // Release application lock while doing the deployment, which is a lengthy task. + + // Carry out deployment without holding the application lock. + ActivateResult result = deploy(job.application(), applicationPackage, zone, platform, endpoints, endpointCertificateMetadata); + + lockApplicationOrThrow(applicationId, application -> + store(application.with(job.application().instance(), + instance -> instance.withNewDeployment(zone, revision, platform, + clock.instant(), warningsFrom(result))))); + return result; + } + } + + private ApplicationPackage getApplicationPackage(ApplicationId application, ZoneId zone, ApplicationVersion revision) { + return new ApplicationPackage(revision.isUnknown() ? applicationStore.getDev(application, zone) + : applicationStore.get(application.tenant(), application.application(), revision)); + } + public ActivateResult deploy(ApplicationId instanceId, ZoneId zone, Optional applicationPackageFromDeployer, Optional applicationVersionFromDeployer, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 861c8089ed5..628a7e41eb2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -173,34 +173,20 @@ public class InternalStepRunner implements StepRunner { versions.sourcePlatform().orElse(versions.targetPlatform()) + " and application version " + versions.sourceApplication().orElse(versions.targetApplication()).id() + " ..."); - return deployReal(id, true, versions, logger); + return deployReal(id, true, logger); } private Optional deployReal(RunId id, DualLogger logger) { Versions versions = controller.jobController().run(id).get().versions(); logger.log("Deploying platform version " + versions.targetPlatform() + " and application version " + versions.targetApplication().id() + " ..."); - return deployReal(id, false, versions, logger); + return deployReal(id, false, logger); } - private Optional deployReal(RunId id, boolean setTheStage, Versions versions, DualLogger logger) { - Optional applicationPackage = id.type().environment().isManuallyDeployed() - ? Optional.of(new ApplicationPackage(controller.applications().applicationStore() - .getDev(id.application(), id.type().zone(controller.system())))) - : Optional.empty(); - - Optional vespaVersion = id.type().environment().isManuallyDeployed() - ? Optional.of(versions.targetPlatform()) // TODO jonmv: This makes it impossible to deploy on older majors — fix. - : Optional.empty(); + private Optional deployReal(RunId id, boolean setTheStage, DualLogger logger) { return deploy(id.application(), id.type(), - () -> controller.applications().deploy(id.application(), - id.type().zone(controller.system()), - applicationPackage, - new DeployOptions(false, - vespaVersion, - false, - setTheStage)), + () -> controller.applications().deploy2(id.job(), setTheStage), controller.jobController().run(id).get() .stepInfo(setTheStage ? deployInitialReal : deployReal).get() .startTime().get(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 007654b2a7c..5d8ad6594df 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -418,9 +418,6 @@ public class JobController { /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void start(ApplicationId id, JobType type, Versions versions, JobProfile profile) { - if (profile != JobProfile.development && versions.targetApplication().isUnknown()) - throw new IllegalArgumentException(" Target application must be a valid reference"); - controller.applications().lockApplicationIfPresent(TenantAndApplicationId.from(id), application -> { locked(id, type, __ -> { Optional last = last(id, type); @@ -436,9 +433,6 @@ public class JobController { /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment. */ public void deploy(ApplicationId id, JobType type, Optional platform, ApplicationPackage applicationPackage) { - if ( ! type.environment().isManuallyDeployed()) - throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments."); - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { if ( ! application.get().instances().containsKey(id.instance())) application = controller.applications().withNewInstance(application, id); @@ -546,6 +540,7 @@ public class JobController { application.get().productionDeployments().values().stream() .flatMap(List::stream) .map(Deployment::applicationVersion) + .filter(version -> ! version.isUnknown()) .min(Comparator.comparingLong(applicationVersion -> applicationVersion.buildNumber().getAsLong())) .ifPresent(oldestDeployed -> { controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 1c8279f6d8d..7b2e1ad7a8e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1482,6 +1482,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse jobDeploy(ApplicationId id, JobType type, HttpRequest request) { + if ( ! type.environment().isManuallyDeployed() && ! isOperator(request)) + throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments."); + Map dataParts = parseDataParts(request); if ( ! dataParts.containsKey("applicationZip")) throw new IllegalArgumentException("Missing required form part 'applicationZip'"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index a1c8ccffaf0..e684d479158 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -3,13 +3,19 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Test; @@ -22,11 +28,14 @@ import java.util.Optional; import java.util.OptionalLong; import java.util.stream.Collectors; +import static com.yahoo.config.provision.SystemName.cd; import static com.yahoo.config.provision.SystemName.main; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApNortheast1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApNortheast2; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApSoutheast1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionAwsUsEast1a; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdAwsUsEast1a; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdUsCentral1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionEuWest1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; @@ -57,7 +66,7 @@ import static org.junit.Assert.assertTrue; */ public class DeploymentTriggerTest { - private final DeploymentTester tester = new DeploymentTester(); + private DeploymentTester tester = new DeploymentTester(); @Test public void testTriggerFailing() { @@ -1066,4 +1075,58 @@ public class DeploymentTriggerTest { assertEquals(Change.of(app.lastSubmission().get()), app.instance().change()); } + @Test + public void mixedDirectAndPipelineJobsInProduction() { + ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("cd-us-central-1") + .region("cd-aws-us-east-1a") + .build(); + ServiceRegistryMock services = new ServiceRegistryMock(); + var zones = List.of(ZoneApiMock.fromId("test.cd-us-central-1"), + ZoneApiMock.fromId("staging.cd-us-central-1"), + ZoneApiMock.fromId("prod.cd-us-central-1"), + ZoneApiMock.fromId("prod.cd-aws-us-east-1a")); + services.zoneRegistry() + .setSystemName(SystemName.cd) + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.exclusive); + tester = new DeploymentTester(new ControllerTester(services)); + tester.configServer().bootstrap(services.zoneRegistry().zones().all().ids(), SystemApplication.values()); + tester.controllerTester().upgradeSystem(Version.fromString("6.1")); + tester.controllerTester().computeVersionStatus(); + var app = tester.newDeploymentContext(); + + app.runJob(productionCdUsCentral1, cdPackage); + app.submit(cdPackage); + app.runJob(systemTest); + // Staging test requires unknown initial version, and is broken. + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); + app.runJob(productionCdUsCentral1) + .abortJob(stagingTest) // Complete failing run. + .runJob(stagingTest) + .runJob(productionCdAwsUsEast1a); + + app.runJob(productionCdUsCentral1, cdPackage); + var version = new Version("7.1"); + tester.controllerTester().upgradeSystem(version); + tester.upgrader().maintain(); + // System and staging tests both require unknown versions, and are broken. + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); + app.runJob(productionCdUsCentral1) + .abortJob(systemTest) + .abortJob(stagingTest) + .runJob(systemTest) + .runJob(stagingTest) + .runJob(productionCdAwsUsEast1a); + + app.runJob(productionCdUsCentral1, cdPackage); + app.submit(cdPackage); + app.runJob(systemTest); + // Staging test requires unknown initial version, and is broken. + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); + app.runJob(productionCdUsCentral1) + .jobAborted(stagingTest) + .runJob(stagingTest) + .runJob(productionCdAwsUsEast1a); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index e611881e791..a57e302d939 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -371,12 +371,6 @@ public class InternalStepRunnerTest { tester.runner().run(); assertEquals(1, tester.jobs().active().size()); assertEquals(version, tester.instance(app.instanceId()).deployments().get(zone).version()); - - try { - tester.jobs().deploy(app.instanceId(), JobType.productionApNortheast1, Optional.empty(), applicationPackage); - fail("Deployments outside dev should not be allowed."); - } - catch (IllegalArgumentException expected) { } } @Test @@ -420,8 +414,8 @@ public class InternalStepRunnerTest { public void certificateTimeoutAbortsJob() { tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public); var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), - ZoneApiMock.fromId("staging.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-us-east-1c")); + ZoneApiMock.fromId("staging.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-us-east-1c")); tester.controllerTester().zoneRegistry() .setZones(zones) .setRoutingMethod(zones, RoutingMethod.exclusive); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 45d87c6756a..810b9c2550c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -257,8 +257,22 @@ public class ApplicationApiTest extends ControllerContainerTest { ApplicationId id = ApplicationId.from("tenant1", "application1", "instance1"); var app1 = deploymentTester.newDeploymentContext(id); - // POST (deploy) an application to start a manual deployment to dev + // POST (deploy) an application to start a manual deployment in prod is not allowed MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST) + .data(entity) + .userIdentity(USER_ID), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Direct deployments are only allowed to manually deployed environments.\"}", 400); + + // POST (deploy) an application to start a manual deployment in prod is allowed for operators + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST) + .data(entity) + .userIdentity(HOSTED_VESPA_OPERATOR), + "{\"message\":\"Deployment started in run 1 of production-us-east-3 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); + app1.runJob(JobType.productionUsEast3); + tester.controller().applications().deactivate(app1.instanceId(), ZoneId.from("prod", "us-east-3")); + + // POST (deploy) an application to start a manual deployment to dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1/", POST) .data(entity) .userIdentity(USER_ID), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 282c18046d3..fe9d0573ed2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -464,8 +464,8 @@ ], "runs": [ { - "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1", + "id": 2, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/2", "start": "(ignore)", "status": "aborted", "versions": { @@ -491,6 +491,31 @@ "status": "unfinished" } ] + }, + { + "id": 1, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1", + "start": "(ignore)", + "end": "(ignore)", + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": {} + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] } ] }, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json index fd8bc256ac5..3d1c5ade300 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json @@ -13,8 +13,8 @@ "projectId": 123, "deploying": { "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -30,10 +30,10 @@ "success": true, "lastTriggered": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -47,10 +47,10 @@ }, "lastCompleted": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -64,10 +64,10 @@ }, "lastSuccess": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -85,10 +85,10 @@ "success": true, "lastTriggered": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -102,10 +102,10 @@ }, "lastCompleted": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -119,10 +119,10 @@ }, "lastSuccess": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -140,10 +140,10 @@ "success": true, "lastTriggered": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -157,10 +157,10 @@ }, "lastCompleted": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -174,10 +174,10 @@ }, "lastSuccess": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -192,13 +192,13 @@ }, { "type": "production-us-east-3", - "success": false, + "success": true, "lastTriggered": { - "id": 1, - "version": "(ignore)", + "id": 2, + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -209,6 +209,18 @@ }, "reason": "unknown reason", "at": "(ignore)" + }, + "lastCompleted": { + "id": 1, + "version": "6.1.0", + "reason": "unknown reason", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 1, + "version": "6.1.0", + "reason": "unknown reason", + "at": "(ignore)" } }, { @@ -263,7 +275,7 @@ "rotationId": "rotation-id-1", "clusterId": "foo", "status": "IN", - "lastUpdated":"(ignore)" + "lastUpdated": "(ignore)" } ], "environment": "prod", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json index ee75d129241..1b7e7893222 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json @@ -13,8 +13,8 @@ "projectId": 123, "deploying": { "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -30,10 +30,10 @@ "success": true, "lastTriggered": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -47,10 +47,10 @@ }, "lastCompleted": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -64,10 +64,10 @@ }, "lastSuccess": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -85,10 +85,10 @@ "success": true, "lastTriggered": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -102,10 +102,10 @@ }, "lastCompleted": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -119,10 +119,10 @@ }, "lastSuccess": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -140,10 +140,10 @@ "success": true, "lastTriggered": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -157,10 +157,10 @@ }, "lastCompleted": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -174,10 +174,10 @@ }, "lastSuccess": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -192,13 +192,13 @@ }, { "type": "production-us-east-3", - "success": false, + "success": true, "lastTriggered": { - "id": 1, - "version": "(ignore)", + "id": 2, + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -209,6 +209,18 @@ }, "reason": "unknown reason", "at": "(ignore)" + }, + "lastCompleted": { + "id": 1, + "version": "6.1.0", + "reason": "unknown reason", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 1, + "version": "6.1.0", + "reason": "unknown reason", + "at": "(ignore)" } }, { @@ -241,7 +253,7 @@ ] } ], - "compileVersion": "(ignore)", + "compileVersion": "6.0.0", "globalRotations": [ "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 8cd102432d0..b16ca4cc67c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -275,7 +275,7 @@ "us-east-3": { "runs": [ { - "id": 1, + "id": 2, "status": "aborted", "start": "(ignore)", "wantedPlatform": "6.1", @@ -296,6 +296,26 @@ "report": "unfinished" }, "tasks": {}, + "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/2" + }, + { + "id": 1, + "status": "success", + "start": "(ignore)", + "end": "(ignore)", + "wantedPlatform": "6.1", + "wantedApplication": { + "hash": "unknown" + }, + "steps": { + "deployReal": "succeeded", + "installReal": "succeeded", + "copyVespaLogs": "succeeded" + }, + "tasks": { + "deploy": "succeeded", + "install": "succeeded" + }, "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1" } ], -- cgit v1.2.3 From dc10115a4f417d96cbd7aaed4b31374f5bb8d23d Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 3 Mar 2020 12:55:48 +0100 Subject: Expose controller client factory with SSL context argument --- .../java/ai/vespa/hosted/api/ControllerHttpClient.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index 6fba083e607..376719aed1d 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -17,6 +17,7 @@ import com.yahoo.slime.JsonFormat; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; +import javax.net.ssl.SSLContext; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -77,6 +78,11 @@ public abstract class ControllerHttpClient { return new SigningControllerHttpClient(endpoint, privateKeyFile, id); } + /** Creates an HTTP client against the given endpoint, which uses the given SSL context for authentication. */ + public static ControllerHttpClient withSSLContext(URI endpoint, SSLContext sslContext) { + return new MutualTlsControllerHttpClient(endpoint, sslContext); + } + /** Creates an HTTP client against the given endpoint, which uses the given private key and certificate identity. */ public static ControllerHttpClient withKeyAndCertificate(URI endpoint, Path privateKeyFile, Path certificateFile) { var privateKey = unchecked(() -> KeyUtils.fromPemEncodedPrivateKey(Files.readString(privateKeyFile, UTF_8))); @@ -410,14 +416,17 @@ public abstract class ControllerHttpClient { /** Client that uses a given key / certificate identity to authenticate to the remote controller. */ private static class MutualTlsControllerHttpClient extends ControllerHttpClient { + private MutualTlsControllerHttpClient(URI endpoint, SSLContext sslContext) { + super(endpoint, HttpClient.newBuilder().sslContext(sslContext)); + } + private MutualTlsControllerHttpClient(URI endpoint, PrivateKey privateKey, List certs) { - super(endpoint, - HttpClient.newBuilder() - .sslContext(new SslContextBuilder().withKeyStore(privateKey, certs).build())); + this(endpoint, new SslContextBuilder().withKeyStore(privateKey, certs).build()); } } + private static DeploymentLog.Status valueOf(String status) { switch (status) { case "running": return DeploymentLog.Status.running; -- cgit v1.2.3 From 1676c7f550a2b36265951f0f22086b8f0b563863 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 3 Mar 2020 15:33:15 +0100 Subject: Retry on out-of-capacity for 5 minutes in CD --- .../vespa/hosted/controller/deployment/InternalStepRunner.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 628a7e41eb2..dc7fdeb8a98 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -115,6 +115,7 @@ public class InternalStepRunner implements StepRunner { static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); + static final Duration capacityTimeout = Duration.ofMinutes(5); static final Duration endpointTimeout = Duration.ofMinutes(15); static final Duration endpointCertificateTimeout = Duration.ofMinutes(15); static final Duration testerTimeout = Duration.ofMinutes(30); @@ -275,7 +276,9 @@ public class InternalStepRunner implements StepRunner { return result; case OUT_OF_CAPACITY: logger.log(e.getServerMessage()); - return Optional.of(outOfCapacity); + return controller.system().isCd() && startTime.plus(capacityTimeout).isAfter(controller.clock().instant()) + ? Optional.empty() + : Optional.of(outOfCapacity); case INVALID_APPLICATION_PACKAGE: case BAD_REQUEST: logger.log(e.getMessage()); @@ -283,7 +286,8 @@ public class InternalStepRunner implements StepRunner { } throw e; - } catch (EndpointCertificateException e) { + } + catch (EndpointCertificateException e) { switch (e.type()) { case CERT_NOT_AVAILABLE: // Same as CERTIFICATE_NOT_READY above, only from the controller -- cgit v1.2.3 From 1bf286b9b95ea1f3705436ca551a29fa4ecb6fc3 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 3 Mar 2020 15:33:27 +0100 Subject: Allow aggregating deployment log in client --- .../src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java | 7 ++++--- hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java | 9 +++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index 376719aed1d..79c11cf86de 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -156,11 +156,12 @@ public abstract class ControllerHttpClient { public DeploymentLog followDeploymentUntilDone(ApplicationId id, ZoneId zone, long run, Consumer out) { long last = -1; - DeploymentLog log; + DeploymentLog log = null; while (true) { - log = deploymentLog(id, zone, run, last); - for (DeploymentLog.Entry entry : log.entries()) + DeploymentLog update = deploymentLog(id, zone, run, last); + for (DeploymentLog.Entry entry : update.entries()) out.accept(entry); + log = (log == null ? update : log.updatedWith(update)); last = log.last().orElse(last); if ( ! log.isActive()) diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java index 177c72107e0..9eae9a33cff 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java @@ -4,6 +4,7 @@ package ai.vespa.hosted.api; import java.time.Instant; import java.util.List; import java.util.OptionalLong; +import java.util.stream.Stream; import static java.util.Comparator.comparing; import static java.util.stream.Collectors.toUnmodifiableList; @@ -27,6 +28,14 @@ public class DeploymentLog { this.last = last; } + /** Returns this log updated with the content of the other. */ + public DeploymentLog updatedWith(DeploymentLog other) { + return new DeploymentLog(Stream.concat(entries.stream(), other.entries.stream()).collect(toUnmodifiableList()), + other.active, + other.status, + other.last); + } + public List entries() { return entries; } -- cgit v1.2.3