Merge pull request #9328 from vespa-engine/jvenstad/deploy-to-dev-jobs

Jvenstad/deploy to dev jobs

10 files changed, 180 insertions, 38 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 6de356c36a4..472a0c5fb7e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -12,6 +12,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.io.IOUtils;
 import com.yahoo.log.LogLevel;
 import com.yahoo.slime.Cursor;
@@ -33,7 +34,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails;
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
@@ -146,24 +146,31 @@ public class InternalStepRunner implements StepRunner {
                    versions.sourcePlatform().orElse(versions.targetPlatform()) +
                    " and application version " +
                    versions.sourceApplication().orElse(versions.targetApplication()).id() + " ...");
-        return deployReal(id, true, logger);
+        return deployReal(id, true, versions, logger);
     }
 
     private Optional<RunStatus> deployReal(RunId id, DualLogger logger) {
         Versions versions = controller.jobController().run(id).get().versions();
         logger.log("Deploying platform version " + versions.targetPlatform() +
                          " and application version " + versions.targetApplication().id() + " ...");
-        return deployReal(id, false, logger);
+        return deployReal(id, false, versions, logger);
     }
 
-    private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, DualLogger logger) {
+    private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, Versions versions, DualLogger logger) {
+        Optional<ApplicationPackage> applicationPackage = id.type().environment().isManuallyDeployed()
+                ? Optional.of(new ApplicationPackage(controller.applications().applicationStore()
+                                                               .getDev(id.application(), id.type().zone(controller.system()))))
+                : Optional.empty();
+        Optional<Version> vespaVersion = id.type().environment().isManuallyDeployed()
+                ? Optional.of(versions.targetPlatform())
+                : Optional.empty();
         return deploy(id.application(),
                       id.type(),
                       () -> controller.applications().deploy(id.application(),
                                                              id.type().zone(controller.system()),
-                                                             Optional.empty(),
+                                                             applicationPackage,
                                                              new DeployOptions(false,
-                                                                               Optional.empty(),
+                                                                               vespaVersion,
                                                                                false,
                                                                                setTheStage)),
                       logger);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 34e04d966a3..c154feb4abe 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -2,10 +2,12 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.google.common.collect.ImmutableMap;
+import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.LockedApplication;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore;
@@ -22,7 +24,6 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
-import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 
 import java.net.URI;
 import java.util.ArrayList;
@@ -234,19 +235,8 @@ public class JobController {
                                      ApplicationPackage applicationPackage, byte[] testPackageBytes) {
         AtomicReference<ApplicationVersion> version = new AtomicReference<>();
         controller.applications().lockOrThrow(id, application -> {
-            if ( ! application.get().deploymentJobs().deployedInternally()) {
-                // TODO jvenstad: Remove when there are no more SDv3 pipelines.
-                // Copy all current packages to the new application store
-                application.get().productionDeployments().values().stream()
-                           .map(Deployment::applicationVersion)
-                           .distinct()
-                           .forEach(appVersion -> {
-                               byte[] content = controller.applications().artifacts().getApplicationPackage(id, appVersion.id());
-                               controller.applications().applicationStore().put(id, appVersion, content);
-                           });
-                // Make sure any ongoing upgrade is cancelled, since future jobs will require the tester artifact.
-                application = application.withChange(application.get().change().withoutPlatform().withoutApplication());
-            }
+            if ( ! application.get().deploymentJobs().deployedInternally())
+                application = registered(application);
 
             long run = nextBuild(id);
             if (applicationPackage.compileVersion().isPresent() && applicationPackage.buildTime().isPresent())
@@ -264,15 +254,34 @@ public class JobController {
                                                              testPackageBytes);
 
             prunePackages(id);
-            controller.applications().storeWithUpdatedConfig(application.withBuiltInternally(true), applicationPackage);
+            controller.applications().storeWithUpdatedConfig(application, applicationPackage);
 
             controller.applications().deploymentTrigger().notifyOfCompletion(DeploymentJobs.JobReport.ofSubmission(id, projectId, version.get()));
         });
         return version.get();
     }
 
+    /** Registers the given application, copying necessary application packages, and returns the modified version. */
+    private LockedApplication registered(LockedApplication application) {
+                // TODO jvenstad: Remove when there are no more SDv3 pipelines.
+                // Copy all current packages to the new application store
+                application.get().productionDeployments().values().stream()
+                           .map(Deployment::applicationVersion)
+                           .distinct()
+                           .forEach(appVersion -> {
+                               byte[] content = controller.applications().artifacts().getApplicationPackage(application.get().id(), appVersion.id());
+                               controller.applications().applicationStore().put(application.get().id(), appVersion, content);
+                           });
+                // Make sure any ongoing upgrade is cancelled, since future jobs will require the tester artifact.
+        return application.withChange(application.get().change().withoutPlatform().withoutApplication())
+                          .withBuiltInternally(true);
+    }
+
     /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */
     public void start(ApplicationId id, JobType type, Versions versions) {
+        if ( ! type.environment().isManuallyDeployed() && versions.targetApplication().isUnknown())
+            throw new IllegalArgumentException("Target application must be a valid reference.");
+
         controller.applications().lockIfPresent(id, application -> {
             if ( ! application.get().deploymentJobs().deployedInternally())
                 throw new IllegalArgumentException(id + " is not built here!");
@@ -288,6 +297,39 @@ public class JobController {
         });
     }
 
+    /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment. */
+    public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage) {
+        controller.applications().lockOrThrow(id, application -> {
+            if ( ! application.get().deploymentJobs().deployedInternally())
+                controller.applications().store(registered(application));
+        });
+        if ( ! type.environment().isManuallyDeployed())
+            throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments.");
+
+        last(id, type).filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id()));
+        locked(id, type, __ -> {
+            controller.applications().applicationStore().putDev(id, type.zone(controller.system()), applicationPackage.zippedContent());
+            start(id, type, new Versions(platform.orElse(controller.systemVersion()),
+                                         ApplicationVersion.unknown,
+                                         Optional.empty(),
+                                         Optional.empty()));
+        });
+    }
+
+    /** Aborts a run and waits for it complete. */
+    private void abortAndWait(RunId id) {
+        abort(id);
+        while ( ! last(id.application(), id.type()).get().hasEnded()) {
+            try {
+                Thread.sleep(100);
+            }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
     /** Unregisters the given application and makes all associated data eligible for garbage collection. */
     public void unregister(ApplicationId id) {
         controller.applications().lockIfPresent(id, application -> {
@@ -379,7 +421,7 @@ public class JobController {
         });
     }
 
-    /** Locks and modifies the list of historic runs for the given application and job type. */
+    /** Locks all runs and modifies the list of historic runs for the given application and job type. */
     private void locked(ApplicationId id, JobType type, Consumer<SortedMap<RunId, Run>> modifications) {
         try (Lock __ = curator.lock(id, type)) {
             SortedMap<RunId, Run> runs = curator.readHistoricRuns(id, type);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java
index 181ac2cdf96..ed2cd8eaa76 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java
@@ -47,7 +47,11 @@ public enum JobProfile {
                           startTests,
                           endTests),
                EnumSet.of(deactivateTester,
-                          report));
+                          report)),
+
+    development(EnumSet.of(deployReal,
+                           installReal),
+                EnumSet.noneOf(Step.class));
 
 
     private final Set<Step> steps;
@@ -64,6 +68,8 @@ public enum JobProfile {
             case test: return systemTest;
             case staging: return stagingTest;
             case prod: return production;
+            case perf:
+            case dev: return development;
             default: throw new AssertionError("Unexpected environment '" + type.environment() + "'!");
         }
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
index bd0c11d20f6..21fd0ec87d4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
@@ -61,7 +61,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
     @Override
     public void run() {
         try {
-            if (!permittedSystems.contains(controller.system())) {
+            if ( ! permittedSystems.contains(controller.system())) {
                 return;
             }
             if (jobControl.isActive(name())) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index ce757e015b8..f29af1055d0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -122,6 +122,9 @@ class RunSerializer {
     }
 
     private ApplicationVersion applicationVersionFrom(Inspector versionObject) {
+        if ( ! versionObject.field(buildField).valid())
+            return ApplicationVersion.unknown;
+
         SourceRevision revision = new SourceRevision(versionObject.field(repositoryField).asString(),
                                                      versionObject.field(branchField).asString(),
                                                      versionObject.field(commitField).asString());
@@ -175,13 +178,12 @@ class RunSerializer {
 
     private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) {
         versionsObject.setString(platformVersionField, platformVersion.toString());
-        SourceRevision revision = applicationVersion.source()
-                                                    .orElseThrow(() -> new IllegalArgumentException("Source revision must be present in application version."));
-        versionsObject.setString(repositoryField, revision.repository());
-        versionsObject.setString(branchField, revision.branch());
-        versionsObject.setString(commitField, revision.commit());
-        versionsObject.setLong(buildField, applicationVersion.buildNumber()
-                                                             .orElseThrow(() -> new IllegalArgumentException("Build number must be present in application version.")));
+        if ( ! applicationVersion.isUnknown()) {
+            versionsObject.setString(repositoryField, applicationVersion.source().get().repository());
+            versionsObject.setString(branchField, applicationVersion.source().get().branch());
+            versionsObject.setString(commitField, applicationVersion.source().get().commit());
+            versionsObject.setLong(buildField, applicationVersion.buildNumber().getAsLong());
+        }
         applicationVersion.authorEmail().ifPresent(email -> versionsObject.setString(authorEmailField, email));
         applicationVersion.compileVersion().ifPresent(version -> versionsObject.setString(compileVersionField, version.toString()));
         applicationVersion.buildTime().ifPresent(time -> versionsObject.setLong(buildTimeField, time.toEpochMilli()));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 47fa73c3493..d32b3f009f4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -209,6 +209,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobtype}")) return jobDeploy(appIdFromPath(path), jobTypeFromPath(path), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return trigger(appIdFromPath(path), jobTypeFromPath(path), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return pause(appIdFromPath(path), jobTypeFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
@@ -898,6 +899,30 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                                                                  "instance", instanceName));
     }
 
+    private HttpResponse jobDeploy(ApplicationId id, JobType type, HttpRequest request) {
+        Map<String, byte[]> dataParts = parseDataParts(request);
+        if ( ! dataParts.containsKey("applicationZip"))
+            throw new IllegalArgumentException("Missing required form part 'applicationZip'");
+
+        ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP));
+        controller.applications().verifyApplicationIdentityConfiguration(id.tenant(),
+                                                                         applicationPackage,
+                                                                         Optional.of(requireUserPrincipal(request)));
+
+        Optional<Version> version = Optional.ofNullable(dataParts.get("deployOptions"))
+                                            .map(json -> SlimeUtils.jsonToSlime(json).get())
+                                            .flatMap(options -> optional("vespaVersion", options))
+                                            .map(Version::fromString);
+
+        controller.jobController().deploy(id, type, version, applicationPackage);
+        RunId runId = controller.jobController().last(id, type).get().id();
+        Slime slime = new Slime();
+        Cursor rootObject = slime.setObject();
+        rootObject.setString("message", "Deployment started in " + runId);
+        rootObject.setString("location", controller.zoneRegistry().dashboardUrl(runId).toString());
+        return new SlimeJsonResponse(slime);
+    }
+
     private HttpResponse deploy(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
         ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName);
         ZoneId zone = ZoneId.from(environment, region);
@@ -960,11 +985,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         boolean deployDirectly = deployOptions.field("deployDirectly").asBool();
         Optional<Version> vespaVersion = optional("vespaVersion", deployOptions).map(Version::new);
 
-        /*
-         * Deploy direct is when we want to redeploy the current application - retrieve version
-         * info from the application package before deploying
-         */
-        if(deployDirectly && applicationPackage.isEmpty() && applicationVersion.isEmpty() && vespaVersion.isEmpty()) {
+        if (deployDirectly && applicationPackage.isEmpty() && applicationVersion.isEmpty() && vespaVersion.isEmpty()) {
 
             // Redeploy the existing deployment with the same versions.
             Optional<Deployment> deployment = controller.applications().get(applicationId)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 41f57a80ffd..b53a7b39d61 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -5,6 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
 import com.yahoo.vespa.config.SlimeUtils;
@@ -18,7 +19,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
-import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -32,12 +33,15 @@ import java.time.Duration;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.debug;
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.error;
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.info;
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.warning;
 import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.appId;
+import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.applicationPackage;
 import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.testerId;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
@@ -45,6 +49,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinishe
 import static java.util.Collections.singletonList;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 /**
  * @author jonmv
@@ -274,6 +279,42 @@ public class InternalStepRunnerTest {
     }
 
     @Test
+    public void deployToDev() {
+        ZoneId zone = JobType.devUsEast1.zone(tester.tester().controller().system());
+        tester.jobs().deploy(appId, JobType.devUsEast1, Optional.empty(), applicationPackage);
+        tester.runner().run();
+        RunId id = tester.jobs().last(appId, JobType.devUsEast1).get().id();
+        assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installReal));
+
+        Version version = new Version("7.8.9");
+        Future<?> concurrentDeployment = Executors.newSingleThreadExecutor().submit(() -> {
+            tester.jobs().deploy(appId, JobType.devUsEast1, Optional.of(version), applicationPackage);
+        });
+        while ( ! concurrentDeployment.isDone())
+            tester.runner().run();
+        assertEquals(id.number() + 1, tester.jobs().last(appId, JobType.devUsEast1).get().id().number());
+
+        ApplicationPackage otherPackage = new ApplicationPackageBuilder().region("us-central-1").build();
+        tester.jobs().deploy(appId, JobType.perfUsEast3, Optional.empty(), otherPackage);
+
+        tester.runner().run(); // Job run order determined by JobType enum order per application.
+        tester.configServer().convergeServices(appId, zone);
+        assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installReal));
+        assertEquals(otherPackage.hash(), tester.configServer().application(appId).get().applicationPackage().hash());
+        
+        tester.configServer().setVersion(appId, zone, version);
+        tester.runner().run();
+        assertEquals(1, tester.jobs().active().size());
+        assertEquals(version, tester.tester().application(appId).deployments().get(zone).version());
+
+        try {
+            tester.jobs().deploy(appId, JobType.productionApNortheast1, Optional.empty(), applicationPackage);
+            fail("Deployments outside dev should not be allowed.");
+        }
+        catch (IllegalArgumentException expected) { }
+    }
+
+    @Test
     public void notificationIsSent() {
         tester.startSystemTestTests();
         tester.cloud().set(TesterCloud.Status.NOT_STARTED);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java
index bb9602903f4..f59f81e21f2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.integration;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
@@ -20,6 +21,7 @@ import static org.junit.Assert.assertFalse;
 public class ApplicationStoreMock implements ApplicationStore {
 
     private final Map<ApplicationId, Map<ApplicationVersion, byte[]>> store = new ConcurrentHashMap<>();
+    private final Map<ApplicationId, Map<ZoneId, byte[]>> devStore = new ConcurrentHashMap<>();
 
     @Override
     public byte[] get(ApplicationId application, ApplicationVersion applicationVersion) {
@@ -68,4 +70,15 @@ public class ApplicationStoreMock implements ApplicationStore {
         store.remove(tester.id());
     }
 
+    @Override
+    public void putDev(ApplicationId application, ZoneId zone, byte[] applicationPackage) {
+        devStore.putIfAbsent(application, new ConcurrentHashMap<>());
+        devStore.get(application).put(zone, applicationPackage);
+    }
+
+    @Override
+    public byte[] getDev(ApplicationId application, ZoneId zone) {
+        return requireNonNull(devStore.get(application).get(zone));
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 1616ac87642..aeb9d02336a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -535,6 +535,12 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .screwdriverIdentity(SCREWDRIVER_ID),
                               "Deactivated tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default");
 
+        // POST an application package to start a deployment to dev
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/deploy/dev-us-east-1", POST)
+                             .userIdentity(USER_ID)
+                             .data(createApplicationDeployData(applicationPackage, false)),
+                              new File("deployment-job-accepted.json"));
+
         // POST an application package and a test jar, submitting a new application for internal pipeline deployment.
         // First attempt does not have an Athenz service definition in deployment spec, and is accepted.
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST)
@@ -611,7 +617,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
         // DELETE a running job to have it aborted.
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/staging-test", DELETE)
                                       .userIdentity(USER_ID),
-                              "{\"message\":\"Aborting run 1 of stagingTest for tenant1.application1\"}");
+                              "{\"message\":\"Aborting run 1 of staging-test for tenant1.application1\"}");
 
         // DELETE submission to unsubscribe from continuous deployment.
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", DELETE)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json
new file mode 100644
index 00000000000..66fce327701
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json
@@ -0,0 +1,4 @@
+{
+  "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1",
+  "location": "https://dashboard.tld/tenant1.application1/dev-us-east-1/1"
+}
\ No newline at end of file