Merge pull request #7610 from vespa-engine/jvenstad/application-package-gc-2

Jvenstad/application package gc 2

7 files changed, 101 insertions, 100 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 33dfcde9f13..bed1df3a8a7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareRes
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
@@ -328,7 +329,7 @@ public class ApplicationController {
 
                 try {
                     applicationPackage = application.get().deploymentJobs().deployedInternally()
-                            ? new ApplicationPackage(applicationStore.getApplicationPackage(application.get().id(), applicationVersion))
+                            ? new ApplicationPackage(applicationStore.get(application.get().id(), applicationVersion))
                             : new ApplicationPackage(artifactRepository.getApplicationPackage(application.get().id(), applicationVersion.id()));
                 }
                 catch (RuntimeException e) { // If application has switched deployment pipeline, artifacts stored prior to the switch are in the other artifact store.
@@ -336,7 +337,7 @@ public class ApplicationController {
                              + (application.get().deploymentJobs().deployedInternally() ? "internally" : "externally"));
                     applicationPackage = application.get().deploymentJobs().deployedInternally()
                             ? new ApplicationPackage(artifactRepository.getApplicationPackage(application.get().id(), applicationVersion.id()))
-                            : new ApplicationPackage(applicationStore.getApplicationPackage(application.get().id(), applicationVersion));
+                            : new ApplicationPackage(applicationStore.get(application.get().id(), applicationVersion));
                 }
                 validateRun(application.get(), zone, platformVersion, applicationVersion);
             }
@@ -402,12 +403,9 @@ public class ApplicationController {
         }
     }
 
-    /** Assembles and deploys a tester application to the given zone. */
-    public ActivateResult deployTester(ApplicationId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) {
-        if ( ! tester.instance().isTester())
-            throw new IllegalArgumentException("'" + tester + "' is not a tester application!");
-
-        return deploy(tester, applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet());
+    /** Deploys the given tester application to the given zone. */
+    public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) {
+        return deploy(tester.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet());
     }
 
     private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 1f3dd30b8c6..633490b9299 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -156,9 +156,9 @@ public class InternalStepRunner implements StepRunner {
     private Optional<RunStatus> deployTester(RunId id, DualLogger logger) {
         // TODO jvenstad: Consider deploying old version of tester for initial staging feeding?
         logger.log("Deploying the tester container ...");
-        return deploy(JobController.testerOf(id.application()),
+        return deploy(id.tester().id(),
                       id.type(),
-                      () -> controller.applications().deployTester(JobController.testerOf(id.application()),
+                      () -> controller.applications().deployTester(id.tester(),
                                                                    testerPackage(id),
                                                                    id.type().zone(controller.system()),
                                                                    new DeployOptions(true,
@@ -263,7 +263,7 @@ public class InternalStepRunner implements StepRunner {
         }
 
         logger.log("Checking installation of tester container ...");
-        if (servicesConverged(JobController.testerOf(id.application()), id.type(), logger)) {
+        if (servicesConverged(id.tester().id(), id.type(), logger)) {
             logger.log("Tester container successfully installed!");
             return Optional.of(running);
         }
@@ -409,7 +409,7 @@ public class InternalStepRunner implements StepRunner {
 
     private Optional<RunStatus> deactivateTester(RunId id, DualLogger logger) {
         logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ...");
-        controller.jobController().deactivateTester(id.application(), id.type());
+        controller.jobController().deactivateTester(id.tester(), id.type());
         return Optional.of(running);
     }
 
@@ -452,7 +452,7 @@ public class InternalStepRunner implements StepRunner {
     private ApplicationPackage testerPackage(RunId id) {
         ApplicationVersion version = controller.jobController().run(id).get().versions().targetApplication();
 
-        byte[] testPackage = controller.applications().applicationStore().getTesterPackage(JobController.testerOf(id.application()), version);
+        byte[] testPackage = controller.applications().applicationStore().get(id.tester(), version);
         byte[] servicesXml = servicesXml(controller.system());
 
         DeploymentSpec spec = controller.applications().require(id.application()).deploymentSpec();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index cd6921e6c42..da98dd0e165 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstance
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
@@ -233,26 +234,26 @@ public class JobController {
                            .distinct()
                            .forEach(appVersion -> {
                                byte[] content = controller.applications().artifacts().getApplicationPackage(application.get().id(), appVersion.id());
-                               controller.applications().applicationStore().putApplicationPackage(application.get().id(), appVersion, content);
+                               controller.applications().applicationStore().put(application.get().id(), appVersion, content);
                            });
             }
 
             long run = nextBuild(id);
             version.set(ApplicationVersion.from(revision, run));
 
-            controller.applications().applicationStore().putApplicationPackage(id,
-                                                                               version.get(),
-                                                                               packageBytes);
-            controller.applications().applicationStore().putTesterPackage(testerOf(id),
-                                                                          version.get(),
-                                                                          testPackageBytes);
+            controller.applications().applicationStore().put(id,
+                                                             version.get(),
+                                                             packageBytes);
+            controller.applications().applicationStore().put(TesterId.of(id),
+                                                             version.get(),
+                                                             testPackageBytes);
 
             application.get().deployments().values().stream()
                        .map(Deployment::applicationVersion)
                        .min(Comparator.comparingLong(applicationVersion -> applicationVersion.buildNumber().getAsLong()))
                        .ifPresent(oldestDeployed -> {
-                           controller.applications().applicationStore().pruneApplicationPackages(id, oldestDeployed);
-                           controller.applications().applicationStore().pruneTesterPackages(testerOf(id), oldestDeployed);
+                           controller.applications().applicationStore().prune(id, oldestDeployed);
+                           controller.applications().applicationStore().prune(TesterId.of(id), oldestDeployed);
                        });
 
             controller.applications().storeWithUpdatedConfig(application.withBuiltInternally(true), new ApplicationPackage(packageBytes));
@@ -287,48 +288,44 @@ public class JobController {
         });
     }
 
-    /** Deletes stale data and tester deployments for applications which are unknown, or no longer built internally. */
+    /** Deletes run data, packages and tester deployments for applications which are unknown, or no longer built internally. */
     public void collectGarbage() {
         Set<ApplicationId> applicationsToBuild = new HashSet<>(applications());
         curator.applicationsWithJobs().stream()
                .filter(id -> ! applicationsToBuild.contains(id))
                .forEach(id -> {
                    try {
+                       TesterId tester = TesterId.of(id);
                        for (JobType type : jobs(id))
                            locked(id, type, deactivateTester, __ -> {
                                try (Lock ___ = curator.lock(id, type)) {
-                                   deactivateTester(id, type);
+                                   deactivateTester(tester, type);
                                    curator.deleteRunData(id, type);
                                    logs.delete(id);
                                }
                            });
+                       controller.applications().applicationStore().removeAll(id);
+                       controller.applications().applicationStore().removeAll(tester);
                    }
                    catch (TimeoutException e) {
-                       return; // Don't remove the data if we couldn't deactivate all testers.
+                       return; // Don't remove the data if we couldn't clean up all resources.
                    }
                    curator.deleteRunData(id);
                });
     }
 
-    public void deactivateTester(ApplicationId id, JobType type) {
+    public void deactivateTester(TesterId id, JobType type) {
         try {
-            controller.configServer().deactivate(new DeploymentId(testerOf(id), type.zone(controller.system())));
+            controller.configServer().deactivate(new DeploymentId(id.id(), type.zone(controller.system())));
         }
         catch (NoInstanceException ignored) {
             // Already gone -- great!
         }
     }
 
-    /** Returns the application id of the tester application for the real application with the given id. */
-    public static ApplicationId testerOf(ApplicationId id) {
-        return ApplicationId.from(id.tenant().value(),
-                                  id.application().value(),
-                                  id.instance().value() + "-t");
-    }
-
     /** Returns a URI of the tester endpoint retrieved from the routing generator, provided it matches an expected form. */
     Optional<URI> testerEndpoint(RunId id) {
-        ApplicationId tester = testerOf(id.application());
+        ApplicationId tester = id.tester().id();
         return controller.applications().getDeploymentEndpoints(new DeploymentId(tester, id.type().zone(controller.system())))
                          .flatMap(uris -> uris.stream()
                                               .filter(uri -> uri.getHost().contains(String.format("%s--%s--%s.",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
index bb0261fa0a7..9e8ee16e0c1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
@@ -29,7 +30,6 @@ import java.util.Collections;
 import java.util.Optional;
 import java.util.logging.Logger;
 
-import static com.yahoo.vespa.hosted.controller.deployment.JobController.testerOf;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
 import static org.junit.Assert.assertEquals;
@@ -45,6 +45,7 @@ public class InternalDeploymentTester {
             .parallel("us-west-1", "us-east-3")
             .build();
     public static final ApplicationId appId = ApplicationId.from("tenant", "application", "default");
+    public static final TesterId testerId = TesterId.of(appId);
     public static final String athenzDomain = "domain";
 
     private final DeploymentTester tester;
@@ -87,8 +88,8 @@ public class InternalDeploymentTester {
      */
     public ApplicationVersion newSubmission() {
         ApplicationVersion version = jobs.submit(appId, BuildJob.defaultSourceRevision, 2, applicationPackage.zippedContent(), new byte[0]);
-        tester.applicationStore().putApplicationPackage(appId, version, applicationPackage.zippedContent());
-        tester.applicationStore().putTesterPackage(testerOf(appId), version, new byte[0]);
+        tester.applicationStore().put(appId, version, applicationPackage.zippedContent());
+        tester.applicationStore().put(testerId, version, new byte[0]);
         return version;
     }
 
@@ -190,13 +191,13 @@ public class InternalDeploymentTester {
         assertEquals(Step.Status.succeeded, jobs.active(run.id()).get().steps().get(Step.installReal));
 
         assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.installTester));
-        tester.configServer().convergeServices(testerOf(appId), zone);
+        tester.configServer().convergeServices(testerId.id(), zone);
         runner.run();
         assertEquals(Step.Status.succeeded, jobs.active(run.id()).get().steps().get(Step.installTester));
 
         // All installation is complete. We now need endpoints, and the tests will then run, and cleanup finish.
         assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.startTests));
-        setEndpoints(testerOf(appId), zone);
+        setEndpoints(testerId.id(), zone);
         runner.run();
         if (!run.versions().sourceApplication().isPresent() || !type.isProduction()) {
             assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.startTests));
@@ -211,20 +212,20 @@ public class InternalDeploymentTester {
         assertTrue(jobs.run(run.id()).get().hasEnded());
         assertFalse(jobs.run(run.id()).get().hasFailed());
         assertEquals(type.isProduction(), app().deployments().containsKey(zone));
-        assertTrue(tester.configServer().nodeRepository().list(zone, testerOf(appId)).isEmpty());
+        assertTrue(tester.configServer().nodeRepository().list(zone, testerId.id()).isEmpty());
 
         if (!app().deployments().containsKey(zone))
             routing.removeEndpoints(deployment);
-        routing.removeEndpoints(new DeploymentId(testerOf(appId), zone));
+        routing.removeEndpoints(new DeploymentId(testerId.id(), zone));
     }
 
     public RunId startSystemTestTests() {
         RunId id = newRun(JobType.systemTest);
         runner.run();
         tester.configServer().convergeServices(appId, JobType.systemTest.zone(tester.controller().system()));
-        tester.configServer().convergeServices(testerOf(appId), JobType.systemTest.zone(tester.controller().system()));
+        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(tester.controller().system()));
         setEndpoints(appId, JobType.systemTest.zone(tester.controller().system()));
-        setEndpoints(testerOf(appId), JobType.systemTest.zone(tester.controller().system()));
+        setEndpoints(testerId.id(), JobType.systemTest.zone(tester.controller().system()));
         runner.run();
         assertEquals(unfinished, jobs.run(id).get().steps().get(Step.endTests));
         return id;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index e1a557f26f1..8b8e21f4d33 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -6,9 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.slime.ArrayTraverser;
-import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
-import com.yahoo.slime.Slime;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction;
@@ -37,13 +35,13 @@ import java.util.Optional;
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.debug;
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.error;
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.info;
-import static com.yahoo.vespa.hosted.controller.deployment.JobController.testerOf;
+import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.appId;
+import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.testerId;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 /**
  * @author jonmv
@@ -69,15 +67,15 @@ public class InternalStepRunnerTest {
     public void canSwitchFromScrewdriverAndBackAgain() {
         // Deploys a default application package with default build number.
         tester.tester().deployCompletely(tester.app(), InternalDeploymentTester.applicationPackage);
-        tester.setEndpoints(InternalDeploymentTester.appId, JobType.productionUsCentral1.zone(tester.tester().controller().system()));
-        tester.setEndpoints(InternalDeploymentTester.appId, JobType.productionUsWest1.zone(tester.tester().controller().system()));
-        tester.setEndpoints(InternalDeploymentTester.appId, JobType.productionUsEast3.zone(tester.tester().controller().system()));
+        tester.setEndpoints(appId, JobType.productionUsCentral1.zone(tester.tester().controller().system()));
+        tester.setEndpoints(appId, JobType.productionUsWest1.zone(tester.tester().controller().system()));
+        tester.setEndpoints(appId, JobType.productionUsEast3.zone(tester.tester().controller().system()));
 
         tester.deployNewSubmission();
 
         tester.deployNewPlatform(new Version("7.1"));
 
-        tester.jobs().unregister(InternalDeploymentTester.appId);
+        tester.jobs().unregister(appId);
         try {
             tester.tester().deployCompletely(tester.app(), InternalDeploymentTester.applicationPackage, BuildJob.defaultBuildNumber + 1);
             throw new IllegalStateException("Component job should get ahead again with build numbers to produce a change.");
@@ -90,7 +88,7 @@ public class InternalStepRunnerTest {
     public void testerHasAthenzIdentity() {
         tester.newRun(JobType.stagingTest);
         tester.runner().run();
-        DeploymentSpec spec = tester.configServer().application(testerOf(InternalDeploymentTester.appId)).get().applicationPackage().deploymentSpec();
+        DeploymentSpec spec = tester.configServer().application(InternalDeploymentTester.testerId.id()).get().applicationPackage().deploymentSpec();
         assertEquals("domain", spec.athenzDomain().get().value());
         ZoneId zone = JobType.stagingTest.zone(tester.tester().controller().system());
         assertEquals("service", spec.athenzService(zone.environment(), zone.region()).get().value());
@@ -115,7 +113,7 @@ public class InternalStepRunnerTest {
     public void restartsServicesAndWaitsForRestartAndReboot() {
         RunId id = tester.newRun(JobType.productionUsCentral1);
         ZoneId zone = id.type().zone(tester.tester().controller().system());
-        HostName host = tester.configServer().hostFor(InternalDeploymentTester.appId, zone);
+        HostName host = tester.configServer().hostFor(appId, zone);
         tester.configServer().setConfigChangeActions(new ConfigChangeActions(Collections.singletonList(new RestartAction("cluster",
                                                                                                                                   "container",
                                                                                                                                   "search",
@@ -128,11 +126,11 @@ public class InternalStepRunnerTest {
         tester.runner().run();
         assertEquals(succeeded, tester.jobs().run(id).get().steps().get(Step.deployReal));
 
-        tester.configServer().convergeServices(InternalDeploymentTester.appId, zone);
+        tester.configServer().convergeServices(appId, zone);
         assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installReal));
 
-        tester.configServer().nodeRepository().doRestart(new DeploymentId(InternalDeploymentTester.appId, zone), Optional.of(host));
-        tester.configServer().nodeRepository().requestReboot(new DeploymentId(InternalDeploymentTester.appId, zone), Optional.of(host));
+        tester.configServer().nodeRepository().doRestart(new DeploymentId(appId, zone), Optional.of(host));
+        tester.configServer().nodeRepository().requestReboot(new DeploymentId(appId, zone), Optional.of(host));
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installReal));
 
@@ -146,50 +144,50 @@ public class InternalStepRunnerTest {
         tester.newRun(JobType.systemTest);
 
         tester.runner().run();
-        tester.configServer().convergeServices(InternalDeploymentTester.appId, JobType.stagingTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(tester.tester().controller().system()));
         tester.runner().run();
-        tester.configServer().convergeServices(InternalDeploymentTester.appId, JobType.systemTest.zone(tester.tester().controller().system()));
-        tester.configServer().convergeServices(testerOf(InternalDeploymentTester.appId), JobType.systemTest.zone(tester.tester().controller().system()));
-        tester.configServer().convergeServices(InternalDeploymentTester.appId, JobType.stagingTest.zone(tester.tester().controller().system()));
-        tester.configServer().convergeServices(testerOf(InternalDeploymentTester.appId), JobType.stagingTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(appId, JobType.stagingTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(testerId.id(), JobType.stagingTest.zone(tester.tester().controller().system()));
         tester.runner().run();
 
         // Tester fails to show up for system tests, and the real deployment for staging tests.
-        tester.setEndpoints(InternalDeploymentTester.appId, JobType.systemTest.zone(tester.tester().controller().system()));
-        tester.setEndpoints(testerOf(InternalDeploymentTester.appId), JobType.stagingTest.zone(tester.tester().controller().system()));
+        tester.setEndpoints(appId, JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.setEndpoints(testerId.id(), JobType.stagingTest.zone(tester.tester().controller().system()));
 
         tester.clock().advance(InternalStepRunner.endpointTimeout.plus(Duration.ofSeconds(1)));
         tester.runner().run();
-        assertEquals(failed, tester.jobs().last(InternalDeploymentTester.appId, JobType.systemTest).get().steps().get(Step.startTests));
-        assertEquals(failed, tester.jobs().last(InternalDeploymentTester.appId, JobType.stagingTest).get().steps().get(Step.startTests));
+        assertEquals(failed, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.startTests));
+        assertEquals(failed, tester.jobs().last(appId, JobType.stagingTest).get().steps().get(Step.startTests));
     }
 
     @Test
     public void installationFailsIfDeploymentExpires() {
         tester.newRun(JobType.systemTest);
         tester.runner().run();
-        tester.configServer().convergeServices(InternalDeploymentTester.appId, JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(tester.tester().controller().system()));
         tester.runner().run();
-        assertEquals(succeeded, tester.jobs().last(InternalDeploymentTester.appId, JobType.systemTest).get().steps().get(Step.installReal));
+        assertEquals(succeeded, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installReal));
 
-        tester.applications().deactivate(InternalDeploymentTester.appId, JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.applications().deactivate(appId, JobType.systemTest.zone(tester.tester().controller().system()));
         tester.runner().run();
-        assertEquals(failed, tester.jobs().last(InternalDeploymentTester.appId, JobType.systemTest).get().steps().get(Step.installTester));
-        assertTrue(tester.jobs().last(InternalDeploymentTester.appId, JobType.systemTest).get().hasEnded());
-        assertTrue(tester.jobs().last(InternalDeploymentTester.appId, JobType.systemTest).get().hasFailed());
+        assertEquals(failed, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installTester));
+        assertTrue(tester.jobs().last(appId, JobType.systemTest).get().hasEnded());
+        assertTrue(tester.jobs().last(appId, JobType.systemTest).get().hasFailed());
     }
 
     @Test
     public void startTestsFailsIfDeploymentExpires() {
         tester.newRun(JobType.systemTest);
         tester.runner().run();
-        tester.configServer().convergeServices(InternalDeploymentTester.appId, JobType.systemTest.zone(tester.tester().controller().system()));
-        tester.configServer().convergeServices(testerOf(InternalDeploymentTester.appId), JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(appId, JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(tester.tester().controller().system()));
         tester.runner().run();
 
-        tester.applications().deactivate(InternalDeploymentTester.appId, JobType.systemTest.zone(tester.tester().controller().system()));
+        tester.applications().deactivate(appId, JobType.systemTest.zone(tester.tester().controller().system()));
         tester.runner().run();
-        assertEquals(unfinished, tester.jobs().last(InternalDeploymentTester.appId, JobType.systemTest).get().steps().get(Step.startTests));
+        assertEquals(unfinished, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.startTests));
     }
 
     @Test
@@ -233,16 +231,16 @@ public class InternalStepRunnerTest {
         RunId id = tester.startSystemTestTests();
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.endTests));
-        assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(testerOf(InternalDeploymentTester.appId), JobType.systemTest.zone(tester.tester().controller().system()))).get(0).getEndpoint()),
+        assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(testerId.id(), JobType.systemTest.zone(tester.tester().controller().system()))).get(0).getEndpoint()),
                      tester.cloud().testerUrl());
         Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get();
-        assertEquals(InternalDeploymentTester.appId.serializedForm(), configObject.field("application").asString());
+        assertEquals(appId.serializedForm(), configObject.field("application").asString());
         assertEquals(JobType.systemTest.zone(tester.tester().controller().system()).value(), configObject.field("zone").asString());
         assertEquals(tester.tester().controller().system().name(), configObject.field("system").asString());
         assertEquals(1, configObject.field("endpoints").children());
         assertEquals(1, configObject.field("endpoints").field(JobType.systemTest.zone(tester.tester().controller().system()).value()).entries());
         configObject.field("endpoints").field(JobType.systemTest.zone(tester.tester().controller().system()).value()).traverse((ArrayTraverser) (__, endpoint) ->
-                assertEquals(tester.routing().endpoints(new DeploymentId(InternalDeploymentTester.appId, JobType.systemTest.zone(tester.tester().controller().system()))).get(0).getEndpoint(), endpoint.asString()));
+                assertEquals(tester.routing().endpoints(new DeploymentId(appId, JobType.systemTest.zone(tester.tester().controller().system()))).get(0).getEndpoint(), endpoint.asString()));
 
         long lastId = tester.jobs().details(id).get().lastId().getAsLong();
         tester.cloud().add(new LogEntry(0, 123, info, "Ready!"));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java
index 2640360f10d..bb9602903f4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java
@@ -4,13 +4,13 @@ package com.yahoo.vespa.hosted.controller.integration;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
 
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 import static java.util.Objects.requireNonNull;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
 
 /**
  * Threadsafe.
@@ -22,43 +22,50 @@ public class ApplicationStoreMock implements ApplicationStore {
     private final Map<ApplicationId, Map<ApplicationVersion, byte[]>> store = new ConcurrentHashMap<>();
 
     @Override
-    public byte[] getApplicationPackage(ApplicationId application, ApplicationVersion applicationVersion) {
+    public byte[] get(ApplicationId application, ApplicationVersion applicationVersion) {
         assertFalse(application.instance().isTester());
         return requireNonNull(store.get(application).get(applicationVersion));
     }
 
     @Override
-    public void putApplicationPackage(ApplicationId application, ApplicationVersion applicationVersion, byte[] applicationPackage) {
+    public void put(ApplicationId application, ApplicationVersion applicationVersion, byte[] applicationPackage) {
         assertFalse(application.instance().isTester());
         store.putIfAbsent(application, new ConcurrentHashMap<>());
         store.get(application).put(applicationVersion, applicationPackage);
     }
 
     @Override
-    public void putTesterPackage(ApplicationId tester, ApplicationVersion applicationVersion, byte[] testerPackage) {
-        assertTrue(tester.instance().isTester());
-        store.putIfAbsent(tester, new ConcurrentHashMap<>());
-        store.get(tester).put(applicationVersion, testerPackage);
+    public boolean prune(ApplicationId application, ApplicationVersion oldestToRetain) {
+        assertFalse(application.instance().isTester());
+        return    store.containsKey(application)
+               && store.get(application).keySet().removeIf(version -> version.compareTo(oldestToRetain) < 0);
     }
 
     @Override
-    public byte[] getTesterPackage(ApplicationId tester, ApplicationVersion applicationVersion) {
-        assertTrue(tester.instance().isTester());
-        return requireNonNull(store.get(tester).get(applicationVersion));
+    public void removeAll(ApplicationId application) {
+        store.remove(application);
     }
 
     @Override
-    public boolean pruneApplicationPackages(ApplicationId application, ApplicationVersion oldestToRetain) {
-        assertFalse(application.instance().isTester());
-        return    store.containsKey(application)
-               && store.get(application).keySet().removeIf(version -> version.compareTo(oldestToRetain) < 0);
+    public byte[] get(TesterId tester, ApplicationVersion applicationVersion) {
+        return requireNonNull(store.get(tester.id()).get(applicationVersion));
+    }
+
+    @Override
+    public void put(TesterId tester, ApplicationVersion applicationVersion, byte[] testerPackage) {
+        store.putIfAbsent(tester.id(), new ConcurrentHashMap<>());
+        store.get(tester.id()).put(applicationVersion, testerPackage);
+    }
+
+    @Override
+    public boolean prune(TesterId tester, ApplicationVersion oldestToRetain) {
+        return    store.containsKey(tester.id())
+               && store.get(tester.id()).keySet().removeIf(version -> version.compareTo(oldestToRetain) < 0);
     }
 
     @Override
-    public boolean pruneTesterPackages(ApplicationId tester, ApplicationVersion oldestToRetain) {
-        assertTrue(tester.instance().isTester());
-        return    store.containsKey(tester)
-               && store.get(tester).keySet().removeIf(version -> version.compareTo(oldestToRetain) < 0);
+    public void removeAll(TesterId tester) {
+        store.remove(tester.id());
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index 3d8486ad5bc..5fe725923a5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -28,7 +28,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.FAILURE;
 import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.appId;
-import static com.yahoo.vespa.hosted.controller.deployment.JobController.testerOf;
+import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.testerId;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running;
@@ -66,9 +66,9 @@ public class JobControllerApiHandlerHelperTest {
 
         ZoneId usWest1 = productionUsWest1.zone(tester.tester().controller().system());
         tester.configServer().convergeServices(appId, usWest1);
-        tester.configServer().convergeServices(testerOf(appId), usWest1);
+        tester.configServer().convergeServices(testerId.id(), usWest1);
         tester.setEndpoints(appId, usWest1);
-        tester.setEndpoints(testerOf(appId), usWest1);
+        tester.setEndpoints(testerId.id(), usWest1);
         tester.runner().run();
         tester.cloud().set(FAILURE);
         tester.runner().run();