diff options
author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2018-11-08 20:40:10 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-11-08 20:40:10 +0100 |
commit | a7ae88ea29f8307730535f2cf0d97354e328bb5e (patch) | |
tree | cd9c89ebab8669160258e746123639ffcb5aabb7 /controller-server | |
parent | b4cc9edbc9c4d157eef620d54c7cdc50c92255a5 (diff) | |
parent | 8eb24fb0531e4379058d1bdbce550976de6bd748 (diff) |
Merge pull request #7604 from vespa-engine/jvenstad/application-store-GC
Jvenstad/application store gc
Diffstat (limited to 'controller-server')
30 files changed, 94 insertions, 219 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 0c33307bf80..2705debf8ba 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.Applicat import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationActivity; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index c69fe5021e7..33dfcde9f13 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -38,7 +38,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; @@ -328,7 +328,7 @@ public class ApplicationController { try { applicationPackage = application.get().deploymentJobs().deployedInternally() - ? new ApplicationPackage(applicationStore.getApplicationPackage(application.get().id(), applicationVersion.id())) + ? new ApplicationPackage(applicationStore.getApplicationPackage(application.get().id(), applicationVersion)) : new ApplicationPackage(artifactRepository.getApplicationPackage(application.get().id(), applicationVersion.id())); } catch (RuntimeException e) { // If application has switched deployment pipeline, artifacts stored prior to the switch are in the other artifact store. @@ -336,7 +336,7 @@ public class ApplicationController { + (application.get().deploymentJobs().deployedInternally() ? "internally" : "externally")); applicationPackage = application.get().deploymentJobs().deployedInternally() ? new ApplicationPackage(artifactRepository.getApplicationPackage(application.get().id(), applicationVersion.id())) - : new ApplicationPackage(applicationStore.getApplicationPackage(application.get().id(), applicationVersion.id())); + : new ApplicationPackage(applicationStore.getApplicationPackage(application.get().id(), applicationVersion)); } validateRun(application.get(), zone, platformVersion, applicationVersion); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index bd10a884213..5951ceb4792 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.Applicat import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationVersion.java deleted file mode 100644 index 703a198be1e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationVersion.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; - -/** - * An application package version, identified by a source revision and a build number. - * - * @author bratseth - * @author mpolden - */ -public class ApplicationVersion implements Comparable<ApplicationVersion> { - - /** - * Used in cases where application version cannot be determined, such as manual deployments (e.g. in dev - * environment) - */ - public static final ApplicationVersion unknown = new ApplicationVersion(Optional.empty(), OptionalLong.empty()); - - // This never changes and is only used to create a valid semantic version number, as required by application bundles - private static final String majorVersion = "1.0"; - - private final Optional<SourceRevision> source; - private final OptionalLong buildNumber; - - private ApplicationVersion(Optional<SourceRevision> source, OptionalLong buildNumber) { - Objects.requireNonNull(source, "source cannot be null"); - Objects.requireNonNull(buildNumber, "buildNumber cannot be null"); - if (source.isPresent() != buildNumber.isPresent()) { - throw new IllegalArgumentException("both buildNumber and source must be set together"); - } - if (buildNumber.isPresent() && buildNumber.getAsLong() <= 0) { - throw new IllegalArgumentException("buildNumber must be > 0"); - } - this.source = source; - this.buildNumber = buildNumber; - } - - /** Create an application package version from a completed build */ - public static ApplicationVersion from(SourceRevision source, long buildNumber) { - return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber)); - } - - /** Returns an unique identifier for this version or "unknown" if version is not known */ - public String id() { - if (isUnknown()) { - return "unknown"; - } - return String.format("%s.%d-%s", majorVersion, buildNumber.getAsLong(), abbreviateCommit(source.get().commit())); - } - - /** - * Returns information about the source of this revision, or empty if the source is not know/defined - * (which is the case for command-line deployment from developers, but never for deployment jobs) - */ - public Optional<SourceRevision> source() { return source; } - - /** Returns the build number that built this version */ - public OptionalLong buildNumber() { return buildNumber; } - - /** Returns whether this is unknown */ - public boolean isUnknown() { - return this.equals(unknown); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ApplicationVersion that = (ApplicationVersion) o; - return Objects.equals(source, that.source) && - Objects.equals(buildNumber, that.buildNumber); - } - - @Override - public int hashCode() { - return Objects.hash(source, buildNumber); - } - - @Override - public String toString() { - return "Application package version: " + id() + source.map(s -> ", " + s.toString()).orElse(""); - } - - /** Abbreviate given commit hash to 9 characters */ - private static String abbreviateCommit(String hash) { - return hash.length() <= 9 ? hash : hash.substring(0, 9); - } - - @Override - public int compareTo(ApplicationVersion o) { - if ( ! buildNumber().isPresent() || ! o.buildNumber().isPresent()) - return Boolean.compare(buildNumber().isPresent(), o.buildNumber.isPresent()); // Application package hash sorts first - - return Long.compare(buildNumber().getAsLong(), o.buildNumber().getAsLong()); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java index 0fb6459611c..ab16f84f628 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import java.util.Objects; import java.util.Optional; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 1016a5adf65..ed06ff5bddc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; import com.yahoo.config.provision.ClusterSpec.Id; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index 6e9318042b5..7b02a45d23f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java index cd15556ba9b..dce40f1c583 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java index 59220d38821..5306cc4ae2b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SourceRevision.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SourceRevision.java deleted file mode 100644 index 9c10e0dc153..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SourceRevision.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import java.util.Objects; - -/** - * A revision in a source repository - * - * @author bratseth - */ -public class SourceRevision { - - private final String repository; - private final String branch; - private final String commit; - - public SourceRevision(String repository, String branch, String commit) { - Objects.requireNonNull(repository, "repository cannot be null"); - Objects.requireNonNull(branch, "branch cannot be null"); - Objects.requireNonNull(commit, "commit cannot be null"); - this.repository = repository; - this.branch = branch; - this.commit = commit; - } - - public String repository() { return repository; } - public String branch() { return branch; } - public String commit() { return commit; } - - @Override - public int hashCode() { return Objects.hash(repository, branch, commit); } - - @Override - public boolean equals(Object o) { - if (o == this) return true; - if ( ! (o instanceof SourceRevision)) return false; - - SourceRevision other = (SourceRevision)o; - return this.repository.equals(other.repository) && - this.branch.equals(other.branch) && - this.commit.equals(other.commit); - } - - @Override - public String toString() { return "source revision of repository '" + repository + - "', branch '" + branch + "' with commit '" + commit + "'"; } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 69a42866a9e..d6182611fd5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.BuildService.JobState; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index da1c2ec753b..1f3dd30b8c6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -28,7 +28,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.yolean.Exceptions; @@ -452,7 +452,7 @@ public class InternalStepRunner implements StepRunner { private ApplicationPackage testerPackage(RunId id) { ApplicationVersion version = controller.jobController().run(id).get().versions().targetApplication(); - byte[] testPackage = controller.applications().applicationStore().getTesterPackage(JobController.testerOf(id.application()), version.id()); + byte[] testPackage = controller.applications().applicationStore().getTesterPackage(JobController.testerOf(id.application()), version); byte[] servicesXml = servicesXml(controller.system()); DeploymentSpec spec = controller.applications().require(id.application()).deploymentSpec(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index a75c3d2f8f3..cd6921e6c42 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -14,16 +14,17 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.net.URI; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -225,13 +226,14 @@ public class JobController { AtomicReference<ApplicationVersion> version = new AtomicReference<>(); controller.applications().lockOrThrow(id, application -> { if ( ! application.get().deploymentJobs().deployedInternally()) { + // TODO jvenstad: Remove when there are no more SDv3 pipelines. // Copy all current packages to the new application store application.get().deployments().values().stream() .map(Deployment::applicationVersion) .distinct() .forEach(appVersion -> { byte[] content = controller.applications().artifacts().getApplicationPackage(application.get().id(), appVersion.id()); - controller.applications().applicationStore().putApplicationPackage(application.get().id(), appVersion.id(), content); + controller.applications().applicationStore().putApplicationPackage(application.get().id(), appVersion, content); }); } @@ -239,12 +241,20 @@ public class JobController { version.set(ApplicationVersion.from(revision, run)); controller.applications().applicationStore().putApplicationPackage(id, - version.get().id(), + version.get(), packageBytes); controller.applications().applicationStore().putTesterPackage(testerOf(id), - version.get().id(), + version.get(), testPackageBytes); + application.get().deployments().values().stream() + .map(Deployment::applicationVersion) + .min(Comparator.comparingLong(applicationVersion -> applicationVersion.buildNumber().getAsLong())) + .ifPresent(oldestDeployed -> { + controller.applications().applicationStore().pruneApplicationPackages(id, oldestDeployed); + controller.applications().applicationStore().pruneTesterPackages(testerOf(id), oldestDeployed); + }); + controller.applications().storeWithUpdatedConfig(application.withBuiltInternally(true), new ApplicationPackage(packageBytes)); notifyOfNewSubmission(id, projectId, revision, run); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java index f146dd57885..b6b452bf606 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobStatus; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 9cbede66bf1..c69e77a43e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -17,7 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.Applicat import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -28,7 +28,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 5103dc244ed..0e5041215cb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -10,8 +10,8 @@ import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 03d149b14a6..3a83ecfb9e2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -53,7 +53,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterCost; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -63,7 +63,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; @@ -97,7 +97,6 @@ import java.util.Map; import java.util.Optional; import java.util.Scanner; import java.util.logging.Level; -import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL; import static java.util.stream.Collectors.joining; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index fa5d713d6b3..21640481e86 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -15,11 +15,11 @@ import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 3351efff2dd..7a5c6ae1170 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -19,11 +19,11 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BuildJob.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BuildJob.java index 0a36787e8f7..8c58e491bc4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BuildJob.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BuildJob.java @@ -6,9 +6,9 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import java.util.Objects; import java.util.Optional; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index db58ef1830f..c75e5137bb9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -2,9 +2,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.slime.Slime; import com.yahoo.test.ManualClock; @@ -18,11 +16,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -34,7 +31,6 @@ import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; import java.util.List; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java index a38bd1caf24..bb0261fa0a7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java @@ -17,7 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.integration.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; @@ -87,8 +87,8 @@ public class InternalDeploymentTester { */ public ApplicationVersion newSubmission() { ApplicationVersion version = jobs.submit(appId, BuildJob.defaultSourceRevision, 2, applicationPackage.zippedContent(), new byte[0]); - tester.applicationStore().putApplicationPackage(appId, version.id(), applicationPackage.zippedContent()); - tester.applicationStore().putTesterPackage(testerOf(appId), version.id(), new byte[0]); + tester.applicationStore().putApplicationPackage(appId, version, applicationPackage.zippedContent()); + tester.applicationStore().putTesterPackage(testerOf(appId), version, new byte[0]); return version; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java index dc6042e669b..2640360f10d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java @@ -3,35 +3,62 @@ package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import static java.util.Objects.requireNonNull; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Threadsafe. + * + * @author jonmv + */ public class ApplicationStoreMock implements ApplicationStore { - Map<String, byte[]> store = new HashMap<>(); + private final Map<ApplicationId, Map<ApplicationVersion, byte[]>> store = new ConcurrentHashMap<>(); + + @Override + public byte[] getApplicationPackage(ApplicationId application, ApplicationVersion applicationVersion) { + assertFalse(application.instance().isTester()); + return requireNonNull(store.get(application).get(applicationVersion)); + } @Override - public byte[] getApplicationPackage(ApplicationId application, String applicationVersion) { - return store.get(path(application, applicationVersion)); + public void putApplicationPackage(ApplicationId application, ApplicationVersion applicationVersion, byte[] applicationPackage) { + assertFalse(application.instance().isTester()); + store.putIfAbsent(application, new ConcurrentHashMap<>()); + store.get(application).put(applicationVersion, applicationPackage); } @Override - public void putApplicationPackage(ApplicationId application, String applicationVersion, byte[] applicationPackage) { - store.put(path(application, applicationVersion), applicationPackage); + public void putTesterPackage(ApplicationId tester, ApplicationVersion applicationVersion, byte[] testerPackage) { + assertTrue(tester.instance().isTester()); + store.putIfAbsent(tester, new ConcurrentHashMap<>()); + store.get(tester).put(applicationVersion, testerPackage); } @Override - public void putTesterPackage(ApplicationId tester, String applicationVersion, byte[] testerPackage) { - store.put(path(tester, applicationVersion), testerPackage); + public byte[] getTesterPackage(ApplicationId tester, ApplicationVersion applicationVersion) { + assertTrue(tester.instance().isTester()); + return requireNonNull(store.get(tester).get(applicationVersion)); } @Override - public byte[] getTesterPackage(ApplicationId tester, String applicationVersion) { - return store.get(path(tester, applicationVersion)); + public boolean pruneApplicationPackages(ApplicationId application, ApplicationVersion oldestToRetain) { + assertFalse(application.instance().isTester()); + return store.containsKey(application) + && store.get(application).keySet().removeIf(version -> version.compareTo(oldestToRetain) < 0); } - String path(ApplicationId tester, String applicationVersion) { - return tester.toString() + applicationVersion; + @Override + public boolean pruneTesterPackages(ApplicationId tester, ApplicationVersion oldestToRetain) { + assertTrue(tester.instance().isTester()); + return store.containsKey(tester) + && store.get(tester).keySet().removeIf(version -> version.compareTo(oldestToRetain) < 0); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java index 955980e89d6..04c670cf136 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java @@ -47,21 +47,6 @@ public class ArtifactRepositoryMock extends AbstractComponent implements Artifac } @Override - public void putApplicationPackage(ApplicationId application, String applicationVersion, byte[] applicationPackage) { - repository.put(artifactHash(application, applicationVersion), new Artifact(applicationPackage)); - } - - @Override - public byte[] getTesterPackage(ApplicationId tester, String applicationVersion) { - return getApplicationPackage(tester, applicationVersion); - } - - @Override - public void putTesterPackage(ApplicationId tester, String applicationVersion, byte[] testerPackage) { - putApplicationPackage(tester, applicationVersion, testerPackage); - } - - @Override public byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version) { return new byte[0]; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index ca31eb52979..792eaab20e2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -5,8 +5,8 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.Run; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index bc22df7bf46..d9846dd37ee 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 3e19ff17d86..29ba2a5b6fd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -24,7 +24,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import org.junit.Test; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index a6937a4f97e..9aefbc712ae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -7,8 +7,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 293a1f8a746..3d8486ad5bc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -5,7 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester; import org.json.JSONException; import org.json.JSONObject; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java index 6e5e8a25e22..9f03df79cfe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java @@ -5,7 +5,7 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; -import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import org.junit.Test; |