diff options
author | Martin Polden <mpolden@mpolden.no> | 2018-01-26 08:59:34 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-01-26 08:59:34 +0100 |
commit | 15c3ce8c18d1208eec5c4a2a1f38be3b015ee8fe (patch) | |
tree | 2b89c14513cca556afbefee67f66cb7889f8aa30 /controller-server | |
parent | f9c5d837359ad447220eb8780f804f2c3f4c52be (diff) | |
parent | b87cbc288ed5eb12d9a9c368c2f501523e4344ab (diff) |
Merge pull request #4702 from vespa-engine/mpolden/application-version
Introduce application version number
Diffstat (limited to 'controller-server')
30 files changed, 627 insertions, 344 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index b75f80917a9..c054d81db35 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.application.ApplicationRotation; -import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Change.VersionChange; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -164,17 +164,17 @@ public class Application { .orElse(oldestDeployedVersion().orElse(controller.systemVersion())); } - /** Returns the revision a new deployment to this zone should use for this application, or empty if we don't know */ - public Optional<ApplicationRevision> deployRevisionIn(ZoneId zone) { + /** Returns the application version a deployment to this zone should use, or empty if we don't know */ + public Optional<ApplicationVersion> deployApplicationVersionIn(ZoneId zone) { if (deploying().isPresent() && deploying().get() instanceof Change.ApplicationChange) - return ((Change.ApplicationChange) deploying().get()).revision(); + return ((Change.ApplicationChange) deploying().get()).version(); - return revisionIn(zone); + return applicationVersionIn(zone); } - /** Returns the revision this application is or should be deployed with in the given zone, or empty if unknown. */ - public Optional<ApplicationRevision> revisionIn(ZoneId zone) { - return Optional.ofNullable(deployments().get(zone)).map(Deployment::revision); + /** Returns the application version that is or should be deployed with in the given zone, or empty if unknown. */ + public Optional<ApplicationVersion> applicationVersionIn(ZoneId zone) { + return Optional.ofNullable(deployments().get(zone)).map(Deployment::applicationVersion); } /** Returns the global rotation of this, if present */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 2c2dcfe549b..89b134cc228 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -7,7 +7,7 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.InstanceEndpoints; @@ -22,13 +22,13 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; @@ -36,8 +36,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; @@ -90,6 +91,7 @@ public class ApplicationController { /** For working memory storage and sharing between controllers */ private final CuratorDb curator; + private final ArtifactRepository artifactRepository; private final RotationRepository rotationRepository; private final AthenzClientFactory zmsClientFactory; private final NameService nameService; @@ -102,6 +104,7 @@ public class ApplicationController { ApplicationController(Controller controller, ControllerDb db, CuratorDb curator, AthenzClientFactory zmsClientFactory, RotationsConfig rotationsConfig, NameService nameService, ConfigServerClient configserverClient, + ArtifactRepository artifactRepository, RoutingGenerator routingGenerator, Clock clock) { this.controller = controller; this.db = db; @@ -112,6 +115,7 @@ public class ApplicationController { this.routingGenerator = routingGenerator; this.clock = clock; + this.artifactRepository = artifactRepository; this.rotationRepository = new RotationRepository(rotationsConfig, this, curator); this.deploymentTrigger = new DeploymentTrigger(controller, curator, clock); @@ -271,7 +275,8 @@ public class ApplicationController { /** Deploys an application. If the application does not exist it is created. */ // TODO: Get rid of the options arg public ActivateResult deployApplication(ApplicationId applicationId, ZoneId zone, - ApplicationPackage applicationPackage, DeployOptions options) { + Optional<ApplicationPackage> applicationPackageFromDeployer, + DeployOptions options) { try (Lock lock = lock(applicationId)) { // TODO: Move application creation outside, to the deploy call in the handler. LockedApplication application = get(applicationId) @@ -281,40 +286,66 @@ public class ApplicationController { return new LockedApplication(new Application(applicationId), lock); }); - // Determine what we are doing + // Determine Vespa version to use Version version; - if (options.deployCurrentVersion) + if (options.deployCurrentVersion) { version = application.versionIn(zone, controller); - else if (canDeployDirectlyTo(zone, options)) + } else if (canDeployDirectlyTo(zone, options)) { version = options.vespaVersion.map(Version::new).orElse(controller.systemVersion()); - else if ( ! application.deploying().isPresent() && ! zone.environment().isManuallyDeployed()) - return unexpectedDeployment(applicationId, zone, applicationPackage); - else + } else if ( ! application.deploying().isPresent() && ! zone.environment().isManuallyDeployed()) { + return unexpectedDeployment(applicationId, zone, applicationPackageFromDeployer); + } else { version = application.deployVersionIn(zone, controller); + } Optional<DeploymentJobs.JobType> jobType = DeploymentJobs.JobType.from(controller.system(), zone); - ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob); + if (!jobType.isPresent() && !applicationPackageFromDeployer.isPresent()) { + throw new IllegalArgumentException("Unable to determine job type from zone '" + zone + + "' and no application package was given"); + } - if ( ! options.deployCurrentVersion) { - // Add missing information to application (unless we're deploying the previous version (initial staging step) - application = application.with(applicationPackage.deploymentSpec()); - application = application.with(applicationPackage.validationOverrides()); - if (options.screwdriverBuildJob.isPresent() && options.screwdriverBuildJob.get().screwdriverId != null) - application = application.withProjectId(options.screwdriverBuildJob.get().screwdriverId.value()); - if (application.deploying().isPresent() && application.deploying().get() instanceof Change.ApplicationChange) - application = application.withDeploying(Optional.of(Change.ApplicationChange.of(revision))); - if ( ! canDeployDirectlyTo(zone, options) && jobType.isPresent()) { + // Determine which application package to use + ApplicationPackage applicationPackage; + ApplicationVersion applicationVersion; + if (applicationPackageFromDeployer.isPresent()) { + applicationVersion = toApplicationPackageRevision(applicationPackageFromDeployer.get(), + options.screwdriverBuildJob); + applicationPackage = applicationPackageFromDeployer.get(); + } else { + applicationVersion = application.deployApplicationVersion(jobType.get(), controller) + .orElseThrow(() -> new IllegalArgumentException("Cannot determine application version to use in " + zone)); + applicationPackage = new ApplicationPackage(artifactRepository.getApplicationPackage( + applicationId, applicationVersion.id()) + ); + } + + validate(applicationPackage.deploymentSpec()); + + // TODO: Remove after introducing new application version number + if (!options.deployCurrentVersion && applicationPackageFromDeployer.isPresent()) { + if (application.deploying().isPresent() && application.deploying().get() instanceof Change.ApplicationChange) { + application = application.withDeploying(Optional.of(Change.ApplicationChange.of(applicationVersion))); + } + if (!canDeployDirectlyTo(zone, options) && jobType.isPresent()) { // Update with (potentially) missing information about what we triggered: // * When someone else triggered the job, we need to store a stand-in triggering event. - // * When this is the system test job, we need to record the new revision, for future use. + // * When this is the system test job, we need to record the new application version, + // for future use. JobStatus.JobRun triggering = getOrCreateTriggering(application, version, jobType.get()); application = application.withJobTriggering(jobType.get(), application.deploying(), triggering.at(), version, - Optional.of(revision), + Optional.of(applicationVersion), triggering.reason()); } + } + + // Update application with information from application package + if (!options.deployCurrentVersion) { + // Store information about application package + application = application.with(applicationPackage.deploymentSpec()); + application = application.with(applicationPackage.validationOverrides()); // Delete zones not listed in DeploymentSpec, if allowed // We do this at deployment time to be able to return a validation failure message when necessary @@ -326,15 +357,19 @@ public class ApplicationController { store(application); // store missing information even if we fail deployment below } - if ( ! canDeployDirectlyTo(zone, options)) { // validate automated deployment - if ( ! application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) + // Validate automated deployment + if (!canDeployDirectlyTo(zone, options)) { + if (!application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) { throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + " as " + application.deploying().get() + " is not tested"); + } Deployment existingDeployment = application.deployments().get(zone); - if (zone.environment().isProduction() && existingDeployment != null && existingDeployment.version().isAfter(version)) + if (zone.environment().isProduction() && existingDeployment != null && + existingDeployment.version().isAfter(version)) { throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + " as the requested version " + version + " is older than" + " the current version " + existingDeployment.version()); + } } application = withRotation(application, zone); @@ -353,11 +388,12 @@ public class ApplicationController { configserverClient.prepare(new DeploymentId(applicationId, zone), options, cnames, rotationNames, applicationPackage.zippedContent()); preparedApplication.activate(); - application = application.withNewDeployment(zone, revision, version, clock.instant()); + application = application.withNewDeployment(zone, applicationVersion, version, clock.instant()); store(application); - return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse()); + return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), + applicationPackage.zippedContent().length); } } @@ -376,7 +412,8 @@ public class ApplicationController { return application; } - private ActivateResult unexpectedDeployment(ApplicationId applicationId, ZoneId zone, ApplicationPackage applicationPackage) { + private ActivateResult unexpectedDeployment(ApplicationId applicationId, ZoneId zone, + Optional<ApplicationPackage> applicationPackage) { Log logEntry = new Log(); logEntry.level = "WARNING"; logEntry.time = clock.instant().toEpochMilli(); @@ -384,7 +421,9 @@ public class ApplicationController { PrepareResponse prepareResponse = new PrepareResponse(); prepareResponse.log = Collections.singletonList(logEntry); prepareResponse.configChangeActions = new ConfigChangeActions(Collections.emptyList(), Collections.emptyList()); - return new ActivateResult(new RevisionId(applicationPackage.hash()), prepareResponse); + return new ActivateResult(new RevisionId(applicationPackage.map(ApplicationPackage::hash) + .orElse("0")), prepareResponse, + applicationPackage.map(a -> a.zippedContent().length).orElse(0)); } private LockedApplication deleteRemovedDeployments(LockedApplication application) { @@ -445,18 +484,18 @@ public class ApplicationController { options.deployCurrentVersion); } - private ApplicationRevision toApplicationPackageRevision(ApplicationPackage applicationPackage, - Optional<ScrewdriverBuildJob> screwDriverBuildJob) { - if ( ! screwDriverBuildJob.isPresent()) - return ApplicationRevision.from(applicationPackage.hash()); + private ApplicationVersion toApplicationPackageRevision(ApplicationPackage applicationPackage, + Optional<ScrewdriverBuildJob> buildJob) { + if ( ! buildJob.isPresent()) + return ApplicationVersion.from(applicationPackage.hash()); - GitRevision gitRevision = screwDriverBuildJob.get().gitRevision; + GitRevision gitRevision = buildJob.get().gitRevision; if (gitRevision.repository == null || gitRevision.branch == null || gitRevision.commit == null) - return ApplicationRevision.from(applicationPackage.hash()); + return ApplicationVersion.from(applicationPackage.hash()); - return ApplicationRevision.from(applicationPackage.hash(), new SourceRevision(gitRevision.repository.id(), - gitRevision.branch.id(), - gitRevision.commit.id())); + return ApplicationVersion.from(applicationPackage.hash(), new SourceRevision(gitRevision.repository.id(), + gitRevision.branch.id(), + gitRevision.commit.id())); } /** Register a DNS name for rotation */ @@ -661,7 +700,7 @@ public class ApplicationController { } /** Verify that each of the production zones listed in the deployment spec exist in this system. */ - public void validate(DeploymentSpec deploymentSpec) { + private void validate(DeploymentSpec deploymentSpec) { deploymentSpec.zones().stream() .filter(zone -> zone.environment() == Environment.prod) .forEach(zone -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 0e13f4181c4..0ec00f61311 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -8,8 +8,6 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; @@ -17,13 +15,16 @@ import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.api.integration.github.GitHub; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.organization.Organization; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -85,11 +86,12 @@ public class Controller extends AbstractComponent { GlobalRoutingService globalRoutingService, ZoneRegistry zoneRegistry, ConfigServerClient configServerClient, NodeRepositoryClientInterface nodeRepositoryClient, MetricsService metricsService, NameService nameService, - RoutingGenerator routingGenerator, Chef chefClient, AthenzClientFactory athenzClientFactory) { + RoutingGenerator routingGenerator, Chef chefClient, AthenzClientFactory athenzClientFactory, + ArtifactRepository artifactRepository) { this(db, curator, rotationsConfig, gitHub, entityService, organization, globalRoutingService, zoneRegistry, configServerClient, nodeRepositoryClient, metricsService, nameService, routingGenerator, chefClient, - Clock.systemUTC(), athenzClientFactory); + Clock.systemUTC(), athenzClientFactory, artifactRepository); } public Controller(ControllerDb db, CuratorDb curator, RotationsConfig rotationsConfig, @@ -98,7 +100,7 @@ public class Controller extends AbstractComponent { ZoneRegistry zoneRegistry, ConfigServerClient configServerClient, NodeRepositoryClientInterface nodeRepositoryClient, MetricsService metricsService, NameService nameService, RoutingGenerator routingGenerator, Chef chefClient, Clock clock, - AthenzClientFactory athenzClientFactory) { + AthenzClientFactory athenzClientFactory, ArtifactRepository artifactRepository) { Objects.requireNonNull(db, "Controller db cannot be null"); Objects.requireNonNull(curator, "Curator cannot be null"); Objects.requireNonNull(rotationsConfig, "RotationsConfig cannot be null"); @@ -115,6 +117,7 @@ public class Controller extends AbstractComponent { Objects.requireNonNull(chefClient, "ChefClient cannot be null"); Objects.requireNonNull(clock, "Clock cannot be null"); Objects.requireNonNull(athenzClientFactory, "Athens cannot be null"); + Objects.requireNonNull(artifactRepository, "ArtifactRepository cannot be null"); this.curator = curator; this.gitHub = gitHub; @@ -131,7 +134,8 @@ public class Controller extends AbstractComponent { applicationController = new ApplicationController(this, db, curator, athenzClientFactory, rotationsConfig, - nameService, configServerClient, routingGenerator, clock); + nameService, configServerClient, artifactRepository, + routingGenerator, clock); tenantController = new TenantController(this, db, curator, entityService, athenzClientFactory); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index debd96016e4..b50ecb82c50 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.application.ApplicationRotation; -import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -42,7 +42,7 @@ public class LockedApplication extends Application { * @param application The application to lock. * @param lock The lock for the application. */ - LockedApplication(Application application, Lock lock) { + LockedApplication(Application application, @SuppressWarnings("unused") Lock lock) { this(new Builder(application)); } @@ -65,14 +65,17 @@ public class LockedApplication extends Application { } public LockedApplication withJobTriggering(JobType type, Optional<Change> change, Instant triggerTime, - Version version, Optional<ApplicationRevision> revision, String reason) { - return new LockedApplication(new Builder(this).with(deploymentJobs().withTriggering(type, change, version, revision, reason, triggerTime))); + Version version, Optional<ApplicationVersion> applicationVersion, + String reason) { + return new LockedApplication(new Builder(this).with(deploymentJobs().withTriggering(type, change, version, applicationVersion, reason, triggerTime))); } - public LockedApplication withNewDeployment(ZoneId zone, ApplicationRevision revision, Version version, Instant instant) { + public LockedApplication withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, + Instant instant) { // Use info from previous deployment if available, otherwise create a new one. - Deployment previousDeployment = deployments().getOrDefault(zone, new Deployment(zone, revision, version, instant)); - Deployment newDeployment = new Deployment(zone, revision, version, instant, + Deployment previousDeployment = deployments().getOrDefault(zone, new Deployment(zone, applicationVersion, + version, instant)); + Deployment newDeployment = new Deployment(zone, applicationVersion, version, instant, previousDeployment.clusterUtils(), previousDeployment.clusterInfo(), previousDeployment.metrics()); @@ -142,10 +145,10 @@ public class LockedApplication extends Application { : deployVersionIn(jobType.zone(controller.system()).get(), controller); } - public Optional<ApplicationRevision> deployRevisionFor(DeploymentJobs.JobType jobType, Controller controller) { + public Optional<ApplicationVersion> deployApplicationVersion(DeploymentJobs.JobType jobType, Controller controller) { return jobType == JobType.component - ? Optional.empty() - : deployRevisionIn(jobType.zone(controller.system()).get()); + ? Optional.empty() + : deployApplicationVersionIn(jobType.zone(controller.system()).get()); } /** Don't expose non-leaf sub-objects. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java index 6b1c5d56a5f..271942ff9a3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java @@ -2,11 +2,8 @@ package com.yahoo.vespa.hosted.controller.api; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; -import java.util.List; - /** * @author Oyvind Gronnesby */ @@ -14,17 +11,23 @@ public class ActivateResult { private final RevisionId revisionId; private final PrepareResponse prepareResponse; + private final long applicationZipSizeBytes; - public ActivateResult(RevisionId revisionId, PrepareResponse prepareResponse) { + public ActivateResult(RevisionId revisionId, PrepareResponse prepareResponse, long applicationZipSizeBytes) { this.revisionId = revisionId; this.prepareResponse = prepareResponse; + this.applicationZipSizeBytes = applicationZipSizeBytes; + } + + public long applicationZipSizeBytes() { + return applicationZipSizeBytes; } - public RevisionId getRevisionId() { + public RevisionId revisionId() { return revisionId; } - public PrepareResponse getPrepareResponse() { + public PrepareResponse prepareResponse() { return prepareResponse; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 283d6a75178..f8c7319fabc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -54,7 +54,7 @@ public class ApplicationList { /** Returns the subset of applications which are currently upgrading (to any version) */ public ApplicationList upgrading() { - return listOf(list.stream().filter(application -> isUpgrading(application))); + return listOf(list.stream().filter(ApplicationList::isUpgrading)); } /** Returns the subset of applications which are currently upgrading to the given version */ @@ -62,11 +62,6 @@ public class ApplicationList { return listOf(list.stream().filter(application -> isUpgradingTo(version, application))); } - /** Returns the subset of applications which are currently upgrading to a version lower than the given version */ - public ApplicationList upgradingToLowerThan(Version version) { - return listOf(list.stream().filter(application -> isUpgradingToLowerThan(version, application))); - } - /** Returns the subset of applications which are currently not upgrading to the given version */ public ApplicationList notUpgradingTo(Version version) { return listOf(list.stream().filter(application -> ! isUpgradingTo(version, application))); @@ -81,11 +76,6 @@ public class ApplicationList { return notUpgradingTo(version.get()); } - /** Returns the subset of applications which is currently not deploying a new application revision */ - public ApplicationList notDeployingApplication() { - return listOf(list.stream().filter(application -> ! isDeployingApplicationChange(application))); - } - /** Returns the subset of applications which is currently not deploying a change */ public ApplicationList notDeploying() { return listOf(list.stream().filter(application -> ! application.deploying().isPresent())); @@ -178,11 +168,6 @@ public class ApplicationList { return listOf(list.stream().sorted(Comparator.comparing(application -> application.oldestDeployedVersion().orElse(Version.emptyVersion)))); } - /** Returns the subset of applications that are not currently upgrading */ - public ApplicationList notCurrentlyUpgrading(Change.VersionChange change, Instant jobTimeoutLimit) { - return listOf(list.stream().filter(a -> ! currentlyUpgrading(change, a, jobTimeoutLimit))); - } - // ----------------------------------- Internal helpers private static boolean isUpgrading(Application application) { @@ -197,17 +182,6 @@ public class ApplicationList { return ((Change.VersionChange)application.deploying().get()).version().equals(version); } - private static boolean isUpgradingToLowerThan(Version version, Application application) { - if ( ! application.deploying().isPresent()) return false; - if ( ! (application.deploying().get() instanceof Change.VersionChange) ) return false; - return ((Change.VersionChange)application.deploying().get()).version().isBefore(version); - } - - private static boolean isDeployingApplicationChange(Application application) { - if ( ! application.deploying().isPresent()) return false; - return application.deploying().get() instanceof Change.ApplicationChange; - } - private static boolean failingOn(Version version, Application application) { return ! JobList.from(application) .failing() @@ -215,13 +189,6 @@ public class ApplicationList { .isEmpty(); } - private static boolean currentlyUpgrading(Change.VersionChange change, Application application, Instant jobTimeoutLimit) { - return ! JobList.from(application) - .running(jobTimeoutLimit) - .lastTriggered().on(change.version()) - .isEmpty(); - } - private static boolean failingUpgradeToVersionSince(Application application, Version version, Instant threshold) { return ! JobList.from(application) .not().failingApplicationChange() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRevision.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRevision.java deleted file mode 100644 index 1b875f28715..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRevision.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import java.util.Objects; -import java.util.Optional; - -/** - * An identifier of a particular revision (exact content) of an application package, - * optionally with information about the source of the package revision. - * - * @author bratseth - */ -public class ApplicationRevision { - - private final String applicationPackageHash; - - private final Optional<SourceRevision> source; - - private ApplicationRevision(String applicationPackageHash, Optional<SourceRevision> source) { - Objects.requireNonNull(applicationPackageHash, "applicationPackageHash cannot be null"); - this.applicationPackageHash = applicationPackageHash; - this.source = source; - } - - /** Create an application package revision where there is no information about its source */ - public static ApplicationRevision from(String applicationPackageHash) { - return new ApplicationRevision(applicationPackageHash, Optional.empty()); - } - - /** Create an application package revision with a source */ - public static ApplicationRevision from(String applicationPackageHash, SourceRevision source) { - return new ApplicationRevision(applicationPackageHash, Optional.of(source)); - } - - /** Returns a unique, content-based identifier of an application package (a hash of the content) */ - public String id() { return applicationPackageHash; } - - /** - * Returns information about the source of this revision, or empty if the source is not know/defined - * (which is the case for command-line deployment from developers, but never for deployment jobs) - */ - public Optional<SourceRevision> source() { return source; } - - @Override - public int hashCode() { return applicationPackageHash.hashCode(); } - - @Override - public boolean equals(Object other) { - if (this == other) return true; - if ( ! (other instanceof ApplicationRevision)) return false; - return this.applicationPackageHash.equals(((ApplicationRevision)other).applicationPackageHash); - } - - @Override - public String toString() { - return "Application package revision '" + applicationPackageHash + "'" + - (source.isPresent() ? " with " + source.get() : ""); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationVersion.java new file mode 100644 index 00000000000..aee178af275 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationVersion.java @@ -0,0 +1,97 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import java.util.Objects; +import java.util.Optional; + +/** + * An application package version. This represents an build artifact, identified by a source revision and a build + * number. + * + * @author bratseth + * @author mpolden + */ +public class ApplicationVersion { + + // Never changes. Only used to create a valid version number for the bundle + private static final String majorVersion = "1.0"; + + // TODO: Remove after introducing new application version + private final Optional<String> applicationPackageHash; + + private final Optional<SourceRevision> source; + private final Optional<Long> buildNumber; + + private ApplicationVersion(Optional<String> applicationPackageHash, Optional<SourceRevision> source, + Optional<Long> buildNumber) { + Objects.requireNonNull(applicationPackageHash, "applicationPackageHash cannot be null"); + Objects.requireNonNull(source, "source cannot be null"); + Objects.requireNonNull(buildNumber, "buildNumber cannot be null"); + if (buildNumber.isPresent() && !source.isPresent()) { + throw new IllegalArgumentException("both buildNumber and source must be set if buildNumber is set"); + } + if (!buildNumber.isPresent() && !applicationPackageHash.isPresent()) { + throw new IllegalArgumentException("applicationPackageHash must be given if buildNumber is unset"); + } + this.applicationPackageHash = applicationPackageHash; + this.source = source; + this.buildNumber = buildNumber; + } + + /** Create an application package revision where there is no information about its source */ + public static ApplicationVersion from(String applicationPackageHash) { + return new ApplicationVersion(Optional.of(applicationPackageHash), Optional.empty(), Optional.empty()); + } + + /** Create an application package revision with a source */ + public static ApplicationVersion from(String applicationPackageHash, SourceRevision source) { + return new ApplicationVersion(Optional.of(applicationPackageHash), Optional.of(source), Optional.empty()); + } + + /** Create an application package version from a completed build */ + public static ApplicationVersion from(SourceRevision source, long buildNumber) { + return new ApplicationVersion(Optional.empty(), Optional.of(source), Optional.of(buildNumber)); + } + + /** Returns an unique identifier for this version */ + public String id() { + if (applicationPackageHash.isPresent()) { + return applicationPackageHash.get(); + } + return String.format("%s.%d-%s", majorVersion, buildNumber.get(), abbreviateCommit(source.get().commit())); + } + + /** + * Returns information about the source of this revision, or empty if the source is not know/defined + * (which is the case for command-line deployment from developers, but never for deployment jobs) + */ + public Optional<SourceRevision> source() { return source; } + + /** Returns the build number that built this version */ + public Optional<Long> buildNumber() { return buildNumber; } + + @Override + public int hashCode() { return applicationPackageHash.hashCode(); } + + @Override + public boolean equals(Object other) { + if (this == other) return true; + if ( ! (other instanceof ApplicationVersion)) return false; + return this.applicationPackageHash.equals(((ApplicationVersion)other).applicationPackageHash); + } + + @Override + public String toString() { + if (buildNumber.isPresent()) { + return "Application package version: " + abbreviateCommit(source.get().commit()) + "-" + buildNumber.get(); + } + return "Application package revision '" + applicationPackageHash + "'" + + (source.isPresent() ? " with " + source.get() : ""); + } + + /** Abbreviate given commit hash to 9 characters */ + private static String abbreviateCommit(String hash) { + return hash.length() <= 9 ? hash : hash.substring(0, 9); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java index d9c22018d26..08291373656 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -18,18 +18,19 @@ public abstract class Change { /** Returns true if this change is blocked by the given spec at the given instant */ public abstract boolean blockedBy(DeploymentSpec deploymentSpec, Instant instant); - /** A change to the application package revision of an application */ + /** A change to the application package version of an application */ public static class ApplicationChange extends Change { + + // TODO: Make non-optional + private final Optional<ApplicationVersion> version; - private final Optional<ApplicationRevision> revision; - - private ApplicationChange(Optional<ApplicationRevision> revision) { - Objects.requireNonNull(revision, "revision cannot be null"); - this.revision = revision; + private ApplicationChange(Optional<ApplicationVersion> version) { + Objects.requireNonNull(version, "version cannot be null"); + this.version = version; } - /** The revision this changes to, or empty if not known yet */ - public Optional<ApplicationRevision> revision() { return revision; } + /** The application package version in this change, or empty if not known yet */ + public Optional<ApplicationVersion> version() { return version; } @Override public boolean blockedBy(DeploymentSpec deploymentSpec, Instant instant) { @@ -37,13 +38,13 @@ public abstract class Change { } @Override - public int hashCode() { return revision.hashCode(); } + public int hashCode() { return version.hashCode(); } @Override public boolean equals(Object other) { if (this == other) return true; if ( ! (other instanceof ApplicationChange)) return false; - return ((ApplicationChange)other).revision.equals(this.revision); + return ((ApplicationChange)other).version.equals(this.version); } /** @@ -56,13 +57,13 @@ public abstract class Change { return new ApplicationChange(Optional.empty()); } - public static ApplicationChange of(ApplicationRevision revision) { - return new ApplicationChange(Optional.of(revision)); + public static ApplicationChange of(ApplicationVersion version) { + return new ApplicationChange(Optional.of(version)); } @Override public String toString() { - return "application change to " + revision.map(ApplicationRevision::toString).orElse("an unknown revision"); + return "application change to " + version.map(ApplicationVersion::toString).orElse("an unknown version"); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 2364e87b345..8fa0c6da49c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -19,28 +19,28 @@ import java.util.Objects; public class Deployment { private final ZoneId zone; - private final ApplicationRevision revision; + private final ApplicationVersion applicationVersion; private final Version version; private final Instant deployTime; private final Map<Id, ClusterUtilization> clusterUtils; private final Map<Id, ClusterInfo> clusterInfo; private final DeploymentMetrics metrics; - public Deployment(ZoneId zone, ApplicationRevision revision, Version version, Instant deployTime) { - this(zone, revision, version, deployTime, new HashMap<>(), new HashMap<>(), new DeploymentMetrics()); + public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime) { + this(zone, applicationVersion, version, deployTime, new HashMap<>(), new HashMap<>(), new DeploymentMetrics()); } - public Deployment(ZoneId zone, ApplicationRevision revision, Version version, Instant deployTime, + public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime, Map<Id, ClusterUtilization> clusterUtils, Map<Id, ClusterInfo> clusterInfo, DeploymentMetrics metrics) { Objects.requireNonNull(zone, "zone cannot be null"); - Objects.requireNonNull(revision, "revision cannot be null"); + Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null"); Objects.requireNonNull(version, "version cannot be null"); Objects.requireNonNull(deployTime, "deployTime cannot be null"); Objects.requireNonNull(clusterUtils, "clusterUtils cannot be null"); Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null"); Objects.requireNonNull(metrics, "deployment metrics cannot be null"); this.zone = zone; - this.revision = revision; + this.applicationVersion = applicationVersion; this.version = version; this.deployTime = deployTime; this.clusterUtils = clusterUtils; @@ -51,10 +51,10 @@ public class Deployment { /** Returns the zone this was deployed to */ public ZoneId zone() { return zone; } - /** Returns the revision of the application which was deployed */ - public ApplicationRevision revision() { return revision; } + /** Returns the deployed application version */ + public ApplicationVersion applicationVersion() { return applicationVersion; } - /** Returns the Vespa version which was deployed */ + /** Returns the deployed Vespa version */ public Version version() { return version; } /** Returns the time this was deployed */ @@ -69,15 +69,15 @@ public class Deployment { } public Deployment withClusterUtils(Map<Id, ClusterUtilization> clusterUtilization) { - return new Deployment(zone, revision, version, deployTime, clusterUtilization, clusterInfo, metrics); + return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics); } public Deployment withClusterInfo(Map<Id, ClusterInfo> newClusterInfo) { - return new Deployment(zone, revision, version, deployTime, clusterUtils, newClusterInfo, metrics); + return new Deployment(zone, applicationVersion, version, deployTime, clusterUtils, newClusterInfo, metrics); } public Deployment withMetrics(DeploymentMetrics metrics) { - return new Deployment(zone, revision, version, deployTime, clusterUtils, clusterInfo, metrics); + return new Deployment(zone, applicationVersion, version, deployTime, clusterUtils, clusterInfo, metrics); } /** @return Key metrics for the deployment (application level) like QPS and document count */ @@ -107,6 +107,6 @@ public class Deployment { @Override public String toString() { - return "deployment to " + zone + " of " + revision + " on version " + version + " at " + deployTime; + return "deployment to " + zone + " of " + applicationVersion + " on version " + version + " at " + deployTime; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index a728786e2ce..c432503a790 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -67,14 +67,14 @@ public class DeploymentJobs { public DeploymentJobs withTriggering(JobType jobType, Optional<Change> change, Version version, - Optional<ApplicationRevision> revision, + Optional<ApplicationVersion> applicationVersion, String reason, Instant triggerTime) { Map<JobType, JobStatus> status = new LinkedHashMap<>(this.status); status.compute(jobType, (type, job) -> { if (job == null) job = JobStatus.initial(jobType); - return job.withTriggering( version, - revision, + return job.withTriggering(version, + applicationVersion, change.isPresent() && change.get() instanceof Change.VersionChange, reason, triggerTime); @@ -274,7 +274,7 @@ public class DeploymentJobs { public enum JobError { unknown, - outOfCapacity; + outOfCapacity } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java index 3c72fd69e42..932229b6bb6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java @@ -13,8 +13,6 @@ import java.util.Optional; import java.util.function.Function; import java.util.function.Predicate; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity; - /** * A list of deployment jobs that can be filtered in various ways. * @@ -48,10 +46,10 @@ public class JobList { // TODO: Add sorting based on various stuff, such as deployment order, time of last completion, etc.. - /** Returns the jobstatuses in this as an immutable list */ + /** Returns the job statuses in this as an immutable list */ public List<JobStatus> asList() { return list; } - /** Returns the jobstatuses in this as an immutable list after mapping with the given function */ + /** Returns the job statuses in this as an immutable list after mapping with the given function */ public <Type> List<Type> mapToList(Function<JobStatus, Type> mapper) { return ImmutableList.copyOf(list.stream().map(mapper)::iterator); } @@ -68,7 +66,7 @@ public class JobList { } /** Returns the subset of jobs which are current upgrading */ - public JobList upgrading() { // TODO: Centralise and standardise reasoning about upgrades and revisions. + public JobList upgrading() { // TODO: Centralise and standardise reasoning about upgrades and application versions. return filter(job -> job.lastSuccess().isPresent() && job.lastTriggered().isPresent() && ! job.lastTriggered().get().at().isBefore(job.lastCompleted().get().at()) @@ -87,7 +85,7 @@ public class JobList { /** Returns the subset of jobs which must be failing due to an application change */ public JobList failingApplicationChange() { - return filter(job -> failingApplicationChange(job)); + return filter(JobList::failingApplicationChange); } /** Returns the subset of jobs which are failing with the given job error */ @@ -109,22 +107,22 @@ public class JobList { /** Returns the list in a state where the next filter is for the lastTriggered run type */ public JobRunFilter lastTriggered() { - return new JobRunFilter(job -> job.lastTriggered()); + return new JobRunFilter(JobStatus::lastTriggered); } /** Returns the list in a state where the next filter is for the lastCompleted run type */ public JobRunFilter lastCompleted() { - return new JobRunFilter(job -> job.lastCompleted()); + return new JobRunFilter(JobStatus::lastCompleted); } /** Returns the list in a state where the next filter is for the lastSuccess run type */ public JobRunFilter lastSuccess() { - return new JobRunFilter(job -> job.lastSuccess()); + return new JobRunFilter(JobStatus::lastSuccess); } /** Returns the list in a state where the next filter is for the firstFailing run type */ public JobRunFilter firstFailing() { - return new JobRunFilter(job -> job.firstFailing()); + return new JobRunFilter(JobStatus::firstFailing); } @@ -158,7 +156,7 @@ public class JobList { } public JobList upgrade() { - return filter(run -> run.upgrade()); + return filter(JobRun::upgrade); } /** Transforms the JobRun condition to a JobStatus condition, by considering only the JobRun mapped by which, and executes */ @@ -174,9 +172,9 @@ public class JobList { private static boolean failingApplicationChange(JobStatus job) { if ( job.isSuccess()) return false; if ( ! job.lastSuccess().isPresent()) return true; // An application which never succeeded is surely bad. - if ( ! job.lastSuccess().get().revision().isPresent()) return true; // Indicates the component job, which is always an application change. + if ( ! job.lastSuccess().get().applicationVersion().isPresent()) return true; // Indicates the component job, which is always an application change. if ( ! job.firstFailing().get().version().equals(job.lastSuccess().get().version())) return false; // Version change may be to blame. - return ! job.firstFailing().get().revision().equals(job.lastSuccess().get().revision()); // Return whether there is an application change. + return ! job.firstFailing().get().applicationVersion().equals(job.lastSuccess().get().applicationVersion()); // Return whether there is an application change. } /** Returns a new JobList which is the result of filtering with the -- possibly negated -- condition */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java index a7940076277..71c4a380a29 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java @@ -55,20 +55,20 @@ public class JobStatus { return new JobStatus(type, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } - public JobStatus withTriggering(Version version, Optional<ApplicationRevision> revision, + public JobStatus withTriggering(Version version, Optional<ApplicationVersion> applicationVersion, boolean upgrade, String reason, Instant triggerTime) { - return new JobStatus(type, jobError, Optional.of(new JobRun(-1, version, revision, upgrade, reason, triggerTime)), + return new JobStatus(type, jobError, Optional.of(new JobRun(-1, version, applicationVersion, upgrade, reason, triggerTime)), lastCompleted, firstFailing, lastSuccess); } public JobStatus withCompletion(long runId, Optional<DeploymentJobs.JobError> jobError, Instant completionTime, Controller controller) { Version version; - Optional<ApplicationRevision> revision; + Optional<ApplicationVersion> applicationVersion; boolean upgrade; String reason; if (type == DeploymentJobs.JobType.component) { // not triggered by us version = controller.systemVersion(); - revision = Optional.empty(); + applicationVersion = Optional.empty(); upgrade = false; reason = "Application commit"; } @@ -79,12 +79,12 @@ public class JobStatus { } else { version = lastTriggered.get().version(); - revision = lastTriggered.get().revision(); + applicationVersion = lastTriggered.get().applicationVersion(); upgrade = lastTriggered.get().upgrade(); reason = lastTriggered.get().reason(); } - JobRun thisCompletion = new JobRun(runId, version, revision, upgrade, reason, completionTime); + JobRun thisCompletion = new JobRun(runId, version, applicationVersion, upgrade, reason, completionTime); Optional<JobRun> firstFailing = this.firstFailing; if (jobError.isPresent() && ! this.firstFailing.isPresent()) @@ -167,20 +167,21 @@ public class JobStatus { private final long id; private final Version version; - private final Optional<ApplicationRevision> revision; + // TODO: Make non-optional after introducing new application version number + private final Optional<ApplicationVersion> applicationVersion; private final boolean upgrade; private final String reason; private final Instant at; - public JobRun(long id, Version version, Optional<ApplicationRevision> revision, + public JobRun(long id, Version version, Optional<ApplicationVersion> applicationVersion, boolean upgrade, String reason, Instant at) { Objects.requireNonNull(version, "version cannot be null"); - Objects.requireNonNull(revision, "revision cannot be null"); + Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null"); Objects.requireNonNull(reason, "Reason cannot be null"); Objects.requireNonNull(at, "at cannot be null"); this.id = id; this.version = version; - this.revision = revision; + this.applicationVersion = applicationVersion; this.upgrade = upgrade; this.reason = reason; this.at = at; @@ -197,8 +198,8 @@ public class JobStatus { /** Returns the Vespa version used on this run */ public Version version() { return version; } - /** Returns the application revision used for this run, or empty when not known */ - public Optional<ApplicationRevision> revision() { return revision; } + /** Returns the application version used for this run, or empty when not known */ + public Optional<ApplicationVersion> applicationVersion() { return applicationVersion; } /** Returns a human-readable reason for this particular job run */ public String reason() { return reason; } @@ -206,12 +207,12 @@ public class JobStatus { /** Returns the time if this triggering or completion */ public Instant at() { return at; } - // TODO: Consider a version and revision for each JobStatus, to compare against a Target (instead of Change, which is, really, a Target). + // TODO: Consider a version and application version for each JobStatus, to compare against a Target (instead of Change, which is, really, a Target). /** Returns whether the job last completed for the given change */ public boolean lastCompletedWas(Change change) { if (change instanceof Change.ApplicationChange) { Change.ApplicationChange applicationChange = (Change.ApplicationChange) change; - return revision().equals(applicationChange.revision()); + return applicationVersion().equals(applicationChange.version()); } else if (change instanceof Change.VersionChange) { Change.VersionChange versionChange = (Change.VersionChange) change; return version().equals(versionChange.version()); @@ -221,7 +222,7 @@ public class JobStatus { @Override public int hashCode() { - return Objects.hash(version, revision, upgrade, at); + return Objects.hash(version, applicationVersion, upgrade, at); } @Override @@ -231,14 +232,14 @@ public class JobStatus { JobRun jobRun = (JobRun) o; return id == jobRun.id && Objects.equals(version, jobRun.version) && - Objects.equals(revision, jobRun.revision) && + Objects.equals(applicationVersion, jobRun.applicationVersion) && upgrade == jobRun.upgrade && Objects.equals(at, jobRun.at); } @Override public String toString() { return "job run " + id + " of version " + (upgrade() ? "upgrade " : "") + version + " " - + revision + " at " + at; } + + applicationVersion + " at " + at; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 7908f9b095a..09768796445 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Change.VersionChange; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -79,15 +80,25 @@ public class DeploymentTrigger { public void triggerFromCompletion(JobReport report) { applications().lockOrThrow(report.applicationId(), application -> { application = application.withJobCompletion(report, clock.instant(), controller); + application = application.withProjectId(report.projectId()); // Handle successful starting and ending if (report.success()) { if (report.jobType() == JobType.component) { - if (acceptNewRevisionNow(application)) { + if (acceptNewApplicationVersionNow(application)) { // Set this as the change we are doing, unless we are already pushing a platform change if ( ! ( application.deploying().isPresent() && - (application.deploying().get() instanceof Change.VersionChange))) - application = application.withDeploying(Optional.of(Change.ApplicationChange.unknown())); + application.deploying().get() instanceof Change.VersionChange)) { + Change.ApplicationChange applicationChange = Change.ApplicationChange.unknown(); + // TODO: Remove guard when source is always reported by component + if (report.sourceRevision().isPresent()) { + applicationChange = Change.ApplicationChange.of( + ApplicationVersion.from(report.sourceRevision().get(), + report.buildNumber()) + ); + } + application = application.withDeploying(Optional.of(applicationChange)); + } } else { // postpone applications().store(application.withOutstandingChange(true)); @@ -133,11 +144,11 @@ public class DeploymentTrigger { if (change instanceof VersionChange) { if (((VersionChange)change).version().isAfter(deployment.version())) return false; // later is ok } - else if (((Change.ApplicationChange)change).revision().isPresent()) { - if ( ! ((Change.ApplicationChange)change).revision().get().equals(deployment.revision())) return false; + else if (((Change.ApplicationChange)change).version().isPresent()) { + if ( ! ((Change.ApplicationChange)change).version().get().equals(deployment.applicationVersion())) return false; } else { - return false; // If we don't yet know the revision we are changing to, then we are not complete + return false; // If we don't yet know the application version we are deploying, then we are not complete } } @@ -203,8 +214,8 @@ public class DeploymentTrigger { } /** - * Returns true if the previous job has completed successfully with a revision and/or version which is - * newer (different) than the one last completed successfully in next + * Returns true if the previous job has completed successfully with a application version and/or Vespa version + * which is newer (different) than the one last completed successfully in next */ private boolean changesAvailable(Application application, JobStatus previous, JobStatus next) { if ( ! application.deploying().isPresent()) return false; @@ -242,11 +253,11 @@ public class DeploymentTrigger { return true; } - else { // revision changes do not need to handle downgrading + else { // Application version changes do not need to handle downgrading if ( ! previous.lastSuccess().isPresent()) return false; if ( ! next.lastSuccess().isPresent()) return true; - return previous.lastSuccess().get().revision().isPresent() && - ! previous.lastSuccess().get().revision().equals(next.lastSuccess().get().revision()); + return previous.lastSuccess().get().applicationVersion().isPresent() && + ! previous.lastSuccess().get().applicationVersion().equals(next.lastSuccess().get().applicationVersion()); } } @@ -357,7 +368,7 @@ public class DeploymentTrigger { application.deploying(), clock.instant(), application.deployVersionFor(jobType, controller), - application.deployRevisionFor(jobType, controller), + application.deployApplicationVersion(jobType, controller), reason); } @@ -406,7 +417,7 @@ public class DeploymentTrigger { .orElse(false); } - private boolean acceptNewRevisionNow(LockedApplication application) { + private boolean acceptNewApplicationVersionNow(LockedApplication application) { if ( ! application.deploying().isPresent()) return true; if (application.deploying().get() instanceof Change.ApplicationChange) return true; // more changes are ok @@ -415,7 +426,8 @@ public class DeploymentTrigger { if (application.isBlocked(clock.instant())) return true; // allow testing changes while upgrade blocked (debatable) - // Otherwise, the application is currently upgrading, without failures, and we should wait with the revision. + // Otherwise, the application is currently upgrading, without failures, and we should wait with the new + // application version. return false; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java index f165b4e4ea3..314f52ca775 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java @@ -1,19 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.Change; import java.time.Duration; /** - * Deploys application changes which have not made it to production because of a revision change block. + * Trigger ready deployment jobs. This drives jobs through each application's deployment pipeline. * * @author bratseth */ -@SuppressWarnings("unused") public class ReadyJobsTrigger extends Maintainer { public ReadyJobsTrigger(Controller controller, Duration interval, JobControl jobControl) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 9c77ebc4bc3..1514d98610a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; -import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -143,7 +143,7 @@ public class ApplicationSerializer { zoneIdToSlime(deployment.zone(), object.setObject(zoneField)); object.setString(versionField, deployment.version().toString()); object.setLong(deployTimeField, deployment.at().toEpochMilli()); - toSlime(deployment.revision(), object.setObject(applicationPackageRevisionField)); + toSlime(deployment.applicationVersion(), object.setObject(applicationPackageRevisionField)); clusterInfoToSlime(deployment.clusterInfo(), object); clusterUtilsToSlime(deployment.clusterUtils(), object); metricsToSlime(deployment.metrics(), object); @@ -197,10 +197,10 @@ public class ApplicationSerializer { object.setString(regionField, zone.region().value()); } - private void toSlime(ApplicationRevision applicationRevision, Cursor object) { - object.setString(applicationPackageHashField, applicationRevision.id()); - if (applicationRevision.source().isPresent()) - toSlime(applicationRevision.source().get(), object.setObject(sourceRevisionField)); + private void toSlime(ApplicationVersion applicationVersion, Cursor object) { + object.setString(applicationPackageHashField, applicationVersion.id()); + if (applicationVersion.source().isPresent()) + toSlime(applicationVersion.source().get(), object.setObject(sourceRevisionField)); } private void toSlime(SourceRevision sourceRevision, Cursor object) { @@ -236,8 +236,8 @@ public class ApplicationSerializer { Cursor object = parent.setObject(jobRunObjectName); object.setLong(jobRunIdField, jobRun.get().id()); object.setString(versionField, jobRun.get().version().toString()); - if ( jobRun.get().revision().isPresent()) - toSlime(jobRun.get().revision().get(), object.setObject(revisionField)); + if ( jobRun.get().applicationVersion().isPresent()) + toSlime(jobRun.get().applicationVersion().get(), object.setObject(revisionField)); object.setBool(upgradeField, jobRun.get().upgrade()); object.setString(reasonField, jobRun.get().reason()); object.setLong(atField, jobRun.get().at().toEpochMilli()); @@ -249,8 +249,8 @@ public class ApplicationSerializer { Cursor object = parentObject.setObject(deployingField); if (deploying.get() instanceof Change.VersionChange) object.setString(versionField, ((Change.VersionChange)deploying.get()).version().toString()); - else if (((Change.ApplicationChange)deploying.get()).revision().isPresent()) - toSlime(((Change.ApplicationChange)deploying.get()).revision().get(), object); + else if (((Change.ApplicationChange)deploying.get()).version().isPresent()) + toSlime(((Change.ApplicationChange)deploying.get()).version().get(), object); } // ------------------ Deserialization @@ -282,7 +282,7 @@ public class ApplicationSerializer { private Deployment deploymentFromSlime(Inspector deploymentObject) { return new Deployment(zoneIdFromSlime(deploymentObject.field(zoneField)), - applicationRevisionFromSlime(deploymentObject.field(applicationPackageRevisionField)).get(), + applicationVersionFromSlime(deploymentObject.field(applicationPackageRevisionField)).get(), Version.fromString(deploymentObject.field(versionField).asString()), Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()), clusterUtilsMapFromSlime(deploymentObject.field(clusterUtilsField)), @@ -340,12 +340,12 @@ public class ApplicationSerializer { return ZoneId.from(object.field(environmentField).asString(), object.field(regionField).asString()); } - private Optional<ApplicationRevision> applicationRevisionFromSlime(Inspector object) { + private Optional<ApplicationVersion> applicationVersionFromSlime(Inspector object) { if ( ! object.valid()) return Optional.empty(); String applicationPackageHash = object.field(applicationPackageHashField).asString(); Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); - return sourceRevision.isPresent() ? Optional.of(ApplicationRevision.from(applicationPackageHash, sourceRevision.get())) - : Optional.of(ApplicationRevision.from(applicationPackageHash)); + return sourceRevision.isPresent() ? Optional.of(ApplicationVersion.from(applicationPackageHash, sourceRevision.get())) + : Optional.of(ApplicationVersion.from(applicationPackageHash)); } private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) { @@ -369,7 +369,7 @@ public class ApplicationSerializer { if (versionFieldValue.valid()) return Optional.of(new Change.VersionChange(Version.fromString(versionFieldValue.asString()))); else if (object.field(applicationPackageHashField).valid()) - return Optional.of(Change.ApplicationChange.of(applicationRevisionFromSlime(object).get())); + return Optional.of(Change.ApplicationChange.of(applicationVersionFromSlime(object).get())); else return Optional.of(Change.ApplicationChange.unknown()); } @@ -398,7 +398,7 @@ public class ApplicationSerializer { if ( ! object.valid()) return Optional.empty(); return Optional.of(new JobStatus.JobRun(optionalLong(object.field(jobRunIdField)).orElse(-1L), // TODO: Make non-optional after November 2017 -- what about lastTriggered? new Version(object.field(versionField).asString()), - applicationRevisionFromSlime(object.field(revisionField)), + applicationVersionFromSlime(object.field(revisionField)), object.field(upgradeField).asBool(), optionalString(object.field(reasonField)).orElse(""), // TODO: Make non-optional after November 2017 Instant.ofEpochMilli(object.field(atField).asLong()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index dc816d70b7f..a489e1d9f63 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -54,7 +54,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterCost; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -352,8 +352,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Cursor deployingObject = object.setObject("deploying"); if (application.deploying().get() instanceof Change.VersionChange) deployingObject.setString("version", ((Change.VersionChange)application.deploying().get()).version().toString()); - else if (((Change.ApplicationChange)application.deploying().get()).revision().isPresent()) - toSlime(((Change.ApplicationChange)application.deploying().get()).revision().get(), deployingObject.setObject("revision")); + else if (((Change.ApplicationChange)application.deploying().get()).version().isPresent()) + toSlime(((Change.ApplicationChange)application.deploying().get()).version().get(), deployingObject.setObject("revision")); } // Jobs sorted according to deployment spec @@ -453,14 +453,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); response.setString("version", deployment.version().toFullString()); - response.setString("revision", deployment.revision().id()); + response.setString("revision", deployment.applicationVersion().id()); response.setLong("deployTimeEpochMs", deployment.at().toEpochMilli()); controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zoneId()) .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", deployment.at().plus(deploymentTimeToLive).toEpochMilli())); controller.applications().get(deploymentId.applicationId()).flatMap(application -> application.deploymentJobs().projectId()) .ifPresent(i -> response.setString("screwdriverId", String.valueOf(i))); - sourceRevisionToSlime(deployment.revision().source(), response); + sourceRevisionToSlime(deployment.applicationVersion().source(), response); // Cost DeploymentCost appCost = deployment.calculateCost(); @@ -477,10 +477,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { metricsObject.setDouble("writeLatencyMillis", metrics.writeLatencyMillis()); } - private void toSlime(ApplicationRevision revision, Cursor object) { - object.setString("hash", revision.id()); - if (revision.source().isPresent()) - sourceRevisionToSlime(revision.source(), object.setObject("source")); + private void toSlime(ApplicationVersion applicationVersion, Cursor object) { + object.setString("hash", applicationVersion.id()); + if (applicationVersion.source().isPresent()) + sourceRevisionToSlime(applicationVersion.source(), object.setObject("source")); } private void sourceRevisionToSlime(Optional<SourceRevision> revision, Cursor object) { @@ -775,12 +775,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Map<String, byte[]> dataParts = new MultipartParser().parse(request); if ( ! dataParts.containsKey("deployOptions")) return ErrorResponse.badRequest("Missing required form part 'deployOptions'"); - if ( ! dataParts.containsKey("applicationZip")) - return ErrorResponse.badRequest("Missing required form part 'applicationZip'"); Inspector deployOptions = SlimeUtils.jsonToSlime(dataParts.get("deployOptions")).get(); - ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get("applicationZip")); + Optional<ApplicationPackage> applicationPackage = Optional.ofNullable(dataParts.get("applicationZip")) + .map(ApplicationPackage::new); DeployAuthorizer deployAuthorizer = new DeployAuthorizer(controller.zoneRegistry(), athenzClientFactory); Tenant tenant = controller.tenants().tenant(new TenantId(tenantName)).orElseThrow(() -> new NotExistsException(new TenantId(tenantName))); Principal principal = authorizer.getPrincipal(request); @@ -791,12 +790,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { optional("vespaVersion", deployOptions).map(Version::new), deployOptions.field("ignoreValidationErrors").asBool(), deployOptions.field("deployCurrentVersion").asBool()); - controller.applications().validate(applicationPackage.deploymentSpec()); ActivateResult result = controller.applications().deployApplication(applicationId, zone, applicationPackage, deployOptionsJsonClass); - return new SlimeJsonResponse(toSlime(result, dataParts.get("applicationZip").length)); + return new SlimeJsonResponse(toSlime(result)); } private HttpResponse deleteTenant(String tenantName, HttpRequest request) { @@ -972,7 +970,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private void toSlime(JobStatus.JobRun jobRun, Cursor object) { object.setLong("id", jobRun.id()); object.setString("version", jobRun.version().toFullString()); - jobRun.revision().ifPresent(revision -> toSlime(revision, object.setObject("revision"))); + jobRun.applicationVersion().ifPresent(applicationVersion -> toSlime(applicationVersion, + object.setObject("revision"))); object.setString("reason", jobRun.reason()); object.setLong("at", jobRun.at().toEpochMilli()); } @@ -1027,14 +1026,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { "/application/" + application.id().application().value(), request.getUri()).toString()); } - private Slime toSlime(ActivateResult result, long applicationZipSizeBytes) { + private Slime toSlime(ActivateResult result) { Slime slime = new Slime(); Cursor object = slime.setObject(); - object.setString("revisionId", result.getRevisionId().id()); - object.setLong("applicationZipSize", applicationZipSizeBytes); + object.setString("revisionId", result.revisionId().id()); + object.setLong("applicationZipSize", result.applicationZipSizeBytes()); Cursor logArray = object.setArray("prepareMessages"); - if (result.getPrepareResponse().log != null) { - for (Log logMessage : result.getPrepareResponse().log) { + if (result.prepareResponse().log != null) { + for (Log logMessage : result.prepareResponse().log) { Cursor logObject = logArray.addObject(); logObject.setLong("time", logMessage.time); logObject.setString("level", logMessage.level); @@ -1045,7 +1044,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Cursor changeObject = object.setObject("configChangeActions"); Cursor restartActionsArray = changeObject.setArray("restart"); - for (RestartAction restartAction : result.getPrepareResponse().configChangeActions.restartActions) { + for (RestartAction restartAction : result.prepareResponse().configChangeActions.restartActions) { Cursor restartActionObject = restartActionsArray.addObject(); restartActionObject.setString("clusterName", restartAction.clusterName); restartActionObject.setString("clusterType", restartAction.clusterType); @@ -1055,7 +1054,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } Cursor refeedActionsArray = changeObject.setArray("refeed"); - for (RefeedAction refeedAction : result.getPrepareResponse().configChangeActions.refeedActions) { + for (RefeedAction refeedAction : result.prepareResponse().configChangeActions.refeedActions) { Cursor refeedActionObject = refeedActionsArray.addObject(); refeedActionObject.setString("name", refeedAction.name); refeedActionObject.setBool("allowed", refeedAction.allowed); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java index 323da24b47d..ee8deef7256 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.athenz.api.AthenzDomain; @@ -16,6 +17,7 @@ import javax.ws.rs.ForbiddenException; import javax.ws.rs.NotAuthorizedException; import java.security.Principal; import java.util.Objects; +import java.util.Optional; import java.util.logging.Logger; import static com.yahoo.vespa.hosted.controller.api.integration.athenz.HostedAthenzIdentities.SCREWDRIVER_DOMAIN; @@ -41,9 +43,10 @@ public class DeployAuthorizer { Environment environment, Tenant tenant, ApplicationId applicationId, - ApplicationPackage applicationPackage) { + Optional<ApplicationPackage> applicationPackage) { // Validate that domain in identity configuration (deployment.xml) is same as tenant domain - applicationPackage.deploymentSpec().athenzDomain().ifPresent(identityDomain -> { + applicationPackage.map(ApplicationPackage::deploymentSpec).flatMap(DeploymentSpec::athenzDomain) + .ifPresent(identityDomain -> { AthenzDomain tenantDomain = tenant.getAthensDomain().orElseThrow(() -> new IllegalArgumentException("Identity provider only available to Athenz onboarded tenants")); if (! Objects.equals(tenantDomain.getName(), identityDomain.value())) { throw new ForbiddenException( diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ArtifactRepositoryMock.java new file mode 100644 index 00000000000..efbc10e8deb --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ArtifactRepositoryMock.java @@ -0,0 +1,39 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * @author mpolden + */ +public class ArtifactRepositoryMock implements ArtifactRepository { + + private final Map<Integer, byte[]> repository = new HashMap<>(); + + public ArtifactRepositoryMock put(ApplicationId applicationId, ApplicationPackage applicationPackage, + String applicationVersion) { + repository.put(artifactHash(applicationId, applicationVersion), applicationPackage.zippedContent()); + return this; + } + + @Override + public byte[] getApplicationPackage(ApplicationId applicationId, String applicationVersion) { + int artifactHash = artifactHash(applicationId, applicationVersion); + if (!repository.containsKey(artifactHash)) { + throw new IllegalArgumentException("No application package found for " + applicationId + " with version " + + applicationVersion); + } + return repository.get(artifactHash); + } + + private static int artifactHash(ApplicationId applicationId, String applicationVersion) { + return Objects.hash(applicationId, applicationVersion); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index ceed52d2dad..1127e739689 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -10,11 +10,11 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; -import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; @@ -26,12 +26,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildSystem; @@ -100,21 +101,21 @@ public class ControllerTest { Version version1 = Version.fromString("6.1"); // Set in config server mock Application app1 = tester.createApplication("app1", "tenant1", 1, 11L); tester.notifyJobCompletion(component, app1, true); - assertFalse("Revision is currently not known", - ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision().isPresent()); + assertFalse("Application version is currently not known", + ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).version().isPresent()); tester.deployAndNotify(app1, applicationPackage, true, systemTest); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); - Optional<ApplicationRevision> revision = ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision(); - assertTrue("Revision has been set during deployment", revision.isPresent()); + Optional<ApplicationVersion> applicationVersion = ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).version(); + assertTrue("Application version has been set during deployment", applicationVersion.isPresent()); assertStatus(JobStatus.initial(stagingTest) - .withTriggering(version1, revision, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) + .withTriggering(version1, applicationVersion, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); // Causes first deployment job to be triggered assertStatus(JobStatus.initial(productionCorpUsEast1) - .withTriggering(version1, revision, false, "", tester.clock().instant()), app1.id(), tester.controller()); + .withTriggering(version1, applicationVersion, false, "", tester.clock().instant()), app1.id(), tester.controller()); tester.clock().advance(Duration.ofSeconds(1)); // production job (failing) @@ -122,9 +123,9 @@ public class ControllerTest { assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); JobStatus expectedJobStatus = JobStatus.initial(productionCorpUsEast1) - .withTriggering(version1, revision, false, "", tester.clock().instant()) // Triggered first without revision info + .withTriggering(version1, applicationVersion, false, "", tester.clock().instant()) // Triggered first without application version info .withCompletion(42, Optional.of(JobError.unknown), tester.clock().instant(), tester.controller()) - .withTriggering(version1, revision, false, "", tester.clock().instant()); // Re-triggering (due to failure) has revision info + .withTriggering(version1, applicationVersion, false, "", tester.clock().instant()); // Re-triggering (due to failure) has application version info assertStatus(expectedJobStatus, app1.id(), tester.controller()); @@ -147,20 +148,20 @@ public class ControllerTest { tester.notifyJobCompletion(component, app1, true); tester.deployAndNotify(app1, applicationPackage, true, false, systemTest); assertStatus(JobStatus.initial(systemTest) - .withTriggering(version1, revision, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) + .withTriggering(version1, applicationVersion, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); // production job succeeding now tester.deployAndNotify(app1, applicationPackage, true, productionCorpUsEast1); expectedJobStatus = expectedJobStatus - .withTriggering(version1, revision, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) + .withTriggering(version1, applicationVersion, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()); assertStatus(expectedJobStatus, app1.id(), tester.controller()); // causes triggering of next production job assertStatus(JobStatus.initial(productionUsEast3) - .withTriggering(version1, revision, false, "", tester.clock().instant()), + .withTriggering(version1, applicationVersion, false, "", tester.clock().instant()), app1.id(), tester.controller()); tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); @@ -198,6 +199,132 @@ public class ControllerTest { applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(SystemName.main).get())); assertNull("Deployment job was removed", applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1)); } + + // TODO: Replace above test with this one after introducing new application version number + @Test + public void testDeploymentWithApplicationVersion() { + // Setup system + DeploymentTester tester = new DeploymentTester(); + ApplicationController applications = tester.controller().applications(); + Version version1 = Version.fromString("6.1"); // Set in config server mock + Application app1 = tester.createApplication("app1", "tenant1", 1, 11L); + + // Component runs, uploads artifact and notifies completion + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .region("us-east-3") + .build(); + SourceRevision source = new SourceRevision("repo", "branch", "deadbeef"); + String expectedVersionString = "1.0.37-deadbeef"; + tester.artifactRepository().put(app1.id(), applicationPackage, expectedVersionString); + tester.notifyJobCompletion(component, app1, Optional.empty(), Optional.of(source), 37); + ApplicationVersion expectedVersion = ApplicationVersion.from(source, 37); + assertEquals(expectedVersionString, ((Change.ApplicationChange) tester.controller().applications() + .require(app1.id()) + .deploying() + .get()).version().get().id()); + + // Deploy without application package + tester.deployAndNotify(app1, true, systemTest); + tester.deployAndNotify(app1, true, stagingTest); + assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); + assertStatus(JobStatus.initial(stagingTest) + .withTriggering(version1, Optional.of(expectedVersion), false, "", tester.clock().instant().minus(Duration.ofMillis(1))) + .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); + + // Causes first deployment job to be triggered + assertStatus(JobStatus.initial(productionCorpUsEast1) + .withTriggering(version1, Optional.of(expectedVersion), false, "", tester.clock().instant()), app1.id(), tester.controller()); + tester.clock().advance(Duration.ofSeconds(1)); + + // production job (failing) + tester.deployAndNotify(app1, false, productionCorpUsEast1); + assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); + + JobStatus expectedJobStatus = JobStatus.initial(productionCorpUsEast1) + .withTriggering(version1, Optional.of(expectedVersion), false, "", tester.clock().instant()) + .withCompletion(42, Optional.of(JobError.unknown), tester.clock().instant(), tester.controller()); + + assertStatus(expectedJobStatus, app1.id(), tester.controller()); + + // Simulate restart + tester.restartController(); + applications = tester.controller().applications(); + + assertNotNull(tester.controller().tenants().tenant(new TenantId("tenant1"))); + assertNotNull(applications.get(ApplicationId.from(TenantName.from("tenant1"), + ApplicationName.from("application1"), + InstanceName.from("default")))); + assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); + + + tester.clock().advance(Duration.ofHours(1)); + + tester.notifyJobCompletion(productionCorpUsEast1, app1, false); // Need to complete the job, or new jobs won't start. + + // Component is triggered again + tester.artifactRepository().put(app1.id(), applicationPackage, "1.0.38-deadbeef"); + tester.notifyJobCompletion(component, app1, Optional.empty(), Optional.of(source), 38); + tester.deployAndNotify(app1, Optional.empty(), true, false, systemTest); + expectedVersion = ApplicationVersion.from(source, 38); + assertStatus(JobStatus.initial(systemTest) + .withTriggering(version1, Optional.of(expectedVersion), false, "", tester.clock().instant().minus(Duration.ofMillis(1))) + .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); + tester.deployAndNotify(app1, Optional.empty(), true, true, stagingTest); + + // production job succeeding now + tester.deployAndNotify(app1, Optional.empty(), true, true, productionCorpUsEast1); + expectedJobStatus = expectedJobStatus + .withTriggering(version1, Optional.of(expectedVersion), false, "", tester.clock().instant().minus(Duration.ofMillis(1))) + .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()); + assertStatus(expectedJobStatus, app1.id(), tester.controller()); + + // causes triggering of next production job + assertStatus(JobStatus.initial(productionUsEast3) + .withTriggering(version1, Optional.of(expectedVersion), false, "", tester.clock().instant()), + app1.id(), tester.controller()); + tester.deployAndNotify(app1, Optional.empty(), true, true, productionUsEast3); + + assertEquals(5, applications.get(app1.id()).get().deploymentJobs().jobStatus().size()); + + // prod zone removal is not allowed + applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-east-3") + .build(); + tester.artifactRepository().put(app1.id(), applicationPackage, "1.0.56-cafed00d"); + source = new SourceRevision("repo", "branch", "cafed00d"); + tester.notifyJobCompletion(component, app1, Optional.empty(), Optional.of(source), 56); + try { + tester.deploy(systemTest, app1, Optional.empty(), false); + fail("Expected exception due to unallowed production deployment removal"); + } + catch (IllegalArgumentException e) { + assertEquals("deployment-removal: application 'tenant1.app1' is deployed in corp-us-east-1, but does not include this zone in deployment.xml", e.getMessage()); + } + assertNotNull("Zone was not removed", + applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(SystemName.main).get())); + JobStatus jobStatus = applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1); + assertNotNull("Deployment job was not removed", jobStatus); + assertEquals(42, jobStatus.lastCompleted().get().id()); + assertEquals("staging-test completed", jobStatus.lastCompleted().get().reason()); + + // prod zone removal is allowed with override + applicationPackage = new ApplicationPackageBuilder() + .allow(ValidationId.deploymentRemoval) + .upgradePolicy("default") + .environment(Environment.prod) + .region("us-east-3") + .build(); + tester.artifactRepository().put(app1.id(), applicationPackage, "1.0.103-c00ffefe"); + source = new SourceRevision("repo", "branch", "c00ffefe"); + tester.notifyJobCompletion(component, app1, Optional.empty(), Optional.of(source), 103); + tester.deployAndNotify(app1, Optional.empty(), true, true, systemTest); + assertNull("Zone was removed", + applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(SystemName.main).get())); + assertNull("Deployment job was removed", applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1)); + } @Test public void testDeployVersion() { @@ -557,6 +684,7 @@ public class ControllerTest { public void testCleanupOfStaleDeploymentData() throws IOException { DeploymentTester tester = new DeploymentTester(); tester.controllerTester().zoneRegistry().setSystem(SystemName.cd); + tester.controllerTester().zoneRegistry().setZones(ZoneId.from("prod", "cd-us-central-1")); Supplier<Map<JobType, JobStatus>> statuses = () -> tester.application(ApplicationId.from("vespa", "canary", "default")).deploymentJobs().jobStatus(); @@ -764,6 +892,7 @@ public class ControllerTest { public void testDeployWithoutProjectId() { DeploymentTester tester = new DeploymentTester(); tester.controllerTester().zoneRegistry().setSystem(SystemName.cd); + tester.controllerTester().zoneRegistry().setZones(ZoneId.from("prod", "cd-us-central-1")); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .region("cd-us-central-1") @@ -777,7 +906,7 @@ public class ControllerTest { // Same options as used in our integration tests DeployOptions options = new DeployOptions(Optional.empty(), Optional.empty(), false, false); - tester.controller().applications().deployApplication(app.id(), zone, applicationPackage, options); + tester.controller().applications().deployApplication(app.id(), zone, Optional.of(applicationPackage), options); assertTrue("Application deployed and activated", tester.controllerTester().configServer().activated().getOrDefault(app.id(), false)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index b1486c8ec00..3b574ac606b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.slime.Slime; import com.yahoo.test.ManualClock; @@ -64,30 +65,32 @@ public final class ControllerTester { private final CuratorDb curator; private final MemoryNameService nameService; private final RotationsConfig rotationsConfig; + private final ArtifactRepositoryMock artifactRepository; private Controller controller; public ControllerTester() { this(new MemoryControllerDb(), new AthenzDbMock(), new ManualClock(), new ConfigServerClientMock(), new ZoneRegistryMock(), new GitHubMock(), new MockCuratorDb(), defaultRotationsConfig(), - new MemoryNameService()); + new MemoryNameService(), new ArtifactRepositoryMock()); } public ControllerTester(ManualClock clock) { this(new MemoryControllerDb(), new AthenzDbMock(), clock, new ConfigServerClientMock(), new ZoneRegistryMock(), new GitHubMock(), new MockCuratorDb(), defaultRotationsConfig(), - new MemoryNameService()); + new MemoryNameService(), new ArtifactRepositoryMock()); } public ControllerTester(RotationsConfig rotationsConfig) { this(new MemoryControllerDb(), new AthenzDbMock(), new ManualClock(), new ConfigServerClientMock(), - new ZoneRegistryMock(), new GitHubMock(), new MockCuratorDb(), rotationsConfig, new MemoryNameService()); + new ZoneRegistryMock(), new GitHubMock(), new MockCuratorDb(), rotationsConfig, new MemoryNameService(), + new ArtifactRepositoryMock()); } private ControllerTester(ControllerDb db, AthenzDbMock athenzDb, ManualClock clock, ConfigServerClientMock configServer, ZoneRegistryMock zoneRegistry, GitHubMock gitHub, CuratorDb curator, RotationsConfig rotationsConfig, - MemoryNameService nameService) { + MemoryNameService nameService, ArtifactRepositoryMock artifactRepository) { this.db = db; this.athenzDb = athenzDb; this.clock = clock; @@ -97,8 +100,9 @@ public final class ControllerTester { this.curator = curator; this.nameService = nameService; this.rotationsConfig = rotationsConfig; + this.artifactRepository = artifactRepository; this.controller = createController(db, curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, - athenzDb, nameService); + athenzDb, nameService, artifactRepository); } public Controller controller() { return controller; } @@ -117,10 +121,12 @@ public final class ControllerTester { public GitHubMock gitHub() { return gitHub; } + public ArtifactRepositoryMock artifactRepository() { return artifactRepository; } + /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */ public final void createNewController() { controller = createController(db, curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, athenzDb, - nameService); + nameService, artifactRepository); } /** Creates the given tenant and application and deploys it */ @@ -209,6 +215,10 @@ public final class ControllerTester { } public void deploy(Application application, ZoneId zone, ApplicationPackage applicationPackage, boolean deployCurrentVersion) { + deploy(application, zone, Optional.of(applicationPackage), deployCurrentVersion); + } + + public void deploy(Application application, ZoneId zone, Optional<ApplicationPackage> applicationPackage, boolean deployCurrentVersion) { ScrewdriverId app1ScrewdriverId = new ScrewdriverId(String.valueOf(application.deploymentJobs().projectId().get())); GitRevision app1RevisionId = new GitRevision(new GitRepository("repo"), new GitBranch("master"), new GitCommit("commit1")); controller().applications().deployApplication(application.id(), @@ -231,7 +241,8 @@ public final class ControllerTester { private static Controller createController(ControllerDb db, CuratorDb curator, RotationsConfig rotationsConfig, ConfigServerClientMock configServerClientMock, ManualClock clock, GitHubMock gitHubClientMock, ZoneRegistryMock zoneRegistryMock, - AthenzDbMock athensDb, MemoryNameService nameService) { + AthenzDbMock athensDb, MemoryNameService nameService, + ArtifactRepository artifactRepository) { Controller controller = new Controller(db, curator, rotationsConfig, @@ -247,7 +258,8 @@ public final class ControllerTester { new MockRoutingGenerator(), new ChefMock(), clock, - new AthenzClientFactoryMock(athensDb)); + new AthenzClientFactoryMock(athensDb), + artifactRepository); controller.updateVersionStatus(VersionStatus.compute(controller)); return controller; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java index c205357c7ef..63751cfaa98 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneFilterMock; import java.net.URI; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -34,9 +35,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Inject public ZoneRegistryMock() { - this.zones.add(ZoneId.from("prod", "corp-us-east-1")); - this.zones.add(ZoneId.from("prod", "us-east-3")); - this.zones.add(ZoneId.from("prod", "us-west-1")); + zones.add(ZoneId.from("prod", "corp-us-east-1")); + zones.add(ZoneId.from("prod", "us-east-3")); + zones.add(ZoneId.from("prod", "us-west-1")); + zones.add(ZoneId.from("prod", "us-central-1")); + zones.add(ZoneId.from("prod", "eu-west-1")); } public ZoneRegistryMock setDeploymentTimeToLive(ZoneId zone, Duration duration) { @@ -54,6 +57,10 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return this; } + public ZoneRegistryMock setZones(ZoneId... zone) { + return setZones(Arrays.asList(zone)); + } + public ZoneRegistryMock setSystem(SystemName system) { this.system = system; return this; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 3311cffa078..9d5fcb31288 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -71,8 +71,8 @@ public class ApplicationPackageBuilder { return this; } - public ApplicationPackageBuilder blockChange(boolean revision, boolean version, - String daySpec, String hourSpec, String zoneSpec) { + public ApplicationPackageBuilder blockChange(boolean revision, boolean version, String daySpec, String hourSpec, + String zoneSpec) { blockChange.append(" <block-change"); blockChange.append(" revision='").append(revision).append("'"); blockChange.append(" version='").append(version).append("'"); @@ -93,7 +93,8 @@ public class ApplicationPackageBuilder { } public ApplicationPackageBuilder athenzIdentity(AthenzDomain domain, AthenzService service) { - this.athenzIdentityAttributes = String.format("athenz-domain='%s' athenz-service='%s'", domain.value(), service.value()); + this.athenzIdentityAttributes = String.format("athenz-domain='%s' athenz-service='%s'", domain.value(), + service.value()); return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index c9f0c6cba1d..8de32b4b531 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.ArtifactRepositoryMock; import com.yahoo.vespa.hosted.controller.ConfigServerClientMock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; @@ -16,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; @@ -43,6 +45,7 @@ public class DeploymentTester { // Set a long interval so that maintainers never do scheduled runs during tests private static final Duration maintenanceInterval = Duration.ofDays(1); + private static final int defaultBuildNumber = 42; private final ControllerTester tester; private final Upgrader upgrader; @@ -81,6 +84,8 @@ public class DeploymentTester { public ConfigServerClientMock configServer() { return tester.configServer(); } + public ArtifactRepositoryMock artifactRepository() { return tester.artifactRepository(); } + public Application application(String name) { return application(ApplicationId.from("tenant1", name, "default")); } @@ -154,16 +159,18 @@ public class DeploymentTester { } public static DeploymentJobs.JobReport jobReport(Application application, JobType jobType, boolean success) { - return jobReport(application, jobType, Optional.ofNullable(success ? null : unknown)); + return jobReport(application, jobType, Optional.ofNullable(success ? null : unknown), Optional.empty(), defaultBuildNumber); } - public static DeploymentJobs.JobReport jobReport(Application application, JobType jobType, Optional<DeploymentJobs.JobError> jobError) { + public static DeploymentJobs.JobReport jobReport(Application application, JobType jobType, + Optional<DeploymentJobs.JobError> jobError, + Optional<SourceRevision> sourceRevision, long buildNumber) { return new DeploymentJobs.JobReport( application.id(), jobType, application.deploymentJobs().projectId().get(), - 42, - Optional.empty(), + buildNumber, + sourceRevision, jobError ); } @@ -204,8 +211,13 @@ public class DeploymentTester { } public void notifyJobCompletion(JobType jobType, Application application, Optional<DeploymentJobs.JobError> jobError) { + notifyJobCompletion(jobType, application, jobError, Optional.empty(), defaultBuildNumber); + } + + public void notifyJobCompletion(JobType jobType, Application application, Optional<DeploymentJobs.JobError> jobError, + Optional<SourceRevision> source, long buildNumber) { clock().advance(Duration.ofMillis(1)); - applications().notifyJobCompletion(jobReport(application, jobType, jobError)); + applications().notifyJobCompletion(jobReport(application, jobType, jobError, source, buildNumber)); } public void completeUpgrade(Application application, Version version, String upgradePolicy) { @@ -233,11 +245,22 @@ public class DeploymentTester { } public void deploy(JobType job, Application application, ApplicationPackage applicationPackage) { - deploy(job, application, applicationPackage, false); + deploy(job, application, Optional.of(applicationPackage), false); + } + + public void deploy(JobType job, Application application, ApplicationPackage applicationPackage, + boolean deployCurrentVersion) { + deploy(job, application, Optional.of(applicationPackage), deployCurrentVersion); } - public void deploy(JobType job, Application application, ApplicationPackage applicationPackage, boolean deployCurrentVersion) { - job.zone(controller().system()).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion)); + public void deploy(JobType job, Application application, Optional<ApplicationPackage> applicationPackage, + boolean deployCurrentVersion) { + job.zone(controller().system()).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, + deployCurrentVersion)); + } + + public void deployAndNotify(Application application, boolean success, JobType... job) { + deployAndNotify(application, Optional.empty(), success, true, job); } public void deployAndNotify(Application application, String upgradePolicy, boolean success, JobType... jobs) { @@ -251,10 +274,15 @@ public class DeploymentTester { public void deployAndNotify(Application application, ApplicationPackage applicationPackage, boolean success, boolean expectOnlyTheseJobs, JobType... jobs) { + deployAndNotify(application, Optional.of(applicationPackage), success, expectOnlyTheseJobs, jobs); + } + + public void deployAndNotify(Application application, Optional<ApplicationPackage> applicationPackage, + boolean success, boolean expectOnlyTheseJobs, JobType... jobs) { consumeJobs(application, expectOnlyTheseJobs, jobs); for (JobType job : jobs) { if (success) { - deploy(job, application, applicationPackage); + deploy(job, application, applicationPackage, false); } notifyJobCompletion(job, application, success); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index b71a9090c79..74d03240ec3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -247,7 +247,7 @@ public class DeploymentTriggerTest { .region("corp-us-east-1") .region("us-central-1") .region("us-west-1") - .region("ap-northeast-1") + .region("eu-west-1") .build(); // Component job finishes @@ -260,7 +260,7 @@ public class DeploymentTriggerTest { tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionCorpUsEast1); tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionUsCentral1); tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionUsWest1); - tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionApNortheast1); + tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionEuWest1); assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); } @@ -277,7 +277,7 @@ public class DeploymentTriggerTest { ApplicationPackageBuilder applicationPackageBuilder = new ApplicationPackageBuilder() .upgradePolicy("canary") - // Block revision changes on tuesday in hours 18 and 19 + // Block application version changes on tuesday in hours 18 and 19 .blockChange(true, false, "tue", "18-19", "UTC") .region("us-west-1") .region("us-central-1") diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index d48f7b84ee6..513e5520d85 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import org.junit.Before; import org.junit.Test; -import java.io.IOException; import java.time.Duration; import java.util.List; import java.util.stream.Collectors; @@ -33,7 +32,7 @@ public class DeploymentExpirerTest { } @Test - public void testDeploymentExpiry() throws IOException, InterruptedException { + public void testDeploymentExpiry() { tester.controllerTester().zoneRegistry().setDeploymentTimeToLive( ZoneId.from(Environment.dev, RegionName.from("us-east-1")), Duration.ofDays(14) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index 62e6d379c60..b200e2d7e18 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -178,6 +179,7 @@ public class FailureRedeployerTest { public void retryIgnoresStaleJobData() throws Exception { DeploymentTester tester = new DeploymentTester(); tester.controllerTester().zoneRegistry().setSystem(SystemName.cd); + tester.controllerTester().zoneRegistry().setZones(ZoneId.from("prod", "cd-us-central-1")); // Current system version, matches version in test data Version version = Version.fromString("6.141.117"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index d7389ca94cd..f42a4c1deb3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -6,14 +6,14 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; -import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -26,7 +26,6 @@ import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import org.junit.Test; -import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; @@ -60,10 +59,11 @@ public class ApplicationSerializerTest { "</validation-overrides>"); List<Deployment> deployments = new ArrayList<>(); - ApplicationRevision revision1 = ApplicationRevision.from("appHash1"); - ApplicationRevision revision2 = ApplicationRevision.from("appHash2", new SourceRevision("repo1", "branch1", "commit1")); - deployments.add(new Deployment(zone1, revision1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils - deployments.add(new Deployment(zone2, revision2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), + ApplicationVersion applicationVersion1 = ApplicationVersion.from("appHash1"); + ApplicationVersion applicationVersion2 = ApplicationVersion + .from("appHash2", new SourceRevision("repo1", "branch1", "commit1")); + deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils + deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), createClusterUtils(3, 0.2), createClusterInfo(3, 4),new DeploymentMetrics(2,3,4,5,6))); Optional<Long> projectId = Optional.of(123L); @@ -96,8 +96,8 @@ public class ApplicationSerializerTest { assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm()); assertEquals(2, serialized.deployments().size()); - assertEquals(original.deployments().get(zone1).revision(), serialized.deployments().get(zone1).revision()); - assertEquals(original.deployments().get(zone2).revision(), serialized.deployments().get(zone2).revision()); + assertEquals(original.deployments().get(zone1).applicationVersion(), serialized.deployments().get(zone1).applicationVersion()); + assertEquals(original.deployments().get(zone2).applicationVersion(), serialized.deployments().get(zone2).applicationVersion()); assertEquals(original.deployments().get(zone1).version(), serialized.deployments().get(zone1).version()); assertEquals(original.deployments().get(zone2).version(), serialized.deployments().get(zone2).version()); assertEquals(original.deployments().get(zone1).at(), serialized.deployments().get(zone1).at()); @@ -145,18 +145,20 @@ public class ApplicationSerializerTest { assertEquals(6, serialized.deployments().get(zone2).metrics().writeLatencyMillis(), Double.MIN_VALUE); { // test more deployment serialization cases - Application original2 = writable(original).withDeploying(Optional.of(Change.ApplicationChange.of(ApplicationRevision.from("hash1")))); + Application original2 = writable(original).withDeploying(Optional.of(Change.ApplicationChange.of(ApplicationVersion + .from("hash1")))); Application serialized2 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original2)); assertEquals(original2.deploying(), serialized2.deploying()); - assertEquals(((Change.ApplicationChange)serialized2.deploying().get()).revision().get().source(), - ((Change.ApplicationChange)original2.deploying().get()).revision().get().source()); + assertEquals(((Change.ApplicationChange)serialized2.deploying().get()).version().get().source(), + ((Change.ApplicationChange)original2.deploying().get()).version().get().source()); - Application original3 = writable(original).withDeploying(Optional.of(Change.ApplicationChange.of(ApplicationRevision.from("hash1", - new SourceRevision("a", "b", "c"))))); + Application original3 = writable(original).withDeploying(Optional.of(Change.ApplicationChange.of(ApplicationVersion + .from("hash1", + new SourceRevision("a", "b", "c"))))); Application serialized3 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original3)); assertEquals(original3.deploying(), serialized2.deploying()); - assertEquals(((Change.ApplicationChange)serialized3.deploying().get()).revision().get().source(), - ((Change.ApplicationChange)original3.deploying().get()).revision().get().source()); + assertEquals(((Change.ApplicationChange)serialized3.deploying().get()).version().get().source(), + ((Change.ApplicationChange)original3.deploying().get()).version().get().source()); Application original4 = writable(original).withDeploying(Optional.empty()); Application serialized4 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original4)); @@ -195,7 +197,7 @@ public class ApplicationSerializerTest { } @Test - public void testLegacySerialization() throws IOException { + public void testLegacySerialization() { Application applicationWithSuccessfulJob = applicationSerializer.fromSlime(applicationSlime(false)); assertFalse("No job error for successful job", applicationWithSuccessfulJob.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest).jobError().isPresent()); @@ -211,7 +213,7 @@ public class ApplicationSerializerTest { @Test public void testCompleteApplicationDeserialization() { - Application application = applicationSerializer.fromSlime(SlimeUtils.jsonToSlime(longApplicationJson.getBytes(StandardCharsets.UTF_8))); + applicationSerializer.fromSlime(SlimeUtils.jsonToSlime(longApplicationJson.getBytes(StandardCharsets.UTF_8))); // ok if no error } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index fc0147dacef..5b806d580e2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -81,7 +81,7 @@ public class ContainerControllerTester { GitRevision app1RevisionId = new GitRevision(new GitRepository("repo"), new GitBranch("master"), new GitCommit("commit1")); controller().applications().deployApplication(application.id(), zone, - applicationPackage, + Optional.of(applicationPackage), new DeployOptions(Optional.of(new ScrewdriverBuildJob(app1ScrewdriverId, app1RevisionId)), Optional.empty(), false, false)); return application; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 028992e8f7d..abc5f9f8aa1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -67,6 +67,7 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.persistence.MemoryControllerDb'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.restapi.application.MockAuthorizer'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.routing.MockRoutingGenerator'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.ArtifactRepositoryMock'/>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + " <binding>http://*/application/v4/*</binding>\n" + " </handler>\n" + |