diff options
14 files changed, 59 insertions, 58 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java index dd9f8c38802..d072ad595c6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java @@ -21,7 +21,7 @@ import java.util.Optional; public interface ApplicationStore { /** Returns the tenant application package of the given version. */ - byte[] get(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion); + byte[] get(DeploymentId deploymentId, ApplicationVersion applicationVersion); /** Find application package by given build number */ Optional<byte[]> find(TenantName tenant, ApplicationName application, long buildNumber); @@ -48,10 +48,7 @@ public interface ApplicationStore { void removeAllTesters(TenantName tenant, ApplicationName application); /** Stores the given application package as the development package for the given application and zone. */ - void putDev(ApplicationId application, ZoneId zone, byte[] applicationPackage); - - /** Returns the development package for the given application and zone. */ - byte[] getDev(ApplicationId application, ZoneId zone); + void putDev(DeploymentId deploymentId, byte[] applicationPackage); /** Stores the given application meta data with the current time as part of the path. */ void putMeta(TenantName tenant, ApplicationName application, Instant now, byte[] metaZip); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java index a9a3e080109..76d7bdc4f12 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java @@ -89,15 +89,12 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { /** Returns an unique identifier for this version or "unknown" if version is not known */ public String id() { - if (isUnknown()) { - return "unknown"; - } - return String.format("%s.%d-%s", - majorVersion, - buildNumber.getAsLong(), - source.map(SourceRevision::commit).map(ApplicationVersion::abbreviateCommit) - .or(this::commit) - .orElse("unknown")); + if (isUnknown()) return "unknown"; + + return source.map(SourceRevision::commit).map(ApplicationVersion::abbreviateCommit) + .or(this::commit) + .map(commit -> String.format("%s.%d-%s", majorVersion, buildNumber.getAsLong(), commit)) + .orElseGet(() -> majorVersion + "." + buildNumber.getAsLong()); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 6557247e21a..00fb494d779 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.aws.TenantRoles; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; @@ -332,11 +331,6 @@ public class ApplicationController { }); } - /** Fetches the requested application package from the artifact store(s). */ - public ApplicationPackage getApplicationPackage(ApplicationId id, ApplicationVersion version) { - return new ApplicationPackage(applicationStore.get(id.tenant(), id.application(), version)); - } - /** Returns given application with a new instance */ public LockedApplication withNewInstance(LockedApplication application, ApplicationId instance) { if (instance.instance().isTester()) @@ -372,7 +366,7 @@ public class ApplicationController { Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform()); ApplicationVersion revision = run.versions().sourceApplication().filter(__ -> deploySourceVersions).orElse(run.versions().targetApplication()); - ApplicationPackage applicationPackage = getApplicationPackage(job.application(), zone, revision); + ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(new DeploymentId(job.application(), zone), revision)); try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); @@ -828,11 +822,6 @@ public class ApplicationController { return DeploymentQuotaCalculator.calculateQuotaUsage(application); } - private ApplicationPackage getApplicationPackage(ApplicationId application, ZoneId zone, ApplicationVersion revision) { - return new ApplicationPackage(revision.isUnknown() ? applicationStore.getDev(application, zone) - : applicationStore.get(application.tenant(), application.application(), revision)); - } - /* * Get the AthenzUser from this principal or Optional.empty if this does not represent a user. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index bef6e88c2de..aab193c1a0c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -478,16 +478,21 @@ public class JobController { controller.applications().store(application); }); - last(id, type).filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); + Optional<Run> lastRun = last(id, type); + lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); + + long build = 1 + lastRun.map(run -> run.versions().targetApplication().buildNumber().orElse(0)).orElse(0L); + ApplicationVersion version = ApplicationVersion.from(Optional.empty(), build, Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty(), Optional.empty(), true); controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - controller.applications().applicationStore().putDev(id, type.zone(controller.system()), applicationPackage.zippedContent()); + controller.applications().applicationStore().putDev(new DeploymentId(id, type.zone(controller.system())), applicationPackage.zippedContent()); start(id, type, new Versions(platform.orElse(applicationPackage.deploymentSpec().majorVersion() .flatMap(controller.applications()::lastCompatibleVersion) .orElseGet(controller::readSystemVersion)), - ApplicationVersion.unknown, + version, Optional.empty(), Optional.empty()), false, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index a69af024b96..0039ce2320e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -1,13 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.yolean.Exceptions; @@ -32,7 +33,8 @@ public class DeploymentUpgrader extends ControllerMaintainer { protected double maintain() { AtomicInteger attempts = new AtomicInteger(); AtomicInteger failures = new AtomicInteger(); - Versions target = new Versions(controller().readSystemVersion(), ApplicationVersion.unknown, Optional.empty(), Optional.empty()); + Version systemVersion = controller().readSystemVersion(); + for (Application application : controller().applications().readable()) for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) @@ -40,8 +42,11 @@ public class DeploymentUpgrader extends ControllerMaintainer { attempts.incrementAndGet(); JobId job = new JobId(instance.id(), JobType.from(controller().system(), deployment.zone()).get()); if ( ! deployment.zone().environment().isManuallyDeployed()) continue; + + Run last = controller().jobController().last(job).get(); + Versions target = new Versions(systemVersion, last.versions().targetApplication(), Optional.empty(), Optional.empty()); if ( ! deployment.version().isBefore(target.targetPlatform())) continue; - if ( controller().clock().instant().isBefore(controller().jobController().last(job).get().start().plus(Duration.ofDays(1)))) continue; + if ( controller().clock().instant().isBefore(last.start().plus(Duration.ofDays(1)))) continue; if ( ! isLikelyNightFor(job)) continue; log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 65a9e2ae282..0449ebfd348 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -592,13 +592,13 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { throw new IllegalArgumentException("Only manually deployed zones have dev packages"); ZoneId zone = type.zone(controller.system()); - byte[] applicationPackage = controller.applications().applicationStore().getDev(id, zone); + ApplicationVersion version = controller.jobController().last(id, type).get().versions().targetApplication(); + byte[] applicationPackage = controller.applications().applicationStore().get(new DeploymentId(id, zone), version); return new ZipResponse(id.toFullString() + "." + zone.value() + ".zip", applicationPackage); } private HttpResponse applicationPackage(String tenantName, String applicationName, HttpRequest request) { var tenantAndApplication = TenantAndApplicationId.from(tenantName, applicationName); - var applicationId = ApplicationId.from(tenantName, applicationName, InstanceName.defaultName().value()); long buildNumber; var requestedBuild = Optional.ofNullable(request.getProperty("build")).map(build -> { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index d4c9425fc03..3e3c06be0ab 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -61,7 +61,7 @@ import static org.junit.Assert.assertTrue; */ public class DeploymentTriggerTest { - private DeploymentTester tester = new DeploymentTester(); + private final DeploymentTester tester = new DeploymentTester(); @Test public void testTriggerFailing() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java index 59e2b6c04d8..7db7268e4bf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java @@ -5,11 +5,11 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import java.time.Instant; import java.util.Map; @@ -30,7 +30,7 @@ public class ApplicationStoreMock implements ApplicationStore { private static final byte[] tombstone = new byte[0]; private final Map<ApplicationId, Map<ApplicationVersion, byte[]>> store = new ConcurrentHashMap<>(); - private final Map<ApplicationId, Map<ZoneId, byte[]>> devStore = new ConcurrentHashMap<>(); + private final Map<DeploymentId, byte[]> devStore = new ConcurrentHashMap<>(); private final Map<ApplicationId, NavigableMap<Instant, byte[]>> meta = new ConcurrentHashMap<>(); private final Map<DeploymentId, NavigableMap<Instant, byte[]>> metaManual = new ConcurrentHashMap<>(); @@ -43,10 +43,14 @@ public class ApplicationStoreMock implements ApplicationStore { } @Override - public byte[] get(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion) { - byte[] bytes = store.get(appId(tenant, application)).get(applicationVersion); + public byte[] get(DeploymentId deploymentId, ApplicationVersion applicationVersion) { + if (applicationVersion.isDeployedDirectly()) + return requireNonNull(devStore.get(deploymentId)); + + TenantAndApplicationId tenantAndApplicationId = TenantAndApplicationId.from(deploymentId.applicationId()); + byte[] bytes = store.get(appId(tenantAndApplicationId.tenant(), tenantAndApplicationId.application())).get(applicationVersion); if (bytes == null) - throw new IllegalArgumentException("No application package found for " + tenant + "." + application + + throw new IllegalArgumentException("No application package found for " + tenantAndApplicationId + " with version " + applicationVersion.id()); return bytes; } @@ -61,8 +65,8 @@ public class ApplicationStoreMock implements ApplicationStore { @Override public void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage) { - store.putIfAbsent(appId(tenant, application), new ConcurrentHashMap<>()); - store.get(appId(tenant, application)).put(applicationVersion, applicationPackage); + store.computeIfAbsent(appId(tenant, application), __ -> new ConcurrentHashMap<>()) + .put(applicationVersion, applicationPackage); } @Override @@ -83,8 +87,8 @@ public class ApplicationStoreMock implements ApplicationStore { @Override public void putTester(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] testerPackage) { - store.putIfAbsent(testerId(tenant, application), new ConcurrentHashMap<>()); - store.get(testerId(tenant, application)).put(applicationVersion, testerPackage); + store.computeIfAbsent(testerId(tenant, application), key -> new ConcurrentHashMap<>()) + .put(applicationVersion, testerPackage); } @Override @@ -99,14 +103,8 @@ public class ApplicationStoreMock implements ApplicationStore { } @Override - public void putDev(ApplicationId application, ZoneId zone, byte[] applicationPackage) { - devStore.putIfAbsent(application, new ConcurrentHashMap<>()); - devStore.get(application).put(zone, applicationPackage); - } - - @Override - public byte[] getDev(ApplicationId application, ZoneId zone) { - return requireNonNull(devStore.get(application).get(zone)); + public void putDev(DeploymentId deploymentId, byte[] applicationPackage) { + devStore.put(deploymentId, applicationPackage); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 2813bd0ab7d..abe3d4100d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -502,7 +502,9 @@ "status": "success", "versions": { "targetPlatform": "6.1.0", - "targetApplication": {} + "targetApplication": { + "build": 1 + } }, "steps": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json index acde58f2a28..dce73ad56cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json @@ -8,7 +8,9 @@ "status": "success", "versions": { "targetPlatform": "7.1.0", - "targetApplication": {} + "targetApplication": { + "build": 1 + } }, "steps": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json index e3beb371acd..92a823bdfc2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -5,7 +5,9 @@ "runs": [ { "versions": { - "targetApplication": {}, + "targetApplication": { + "build": 1 + }, "targetPlatform": "6.1.0" }, "start": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json index 72411d155c7..3ef993c6589 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json @@ -6,7 +6,7 @@ { "at": 0, "type": "info", - "message": "Deploying platform version 6.1 and application version unknown ..." + "message": "Deploying platform version 6.1 and application version 1.0.1 ..." }, { "at": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 9a742a9b176..7ebc2d24fe9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -11,7 +11,9 @@ "status": "success", "versions": { "targetPlatform": "6.1.0", - "targetApplication": {} + "targetApplication": { + "build": 1 + } }, "steps": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index 2601937faee..f8aba54356b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -5,7 +5,9 @@ "runs": [ { "versions": { - "targetApplication": {}, + "targetApplication": { + "build": 1 + }, "targetPlatform": "7.1.0" }, "start": 14503000, |