diff options
Diffstat (limited to 'controller-server')
22 files changed, 160 insertions, 59 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java index 40e2e4a92d1..a0b4a888727 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java @@ -1,14 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; import org.apache.commons.codec.digest.DigestUtils; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; +import java.time.Instant; import java.util.Objects; import java.util.Optional; @@ -27,10 +32,12 @@ public class ApplicationPackage { private final byte[] zippedContent; private final DeploymentSpec deploymentSpec; private final ValidationOverrides validationOverrides; + private final Optional<Version> compileVersion; + private final Optional<Instant> buildTime; /** * Creates an application package from its zipped content. - * This <b>assigns ownership</b> of the given byte array to this class: + * This <b>assigns ownership</b> of the given byte array to this class; * it must not be further changed by the caller. */ public ApplicationPackage(byte[] zippedContent) { @@ -38,6 +45,10 @@ public class ApplicationPackage { this.contentHash = DigestUtils.shaHex(zippedContent); this.deploymentSpec = extractFile("deployment.xml", zippedContent).map(DeploymentSpec::fromXml).orElse(DeploymentSpec.empty); this.validationOverrides = extractFile("validation-overrides.xml", zippedContent).map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); + Optional<Inspector> buildMetaObject = extractFileBytes("build-meta.json", zippedContent) + .map(SlimeUtils::jsonToSlime).map(Slime::get); + this.compileVersion = buildMetaObject.map(object -> Version.fromString(object.field("compileVersion").asString())); + this.buildTime = buildMetaObject.map(object -> Instant.ofEpochMilli(object.field("buildTime").asLong())); } /** Returns a hash of the content of this package */ @@ -58,12 +69,18 @@ public class ApplicationPackage { */ public ValidationOverrides validationOverrides() { return validationOverrides; } - private static Optional<Reader> extractFile(String fileName, byte[] zippedContent) { + /** Returns the platform version which package was compiled against, if known. */ + public Optional<Version> compileVersion() { return compileVersion; } + + /** Returns the time this package was built, if known. */ + public Optional<Instant> buildTime() { return buildTime; } + + private static Optional<byte[]> extractFileBytes(String fileName, byte[] zippedContent) { try (ByteArrayInputStream stream = new ByteArrayInputStream(zippedContent)) { ZipStreamReader reader = new ZipStreamReader(stream); for (ZipStreamReader.ZipEntryWithContent entry : reader.entries()) if (entry.zipEntry().getName().equals(fileName) || entry.zipEntry().getName().equals("application/" + fileName)) // TODO: Remove application/ directory support - return Optional.of(new InputStreamReader(new ByteArrayInputStream(entry.content()))); + return Optional.of(entry.content()); return Optional.empty(); } catch (IOException e) { @@ -71,4 +88,8 @@ public class ApplicationPackage { } } + private static Optional<Reader> extractFile(String fileName, byte[] zippedContent) { + return extractFileBytes(fileName, zippedContent).map(ByteArrayInputStream::new).map(InputStreamReader::new); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 14222ca96d2..81a4e30feac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -504,7 +504,7 @@ public class InternalStepRunner implements StepRunner { DeploymentSpec spec = controller.applications().require(id.application()).deploymentSpec(); ZoneId zone = id.type().zone(controller.system()); - byte[] deploymentXml = deploymentXml(spec.athenzDomain().get(), spec.athenzService(zone.environment(), zone.region()).get()); + byte[] deploymentXml = deploymentXml(spec.athenzDomain(), spec.athenzService(zone.environment(), zone.region())); try (ZipBuilder zipBuilder = new ZipBuilder(testPackage.length + servicesXml.length + 1000)) { zipBuilder.add(testPackage); @@ -591,11 +591,14 @@ public class InternalStepRunner implements StepRunner { return servicesXml.getBytes(); } - /** Returns a dummy deployment xml which sets up the service identity for the tester. */ - static byte[] deploymentXml(AthenzDomain domain, AthenzService service) { + /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */ + private static byte[] deploymentXml(Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) { String deploymentSpec = "<?xml version='1.0' encoding='UTF-8'?>\n" + - "<deployment version=\"1.0\" athenz-domain=\"" + domain.value() + "\" athenz-service=\"" + service.value() + "\" />"; + "<deployment version=\"1.0\" " + + athenzDomain.map(domain -> "athenz-domain=\"" + domain.value() + "\" ").orElse("") + + athenzService.map(service -> "athenz-service=\"" + service.value() + "\" ").orElse("") + + "/>"; return deploymentSpec.getBytes(StandardCharsets.UTF_8); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 5c56c7d2280..fcead86893a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -10,25 +10,23 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.net.URI; -import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.Deque; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -41,7 +39,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.logging.Level; -import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; @@ -229,7 +226,7 @@ public class JobController { * Accepts and stores a new application package and test jar pair under a generated application version key. */ public ApplicationVersion submit(ApplicationId id, SourceRevision revision, String authorEmail, long projectId, - byte[] packageBytes, byte[] testPackageBytes) { + ApplicationPackage applicationPackage, byte[] testPackageBytes) { AtomicReference<ApplicationVersion> version = new AtomicReference<>(); controller.applications().lockOrThrow(id, application -> { if ( ! application.get().deploymentJobs().deployedInternally()) { @@ -245,17 +242,22 @@ public class JobController { } long run = nextBuild(id); - version.set(ApplicationVersion.from(revision, run, authorEmail)); + if (applicationPackage.compileVersion().isPresent() && applicationPackage.buildTime().isPresent()) + version.set(ApplicationVersion.from(revision, run, authorEmail, + applicationPackage.compileVersion().get(), + applicationPackage.buildTime().get())); + else + version.set(ApplicationVersion.from(revision, run, authorEmail)); controller.applications().applicationStore().put(id, version.get(), - packageBytes); + applicationPackage.zippedContent()); controller.applications().applicationStore().put(TesterId.of(id), version.get(), testPackageBytes); prunePackages(id); - controller.applications().storeWithUpdatedConfig(application.withBuiltInternally(true), new ApplicationPackage(packageBytes)); + controller.applications().storeWithUpdatedConfig(application.withBuiltInternally(true), applicationPackage); controller.applications().deploymentTrigger().notifyOfCompletion(DeploymentJobs.JobReport.ofSubmission(id, projectId, version.get())); }); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 7924ea34475..08e5be2ea1f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -72,7 +72,7 @@ public class ControllerMaintenance extends AbstractComponent { readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofMinutes(1), jobControl); clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepositoryClient); clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl); - deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl); + deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(5), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService); systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 305241a7d0e..2ae38fabb94 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.chef.AttributeMapping; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode; import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; @@ -37,6 +38,7 @@ public class MetricsReporter extends Maintainer { public static final String deploymentFailMetric = "deployment.failurePercentage"; public static final String deploymentAverageDuration = "deployment.averageDuration"; public static final String deploymentFailingUpgrades = "deployment.failingUpgrades"; + public static final String deploymentBuildAgeSeconds = "deployment.buildAgeSeconds"; public static final String remainingRotations = "remaining_rotations"; private final Metric metric; @@ -128,6 +130,14 @@ public class MetricsReporter extends Maintainer { deploymentsFailingUpgrade(applications).forEach((application, failingJobs) -> { metric.set(deploymentFailingUpgrades, failingJobs, metric.createContext(dimensions(application))); }); + + for (Application application : applications.asList()) + application.deploymentJobs().statusOf(JobType.component) + .flatMap(JobStatus::lastSuccess) + .flatMap(run -> run.application().buildTime()) + .ifPresent(buildTime -> metric.set(deploymentBuildAgeSeconds, + controller().clock().instant().getEpochSecond() - buildTime.getEpochSecond(), + metric.createContext(dimensions(application.id())))); } private static double deploymentFailRatio(ApplicationList applicationList) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 69ea36c7e3a..147b2edee3e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -81,6 +81,8 @@ public class ApplicationSerializer { private final String branchField = "branchField"; private final String commitField = "commitField"; private final String authorEmailField = "authorEmailField"; + private final String compileVersionField = "compileVersion"; + private final String buildTimeField = "buildTime"; private final String lastQueriedField = "lastQueried"; private final String lastWrittenField = "lastWritten"; private final String lastQueriesPerSecondField = "lastQueriesPerSecond"; @@ -231,6 +233,8 @@ public class ApplicationSerializer { object.setLong(applicationBuildNumberField, applicationVersion.buildNumber().getAsLong()); toSlime(applicationVersion.source().get(), object.setObject(sourceRevisionField)); applicationVersion.authorEmail().ifPresent(email -> object.setString(authorEmailField, email)); + applicationVersion.compileVersion().ifPresent(version -> object.setString(compileVersionField, version.toString())); + applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli())); } } @@ -406,12 +410,21 @@ public class ApplicationSerializer { if ( ! object.valid()) return ApplicationVersion.unknown; OptionalLong applicationBuildNumber = optionalLong(object.field(applicationBuildNumberField)); Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); - if (!sourceRevision.isPresent() || !applicationBuildNumber.isPresent()) { + if ( ! sourceRevision.isPresent() || ! applicationBuildNumber.isPresent()) { return ApplicationVersion.unknown; } - return object.field(authorEmailField).valid() - ? ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong(), object.field(authorEmailField).asString()) - : ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong()); + Optional<String> authorEmail = optionalString(object.field(authorEmailField)); + Optional<Version> compileVersion = optionalString(object.field(compileVersionField)).map(Version::fromString); + Optional<Instant> buildTime = optionalInstant(object.field(buildTimeField)); + + if ( ! authorEmail.isPresent()) + return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong()); + + if ( ! compileVersion.isPresent() || ! buildTime.isPresent()) + return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong(), authorEmail.get()); + + return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong(), authorEmail.get(), + compileVersion.get(), buildTime.get()); } private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index ef91a2e5a15..32c92e6f135 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -68,6 +68,8 @@ class RunSerializer { private static final String branchField = "branch"; private static final String commitField = "commit"; private static final String authorEmailField = "authorEmail"; + private static final String compileVersionField = "compileVersion"; + private static final String buildTimeField = "buildTime"; private static final String buildField = "build"; private static final String sourceField = "source"; private static final String lastTestRecordField = "lastTestRecord"; @@ -123,9 +125,16 @@ class RunSerializer { versionObject.field(branchField).asString(), versionObject.field(commitField).asString()); long buildNumber = versionObject.field(buildField).asLong(); - return versionObject.field(authorEmailField).valid() - ? ApplicationVersion.from(revision, buildNumber, versionObject.field(authorEmailField).asString()) - : ApplicationVersion.from(revision, buildNumber); + + if ( ! versionObject.field(authorEmailField).valid()) + return ApplicationVersion.from(revision, buildNumber); + + if ( ! versionObject.field(compileVersionField).valid() || ! versionObject.field(buildTimeField).valid()) + return ApplicationVersion.from(revision, buildNumber, versionObject.field(authorEmailField).asString()); + + return ApplicationVersion.from(revision, buildNumber, versionObject.field(authorEmailField).asString(), + Version.fromString(versionObject.field(compileVersionField).asString()), + Instant.ofEpochMilli(versionObject.field(buildTimeField).asLong())); } Slime toSlime(Iterable<Run> runs) { @@ -173,6 +182,8 @@ class RunSerializer { versionsObject.setLong(buildField, applicationVersion.buildNumber() .orElseThrow(() -> new IllegalArgumentException("Build number must be present in application version."))); applicationVersion.authorEmail().ifPresent(email -> versionsObject.setString(authorEmailField, email)); + applicationVersion.compileVersion().ifPresent(version -> versionsObject.setString(compileVersionField, version.toString())); + applicationVersion.buildTime().ifPresent(time -> versionsObject.setLong(buildTimeField, time.toEpochMilli())); } static String valueOf(Step step) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 12f5fa88af9..ba74ae18fda 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -801,7 +801,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { StringBuilder response = new StringBuilder(); controller.applications().lockOrThrow(id, application -> { Change change = application.get().change(); - if ( ! change.isPresent()) { + if ( ! change.isPresent() && ! change.isPinned()) { response.append("No deployment in progress for " + application + " at this time"); return; } @@ -1298,9 +1298,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { long projectId = Math.max(1, submitOptions.field("projectId").asLong()); ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP)); - if ( ! applicationPackage.deploymentSpec().athenzDomain().isPresent()) - throw new IllegalArgumentException("Application must define an Athenz service in deployment.xml!"); - controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant), applicationPackage, Optional.of(getUserPrincipal(request).getIdentity())); + controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant), + applicationPackage, + Optional.of(getUserPrincipal(request).getIdentity())); return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, @@ -1308,7 +1308,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { sourceRevision, authorEmail, projectId, - applicationPackage.zippedContent(), + applicationPackage, dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index a3dc3f1e706..7d49ffe8139 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobStatus; @@ -27,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Versions; +import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -398,19 +400,16 @@ class JobControllerApiHandlerHelper { * @return Response with the new application version */ static HttpResponse submitResponse(JobController jobController, String tenant, String application, - SourceRevision sourceRevision, String authorEmail, - long projectId, byte[] appPackage, byte[] testPackage) { + SourceRevision sourceRevision, String authorEmail, long projectId, + ApplicationPackage applicationPackage, byte[] testPackage) { ApplicationVersion version = jobController.submit(ApplicationId.from(tenant, application, "default"), sourceRevision, authorEmail, projectId, - appPackage, + applicationPackage, testPackage); - Slime slime = new Slime(); - Cursor responseObject = slime.setObject(); - responseObject.setString("version", version.id()); - return new SlimeJsonResponse(slime); + return new MessageResponse(version.toString()); } /** Aborts any job of the given type. */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index b7a881c672e..1199f0229b6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -155,7 +155,8 @@ public class ControllerTest { .region("deep-space-9") .build(); try { - tester.controller().jobController().submit(app1.id(), BuildJob.defaultSourceRevision, "a@b", 2, applicationPackage.zippedContent(), new byte[0]); + tester.controller().jobController().submit(app1.id(), BuildJob.defaultSourceRevision, "a@b", + 2, applicationPackage, new byte[0]); fail("Expected exception due to illegal deployment spec."); } catch (IllegalArgumentException e) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 0499a92d05f..44a797687d4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -170,6 +170,10 @@ public class ApplicationPackageBuilder { return searchDefinition.getBytes(StandardCharsets.UTF_8); } + private byte[] buildMeta() { + return "{\"compileVersion\":\"6.1\",\"buildTime\":1000}".getBytes(StandardCharsets.UTF_8); + } + public ApplicationPackage build() { ByteArrayOutputStream zip = new ByteArrayOutputStream(); try (ZipOutputStream out = new ZipOutputStream(zip)) { @@ -182,6 +186,9 @@ public class ApplicationPackageBuilder { out.putNextEntry(new ZipEntry("search-definitions/test.sd")); out.write(searchDefinition()); out.closeEntry(); + out.putNextEntry(new ZipEntry("build-meta.json")); + out.write(buildMeta()); + out.closeEntry(); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java index 80cbbfa2c30..35e7aea5efe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java @@ -89,7 +89,7 @@ public class InternalDeploymentTester { * Submits a new application, and returns the version of the new submission. */ public ApplicationVersion newSubmission() { - ApplicationVersion version = jobs.submit(appId, BuildJob.defaultSourceRevision, "a@b", 2, applicationPackage.zippedContent(), new byte[0]); + ApplicationVersion version = jobs.submit(appId, BuildJob.defaultSourceRevision, "a@b", 2, applicationPackage, new byte[0]); tester.applicationStore().put(appId, version, applicationPackage.zippedContent()); tester.applicationStore().put(testerId, version, new byte[0]); return version; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 7e6e892891a..8ebb8c108c0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.Run; @@ -64,6 +65,7 @@ import static org.junit.Assert.fail; */ public class JobRunnerTest { + private static final ApplicationPackage applicationPackage = new ApplicationPackage(new byte[0]); private static final Versions versions = new Versions(Version.fromString("1.2.3"), ApplicationVersion.from(new SourceRevision("repo", "branch", @@ -82,7 +84,7 @@ public class JobRunnerTest { phasedExecutor(phaser), stepRunner); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, applicationPackage, new byte[0]); jobs.start(id, systemTest, versions); try { @@ -113,7 +115,7 @@ public class JobRunnerTest { inThreadExecutor(), mappedRunner(outcomes)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, applicationPackage, new byte[0]); Supplier<Run> run = () -> jobs.last(id, systemTest).get(); jobs.start(id, systemTest, versions); @@ -197,7 +199,7 @@ public class JobRunnerTest { Executors.newFixedThreadPool(32), waitingRunner(barrier)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, applicationPackage, new byte[0]); RunId runId = new RunId(id, systemTest, 1); jobs.start(id, systemTest, versions); @@ -233,7 +235,7 @@ public class JobRunnerTest { inThreadExecutor(), (id, step) -> Optional.of(running)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, applicationPackage, new byte[0]); for (int i = 0; i < jobs.historyLength(); i++) { jobs.start(id, systemTest, versions); @@ -261,7 +263,7 @@ public class JobRunnerTest { inThreadExecutor(), mappedRunner(outcomes)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, applicationPackage, new byte[0]); jobs.start(id, systemTest, versions); tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 7ab858f3081..9c01e526a50 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -11,9 +11,11 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import com.yahoo.vespa.hosted.controller.integration.MetricsMock.MapContext; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; @@ -204,6 +206,18 @@ public class MetricsReporterTest { assertEquals(0, getDeploymentsFailingUpgrade(app)); } + @Test + public void testBuildTimeReporting() { + InternalDeploymentTester tester = new InternalDeploymentTester(); + ApplicationVersion version = tester.deployNewSubmission(); + assertEquals(1000, version.buildTime().get().toEpochMilli()); + + MetricsReporter reporter = createReporter(tester.tester().controller(), metrics, SystemName.main); + reporter.maintain(); + assertEquals(tester.clock().instant().getEpochSecond() - 1, + getMetric(MetricsReporter.deploymentBuildAgeSeconds, tester.app())); + } + private Duration getAverageDeploymentDuration(Application application) { return Duration.ofSeconds(getMetric(MetricsReporter.deploymentAverageDuration, application).longValue()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 774caea97b0..0b337eb5380 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -10,11 +10,12 @@ import com.yahoo.config.provision.HostName; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -25,7 +26,6 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import org.junit.Test; @@ -70,7 +70,8 @@ public class ApplicationSerializerTest { List<Deployment> deployments = new ArrayList<>(); ApplicationVersion applicationVersion1 = ApplicationVersion.from(new SourceRevision("repo1", "branch1", "commit1"), 31); ApplicationVersion applicationVersion2 = ApplicationVersion - .from(new SourceRevision("repo1", "branch1", "commit1"), 32, "a@b"); + .from(new SourceRevision("repo1", "branch1", "commit1"), 32, "a@b", + Version.fromString("6.3.1"), Instant.ofEpochMilli(496)); Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index d6334a9ea86..03b432588bd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -19,6 +19,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.Optional; @@ -78,7 +79,9 @@ public class RunSerializerTest { "master", "f00bad"), 123, - "a@b"), + "a@b", + Version.fromString("6.3.1"), + Instant.ofEpochMilli(100)), run.versions().targetApplication()); assertEquals(new Version(1, 2, 2), run.versions().sourcePlatform().get()); assertEquals(ApplicationVersion.from(new SourceRevision("git@github.com:user/repo.git", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 75bbea6861d..cda3834d47d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -26,6 +26,8 @@ "commit": "f00bad", "build": 123, "authorEmail": "a@b", + "compileVersion": "6.3.1", + "buildTime": 100, "source": { "platform": "1.2.2", "repository": "git@github.com:user/repo.git", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index d267dabe9a6..cfe2dcba214 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -361,24 +361,40 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE (cancel) ongoing change tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE) .userIdentity(HOSTED_VESPA_OPERATOR), - new File("application-deployment-cancelled.json")); + "{\"message\":\"Changed deployment from 'application change to 1.0.42-commit1' to 'no change' for application 'tenant1.application1'\"}"); // DELETE (cancel) again is a no-op tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE) .userIdentity(USER_ID) .data("{\"cancel\":\"all\"}"), - new File("application-deployment-cancelled-no-op.json")); + "{\"message\":\"No deployment in progress for application 'tenant1.application1' at this time\"}"); // POST pinning to a given version to an application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", POST) .userIdentity(USER_ID) .data("6.1.0"), - new File("application-deployment.json")); + "{\"message\":\"Triggered pin to 6.1 for tenant1.application1\"}"); // DELETE only the pin to a given version tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", DELETE) .userIdentity(USER_ID), - new File("application-pin-cancelled.json")); + "{\"message\":\"Changed deployment from 'pin to 6.1' to 'upgrade to 6.1' for application 'tenant1.application1'\"}"); + + // POST pinning again + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", POST) + .userIdentity(USER_ID) + .data("6.1"), + "{\"message\":\"Triggered pin to 6.1 for tenant1.application1\"}"); + + // DELETE only the version, but leave the pin + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/platform", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"Changed deployment from 'pin to 6.1' to 'pin to current platform' for application 'tenant1.application1'\"}"); + + // DELETE also the pin to a given version + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"Changed deployment from 'pin to current platform' to 'no change' for application 'tenant1.application1'\"}"); // POST a pause to a production job tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1/pause", POST) @@ -435,13 +451,13 @@ public class ApplicationApiTest extends ControllerContainerTest { "Deactivated tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default"); // POST an application package and a test jar, submitting a new application for internal pipeline deployment. - // First attempt does not have an Athenz service definition in deployment spec, and fails. + // First attempt does not have an Athenz service definition in deployment spec, and is accepted. tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(applicationPackage)), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Application must define an Athenz service in deployment.xml!\"}", 400); + "{\"message\":\"Application package version: 1.0.43-d00d, source revision of repository 'repo', branch 'master' with commit 'd00d', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); - // Second attempt has a service under a different domain than the tenant of the application, and fails as well. + // Second attempt has a service under a different domain than the tenant of the application, and fails. ApplicationPackage packageWithServiceForWrongDomain = new ApplicationPackageBuilder() .environment(Environment.prod) .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from(ATHENZ_TENANT_DOMAIN_2.getName()), AthenzService.from("service")) @@ -461,7 +477,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(packageWithService)), - "{\"version\":\"1.0.43-d00d\"}"); + "{\"message\":\"Application package version: 1.0.44-d00d, source revision of repository 'repo', branch 'master' with commit 'd00d', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/jobreport", POST) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled-no-op.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled-no-op.json deleted file mode 100644 index 91d3e64d6db..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled-no-op.json +++ /dev/null @@ -1 +0,0 @@ -{"message":"No deployment in progress for application 'tenant1.application1' at this time"} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json deleted file mode 100644 index efca5831256..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json +++ /dev/null @@ -1 +0,0 @@ -{"message":"Changed deployment from 'application change to 1.0.42-commit1' to 'no change' for application 'tenant1.application1'"} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json deleted file mode 100644 index fe68f3d94a3..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json +++ /dev/null @@ -1 +0,0 @@ -{"message":"Triggered pin to 6.1 for tenant1.application1"}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-pin-cancelled.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-pin-cancelled.json deleted file mode 100644 index 62360458ce4..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-pin-cancelled.json +++ /dev/null @@ -1 +0,0 @@ -{"message":"Changed deployment from 'pin to 6.1' to 'upgrade to 6.1' for application 'tenant1.application1'"} |