diff options
15 files changed, 453 insertions, 206 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 364e91f1828..9ad03522179 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.function.Function; +import java.util.stream.Collector; import java.util.stream.Collectors; /** @@ -98,7 +99,8 @@ public class Application { public Map<Zone, Deployment> productionDeployments() { return deployments.values().stream() .filter(deployment -> deployment.zone().environment() == Environment.prod) - .collect(Collectors.toMap(Deployment::zone, Function.identity())); + .collect(Collectors.collectingAndThen(Collectors.toMap(Deployment::zone, Function.identity()), + ImmutableMap::copyOf)); } public DeploymentJobs deploymentJobs() { return deploymentJobs; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 26debf3083f..66bb076e099 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -303,7 +303,7 @@ public class ApplicationController { else version = application.currentDeployVersion(controller, zone); - DeploymentJobs.JobType jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone); + Optional<DeploymentJobs.JobType> jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone); ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob); if ( ! options.deployCurrentVersion) { @@ -314,11 +314,11 @@ public class ApplicationController { application = application.withProjectId(options.screwdriverBuildJob.get().screwdriverId.value()); if (application.deploying().isPresent() && application.deploying().get() instanceof Change.ApplicationChange) application = application.withDeploying(Optional.of(Change.ApplicationChange.of(revision))); - if ( ! canDeployDirectlyTo(zone, options) && jobType != null) { + if ( ! canDeployDirectlyTo(zone, options) && jobType.isPresent()) { // Update with (potentially) missing information about what we triggered - JobStatus.JobRun triggering = getOrCreateTriggering(application, version, jobType); + JobStatus.JobRun triggering = getOrCreateTriggering(application, version, jobType.get()); application = application.with(application.deploymentJobs() - .withTriggering(jobType, + .withTriggering(jobType.get(), application.deploying(), triggering.id(), version, @@ -423,10 +423,9 @@ public class ApplicationController { * This is needed (only) in the case where some external entity triggers a job. */ private JobStatus.JobRun getOrCreateTriggering(Application application, Version version, DeploymentJobs.JobType jobType) { - if (jobType == null) return incompleteTriggeringEvent(version); JobStatus status = application.deploymentJobs().jobStatus().get(jobType); - if (status == null) return incompleteTriggeringEvent(version); - if ( ! status.lastTriggered().isPresent()) return incompleteTriggeringEvent(version); + if (status == null) return incompleteTriggeringEvent(version); + if ( ! status.lastTriggered().isPresent()) return incompleteTriggeringEvent(version); return status.lastTriggered().get(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 710d2ad6492..ae015c36aac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import java.time.Instant; +import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -23,17 +24,17 @@ public class ApplicationList { private final ImmutableList<Application> list; - private ApplicationList(List<Application> applications) { + private ApplicationList(Iterable<Application> applications) { this.list = ImmutableList.copyOf(applications); } // ----------------------------------- Factories - public static ApplicationList from(List<Application> applications) { + public static ApplicationList from(Iterable<Application> applications) { return new ApplicationList(applications); } - public static ApplicationList from(List<ApplicationId> ids, ApplicationController applications) { + public static ApplicationList from(Collection<ApplicationId> ids, ApplicationController applications) { return listOf(ids.stream().map(applications::require)); } @@ -166,7 +167,7 @@ public class ApplicationList { /** Returns the subset of applications that are not currently upgrading */ public ApplicationList notCurrentlyUpgrading(Change.VersionChange change, Instant jobTimeoutLimit) { - return listOf(list.stream().filter(a -> !currentlyUpgrading(change, a, jobTimeoutLimit))); + return listOf(list.stream().filter(a -> ! currentlyUpgrading(change, a, jobTimeoutLimit))); } // ----------------------------------- Internal helpers @@ -193,56 +194,39 @@ public class ApplicationList { if ( ! application.deploying().isPresent()) return false; return application.deploying().get() instanceof Change.ApplicationChange; } - + private static boolean failingOn(Version version, Application application) { - for (JobStatus jobStatus : application.deploymentJobs().jobStatus().values()) - if ( ! jobStatus.isSuccess() && jobStatus.lastCompleted().get().version().equals(version)) return true; - return false; + return JobList.from(application) + .failing() + .lastCompleted().on(version) + .anyMatch(); } private static boolean currentlyUpgrading(Change.VersionChange change, Application application, Instant jobTimeoutLimit) { - return application.deploymentJobs().jobStatus().values().stream() - .filter(status -> status.isRunning(jobTimeoutLimit)) - .filter(status -> status.lastTriggered().isPresent()) - .map(status -> status.lastTriggered().get()) - .anyMatch(jobRun -> jobRun.version().equals(change.version())); + return JobList.from(application) + .running(jobTimeoutLimit) + .lastTriggered().on(change.version()) + .anyMatch(); } private static boolean failingUpgradeToVersionSince(Application application, Version version, Instant threshold) { - return application.deploymentJobs().jobStatus().values().stream() - .filter(job -> isUpgradeFailure(job)) - .filter(job -> job.firstFailing().get().at().isBefore(threshold)) - .anyMatch(job -> job.lastCompleted().get().version().equals(version)); + return JobList.from(application) + .not().failingApplicationChange() + .firstFailing().before(threshold) + .lastCompleted().on(version) + .anyMatch(); } private static boolean failingApplicationChangeSince(Application application, Instant threshold) { - return application.deploymentJobs().jobStatus().values().stream() - .filter(job -> isApplicationChangeFailure(job)) - .anyMatch(job -> job.firstFailing().get().at().isBefore(threshold)); - } - - private static boolean isUpgradeFailure(JobStatus job) { - if ( job.isSuccess()) return false; - if ( ! job.lastSuccess().isPresent()) return false; // An application which never succeeded is surely bad. - if ( ! job.lastSuccess().get().revision().isPresent()) return false; // Indicates the component job, which is not an upgrade. - if ( ! job.firstFailing().get().revision().equals(job.lastSuccess().get().revision())) return false; // Application change may be to blame. - return ! job.firstFailing().get().version().equals(job.lastSuccess().get().version()); // Return whether there is a version change. + return JobList.from(application) + .failingApplicationChange() + .firstFailing().before(threshold) + .anyMatch(); } - private static boolean isApplicationChangeFailure(JobStatus job) { - if ( job.isSuccess()) return false; - if ( ! job.lastSuccess().isPresent()) return true; // An application which never succeeded is surely bad. - if ( ! job.lastSuccess().get().revision().isPresent()) return true; // Indicates the component job, which is always an application change. - if ( ! job.firstFailing().get().version().equals(job.lastSuccess().get().version())) return false; // Version change may be to blame. - return ! job.firstFailing().get().revision().equals(job.lastSuccess().get().revision()); // Return whether there is an application change. - } - - /** Convenience converter from a stream to an ApplicationList */ private static ApplicationList listOf(Stream<Application> applications) { - ImmutableList.Builder<Application> b = new ImmutableList.Builder<>(); - applications.forEach(b::add); - return new ApplicationList(b.build()); + return from(applications::iterator); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index 8f654c66871..f94b0f7284b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -13,12 +13,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import java.time.Instant; import java.util.Collection; -import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Information about which deployment jobs an application should run and their current status. @@ -148,13 +150,11 @@ public class DeploymentJobs { /** Returns the oldest failingSince time of the jobs of this, or null if none are failing */ public Instant failingSince() { - Instant failingSince = null; - for (JobStatus jobStatus : jobStatus().values()) { - if (jobStatus.isSuccess()) continue; - if (failingSince == null || failingSince.isAfter(jobStatus.firstFailing().get().at())) - failingSince = jobStatus.firstFailing().get().at(); - } - return failingSince; + return JobList.from(jobStatus().values()) + .failing() + .mapToList(job -> job.firstFailing().get().at()) + .stream() + .min(Comparator.naturalOrder()).orElse(null); } /** @@ -184,18 +184,12 @@ public class DeploymentJobs { productionCdUsCentral2("production-cd-us-central-2", zone(SystemName.cd, "prod", "cd-us-central-2")); private final String id; - private final Map<SystemName, Zone> zones; + private final ImmutableMap<SystemName, Zone> zones; - JobType(String id, Zone... zone) { + JobType(String id, Zone... zones) { this.id = id; - Map<SystemName, Zone> zones = new HashMap<>(); - for (Zone z : zone) { - if (zones.containsKey(z.system())) { - throw new IllegalArgumentException("A job can only map to a single zone per system"); - } - zones.put(z.system(), z); - } - this.zones = Collections.unmodifiableMap(zones); + this.zones = ImmutableMap.copyOf(Stream.of(zones).collect(Collectors.toMap(zone -> zone.system(), + zone -> zone))); } public String id() { return id; } @@ -242,23 +236,23 @@ public class DeploymentJobs { } } - /** Returns the job type for the given zone, or null if none */ - public static JobType from(SystemName system, com.yahoo.config.provision.Zone zone) { + /** Returns the job type for the given zone */ + public static Optional<JobType> from(SystemName system, Zone zone) { for (JobType job : values()) { - Optional<com.yahoo.config.provision.Zone> jobZone = job.zone(system); + Optional<Zone> jobZone = job.zone(system); if (jobZone.isPresent() && jobZone.get().equals(zone)) - return job; + return Optional.of(job); } - return null; + return Optional.empty(); } /** Returns the job job type for the given environment and region or null if none */ - public static JobType from(SystemName system, Environment environment, RegionName region) { + public static Optional<JobType> from(SystemName system, Environment environment, RegionName region) { switch (environment) { - case test: return systemTest; - case staging: return stagingTest; + case test: return Optional.of(systemTest); + case staging: return Optional.of(stagingTest); } - return from(system, new com.yahoo.config.provision.Zone(environment, region)); + return from(system, new Zone(environment, region)); } private static Zone zone(SystemName system, String environment, String region) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java new file mode 100644 index 00000000000..19c7bbe98b4 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java @@ -0,0 +1,179 @@ +package com.yahoo.vespa.hosted.controller.application; + +import com.google.common.collect.ImmutableList; +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; + +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Predicate; + +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity; + +/** + * A list of deployment jobs that can be filtered in various ways. + * + * @author jvenstad + */ +public class JobList { + + private final ImmutableList<JobStatus> list; + private final boolean negate; + + private JobList(Iterable<JobStatus> jobs, boolean negate) { + this.list = ImmutableList.copyOf(jobs); + this.negate = negate; + } + + private JobList(Iterable<JobStatus> jobs) { + this(jobs, false); + } + + // ----------------------------------- Factories + + public static JobList from(Iterable<JobStatus> jobs) { + return new JobList(jobs); + } + + public static JobList from(Application application) { + return from(application.deploymentJobs().jobStatus().values()); + } + + // ----------------------------------- Accessors + + /** Returns the jobstatuses in this as an immutable list */ + public List<JobStatus> asList() { return list; } + + /** Returns the jobstatuses in this as an immutable list after mapping with the given function */ + public <Type> List<Type> mapToList(Function<JobStatus, Type> mapper) { + return ImmutableList.copyOf(list.stream().map(mapper)::iterator); + } + + public boolean isEmpty() { return list.isEmpty(); } + + public boolean anyMatch() { return ! isEmpty(); } + + public int size() { return list.size(); } + + // ----------------------------------- Basic filters + + /** Negates the next filter operation */ + public JobList not() { + return new JobList(list, ! negate); + } + + /** Returns the subset of jobs which are currently running, according to the given timeout */ + public JobList running(Instant timeoutLimit) { + return filter(job -> job.isRunning(timeoutLimit)); + } + + /** Returns the subset of jobs which are currently failing */ + public JobList failing() { + return filter(job -> ! job.isSuccess()); + } + + /** Returns the subset of jobs which must be failing due to an application change */ + public JobList failingApplicationChange() { + return filter(job -> failingApplicationChange(job)); + } + + /** Returns the subset of jobs which are failing with the given job error */ + public JobList failingBecause(DeploymentJobs.JobError error) { + return filter(job -> job.jobError().filter(error::equals).isPresent()); + } + + /** Returns the subset of jobs of the given type -- most useful when negated */ + public JobList type(JobType type) { + return filter(job -> job.type() == type); + } + + /** Returns the subset of jobs of which are production jobs */ + public JobList production() { + return filter(job -> job.type().isProduction()); + } + + // ----------------------------------- JobRun filtering + + /** Returns the list in a state where the next filter is for the lastTriggered run type */ + public JobRunFilter lastTriggered() { + return new JobRunFilter(job -> job.lastTriggered()); + } + + /** Returns the list in a state where the next filter is for the lastCompleted run type */ + public JobRunFilter lastCompleted() { + return new JobRunFilter(job -> job.lastCompleted()); + } + + /** Returns the list in a state where the next filter is for the lastSuccess run type */ + public JobRunFilter lastSuccess() { + return new JobRunFilter(job -> job.lastSuccess()); + } + + /** Returns the list in a state where the next filter is for the firstFailing run type */ + public JobRunFilter firstFailing() { + return new JobRunFilter(job -> job.firstFailing()); + } + + + /** Allows sub-filters for runs of the given kind */ + public class JobRunFilter { + + private final Function<JobStatus, Optional<JobRun>> which; + + private JobRunFilter(Function<JobStatus, Optional<JobRun>> which) { + this.which = which; + } + + /** Returns the subset of jobs where the run of the given type exists */ + public JobList present() { + return filter(run -> true); + } + + /** Returns the subset of jobs where the run of the given type occurred before the given instant */ + public JobList before(Instant threshold) { + return filter(run -> run.at().isBefore(threshold)); + } + + /** Returns the subset of jobs where the run of the given type occurred after the given instant */ + public JobList after(Instant threshold) { + return filter(run -> run.at().isAfter(threshold)); + } + + /** Returns the subset of jobs where the run of the given type was on the given version */ + public JobList on(Version version) { + return filter(run -> run.version().equals(version)); + } + + public JobList upgrade() { + return filter(run -> run.upgrade()); + } + + /** Transforms the JobRun condition to a JobStatus condition, by considering only the JobRun mapped by which, and executes */ + private JobList filter(Predicate<JobRun> condition) { + return JobList.this.filter(job -> which.apply(job).filter(condition).isPresent()); + } + + } + + + // ----------------------------------- Internal helpers + + private static boolean failingApplicationChange(JobStatus job) { + if ( job.isSuccess()) return false; + if ( ! job.lastSuccess().isPresent()) return true; // An application which never succeeded is surely bad. + if ( ! job.lastSuccess().get().revision().isPresent()) return true; // Indicates the component job, which is always an application change. + if ( ! job.firstFailing().get().version().equals(job.lastSuccess().get().version())) return false; // Version change may be to blame. + return ! job.firstFailing().get().revision().equals(job.lastSuccess().get().revision()); // Return whether there is an application change. + } + + /** Returns a new JobList which is the result of filtering with the -- possibly negated -- condition */ + private JobList filter(Predicate<JobStatus> condition) { + return from(list.stream().filter(negate ? condition.negate() : condition)::iterator); + } + +} + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java index bb84c9e17d4..8fb01577e2c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java @@ -162,7 +162,8 @@ public class DeploymentOrder { /** Resolve job from deployment step */ private JobType toJob(DeploymentSpec.DeclaredZone zone) { - return JobType.from(controller.system(), zone.environment(), zone.region().orElse(null)); + return JobType.from(controller.system(), zone.environment(), zone.region().orElse(null)) + .orElseThrow(() -> new IllegalArgumentException("Invalid zone " + zone)); } /** Returns whether deployment should be postponed according to delay */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 81c3bb963db..6b60b49e1ef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; @@ -40,6 +41,7 @@ public class VersionStatusSerializer { private static final String versionField = "version"; private static final String failingField = "failing"; private static final String productionField = "production"; + private static final String deployingField = "deploying"; public Slime toSlime(VersionStatus status) { Slime slime = new Slime(); @@ -74,9 +76,10 @@ public class VersionStatusSerializer { object.setString(versionField, statistics.version().toString()); applicationsToSlime(statistics.failing(), object.setArray(failingField)); applicationsToSlime(statistics.production(), object.setArray(productionField)); + applicationsToSlime(statistics.deploying(), object.setArray(deployingField)); } - private void applicationsToSlime(List<ApplicationId> applications, Cursor array) { + private void applicationsToSlime(Collection<ApplicationId> applications, Cursor array) { applications.forEach(application -> array.addString(application.serializedForm())); } @@ -105,7 +108,8 @@ public class VersionStatusSerializer { private DeploymentStatistics deploymentStatisticsFromSlime(Inspector object) { return new DeploymentStatistics(Version.fromString(object.field(versionField).asString()), applicationsFromSlime(object.field(failingField)), - applicationsFromSlime(object.field(productionField))); + applicationsFromSlime(object.field(productionField)), + applicationsFromSlime(object.field(deployingField))); } private List<ApplicationId> applicationsFromSlime(Inspector array) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 8a5f1e4639a..bcc72245f0b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.deployment; +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; @@ -11,6 +12,8 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.JobList; +import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; @@ -19,15 +22,17 @@ import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse; import com.yahoo.vespa.hosted.controller.restapi.Path; import com.yahoo.yolean.Exceptions; -import java.time.Instant; import java.util.Optional; import java.util.concurrent.Executor; import java.util.logging.Level; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity; +import static java.util.Comparator.comparing; + /** * This implements the deployment/v1 API which provides information about the status of Vespa platform and * application deployments. - * + * * @author bratseth */ public class DeploymentApiHandler extends LoggingRequestHandler { @@ -56,7 +61,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); } } - + private HttpResponse handleGET(HttpRequest request) { Path path = new Path(request.getUri().getPath()); if (path.matches("/deployment/v1/")) return root(request); @@ -70,7 +75,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { response.headers().put("Allow", "GET,OPTIONS"); return response; } - + private HttpResponse root(HttpRequest request) { Slime slime = new Slime(); Cursor root = slime.setObject(); @@ -83,7 +88,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { versionObject.setLong("date", version.releasedAt().toEpochMilli()); versionObject.setBool("controllerVersion", version.isSelfVersion()); versionObject.setBool("systemVersion", version.isCurrentSystemVersion()); - + Cursor configServerArray = versionObject.setArray("configServers"); for (String configServerHostnames : version.configServerHostnames()) { Cursor configServerObject = configServerArray.addObject(); @@ -92,29 +97,42 @@ public class DeploymentApiHandler extends LoggingRequestHandler { Cursor failingArray = versionObject.setArray("failingApplications"); for (ApplicationId id : version.statistics().failing()) { - Optional<Application> application = controller.applications().get(id); - if ( ! application.isPresent()) continue; // deleted just now - - Instant failingSince = application.get().deploymentJobs().failingSince(); - if (failingSince == null) continue; // started working just now - - Cursor applicationObject = failingArray.addObject(); - toSlime(application.get(), applicationObject, request); - applicationObject.setLong("failingSince", failingSince.toEpochMilli()); - + controller.applications().get(id).ifPresent(application -> { + firstFailingOn(version.versionNumber(), application).ifPresent(firstFailing -> { + Cursor applicationObject = failingArray.addObject(); + toSlime(applicationObject, application, request); + applicationObject.setString("failing", firstFailing.type().id()); + }); + }); } Cursor productionArray = versionObject.setArray("productionApplications"); for (ApplicationId id : version.statistics().production()) { - Optional<Application> application = controller.applications().get(id); - if ( ! application.isPresent()) continue; // deleted just now - toSlime(application.get(), productionArray.addObject(), request); + controller.applications().get(id).ifPresent(application -> { + int successes = productionSuccessesFor(version.versionNumber(), application); + if (successes == 0) return; // Just upgraded to a newer version. + Cursor applicationObject = productionArray.addObject(); + toSlime(applicationObject, application, request); + applicationObject.setLong("productionJobs", productionJobsFor(application)); + applicationObject.setLong("productionSuccesses", productionSuccessesFor(version.versionNumber(), application)); + }); + } + + Cursor runningArray = versionObject.setArray("deployingApplications"); + for (ApplicationId id : version.statistics().deploying()) { + controller.applications().get(id).ifPresent(application -> { + lastDeployingTo(version.versionNumber(), application).ifPresent(lastDeploying -> { + Cursor applicationObject = runningArray.addObject(); + toSlime(applicationObject, application, request); + applicationObject.setString("running", lastDeploying.type().id()); + }); + }); } } return new SlimeJsonResponse(slime); } - private void toSlime(Application application, Cursor object, HttpRequest request) { + private void toSlime(Cursor object, Application application, HttpRequest request) { object.setString("tenant", application.id().tenant().value()); object.setString("application", application.id().application().value()); object.setString("instance", application.id().instance().value()); @@ -132,4 +150,41 @@ public class DeploymentApiHandler extends LoggingRequestHandler { return upgradePolicy.name(); } + // ----------------------------- Utilities to pick out the relevant JobStatus -- filter chains should mirror the ones in VersionStatus + + /** The first upgrade job to fail on this version, for this application */ + private Optional<JobStatus> firstFailingOn(Version version, Application application) { + return JobList.from(application) + .failing() + .not().failingApplicationChange() + .not().failingBecause(outOfCapacity) + .lastCompleted().on(version) + .asList().stream() + .min(comparing(job -> job.lastCompleted().get().at())); + } + + /** The number of production jobs for this application */ + private int productionJobsFor(Application application) { + return JobList.from(application) + .production() + .size(); + } + + /** The number of production jobs with last success on the given version, for this application */ + private int productionSuccessesFor(Version version, Application application) { + return JobList.from(application) + .production() + .lastSuccess().on(version) + .size(); + } + + /** The last triggered upgrade to this version, for this application */ + private Optional<JobStatus> lastDeployingTo(Version version, Application application) { + return JobList.from(application) + .running(controller.applications().deploymentTrigger().jobTimeoutLimit()) + .lastTriggered().upgrade() + .asList().stream() + .max(comparing(job -> job.lastTriggered().get().at())); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java index 6174a017a54..ae7223489c2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java @@ -1,12 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import java.util.List; +import java.util.Collection; import java.util.Objects; +import java.util.Set; /** * Statistics about deployments on a platform version. This is immutable. @@ -16,20 +17,22 @@ import java.util.Objects; public class DeploymentStatistics { private final Version version; - private final ImmutableList<ApplicationId> failing; - private final ImmutableList<ApplicationId> production; + private final ImmutableSet<ApplicationId> failing; + private final ImmutableSet<ApplicationId> production; + private final ImmutableSet<ApplicationId> deploying; /** DO NOT USE. Public for serialization purposes */ - public DeploymentStatistics(Version version, List<ApplicationId> failingApplications, - List<ApplicationId> production) { + public DeploymentStatistics(Version version, Collection<ApplicationId> failingApplications, + Collection<ApplicationId> production, Collection<ApplicationId> deploying) { this.version = version; - this.failing = ImmutableList.copyOf(failingApplications); - this.production = ImmutableList.copyOf(production); + this.failing = ImmutableSet.copyOf(failingApplications); + this.production = ImmutableSet.copyOf(production); + this.deploying = ImmutableSet.copyOf(deploying); } /** Returns a statistics instance with the values as 0 */ public static DeploymentStatistics empty(Version version) { - return new DeploymentStatistics(version, ImmutableList.of(), ImmutableList.of()); + return new DeploymentStatistics(version, ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of()); } /** Returns the version these statistics are for */ @@ -39,23 +42,31 @@ public class DeploymentStatistics { * Returns the applications which have at least one job (of any type) which fails on this version, * excluding errors known to not be caused by this version */ - public List<ApplicationId> failing() { return failing; } + public Set<ApplicationId> failing() { return failing; } /** Returns the applications which have this version in production in at least one zone */ - public List<ApplicationId> production() { return production; } - + public Set<ApplicationId> production() { return production; } + + /** Returns the applications which are currently upgrading to this version */ + public Set<ApplicationId> deploying() { return deploying; } + /** Returns a version of this with the given failing application added */ public DeploymentStatistics withFailing(ApplicationId application) { - return new DeploymentStatistics(version, add(application, failing), production); + return new DeploymentStatistics(version, add(application, failing), production, deploying); } /** Returns a version of this with the given production application added */ public DeploymentStatistics withProduction(ApplicationId application) { - return new DeploymentStatistics(version, failing, add(application, production)); + return new DeploymentStatistics(version, failing, add(application, production), deploying); + } + + /** Returns a version of this with the given deploying application added */ + public DeploymentStatistics withDeploying(ApplicationId application) { + return new DeploymentStatistics(version, failing, production, add(application, deploying)); } - private ImmutableList<ApplicationId> add(ApplicationId application, ImmutableList<ApplicationId> list) { - ImmutableList.Builder<ApplicationId> b = new ImmutableList.Builder<>(); + private ImmutableSet<ApplicationId> add(ApplicationId application, ImmutableSet<ApplicationId> list) { + ImmutableSet.Builder<ApplicationId> b = new ImmutableSet.Builder<>(); b.addAll(list); b.add(application); return b.build(); @@ -75,4 +86,5 @@ public class DeploymentStatistics { public int hashCode() { return Objects.hash(version, failing, production); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 73e4eb4d527..3de4a7539f8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.JobList; import java.net.URI; import java.time.Instant; @@ -28,6 +28,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity; + /** * Information about the current platform versions in use. * The versions in use are the set of all versions running in current applications, versions @@ -92,7 +94,8 @@ public class VersionStatus { Version systemVersion = infrastructureVersions.stream().sorted().findFirst().get(); Collection<DeploymentStatistics> deploymentStatistics = computeDeploymentStatistics(infrastructureVersions, - controller.applications().asList()); + controller.applications().asList(), + controller.applications().deploymentTrigger().jobTimeoutLimit()); List<VespaVersion> versions = new ArrayList<>(); for (DeploymentStatistics statistics : deploymentStatistics) { @@ -126,7 +129,8 @@ public class VersionStatus { } private static Collection<DeploymentStatistics> computeDeploymentStatistics(Set<Version> infrastructureVersions, - List<Application> applications) { + List<Application> applications, + Instant jobTimeoutLimit) { Map<Version, DeploymentStatistics> versionMap = new HashMap<>(); for (Version infrastructureVersion : infrastructureVersions) { @@ -142,41 +146,39 @@ public class VersionStatus { versionMap.computeIfAbsent(deployment.version(), DeploymentStatistics::empty); } - // List versions which have failing jobs, and versions which are in production + // List versions which have failing jobs, versions which are in production, and versions for which there are running deployment jobs // Failing versions - Map<Version, List<JobStatus>> failingJobsByVersion = jobs.jobStatus().values().stream() - .filter(jobStatus -> jobStatus.lastCompleted().isPresent()) - .filter(jobStatus -> jobStatus.lastCompleted().get().upgrade()) - .filter(jobStatus -> jobStatus.jobError().isPresent()) - .filter(jobStatus -> jobStatus.jobError().get() != DeploymentJobs.JobError.outOfCapacity) - .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastCompleted().get().version())); - for (Version v : failingJobsByVersion.keySet()) { - versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withFailing(application.id())); - } + JobList.from(application) + .failing() + .not().failingApplicationChange() + .not().failingBecause(outOfCapacity) + .mapToList(job -> job.lastCompleted().get().version()) + .forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withFailing(application.id()))); // Succeeding versions - Map<Version, List<JobStatus>> succeedingJobsByVersions = jobs.jobStatus().values().stream() - .filter(jobStatus -> jobStatus.lastSuccess().isPresent()) - .filter(jobStatus -> jobStatus.type().isProduction()) - .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastSuccess().get().version())); - for (Version v : succeedingJobsByVersions.keySet()) { - versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withProduction(application.id())); - } + JobList.from(application) + .lastSuccess().present() + .production() + .mapToList(job -> job.lastSuccess().get().version()) + .forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withProduction(application.id()))); + + // Deploying versions + JobList.from(application) + .running(jobTimeoutLimit) + .lastTriggered().upgrade() + .mapToList(job -> job.lastTriggered().get().version()) + .forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withDeploying(application.id()))); } return versionMap.values(); } - private static DeploymentStatistics emptyIfMissing(Version version, DeploymentStatistics statistics) { - return statistics == null ? DeploymentStatistics.empty(version) : statistics; - } - - private static VespaVersion createVersion(DeploymentStatistics statistics, + private static VespaVersion createVersion(DeploymentStatistics statistics, boolean isSystemVersion, Collection<String> configServerHostnames, Controller controller) { GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString()); - Instant releasedAt = Instant.ofEpochMilli(gitSha.commit.author.date.getTime()); + Instant releasedAt = Instant.ofEpochMilli(gitSha.commit.author.date.getTime()); // commitedAt ... VespaVersion.Confidence confidence; // Always compute confidence for system version if (isSystemVersion) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 79fd717a24f..9e228e0becb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import java.time.Duration; import java.util.List; +import java.util.NoSuchElementException; import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; @@ -225,21 +226,20 @@ public class DeploymentTester { /** Assert that the sceduled jobs of this application are exactly those given, and take them */ private void consumeJobs(Application application, boolean expectOnlyTheseJobs, JobType... jobs) { for (JobType job : jobs) { - Optional<BuildService.BuildJob> buildJob = findJob(application, job); - assertTrue(String.format("Job %s is scheduled for %s", job, application), buildJob.isPresent()); - assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.get().projectId()); - assertEquals(job.id(), buildJob.get().jobName()); + BuildService.BuildJob buildJob = findJob(application, job); + assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.projectId()); + assertEquals(job.id(), buildJob.jobName()); } if (expectOnlyTheseJobs) assertEquals(jobs.length, countJobsOf(application)); buildSystem().removeJobs(application.id()); } - private Optional<BuildService.BuildJob> findJob(Application application, JobType jobType) { + private BuildService.BuildJob findJob(Application application, JobType jobType) { for (BuildService.BuildJob job : buildSystem().jobs()) if (job.projectId() == application.deploymentJobs().projectId().get() && job.jobName().equals(jobType.id())) - return Optional.of(job); - return Optional.empty(); + return job; + throw new NoSuchElementException(jobType + " is not scheduled for " + application); } private int countJobsOf(Application application) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index f5f43265cb8..189b3a97a80 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -27,6 +27,8 @@ public class VersionStatusSerializerTest { Version.fromString("5.0"), Arrays.asList(ApplicationId.from("tenant1", "failing1", "default")), Arrays.asList(ApplicationId.from("tenant2", "success1", "default"), + ApplicationId.from("tenant2", "success2", "default")), + Arrays.asList(ApplicationId.from("tenant1", "failing1", "default"), ApplicationId.from("tenant2", "success2", "default")) ); vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index 7fd000b82c5..5f7fedfd75f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -1,27 +1,26 @@ { "versions":[ { - "version":"(ignore)", - "confidence":"high", - "commit":"(ignore)", - "date":0, - "controllerVersion":false, - "systemVersion":false, - "configServers":[ - - ], - "failingApplications":[ - - ], - "productionApplications":[ + "version": "(ignore)", + "confidence": "high", + "commit": "(ignore)", + "date": 0, + "controllerVersion": false, + "systemVersion": false, + "configServers": [ ], + "failingApplications": [ ], + "productionApplications": [ { - "tenant":"tenant1", - "application":"application1", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", - "upgradePolicy":"default" + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default", + "productionJobs": 1, + "productionSuccesses": 1 } - ] + ], + "deployingApplications": [ ] }, { "version":"(ignore)", @@ -40,40 +39,47 @@ ], "failingApplications":[ { - "tenant":"tenant1", - "application":"application1", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", - "upgradePolicy":"default", - "failingSince":"(ignore)" + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default", + "failing": "staging-test" } ], "productionApplications":[ { - "tenant":"tenant2", - "application":"application2", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant2/application/application2", - "upgradePolicy":"default" + "tenant": "tenant2", + "application": "application2", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2", + "upgradePolicy": "default", + "productionJobs": 1, + "productionSuccesses": 1 + } + ], + "deployingApplications": [ + { + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default", + "running": "staging-test" } ] }, { - "version":"(ignore)", - "confidence":"normal", - "commit":"(ignore)", - "date":0, - "controllerVersion":true, - "systemVersion":false, - "configServers":[ - - ], - "failingApplications":[ - - ], - "productionApplications":[ - - ] + "version": "(ignore)", + "confidence": "normal", + "commit": "(ignore)", + "date": 0, + "controllerVersion": true, + "systemVersion": false, + "configServers": [ ], + "failingApplications": [ ], + "productionApplications": [ ], + "deployingApplications": [ ] } ] }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 519c457e73b..1157863f009 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; +import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.provision.Environment; @@ -19,6 +20,7 @@ import org.junit.Test; import java.net.URI; import java.net.URISyntaxException; +import java.util.Collections; import java.util.List; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; @@ -28,6 +30,7 @@ import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobTy import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -95,23 +98,20 @@ public class VersionStatusTest { List<VespaVersion> versions = tester.controller().versionStatus().versions(); assertEquals("The two versions above exist", 2, versions.size()); + System.err.println(tester.controller().applications().deploymentTrigger().jobTimeoutLimit()); + VespaVersion v1 = versions.get(0); assertEquals(version1, v1.versionNumber()); - assertEquals(0, v1.statistics().failing().size()); - // All applications are on v1 in at least one zone - assertEquals(3, v1.statistics().production().size()); - assertTrue(v1.statistics().production().contains(app2.id())); - assertTrue(v1.statistics().production().contains(app1.id())); + assertEquals("No applications are failing on version1.", ImmutableSet.of(), v1.statistics().failing()); + assertEquals("All applications have at least one active production deployment on version 1.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v1.statistics().production()); + assertEquals("No applications have active deployment jobs on version1.", ImmutableSet.of(), v1.statistics().deploying()); VespaVersion v2 = versions.get(1); assertEquals(version2, v2.versionNumber()); - // All applications have failed on v2 in at least one zone - assertEquals(3, v2.statistics().failing().size()); - assertTrue(v2.statistics().failing().contains(app1.id())); - assertTrue(v2.statistics().failing().contains(app3.id())); - // Only one application is on v2 in at least one zone - assertEquals(1, v2.statistics().production().size()); - assertTrue(v2.statistics().production().contains(app2.id())); + assertEquals("All applications have failed on version2 in at least one zone.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v2.statistics().failing()); + assertEquals("Only app2 has successfully deployed to production on version2.", ImmutableSet.of(app2.id()), v2.statistics().production()); + // Should test the below, but can't easily be done with current test framework. This test passes in DeploymentApiTest. + // assertEquals("All applications are being retried on version2.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v2.statistics().deploying()); } @Test @@ -241,7 +241,7 @@ public class VersionStatusTest { } @Test - public void testIgnoreConfigdeince() { + public void testIgnoreConfidence() { DeploymentTester tester = new DeploymentTester(); Version version0 = new Version("5.0"); @@ -270,7 +270,6 @@ public class VersionStatusTest { tester.completeUpgradeWithError(default3, version1, "default", stagingTest); tester.completeUpgradeWithError(default4, version1, "default", stagingTest); tester.updateVersionStatus(); - assertEquals("Canaries have upgraded, 1 of 4 default apps failing: Broken", Confidence.broken, confidence(tester.controller(), version1)); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 906f71fecb4..7413d0f369f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -143,7 +143,15 @@ applyCompactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum, AttributeVector &attr) { if (attr.getStatus().getLastSyncToken() < serialNum) { - attr.compactLidSpace(wantedLidLimit); + /* + * If the attribute is an empty placeholder attribute due to + * later config changes removing the attribute then it might + * be smaller than expected during transaction log replay. + */ + attr.commit(); + if (wantedLidLimit <= attr.getCommittedDocIdLimit()) { + attr.compactLidSpace(wantedLidLimit); + } attr.commit(serialNum, serialNum); } } |