diff options
Diffstat (limited to 'controller-server/src/main/java/com/yahoo/vespa/hosted/controller')
24 files changed, 134 insertions, 158 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 0f7cbcee4ab..613e3413ae8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -296,7 +296,7 @@ public class ApplicationController { /** Reads the oldest installed platform for the given application and zone from the node repo of that zone. */ private Optional<Version> oldestInstalledPlatform(JobId job) { - return configServer.nodeRepository().list(job.type().zone(controller.system()), + return configServer.nodeRepository().list(job.type().zone(), NodeFilter.all() .applications(job.application()) .states(active, reserved)) @@ -454,7 +454,7 @@ public class ApplicationController { throw new IllegalArgumentException("'" + job.application() + "' is a tester application!"); TenantAndApplicationId applicationId = TenantAndApplicationId.from(job.application()); - ZoneId zone = job.type().zone(controller.system()); + ZoneId zone = job.type().zone(); DeploymentId deployment = new DeploymentId(job.application(), zone); try (Mutex deploymentLock = lockForDeployment(job.application(), zone)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java index 9ab5096ec8f..950eaea904a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java @@ -62,7 +62,7 @@ public class ApplicationPackageValidator { /** Verify that each of the production zones listed in the deployment spec exist in this system */ private void validateSteps(DeploymentSpec deploymentSpec) { for (var spec : deploymentSpec.instances()) { - new DeploymentSteps(spec, controller::system).jobs(); + new DeploymentSteps(spec, controller.zoneRegistry()).jobs(); spec.zones().stream() .filter(zone -> zone.environment() == Environment.prod) .forEach(zone -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index e13175806bf..cc7031bab5a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -12,7 +12,6 @@ import com.yahoo.config.application.api.DeploymentSpec.UpgradeRollout; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; @@ -20,6 +19,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -45,8 +45,6 @@ import static com.yahoo.config.application.api.DeploymentSpec.RevisionTarget.nex import static com.yahoo.config.provision.Environment.prod; import static com.yahoo.config.provision.Environment.staging; import static com.yahoo.config.provision.Environment.test; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; import static java.util.Objects.requireNonNull; @@ -69,17 +67,19 @@ public class DeploymentStatus { private final Application application; private final JobList allJobs; - private final SystemName system; + private final JobType systemTest; + private final JobType stagingTest; private final Version systemVersion; private final Function<InstanceName, VersionCompatibility> versionCompatibility; private final Instant now; private final Map<JobId, StepStatus> jobSteps; private final List<StepStatus> allSteps; - public DeploymentStatus(Application application, Function<JobId, JobStatus> allJobs, SystemName system, + public DeploymentStatus(Application application, Function<JobId, JobStatus> allJobs, ZoneRegistry zones, Version systemVersion, Function<InstanceName, VersionCompatibility> versionCompatibility, Instant now) { this.application = requireNonNull(application); - this.system = requireNonNull(system); + this.systemTest = JobType.systemTest(zones); + this.stagingTest = JobType.stagingTest(zones); this.systemVersion = requireNonNull(systemVersion); this.versionCompatibility = versionCompatibility; this.now = requireNonNull(now); @@ -244,7 +244,7 @@ public class DeploymentStatus { public Optional<Deployment> deploymentFor(JobId job) { return Optional.ofNullable(application.require(job.application().instance()) - .deployments().get(job.type().zone(system))); + .deployments().get(job.type().zone())); } /** @@ -388,7 +388,7 @@ public class DeploymentStatus { // For a dual change, where both targets remain, we determine what to run by looking at when the two parts became ready: // for deployments, we look at dependencies; for production tests, this may be overridden by what is already deployed. - JobId deployment = new JobId(job.application(), JobType.from(system, job.type().zone(system)).get()); + JobId deployment = new JobId(job.application(), JobType.deploymentTo(job.type().zone())); UpgradeRollout rollout = application.deploymentSpec().requireInstance(job.application().instance()).upgradeRollout(); if (job.type().isTest()) { Optional<Instant> platformDeployedAt = jobSteps.get(deployment).completedAt(change.withoutApplication(), Optional.of(deployment)); @@ -556,23 +556,20 @@ public class DeploymentStatus { JobId jobId; StepStatus stepStatus; if (step.concerns(test) || step.concerns(staging)) { - jobType = JobType.from(system, ((DeclaredZone) step).environment(), null) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); + jobType = step.concerns(test) ? systemTest : stagingTest; jobId = new JobId(application.id().instance(instance), jobType); stepStatus = JobStepStatus.ofTestDeployment((DeclaredZone) step, List.of(), this, jobs.apply(jobId), true); previous = new ArrayList<>(previous); previous.add(stepStatus); } else if (step.isTest()) { - jobType = JobType.testFrom(system, ((DeclaredTest) step).region()) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); + jobType = JobType.test(((DeclaredTest) step).region()); jobId = new JobId(application.id().instance(instance), jobType); stepStatus = JobStepStatus.ofProductionTest((DeclaredTest) step, previous, this, jobs.apply(jobId)); previous = List.of(stepStatus); } else if (step.concerns(prod)) { - jobType = JobType.from(system, ((DeclaredZone) step).environment(), ((DeclaredZone) step).region().get()) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); + jobType = JobType.prod(((DeclaredZone) step).region().get()); jobId = new JobId(application.id().instance(instance), jobType); stepStatus = JobStepStatus.ofProductionDeployment((DeclaredZone) step, previous, this, jobs.apply(jobId)); previous = List.of(stepStatus); @@ -872,7 +869,7 @@ public class DeploymentStatus { private static JobStepStatus ofProductionTest(DeclaredTest step, List<StepStatus> dependencies, DeploymentStatus status, JobStatus job) { - JobId prodId = new JobId(job.id().application(), JobType.from(status.system, job.id().type().zone(status.system)).get()); + JobId prodId = new JobId(job.id().application(), JobType.deploymentTo(job.id().type().zone())); return new JobStepStatus(StepType.test, step, dependencies, job, status) { @Override Optional<Instant> readyAt(Change change, Optional<JobId> dependent) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java index 7ab895654f3..44079a90097 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Deployment; import java.util.Collection; @@ -29,16 +30,16 @@ import static java.util.stream.Collectors.collectingAndThen; public class DeploymentSteps { private final DeploymentInstanceSpec spec; - private final Supplier<SystemName> system; + private final ZoneRegistry zones; - public DeploymentSteps(DeploymentInstanceSpec spec, Supplier<SystemName> system) { + public DeploymentSteps(DeploymentInstanceSpec spec, ZoneRegistry zones) { this.spec = Objects.requireNonNull(spec, "spec cannot be null"); - this.system = Objects.requireNonNull(system, "system cannot be null"); + this.zones = Objects.requireNonNull(zones, "system cannot be null"); } /** Returns jobs for this, in the order they should run */ public List<JobType> jobs() { - return Stream.concat(production().isEmpty() ? Stream.of() : Stream.of(JobType.systemTest, JobType.stagingTest), + return Stream.concat(production().isEmpty() ? Stream.of() : Stream.of(JobType.systemTest(zones), JobType.stagingTest(zones)), spec.steps().stream().flatMap(step -> toJobs(step).stream())) .distinct() .collect(Collectors.toUnmodifiableList()); @@ -67,7 +68,6 @@ public class DeploymentSteps { public List<JobType> toJobs(DeploymentSpec.Step step) { return step.zones().stream() .map(this::toJob) - .flatMap(Optional::stream) .collect(Collectors.toUnmodifiableList()); } @@ -93,8 +93,13 @@ public class DeploymentSteps { } /** Resolve job from deployment zone */ - private Optional<JobType> toJob(DeploymentSpec.DeclaredZone zone) { - return JobType.from(system.get(), zone.environment(), zone.region().orElse(null)); + private JobType toJob(DeploymentSpec.DeclaredZone zone) { + switch (zone.environment()) { + case prod: return JobType.prod(zone.region().get()); + case test: return JobType.systemTest(zones); + case staging: return JobType.stagingTest(zones); + default: throw new IllegalArgumentException("region must be one with automated deployments, but got: " + zone.environment()); + } } /** Resolve jobs from steps */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index c98b3b76292..be07a2b0cb1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -74,10 +74,7 @@ public class DeploymentTrigger { } public DeploymentSteps steps(DeploymentInstanceSpec spec) { - return new DeploymentSteps(spec, controller::system); - } - - public void notifyOfSubmission(TenantAndApplicationId id, ApplicationVersion version, long projectId) { + return new DeploymentSteps(spec, controller.zoneRegistry()); } /** @@ -268,8 +265,7 @@ public class DeploymentTrigger { /** Retrigger job. If the job is already running, it will be canceled, and retrigger enqueued. */ public Optional<JobId> reTriggerOrAddToQueue(DeploymentId deployment, String reason) { - JobType jobType = JobType.from(controller.system(), deployment.zoneId()) - .orElseThrow(() -> new IllegalArgumentException(Text.format("No job to trigger for (system/zone): %s/%s", controller.system().value(), deployment.zoneId().value()))); + JobType jobType = JobType.deploymentTo(deployment.zoneId()); Optional<Run> existingRun = controller.jobController().active(deployment.applicationId()).stream() .filter(run -> run.id().type().equals(jobType)) .findFirst(); @@ -389,7 +385,7 @@ public class DeploymentTrigger { /** Returns whether the application is healthy in all other production zones. */ private boolean isUnhealthyInAnotherZone(Application application, JobId job) { for (Deployment deployment : application.require(job.application().instance()).productionDeployments().values()) { - if ( ! deployment.zone().equals(job.type().zone(controller.system())) + if ( ! deployment.zone().equals(job.type().zone()) && ! controller.applications().isHealthy(new DeploymentId(job.application(), deployment.zone()))) return true; } @@ -418,9 +414,7 @@ public class DeploymentTrigger { boolean blocked = status.jobs().get(job).get().isRunning(); if ( ! job.type().isTest()) { - Optional<JobStatus> productionTest = JobType.testFrom(controller.system(), job.type().zone(controller.system()).region()) - .map(type -> new JobId(job.application(), type)) - .flatMap(status.jobs()::get); + Optional<JobStatus> productionTest = status.jobs().get(new JobId(job.application(), JobType.productionTestOf(job.type().zone()))); if (productionTest.isPresent()) { abortIfOutdated(status, jobs, productionTest.get().id()); // Production deployments are also blocked by their declared tests, if the next versions to run diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 28d9439b457..9e551c7ce78 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -217,7 +217,7 @@ public class InternalStepRunner implements StepRunner { logger.log("Deploying the tester container on platform " + platform + " ..."); return deploy(() -> controller.applications().deployTester(id.tester(), testerPackage(id), - id.type().zone(controller.system()), + id.type().zone(), platform), controller.jobController().run(id).get() .stepInfo(deployTester).get() @@ -314,19 +314,19 @@ public class InternalStepRunner implements StepRunner { Version platform = setTheStage ? versions.sourcePlatform().orElse(versions.targetPlatform()) : versions.targetPlatform(); Run run = controller.jobController().run(id).get(); - Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone(controller.system())), + Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone()), Optional.of(platform)); if (services.isEmpty()) { logger.log("Config status not currently available -- will retry."); return Optional.empty(); } - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), + List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), NodeFilter.all() .applications(id.application()) .states(active)); Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); - List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), + List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), NodeFilter.all() .hostnames(parentHostnames)); boolean firstTick = run.convergenceSummary().isEmpty(); @@ -357,8 +357,8 @@ public class InternalStepRunner implements StepRunner { } if (summary.converged()) { controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); - if (endpointsAvailable(id.application(), id.type().zone(controller.system()), logger)) { - if (containersAreUp(id.application(), id.type().zone(controller.system()), logger)) { + if (endpointsAvailable(id.application(), id.type().zone(), logger)) { + if (containersAreUp(id.application(), id.type().zone(), logger)) { logger.log("Installation succeeded!"); return Optional.of(running); } @@ -440,7 +440,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> installTester(RunId id, DualLogger logger) { Run run = controller.jobController().run(id).get(); Version platform = testerPlatformVersion(id); - ZoneId zone = id.type().zone(controller.system()); + ZoneId zone = id.type().zone(); ApplicationId testerId = id.tester().id(); Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(testerId, zone), @@ -609,7 +609,7 @@ public class InternalStepRunner implements StepRunner { .productionDeployments().keySet().stream() .map(zone -> new DeploymentId(id.application(), zone)) .collect(Collectors.toSet()); - ZoneId zoneId = id.type().zone(controller.system()); + ZoneId zoneId = id.type().zone(); deployments.add(new DeploymentId(id.application(), zoneId)); logger.log("Attempting to find endpoints ..."); @@ -722,8 +722,8 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deactivateReal(RunId id, DualLogger logger) { try { - logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.applications().deactivate(id.application(), id.type().zone(controller.system())); + logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone() + " ..."); + controller.applications().deactivate(id.application(), id.type().zone()); return Optional.of(running); } catch (RuntimeException e) { @@ -737,7 +737,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deactivateTester(RunId id, DualLogger logger) { try { - logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone() + " ..."); controller.jobController().deactivateTester(id.tester(), id.type()); return Optional.of(running); } @@ -870,7 +870,7 @@ public class InternalStepRunner implements StepRunner { /** Returns the deployment of the real application in the zone of the given job, if it exists. */ private Optional<Deployment> deployment(ApplicationId id, JobType type) { - return Optional.ofNullable(application(id).deployments().get(type.zone(controller.system()))); + return Optional.ofNullable(application(id).deployments().get(type.zone())); } /** Returns the real application with the given id. */ @@ -908,7 +908,7 @@ public class InternalStepRunner implements StepRunner { RevisionId revision = controller.jobController().run(id).get().versions().targetRevision(); DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); - ZoneId zone = id.type().zone(controller.system()); + ZoneId zone = id.type().zone(); boolean useTesterCertificate = useTesterCertificate(id); byte[] servicesXml = servicesXml( ! controller.system().isPublic(), @@ -952,7 +952,7 @@ public class InternalStepRunner implements StepRunner { } private DeploymentId getTesterDeploymentId(RunId runId) { - ZoneId zoneId = runId.type().zone(controller.system()); + ZoneId zoneId = runId.type().zone(); return new DeploymentId(runId.tester().id(), zoneId); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 6a6ed6e3b5d..69d9ba504a5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -30,7 +30,6 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.security.cert.X509Certificate; @@ -51,6 +50,7 @@ import java.util.TreeMap; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.function.UnaryOperator; import java.util.logging.Level; import java.util.logging.Logger; @@ -169,7 +169,7 @@ public class JobController { if ( ! run.hasStep(copyVespaLogs)) return run; - ZoneId zone = id.type().zone(controller.system()); + ZoneId zone = id.type().zone(); Optional<Deployment> deployment = Optional.ofNullable(controller.applications().requireInstance(id.application()) .deployments().get(zone)); if (deployment.isEmpty() || deployment.get().at().isBefore(run.start())) @@ -197,7 +197,7 @@ public class JobController { if (step.isEmpty()) return run; - List<LogEntry> entries = cloud.getLog(new DeploymentId(id.tester().id(), id.type().zone(controller.system())), + List<LogEntry> entries = cloud.getLog(new DeploymentId(id.tester().id(), id.type().zone()), run.lastTestLogEntry()); if (entries.isEmpty()) return run; @@ -209,7 +209,7 @@ public class JobController { public void updateTestReport(RunId id) { locked(id, run -> { - Optional<TestReport> report = cloud.getTestReport(new DeploymentId(id.tester().id(), id.type().zone(controller.system()))); + Optional<TestReport> report = cloud.getTestReport(new DeploymentId(id.tester().id(), id.type().zone())); if (report.isEmpty()) { return run; } @@ -257,10 +257,9 @@ public class JobController { /** Returns when given deployment last started deploying, falling back to time of deployment if it cannot be determined from job runs */ public Instant lastDeploymentStart(ApplicationId instanceId, Deployment deployment) { - return jobStarts(new JobId(instanceId, JobType.from(controller.system(), - deployment.zone()).get())).stream() - .findFirst() - .orElseGet(deployment::at); + return jobStarts(new JobId(instanceId, JobType.deploymentTo(deployment.zone()))).stream() + .findFirst() + .orElseGet(deployment::at); } /** Returns an immutable map of all known runs for the given application and job type. */ @@ -353,7 +352,7 @@ public class JobController { private DeploymentStatus deploymentStatus(Application application, Version systemVersion) { return new DeploymentStatus(application, this::jobStatus, - controller.system(), + controller.zoneRegistry(), systemVersion, instance -> controller.applications().versionCompatibility(application.id().instance(instance)), controller.clock().instant()); @@ -526,7 +525,7 @@ public class JobController { controller.applications().store(application.withRevisions(revisions -> revisions.withoutOlderThan(oldestRevision))); } else { - controller.applications().applicationStore().pruneDevDiffs(new DeploymentId(run.id().application(), run.id().job().type().zone(controller.system())), oldestRevision.number()); + controller.applications().applicationStore().pruneDevDiffs(new DeploymentId(run.id().application(), run.id().job().type().zone()), oldestRevision.number()); controller.applications().store(application.withRevisions(revisions -> revisions.withoutOlderThan(oldestRevision, run.id().job()))); } }); @@ -566,6 +565,9 @@ public class JobController { /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment.*/ public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage, boolean dryRun) { + if ( ! controller.zoneRegistry().hasZone(type.zone())) + throw new IllegalArgumentException(type.zone() + " is not present in this system"); + controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { if ( ! application.get().instances().containsKey(id.instance())) application = controller.applications().withNewInstance(application, id); @@ -573,7 +575,7 @@ public class JobController { controller.applications().store(application); }); - DeploymentId deploymentId = new DeploymentId(id, type.zone(controller.system())); + DeploymentId deploymentId = new DeploymentId(id, type.zone()); Optional<Run> lastRun = last(id, type); lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); @@ -585,7 +587,7 @@ public class JobController { controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { controller.applications().applicationStore().putDev(deploymentId, version.id(), applicationPackage.zippedContent(), diff); - Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, lastRun, id)); + Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, deploymentId, application.get().get(id.instance()))); controller.applications().store(application.withRevisions(revisions -> revisions.with(version))); start(id, type, @@ -615,24 +617,27 @@ public class JobController { .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage)); } - private Version findTargetPlatform(ApplicationPackage applicationPackage, Optional<Run> lastRun, ApplicationId id) { + private Version findTargetPlatform(ApplicationPackage applicationPackage, DeploymentId id, Optional<Instance> instance) { Optional<Integer> major = applicationPackage.deploymentSpec().majorVersion(); if (major.isPresent()) return controller.applications().lastCompatibleVersion(major.get()) .orElseThrow(() -> new IllegalArgumentException("major " + major.get() + " specified in deployment.xml, " + "but no version on this major was found")); - // Prefer previous platform if possible. - VersionStatus versionStatus = controller.readVersionStatus(); - VersionCompatibility compatibility = controller.applications().versionCompatibility(id); - Optional<Version> target = lastRun.map(run -> run.versions().targetPlatform()).filter(versionStatus::isActive); - if (target.isPresent() && compatibility.accept(target.get(), applicationPackage.compileVersion().orElse(target.get()))) - return target.get(); + VersionCompatibility compatibility = controller.applications().versionCompatibility(id.applicationId()); + + // Prefer previous platform if possible. Candidates are all deployable, ascending, with existing version appended; then reversed. + List<Version> versions = controller.readVersionStatus().deployableVersions().stream() + .map(VespaVersion::versionNumber) + .collect(toList()); + instance.map(Instance::deployments) + .map(deployments -> deployments.get(id.zoneId())) + .map(Deployment::version) + .ifPresent(versions::add); - // Otherwise, use newest, compatible version. - for (VespaVersion platform : reversed(versionStatus.deployableVersions())) - if (compatibility.accept(platform.versionNumber(), applicationPackage.compileVersion().orElse(platform.versionNumber()))) - return platform.versionNumber(); + for (Version target : reversed(versions)) + if (applicationPackage.compileVersion().isEmpty() || compatibility.accept(target, applicationPackage.compileVersion().get())) + return target; throw new IllegalArgumentException("no suitable platform version found" + applicationPackage.compileVersion() @@ -686,7 +691,7 @@ public class JobController { } public void deactivateTester(TesterId id, JobType type) { - controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone(controller.system()))); + controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone())); } /** Locks all runs and modifies the list of historic runs for the given application and job type. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java index e95e515685f..14fce806152 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -47,7 +47,7 @@ public class JobMetrics { "tenantName", id.application().tenant().value(), "app", id.application().application().value() + "." + id.application().instance().value(), "test", Boolean.toString(id.type().isTest()), - "zone", id.type().zone(system.get()).value()); + "zone", id.type().zone().value()); } static String valueOf(RunStatus status) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java index 063167647d5..e0c1fef91b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java @@ -24,12 +24,6 @@ public class RetriggerEntrySerializer { private static final String JOB_TYPE_KEY = "jobType"; private static final String MIN_REQUIRED_RUN_ID_KEY = "minimumRunId"; - private final SystemName system; - - public RetriggerEntrySerializer(SystemName system) { - this.system = system; - } - public List<RetriggerEntry> fromSlime(Slime slime) { return SlimeUtils.entriesStream(slime.get().field("entries")) .map(this::deserializeEntry) @@ -48,14 +42,14 @@ public class RetriggerEntrySerializer { Cursor root = array.addObject(); Cursor jobid = root.setObject(JOB_ID_KEY); jobid.setString(APPLICATION_ID_KEY, entry.jobId().application().serializedForm()); - jobid.setString(JOB_TYPE_KEY, entry.jobId().type().serialized(system)); + jobid.setString(JOB_TYPE_KEY, entry.jobId().type().serialized()); root.setLong(MIN_REQUIRED_RUN_ID_KEY, entry.requiredRun()); } private RetriggerEntry deserializeEntry(Inspector inspector) { Inspector jobid = inspector.field(JOB_ID_KEY); ApplicationId applicationId = ApplicationId.fromSerializedForm(require(jobid, APPLICATION_ID_KEY).asString()); - JobType jobType = JobType.fromJobName(require(jobid, JOB_TYPE_KEY).asString()); + JobType jobType = JobType.ofSerialized(require(jobid, JOB_TYPE_KEY).asString()); long minRequiredRunId = require(inspector, MIN_REQUIRED_RUN_ID_KEY).asLong(); return new RetriggerEntry(new JobId(applicationId, jobType), minRequiredRunId); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java index da42a00cd44..1680e064234 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java @@ -37,7 +37,7 @@ public class TestConfigSerializer { Cursor root = slime.setObject(); root.setString("application", id.serializedForm()); - root.setString("zone", type.zone(system).value()); + root.setString("zone", type.zone().value()); root.setString("system", system.value()); root.setBool("isCI", isCI); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index deafcd35e9b..041d0694ca9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; import java.time.Duration; @@ -36,7 +37,7 @@ public class ControllerMaintenance extends AbstractComponent { @Inject @SuppressWarnings("unused") // instantiated by Dependency Injection - public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement) { + public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement, AthenzClientFactory athenzClientFactory) { Intervals intervals = new Intervals(controller.system()); upgrader = new Upgrader(controller, intervals.defaultInterval); maintainers.add(upgrader); @@ -44,7 +45,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new DeploymentExpirer(controller, intervals.defaultInterval)); maintainers.add(new DeploymentUpgrader(controller, intervals.defaultInterval)); maintainers.add(new DeploymentIssueReporter(controller, controller.serviceRegistry().deploymentIssues(), intervals.defaultInterval)); - maintainers.add(new MetricsReporter(controller, metric)); + maintainers.add(new MetricsReporter(controller, metric, athenzClientFactory.createZmsClient())); maintainers.add(new OutstandingChangeDeployer(controller, intervals.outstandingChangeDeployer)); maintainers.add(new VersionStatusUpdater(controller, intervals.versionStatusUpdater)); maintainers.add(new ReadyJobsTrigger(controller, intervals.readyJobsTrigger)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index d33603243e2..97f3f955a20 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -57,10 +57,6 @@ public class DeploymentExpirer extends ControllerMaintainer { Optional<Duration> ttl = controller().zoneRegistry().getDeploymentTimeToLive(deployment.zone()); if (ttl.isEmpty()) return false; - Optional<JobId> jobId = JobType.from(controller().system(), deployment.zone()) - .map(type -> new JobId(instance, type)); - if (jobId.isEmpty()) return false; - return controller().jobController().lastDeploymentStart(instance, deployment) .plus(ttl.get()).isBefore(controller().clock().instant()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index 144c27b9e5e..c86f79ce188 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -47,7 +47,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) try { - JobId job = new JobId(instance.id(), JobType.from(controller().system(), deployment.zone()).get()); + JobId job = new JobId(instance.id(), JobType.deploymentTo(deployment.zone())); if ( ! deployment.zone().environment().isManuallyDeployed()) continue; Run last = controller().jobController().last(job).get(); @@ -60,7 +60,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); attempts.incrementAndGet(); - controller().jobController().start(instance.id(), JobType.from(controller().system(), deployment.zone()).get(), target, true, Optional.of("automated upgrade")); + controller().jobController().start(instance.id(), JobType.deploymentTo(deployment.zone()), target, true, Optional.of("automated upgrade")); } catch (Exception e) { failures.incrementAndGet(); log.log(Level.WARNING, "Failed upgrading " + deployment + " of " + instance + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index faa42e5caef..193fb89eb99 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -104,7 +104,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { controller().applications().getInstance(applicationId) .ifPresent(instance -> instance.productionDeployments().forEach((zone, deployment) -> { if (deployment.at().isBefore(refreshTime)) { - JobType job = JobType.from(controller().system(), zone).orElseThrow(); + JobType job = JobType.deploymentTo(zone); deploymentTrigger.reTrigger(applicationId, job, "re-triggered by EndpointCertificateMaintainer"); log.info("Re-triggering deployment job " + job.jobName() + " for instance " + applicationId.serializedForm() + " to roll out refreshed endpoint certificate"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 74bbf906653..294d5bad42d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -62,17 +63,20 @@ public class MetricsReporter extends ControllerMaintainer { public static final String REMAINING_ROTATIONS = "remaining_rotations"; public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; public static final String OPERATION_PREFIX = "operation."; + public static final String ZMS_QUOTA_USAGE = "zms.quota.usage"; private final Metric metric; private final Clock clock; + private final ZmsClient zmsClient; // Keep track of reported node counts for each version private final ConcurrentHashMap<NodeCountKey, Long> nodeCounts = new ConcurrentHashMap<>(); - public MetricsReporter(Controller controller, Metric metric) { + public MetricsReporter(Controller controller, Metric metric, ZmsClient zmsClient) { super(controller, Duration.ofMinutes(1)); // use fixed rate for metrics this.metric = metric; this.clock = controller.clock(); + this.zmsClient = zmsClient; } @Override @@ -85,6 +89,7 @@ public class MetricsReporter extends ControllerMaintainer { reportAuditLog(); reportBrokenSystemVersion(versionStatus); reportTenantMetrics(); + reportZmsQuotaMetrics(); return 1.0; } @@ -252,6 +257,20 @@ public class MetricsReporter extends ControllerMaintainer { }); } + private void reportZmsQuotaMetrics() { + var quota = zmsClient.getQuotaUsage(); + reportZmsQuota("subdomains", quota.getSubdomainUsage()); + reportZmsQuota("services", quota.getServiceUsage()); + reportZmsQuota("policies", quota.getPolicyUsage()); + reportZmsQuota("roles", quota.getRoleUsage()); + reportZmsQuota("groups", quota.getGroupUsage()); + } + + private void reportZmsQuota(String resourceType, double usage) { + var context = metric.createContext(Map.of("resourceType", resourceType)); + metric.set(ZMS_QUOTA_USAGE, usage, context); + } + private Map<NodeVersion, Duration> platformChangeDurations(VersionStatus versionStatus) { return changeDurations(versionStatus.versions(), VespaVersion::nodeVersions); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index e853dbc0d5a..48d8627f407 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -157,12 +157,6 @@ public class ApplicationSerializer { private static final String deploymentCostField = "cost"; - private final SystemName system; - - public ApplicationSerializer(SystemName system) { - this.system = system; - } - // ------------------ Serialization public Slime toSlime(Application application) { @@ -244,7 +238,7 @@ public class ApplicationSerializer { revisions.development().forEach((job, devRevisions) -> { Cursor devRevisionsObject = devRevisionsArray.addObject(); devRevisionsObject.setString(instanceNameField, job.application().instance().value()); - devRevisionsObject.setString(jobTypeField, job.type().serialized(system)); + devRevisionsObject.setString(jobTypeField, job.type().serialized()); revisionsToSlime(devRevisions, devRevisionsObject.setArray(versionsField)); }); } @@ -284,7 +278,7 @@ public class ApplicationSerializer { Cursor jobStatusArray = cursor.setArray(jobStatusField); jobPauses.forEach((type, until) -> { Cursor jobPauseObject = jobStatusArray.addObject(); - jobPauseObject.setString(jobTypeField, type.serialized(system)); + jobPauseObject.setString(jobTypeField, type.serialized()); jobPauseObject.setLong(pausedUntilField, until.toEpochMilli()); }); } @@ -371,7 +365,7 @@ public class ApplicationSerializer { private JobId jobIdFromSlime(TenantAndApplicationId base, Inspector idObject) { return new JobId(base.instance(idObject.field(instanceNameField).asString()), - JobType.fromJobName(idObject.field(jobTypeField).asString())); + JobType.ofSerialized(idObject.field(jobTypeField).asString())); } private List<ApplicationVersion> revisionsFromSlime(Inspector versionsArray, JobId job) { @@ -414,7 +408,7 @@ public class ApplicationSerializer { private Deployment deploymentFromSlime(Inspector deploymentObject, ApplicationId id) { ZoneId zone = zoneIdFromSlime(deploymentObject.field(zoneField)); return new Deployment(zone, - revisionFromSlime(deploymentObject.field(applicationPackageRevisionField), new JobId(id, JobType.from(system, zone).get())), + revisionFromSlime(deploymentObject.field(applicationPackageRevisionField), new JobId(id, JobType.deploymentTo(zone))), Version.fromString(deploymentObject.field(versionField).asString()), SlimeUtils.instant(deploymentObject.field(deployTimeField)), deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)), @@ -507,9 +501,8 @@ public class ApplicationSerializer { private Map<JobType, Instant> jobPausesFromSlime(Inspector object) { Map<JobType, Instant> jobPauses = new HashMap<>(); object.field(jobStatusField).traverse((ArrayTraverser) (__, jobPauseObject) -> - JobType.fromOptionalJobName(jobPauseObject.field(jobTypeField).asString()) - .ifPresent(jobType -> jobPauses.put(jobType, - SlimeUtils.instant(jobPauseObject.field(pausedUntilField))))); + jobPauses.put(JobType.ofSerialized(jobPauseObject.field(jobTypeField).asString()), + SlimeUtils.instant(jobPauseObject.field(pausedUntilField)))); return jobPauses; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 45b762b1b9c..16398fb1137 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -114,10 +114,10 @@ public class CuratorDb { private final ZoneRoutingPolicySerializer zoneRoutingPolicySerializer = new ZoneRoutingPolicySerializer(routingPolicySerializer); private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer(); private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer(); - private final ApplicationSerializer applicationSerializer; - private final RunSerializer runSerializer; - private final RetriggerEntrySerializer retriggerEntrySerializer; - private final NotificationsSerializer notificationsSerializer; + private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); + private final RunSerializer runSerializer = new RunSerializer(); + private final RetriggerEntrySerializer retriggerEntrySerializer = new RetriggerEntrySerializer(); + private final NotificationsSerializer notificationsSerializer = new NotificationsSerializer(); private final Curator curator; private final Duration tryLockTimeout; @@ -139,10 +139,6 @@ public class CuratorDb { this.curator = curator; this.tryLockTimeout = tryLockTimeout; this.lockScheme = Flags.CONTROLLER_LOCK_SCHEME.bindTo(flagSource); - this.applicationSerializer = new ApplicationSerializer(system); - this.runSerializer = new RunSerializer(system); - this.retriggerEntrySerializer = new RetriggerEntrySerializer(system); - this.notificationsSerializer = new NotificationsSerializer(system); } /** Returns all hostnames configured to be part of this ZooKeeper cluster */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java index 1d5f6d70ca5..c882681632e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java @@ -44,12 +44,6 @@ public class NotificationsSerializer { private static final String jobTypeField = "jobId"; private static final String runNumberField = "runNumber"; - private final SystemName system; - - NotificationsSerializer(SystemName system) { - this.system = system; - } - public Slime toSlime(List<Notification> notifications) { Slime slime = new Slime(); Cursor notificationsArray = slime.setObject().setArray(notificationsFieldName); @@ -66,7 +60,7 @@ public class NotificationsSerializer { notification.source().instance().ifPresent(instance -> notificationObject.setString(instanceField, instance.value())); notification.source().zoneId().ifPresent(zoneId -> notificationObject.setString(zoneField, zoneId.value())); notification.source().clusterId().ifPresent(clusterId -> notificationObject.setString(clusterIdField, clusterId.value())); - notification.source().jobType().ifPresent(jobType -> notificationObject.setString(jobTypeField, jobType.serialized(system))); + notification.source().jobType().ifPresent(jobType -> notificationObject.setString(jobTypeField, jobType.serialized())); notification.source().runNumber().ifPresent(runNumber -> notificationObject.setLong(runNumberField, runNumber)); } @@ -90,7 +84,7 @@ public class NotificationsSerializer { SlimeUtils.optionalString(inspector.field(instanceField)).map(InstanceName::from), SlimeUtils.optionalString(inspector.field(zoneField)).map(ZoneId::from), SlimeUtils.optionalString(inspector.field(clusterIdField)).map(ClusterSpec.Id::from), - SlimeUtils.optionalString(inspector.field(jobTypeField)).map(JobType::fromJobName), + SlimeUtils.optionalString(inspector.field(jobTypeField)).map(jobName -> JobType.ofSerialized(jobName)), SlimeUtils.optionalLong(inspector.field(runNumberField))), SlimeUtils.entriesStream(inspector.field(messagesField)).map(Inspector::asString).collect(Collectors.toUnmodifiableList())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 143aaaeabb8..dd28978d948 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -98,12 +98,6 @@ class RunSerializer { private static final String isDryRunField = "isDryRun"; private static final String reasonField = "reason"; - private final SystemName system; - - RunSerializer(SystemName system) { - this.system = system; - } - Run runFromSlime(Slime slime) { return runFromSlime(slime.get()); } @@ -132,7 +126,7 @@ class RunSerializer { steps.put(typedStep, new StepInfo(typedStep, stepStatusOf(status.asString()), startTime)); }); RunId id = new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), - JobType.fromJobName(runObject.field(jobTypeField).asString()), + JobType.ofSerialized(runObject.field(jobTypeField).asString()), runObject.field(numberField).asLong()); return new Run(id, steps, @@ -212,7 +206,7 @@ class RunSerializer { private void toSlime(Run run, Cursor runObject) { runObject.setString(applicationField, run.id().application().serializedForm()); - runObject.setString(jobTypeField, run.id().type().serialized(system)); + runObject.setString(jobTypeField, run.id().type().serialized()); runObject.setBool(isRedeploymentField, run.isRedeployment()); runObject.setLong(numberField, run.id().number()); runObject.setLong(startField, run.start().toEpochMilli()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index c11a5adf3b9..c07794ea39c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -797,14 +797,14 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private HttpResponse devApplicationPackage(ApplicationId id, JobType type) { - ZoneId zone = type.zone(controller.system()); + ZoneId zone = type.zone(); RevisionId revision = controller.jobController().last(id, type).get().versions().targetRevision(); byte[] applicationPackage = controller.applications().applicationStore().get(new DeploymentId(id, zone), revision); return new ZipResponse(id.toFullString() + "." + zone.value() + ".zip", applicationPackage); } private HttpResponse devApplicationPackageDiff(RunId runId) { - DeploymentId deploymentId = new DeploymentId(runId.application(), runId.job().type().zone(controller.system())); + DeploymentId deploymentId = new DeploymentId(runId.application(), runId.job().type().zone()); return controller.applications().applicationStore().getDevDiff(deploymentId, runId.number()) .map(ByteArrayResponse::new) .orElseThrow(() -> new NotExistsException("No application package diff found for " + runId)); @@ -1102,12 +1102,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { Slime slime = new Slime(); Cursor nodesArray = slime.setObject().setArray("nodes"); for (Node node : nodes) { - Optional<Instant> downAt = node.history().stream() - .filter(event -> "down".equals(event.name())) - .map(Node.Event::at) - .findFirst(); - boolean isUp = downAt.isEmpty() || node.history().stream() - .anyMatch(event -> "up".equals(event.name()) && event.at().isAfter(downAt.get())); Cursor nodeObject = nodesArray.addObject(); nodeObject.setString("hostname", node.hostname().value()); nodeObject.setString("state", valueOf(node.state())); @@ -1118,8 +1112,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { toSlime(node.resources(), nodeObject); nodeObject.setString("clusterId", node.clusterId()); nodeObject.setString("clusterType", valueOf(node.clusterType())); - nodeObject.setBool("down", !isUp); -// nodeObject.setBool("down", node.down()); // TODO (valerijf): Enable when all configservers expose this + nodeObject.setBool("down", node.down()); nodeObject.setBool("retired", node.retired() || node.wantToRetire()); nodeObject.setBool("restarting", node.wantedRestartGeneration() > node.restartGeneration()); nodeObject.setBool("rebooting", node.wantedRebootGeneration() > node.rebootGeneration()); @@ -1393,7 +1386,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { // Deployments sorted according to deployment spec List<Deployment> deployments = deploymentSpec.instance(instance.name()) - .map(spec -> new DeploymentSteps(spec, controller::system)) + .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry())) .map(steps -> steps.sortedDeployments(instance.deployments().values())) .orElse(List.copyOf(instance.deployments().values())); @@ -1481,7 +1474,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { // Deployments sorted according to deployment spec List<Deployment> deployments = application.deploymentSpec().instance(instance.name()) - .map(spec -> new DeploymentSteps(spec, controller::system)) + .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry())) .map(steps -> steps.sortedDeployments(instance.deployments().values())) .orElse(List.copyOf(instance.deployments().values())); Cursor instancesArray = object.setArray("instances"); @@ -1523,7 +1516,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { controller.jobController().active(instance.id()).stream() .map(run -> run.id().job()) .filter(job -> job.type().environment().isManuallyDeployed())) - .map(job -> job.type().zone(controller.system())) + .map(job -> job.type().zone()) .filter(zone -> ! instance.deployments().containsKey(zone)) .forEach(zone -> { Cursor deploymentObject = instancesArray.addObject(); @@ -1635,9 +1628,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (!deployment.zone().environment().isManuallyDeployed()) { DeploymentStatus status = controller.jobController().deploymentStatus(application); - JobType.from(controller.system(), deployment.zone()) - .map(type -> new JobId(instance.id(), type)) - .map(status.jobSteps()::get) + JobId jobId = new JobId(instance.id(), JobType.deploymentTo(deployment.zone())); + Optional.ofNullable(status.jobSteps().get(jobId)) .ifPresent(stepStatus -> { JobControllerApiHandlerHelper.toSlime(response.setObject("applicationVersion"), application.revisions().get(deployment.revision())); if ( ! status.jobsToRun().containsKey(stepStatus.job().get())) @@ -1647,9 +1639,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { else response.setString("status", "running"); }); } else { - var deploymentRun = JobType.from(controller.system(), deploymentId.zoneId()) - .flatMap(jobType -> controller.jobController().last(deploymentId.applicationId(), jobType)); - + var deploymentRun = controller.jobController().last(deploymentId.applicationId(), JobType.deploymentTo(deploymentId.zoneId())); deploymentRun.ifPresent(run -> { response.setString("status", run.hasEnded() ? "complete" : "running"); }); @@ -2066,7 +2056,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP)); controller.applications().verifyApplicationIdentityConfiguration(id.tenant(), Optional.of(id.instance()), - Optional.of(type.zone(controller.system())), + Optional.of(type.zone()), applicationPackage, Optional.of(requireUserPrincipal(request))); @@ -2179,7 +2169,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .flatMap(instance -> instance.productionDeployments().keySet().stream()) .map(zone -> new DeploymentId(prodInstanceId, zone)) .collect(Collectors.toCollection(HashSet::new)); - ZoneId testedZone = type.zone(controller.system()); + ZoneId testedZone = type.zone(); // If a production job is specified, the production deployment of the orchestrated instance is the relevant one, // as user instances should not exist in prod. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 0bbfaea3e6f..fac883299e3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -341,7 +341,7 @@ class JobControllerApiHandlerHelper { "/job/" + job.type().jobName()).normalize(); stepObject.setString("url", baseUriForJob.toString()); stepObject.setString("environment", job.type().environment().value()); - stepObject.setString("region", job.type().zone(controller.system()).value()); + stepObject.setString("region", job.type().zone().value()); if (job.type().isProduction() && job.type().isDeployment()) { status.deploymentFor(job).ifPresent(deployment -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java index 4cfcadd5c41..9b400fdfb78 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java @@ -81,9 +81,7 @@ public class BadgeApiHandler extends ThreadedHttpRequestHandler { () -> { DeploymentStatus status = controller.jobController().deploymentStatus(controller.applications().requireApplication(TenantAndApplicationId.from(id))); Predicate<JobStatus> isDeclaredJob = job -> status.jobSteps().get(job.id()) != null && status.jobSteps().get(job.id()).isDeclared(); - return Badges.overviewBadge(id, - status.jobs().instance(id.instance()).matching(isDeclaredJob), - controller.system()); + return Badges.overviewBadge(id, status.jobs().instance(id.instance()).matching(isDeclaredJob)); }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java index 7b4f2fec853..26a5da45bdb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java @@ -144,7 +144,7 @@ public class Badges { return badge(sections, texts, x); } - static String overviewBadge(ApplicationId id, JobList jobs, SystemName system) { + static String overviewBadge(ApplicationId id, JobList jobs) { // Put production tests right after their deployments, for a more compact rendering. List<Run> runs = new ArrayList<>(jobs.lastTriggered().asList()); boolean anyTest = false; @@ -153,7 +153,7 @@ public class Badges { if (run.id().type().isProduction() && run.id().type().isTest()) { anyTest = true; int j = i; - while ( ! runs.get(j - 1).id().type().zone(system).equals(run.id().type().zone(system))) + while ( ! runs.get(j - 1).id().type().zone().equals(run.id().type().zone())) runs.set(j, runs.get(--j)); runs.set(j, run); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 43037322f22..01bd02fdc13 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -111,7 +111,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/{*}")) { zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobname}")) { - zone = Optional.of(JobType.fromJobName(path.get("jobname"), zones).zone(systemName)); + zone = Optional.of(JobType.fromJobName(path.get("jobname"), zones).zone()); } else { zone = Optional.empty(); } |