diff options
Diffstat (limited to 'controller-server')
30 files changed, 171 insertions, 187 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 5b61996083f..3be5345b377 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.log.LogLevel; import com.yahoo.text.Text; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; @@ -402,7 +403,7 @@ public class ApplicationController { * @throws IllegalArgumentException if the application already exists */ public Application createApplication(TenantAndApplicationId id, Credentials credentials) { - try (Lock lock = lock(id)) { + try (Mutex lock = lock(id)) { if (getApplication(id).isPresent()) throw new IllegalArgumentException("Could not create '" + id + "': Application already exists"); if (getApplication(dashToUnderscore(id)).isPresent()) // VESPA-1945 @@ -456,7 +457,7 @@ public class ApplicationController { ZoneId zone = job.type().zone(controller.system()); DeploymentId deployment = new DeploymentId(job.application(), zone); - try (Lock deploymentLock = lockForDeployment(job.application(), zone)) { + try (Mutex deploymentLock = lockForDeployment(job.application(), zone)) { Set<ContainerEndpoint> containerEndpoints; Optional<EndpointCertificateMetadata> endpointCertificateMetadata; @@ -470,7 +471,7 @@ public class ApplicationController { RevisionId revision = run.versions().sourceRevision().filter(__ -> deploySourceVersions).orElse(run.versions().targetRevision()); ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(deployment, revision)); - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); Instance instance = application.get().require(job.application().instance()); @@ -742,7 +743,7 @@ public class ApplicationController { * @param action Function which acts on the locked application. */ public void lockApplicationIfPresent(TenantAndApplicationId applicationId, Consumer<LockedApplication> action) { - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { getApplication(applicationId).map(application -> new LockedApplication(application, lock)).ifPresent(action); } } @@ -755,7 +756,7 @@ public class ApplicationController { * @throws IllegalArgumentException when application does not exist. */ public void lockApplicationOrThrow(TenantAndApplicationId applicationId, Consumer<LockedApplication> action) { - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { action.accept(new LockedApplication(requireApplication(applicationId), lock)); } } @@ -828,14 +829,14 @@ public class ApplicationController { * Any operation which stores an application need to first acquire this lock, then read, modify * and store the application, and finally release (close) the lock. */ - Lock lock(TenantAndApplicationId application) { + Mutex lock(TenantAndApplicationId application) { return curator.lock(application); } /** * Returns a lock which provides exclusive rights to deploying this application to the given zone. */ - private Lock lockForDeployment(ApplicationId application, ZoneId zone) { + private Mutex lockForDeployment(ApplicationId application, ZoneId zone) { return curator.lockForDeployment(application, zone); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index c28cfb4ec6f..ca8f6a3ae33 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.Metric; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; @@ -189,7 +190,7 @@ public class Controller extends AbstractComponent { /** Remove confidence override for versions matching given filter */ public void removeConfidenceOverride(Predicate<Version> filter) { - try (Lock lock = curator.lockConfidenceOverrides()) { + try (Mutex lock = curator.lockConfidenceOverrides()) { Map<Version, VespaVersion.Confidence> overrides = new LinkedHashMap<>(curator.readConfidenceOverrides()); overrides.keySet().removeIf(filter); curator.writeConfidenceOverrides(overrides); @@ -228,7 +229,7 @@ public class Controller extends AbstractComponent { throw new IllegalArgumentException("Cloud '" + cloudName + "' does not exist in this system"); } Instant scheduledAt = clock.instant(); - try (Lock lock = curator.lockOsVersions()) { + try (Mutex lock = curator.lockOsVersions()) { Map<CloudName, OsVersionTarget> targets = curator.readOsVersionTargets().stream() .collect(Collectors.toMap(t -> t.osVersion().cloud(), Function.identity())); @@ -258,7 +259,7 @@ public class Controller extends AbstractComponent { /** Replace the current OS version status with a new one */ public void updateOsVersionStatus(OsVersionStatus newStatus) { - try (Lock lock = curator.lockOsVersionStatus()) { + try (Mutex lock = curator.lockOsVersionStatus()) { OsVersionStatus currentStatus = curator.readOsVersionStatus(); for (CloudName cloud : clouds()) { Set<Version> newVersions = newStatus.versionsIn(cloud); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 551327fd2e5..3e822415e96 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.InstanceName; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; @@ -30,7 +31,7 @@ import java.util.function.UnaryOperator; */ public class LockedApplication { - private final Lock lock; + private final Mutex lock; private final TenantAndApplicationId id; private final Instant createdAt; private final DeploymentSpec deploymentSpec; @@ -51,7 +52,7 @@ public class LockedApplication { * @param application The application to lock. * @param lock The lock for the application. */ - LockedApplication(Application application, Lock lock) { + LockedApplication(Application application, Mutex lock) { this(Objects.requireNonNull(lock, "lock cannot be null"), application.id(), application.createdAt(), application.deploymentSpec(), application.validationOverrides(), application.deploymentIssueId(), application.ownershipIssueId(), @@ -59,7 +60,7 @@ public class LockedApplication { application.projectId(), application.instances(), application.revisions()); } - private LockedApplication(Lock lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, + private LockedApplication(Mutex lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner, OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index bcc3b9b54c7..7a0e60aacb4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -6,6 +6,7 @@ import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; import com.yahoo.security.KeyUtils; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; @@ -46,7 +47,7 @@ public abstract class LockedTenant { this.lastLoginInfo = requireNonNull(lastLoginInfo); } - static LockedTenant of(Tenant tenant, Lock lock) { + static LockedTenant of(Tenant tenant, Mutex lock) { switch (tenant.type()) { case athenz: return new Athenz((AthenzTenant) tenant); case cloud: return new Cloud((CloudTenant) tenant); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 138bf547655..beba1cdb358 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Text; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -74,7 +75,7 @@ public class TenantController { /** Locks a tenant for modification and applies the given action. */ public <T extends LockedTenant> void lockIfPresent(TenantName name, Class<T> token, Consumer<T> action) { - try (Lock lock = lock(name)) { + try (Mutex lock = lock(name)) { get(name).map(tenant -> LockedTenant.of(tenant, lock)) .map(token::cast) .ifPresent(action); @@ -83,7 +84,7 @@ public class TenantController { /** Lock a tenant for modification and apply action. Throws if the tenant does not exist */ public <T extends LockedTenant> void lockOrThrow(TenantName name, Class<T> token, Consumer<T> action) { - try (Lock lock = lock(name)) { + try (Mutex lock = lock(name)) { action.accept(token.cast(LockedTenant.of(require(name), lock))); } } @@ -111,7 +112,7 @@ public class TenantController { /** Create a tenant, provided the given credentials are valid. */ public void create(TenantSpec tenantSpec, Credentials credentials) { - try (Lock lock = lock(tenantSpec.tenant())) { + try (Mutex lock = lock(tenantSpec.tenant())) { TenantId.validate(tenantSpec.tenant().value()); requireNonExistent(tenantSpec.tenant()); curator.writeTenant(accessControl.createTenant(tenantSpec, controller.clock().instant(), credentials, asList())); @@ -137,7 +138,7 @@ public class TenantController { /** Updates the tenant contained in the given tenant spec with new data. */ public void update(TenantSpec tenantSpec, Credentials credentials) { - try (Lock lock = lock(tenantSpec.tenant())) { + try (Mutex lock = lock(tenantSpec.tenant())) { curator.writeTenant(accessControl.updateTenant(tenantSpec, credentials, asList(), controller.applications().asList(tenantSpec.tenant()))); } @@ -148,7 +149,7 @@ public class TenantController { * new instant is later */ public void updateLastLogin(TenantName tenantName, List<LastLoginInfo.UserLevel> userLevels, Instant loggedInAt) { - try (Lock lock = lock(tenantName)) { + try (Mutex lock = lock(tenantName)) { Tenant tenant = require(tenantName); LastLoginInfo loginInfo = tenant.lastLoginInfo(); for (LastLoginInfo.UserLevel userLevel : userLevels) @@ -161,7 +162,7 @@ public class TenantController { /** Deletes the given tenant. */ public void delete(TenantName tenant, Optional<Credentials> credentials, boolean forget) { - try (Lock lock = lock(tenant)) { + try (Mutex lock = lock(tenant)) { Tenant oldTenant = get(tenant, true) .orElseThrow(() -> new NotExistsException("Could not delete tenant '" + tenant + "': Tenant not found")); @@ -202,7 +203,7 @@ public class TenantController { * Any operation which stores a tenant need to first acquire this lock, then read, modify * and store the tenant, and finally release (close) the lock. */ - private Lock lock(TenantName tenant) { + private Mutex lock(TenantName tenant) { return curator.lock(tenant); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 258884a4d11..ed794a9d929 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -197,11 +197,11 @@ public class ApplicationPackage { RegionName.defaultName()) .run(); // Populates the zip archive cache with files that would be included. } - catch (RuntimeException e) { + catch (IllegalArgumentException e) { throw e; } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalArgumentException(e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java index 78d65766075..34e7955e02a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.auditlog; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -70,7 +71,7 @@ public class AuditLogger { Instant now = clock.instant(); AuditLog.Entry entry = new AuditLog.Entry(now, principal.getName(), method.get(), pathAndQueryOf(request.getUri()), Optional.of(new String(data, StandardCharsets.UTF_8))); - try (Lock lock = db.lockAuditLog()) { + try (Mutex lock = db.lockAuditLog()) { AuditLog auditLog = db.readAuditLog() .pruneBefore(now.minus(entryTtl)) .with(entry) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 814e33f616b..7b53d9c5d99 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -927,7 +927,7 @@ public class DeploymentStatus { private final Change change; public Job(JobType type, Versions versions, Optional<Instant> readyAt, Change change) { - this.versions = type == systemTest ? versions.withoutSources() : versions; + this.versions = type.isSystemTest() ? versions.withoutSources() : versions; this.readyAt = readyAt; this.change = change; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 54703308102..c98b3b76292 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -8,6 +8,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.text.Text; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; @@ -275,7 +276,7 @@ public class DeploymentTrigger { if (existingRun.isPresent()) { Run run = existingRun.get(); - try (Lock lock = controller.curator().lockDeploymentRetriggerQueue()) { + try (Mutex lock = controller.curator().lockDeploymentRetriggerQueue()) { List<RetriggerEntry> retriggerEntries = controller.curator().readRetriggerEntries(); List<RetriggerEntry> newList = new ArrayList<>(retriggerEntries); RetriggerEntry requiredEntry = new RetriggerEntry(new JobId(deployment.applicationId(), jobType), run.id().number() + 1); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index d1c19476929..74730a78e31 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.component.VersionCompatibility; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; @@ -120,7 +121,7 @@ public class JobController { for (ApplicationId id : instances()) for (JobType type : jobs(id)) { locked(id, type, runs -> { // Runs are not modified here, and are written as they were. - curator.readLastRun(id, type).ifPresent(run -> curator.writeLastRun(run, controller.applications().requireApplication(TenantAndApplicationId.from(id)))); + curator.readLastRun(id, type).ifPresent(curator::writeLastRun); }); } } @@ -132,7 +133,7 @@ public class JobController { /** Returns the logged entries for the given run, which are after the given id threshold. */ public Optional<RunLog> details(RunId id, long after) { - try (Lock __ = curator.lock(id.application(), id.type())) { + try (Mutex __ = curator.lock(id.application(), id.type())) { Run run = runs(id.application(), id.type()).get(id); if (run == null) return Optional.empty(); @@ -237,7 +238,7 @@ public class JobController { /** Returns all job types which have been run for the given application. */ private List<JobType> jobs(ApplicationId id) { - return JobType.allIn(controller.system()).stream() + return JobType.allIn(controller.zoneRegistry()).stream() .filter(type -> last(id, type).isPresent()) .collect(toUnmodifiableList()); } @@ -316,20 +317,20 @@ public class JobController { /** Returns a list of all active runs for the given application. */ public List<Run> active(TenantAndApplicationId id) { return controller.applications().requireApplication(id).instances().keySet().stream() - .flatMap(name -> Stream.of(JobType.values()) - .map(type -> last(id.instance(name), type)) - .flatMap(Optional::stream) - .filter(run -> !run.hasEnded())) + .flatMap(name -> JobType.allIn(controller.zoneRegistry()).stream() + .map(type -> last(id.instance(name), type)) + .flatMap(Optional::stream) + .filter(run -> ! run.hasEnded())) .collect(toUnmodifiableList()); } /** Returns a list of all active runs for the given instance. */ public List<Run> active(ApplicationId id) { - return Stream.of(JobType.values()) - .map(type -> last(id, type)) - .flatMap(Optional::stream) - .filter(run -> !run.hasEnded()) - .collect(toUnmodifiableList()); + return JobType.allIn(controller.zoneRegistry()).stream() + .map(type -> last(id, type)) + .flatMap(Optional::stream) + .filter(run -> !run.hasEnded()) + .collect(toUnmodifiableList()); } /** Returns the job status of the given job, possibly empty. */ @@ -378,7 +379,7 @@ public class JobController { * Throws TimeoutException if some step in this job is still being run. */ public void finish(RunId id) throws TimeoutException { - List<Lock> locks = new ArrayList<>(); + List<Mutex> locks = new ArrayList<>(); try { // Ensure no step is still running before we finish the run — report depends transitively on all the other steps. Run unlockedRun = run(id).get(); @@ -426,7 +427,7 @@ public class JobController { }); } finally { - for (Lock lock : locks) + for (Mutex lock : locks) lock.close(); } } @@ -545,7 +546,7 @@ public class JobController { throw new IllegalArgumentException("Cannot start " + type + " for " + id + "; it is already running!"); RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); - curator.writeLastRun(Run.initial(newId, versions, isRedeployment, controller.clock().instant(), profile, reason), controller.applications().requireApplication(TenantAndApplicationId.from(id))); + curator.writeLastRun(Run.initial(newId, versions, isRedeployment, controller.clock().instant(), profile, reason)); metric.jobStarted(newId.job()); }); } @@ -658,7 +659,7 @@ public class JobController { TesterId tester = TesterId.of(id); for (JobType type : jobs(id)) locked(id, type, deactivateTester, __ -> { - try (Lock ___ = curator.lock(id, type)) { + try (Mutex ___ = curator.lock(id, type)) { try { deactivateTester(tester, type); } @@ -683,30 +684,27 @@ public class JobController { /** Locks all runs and modifies the list of historic runs for the given application and job type. */ private void locked(ApplicationId id, JobType type, Consumer<SortedMap<RunId, Run>> modifications) { - Application application = controller.applications().requireApplication(TenantAndApplicationId.from(id)); - try (Lock __ = curator.lock(id, type)) { + try (Mutex __ = curator.lock(id, type)) { SortedMap<RunId, Run> runs = new TreeMap<>(curator.readHistoricRuns(id, type)); modifications.accept(runs); - curator.writeHistoricRuns(id, type, runs.values(), application); + curator.writeHistoricRuns(id, type, runs.values()); } } /** Locks and modifies the run with the given id, provided it is still active. */ public void locked(RunId id, UnaryOperator<Run> modifications) { - Application application = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())); - try (Lock __ = curator.lock(id.application(), id.type())) { + try (Mutex __ = curator.lock(id.application(), id.type())) { active(id).ifPresent(run -> { - run = modifications.apply(run); - curator.writeLastRun(run, application); + curator.writeLastRun(modifications.apply(run)); }); } } /** Locks the given step and checks none of its prerequisites are running, then performs the given actions. */ public void locked(ApplicationId id, JobType type, Step step, Consumer<LockedStep> action) throws TimeoutException { - try (Lock lock = curator.lock(id, type, step)) { + try (Mutex lock = curator.lock(id, type, step)) { for (Step prerequisite : step.allPrerequisites(last(id, type).get().steps().keySet())) // Check that no prerequisite is still running. - try (Lock __ = curator.lock(id, type, prerequisite)) { ; } + try (Mutex __ = curator.lock(id, type, prerequisite)) { ; } action.accept(new LockedStep(lock, step)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java index d46516582be..8147ccb3180 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; /** @@ -9,7 +10,7 @@ import com.yahoo.vespa.curator.Lock; public class LockedStep { private final Step step; - LockedStep(Lock lock, Step step) { this.step = step; } + LockedStep(Mutex lock, Step step) { this.step = step; } public Step get() { return step; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java index 00cd4bd5c6c..80c6552d3d4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java @@ -38,7 +38,7 @@ public class RunList extends AbstractFilteringList<Run, RunList> { private static boolean matchingVersions(Run run, Versions versions) { return versions.targetsMatch(run.versions()) - && (versions.sourcesMatchIfPresent(run.versions()) || run.id().type() == JobType.systemTest); + && (versions.sourcesMatchIfPresent(run.versions()) || run.id().type().isSystemTest()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index aecb1e7a2c1..540e8489e6d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; @@ -79,7 +80,7 @@ public class NameServiceForwarder { } protected void forward(NameServiceRequest request, NameServiceQueue.Priority priority) { - try (Lock lock = db.lockNameServiceQueue()) { + try (Mutex lock = db.lockNameServiceQueue()) { NameServiceQueue queue = db.readNameServiceQueue(); var queued = queue.requests().size(); if (queued >= QUEUE_CAPACITY) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 15f8d6380c0..faa42e5caef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -6,6 +6,7 @@ import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -82,7 +83,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { var refreshedCertificateMetadata = endpointCertificateMetadata .withVersion(latestAvailableVersion.getAsInt()) .withLastRefreshed(clock.instant().getEpochSecond()); - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { if (Optional.of(endpointCertificateMetadata).equals(curator.readEndpointCertificateMetadata(applicationId))) { curator.writeEndpointCertificateMetadata(applicationId, refreshedCertificateMetadata); // Certificate not validated here, but on deploy. } @@ -128,7 +129,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { curator.readAllEndpointCertificateMetadata().forEach((applicationId, storedMetaData) -> { var lastRequested = Instant.ofEpochSecond(storedMetaData.lastRequested()); if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(applicationId)) { - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { if (Optional.of(storedMetaData).equals(curator.readEndpointCertificateMetadata(applicationId))) { log.log(Level.INFO, "Cert for app " + applicationId.serializedForm() + " has not been requested in a month and app has no deployments, deleting from provider and ZK"); @@ -140,7 +141,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { }); } - private Lock lock(ApplicationId applicationId) { + private Mutex lock(ApplicationId applicationId) { return curator.lock(TenantAndApplicationId.from(applicationId)); } @@ -169,7 +170,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { EndpointCertificateMetadata storedAppMetadata = storedAppEntry.getValue(); if (storedAppMetadata.certName().equals(unknownCertDetails.cert_key_keyname())) { matchFound = true; - try (Lock lock = lock(storedApp)) { + try (Mutex lock = lock(storedApp)) { if (Optional.of(storedAppMetadata).equals(curator.readEndpointCertificateMetadata(storedApp))) { log.log(Level.INFO, "Cert for app " + storedApp.serializedForm() + " has a new leafRequestId " + unknownCertDetails.request_id() + ", updating in ZK"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index fa5afa3a216..4ed34a91029 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; @@ -176,7 +177,7 @@ public class Upgrader extends ControllerMaintainer { " for version " + version.toFullString() + ": Version may be in use by applications"); } - try (Lock lock = curator.lockConfidenceOverrides()) { + try (Mutex lock = curator.lockConfidenceOverrides()) { Map<Version, Confidence> overrides = new LinkedHashMap<>(curator.readConfidenceOverrides()); overrides.put(version, confidence); curator.writeConfidenceOverrides(overrides); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java index aa62028749b..7876099cb21 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java @@ -5,6 +5,7 @@ import com.yahoo.collections.Pair; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Text; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; @@ -65,7 +66,7 @@ public class NotificationsDb { */ public void setNotification(NotificationSource source, Type type, Level level, List<String> messages) { Optional<Notification> changed = Optional.empty(); - try (Lock lock = curatorDb.lockNotifications(source.tenant())) { + try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { var existingNotifications = curatorDb.readNotifications(source.tenant()); List<Notification> notifications = existingNotifications.stream() .filter(notification -> !source.equals(notification.source()) || type != notification.type()) @@ -82,7 +83,7 @@ public class NotificationsDb { /** Remove the notification with the given source and type */ public void removeNotification(NotificationSource source, Type type) { - try (Lock lock = curatorDb.lockNotifications(source.tenant())) { + try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { List<Notification> initial = curatorDb.readNotifications(source.tenant()); List<Notification> filtered = initial.stream() .filter(notification -> !source.equals(notification.source()) || type != notification.type()) @@ -94,7 +95,7 @@ public class NotificationsDb { /** Remove all notifications for this source or sources contained by this source */ public void removeNotifications(NotificationSource source) { - try (Lock lock = curatorDb.lockNotifications(source.tenant())) { + try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { if (source.application().isEmpty()) { // Source is tenant curatorDb.deleteNotifications(source.tenant()); return; @@ -130,7 +131,7 @@ public class NotificationsDb { .collect(Collectors.toUnmodifiableList()); NotificationSource deploymentSource = NotificationSource.from(deploymentId); - try (Lock lock = curatorDb.lockNotifications(deploymentSource.tenant())) { + try (Mutex lock = curatorDb.lockNotifications(deploymentSource.tenant())) { List<Notification> initial = curatorDb.readNotifications(deploymentSource.tenant()); List<Notification> updated = Stream.concat( initial.stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index c269463c9fa..4b9df825951 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -190,11 +190,11 @@ public class ApplicationSerializer { for (Instance instance : application.instances().values()) { Cursor instanceObject = array.addObject(); instanceObject.setString(instanceNameField, instance.name().value()); - deploymentsToSlime(instance.deployments().values(), application, instanceObject.setArray(deploymentsField)); + deploymentsToSlime(instance.deployments().values(), instanceObject.setArray(deploymentsField)); toSlime(instance.jobPauses(), instanceObject.setObject(deploymentJobsField)); assignedRotationsToSlime(instance.rotations(), instanceObject); toSlime(instance.rotationStatus(), instanceObject.setArray(rotationStatusField)); - toSlime(instance.change(), application, instanceObject, deployingField); + toSlime(instance.change(), instanceObject, deployingField); } } @@ -202,16 +202,16 @@ public class ApplicationSerializer { deployKeys.forEach(key -> array.addString(KeyUtils.toPem(key))); } - private void deploymentsToSlime(Collection<Deployment> deployments, Application application, Cursor array) { + private void deploymentsToSlime(Collection<Deployment> deployments, Cursor array) { for (Deployment deployment : deployments) - deploymentToSlime(deployment, application.revisions()::get, array.addObject()); + deploymentToSlime(deployment, array.addObject()); } - private void deploymentToSlime(Deployment deployment, Function<RevisionId, ApplicationVersion> compatibility, Cursor object) { + private void deploymentToSlime(Deployment deployment, Cursor object) { zoneIdToSlime(deployment.zone(), object.setObject(zoneField)); object.setString(versionField, deployment.version().toString()); object.setLong(deployTimeField, deployment.at().toEpochMilli()); - toSlime(compatibility.apply(deployment.revision()), object.setObject(applicationPackageRevisionField)); + toSlime(deployment.revision(), object.setObject(applicationPackageRevisionField)); deploymentMetricsToSlime(deployment.metrics(), object); deployment.activity().lastQueried().ifPresent(instant -> object.setLong(lastQueriedField, instant.toEpochMilli())); deployment.activity().lastWritten().ifPresent(instant -> object.setLong(lastWrittenField, instant.toEpochMilli())); @@ -254,6 +254,11 @@ public class ApplicationSerializer { revisions.forEach(version -> toSlime(version, revisionsArray.addObject())); } + private void toSlime(RevisionId revision, Cursor object) { + object.setLong(applicationBuildNumberField, revision.number()); + object.setBool(deployedDirectlyField, ! revision.isProduction()); + } + private void toSlime(ApplicationVersion applicationVersion, Cursor object) { applicationVersion.buildNumber().ifPresent(number -> object.setLong(applicationBuildNumberField, number)); applicationVersion.source().ifPresent(source -> toSlime(source, object.setObject(sourceRevisionField))); @@ -285,14 +290,14 @@ public class ApplicationSerializer { }); } - private void toSlime(Change deploying, Application application, Cursor parentObject, String fieldName) { + private void toSlime(Change deploying, Cursor parentObject, String fieldName) { if (deploying.isEmpty()) return; Cursor object = parentObject.setObject(fieldName); if (deploying.platform().isPresent()) object.setString(versionField, deploying.platform().get().toString()); if (deploying.revision().isPresent()) - toSlime(application.revisions().get(deploying.revision().get()), object); + toSlime(deploying.revision().get(), object); if (deploying.isPinned()) object.setBool(pinnedField, true); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java index 9c3a6aa17d7..9721026c628 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java @@ -110,10 +110,8 @@ public class BufferedLogStore { store.delete(id); } - /** Deletes all logs for the given application. */ + /** Deletes all logs in permanent storage for the given application. */ public void delete(ApplicationId id) { - for (JobType type : JobType.values()) - buffer.deleteLog(id, type); store.delete(id); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 0bfdd53eeab..6190d58e0a1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.curator.MultiplePathsLock; @@ -149,14 +150,15 @@ public class CuratorDb { // -------------- Locks --------------------------------------------------- - public Lock lock(TenantName name) { + public Mutex lock(TenantName name) { return curator.lock(lockPath(name), defaultLockTimeout.multipliedBy(2)); } - public Lock lock(TenantAndApplicationId id) { + public Mutex lock(TenantAndApplicationId id) { switch (lockScheme.value()) { case "BOTH": - return new MultiplePathsLock(lockPath(id), legacyLockPath(id), defaultLockTimeout.multipliedBy(2), curator); + return new MultiplePathsLock(curator.lock(legacyLockPath(id), defaultLockTimeout.multipliedBy(2)), + curator.lock(lockPath(id), defaultLockTimeout.multipliedBy(2))); case "OLD": return curator.lock(legacyLockPath(id), defaultLockTimeout.multipliedBy(2)); case "NEW": @@ -166,10 +168,11 @@ public class CuratorDb { } } - public Lock lockForDeployment(ApplicationId id, ZoneId zone) { + public Mutex lockForDeployment(ApplicationId id, ZoneId zone) { switch (lockScheme.value()) { case "BOTH": - return new MultiplePathsLock(lockPath(id, zone), legacyLockPath(id, zone), deployLockTimeout, curator); + return new MultiplePathsLock(curator.lock(legacyLockPath(id, zone), defaultLockTimeout), + curator.lock(lockPath(id, zone), defaultLockTimeout)); case "OLD": return curator.lock(legacyLockPath(id, zone), deployLockTimeout); case "NEW": @@ -179,10 +182,11 @@ public class CuratorDb { } } - public Lock lock(ApplicationId id, JobType type) { + public Mutex lock(ApplicationId id, JobType type) { switch (lockScheme.value()) { case "BOTH": - return new MultiplePathsLock(lockPath(id, type), legacyLockPath(id, type), defaultLockTimeout, curator); + return new MultiplePathsLock(curator.lock(legacyLockPath(id, type), defaultLockTimeout), + curator.lock(lockPath(id, type), defaultLockTimeout)); case "OLD": return curator.lock(legacyLockPath(id, type), defaultLockTimeout); case "NEW": @@ -192,10 +196,10 @@ public class CuratorDb { } } - public Lock lock(ApplicationId id, JobType type, Step step) throws TimeoutException { + public Mutex lock(ApplicationId id, JobType type, Step step) throws TimeoutException { switch (lockScheme.value()) { case "BOTH": - return tryLock(lockPath(id, type, step), legacyLockPath(id, type, step)); + return tryLock(legacyLockPath(id, type, step), lockPath(id, type, step)); case "OLD": return tryLock(legacyLockPath(id, type, step)); case "NEW": @@ -205,15 +209,15 @@ public class CuratorDb { } } - public Lock lockRotations() { + public Mutex lockRotations() { return curator.lock(lockRoot.append("rotations"), defaultLockTimeout); } - public Lock lockConfidenceOverrides() { + public Mutex lockConfidenceOverrides() { return curator.lock(lockRoot.append("confidenceOverrides"), defaultLockTimeout); } - public Lock lockMaintenanceJob(String jobName) { + public Mutex lockMaintenanceJob(String jobName) { try { return tryLock(lockRoot.append("maintenanceJobLocks").append(jobName)); } catch (TimeoutException e) { @@ -221,52 +225,51 @@ public class CuratorDb { } } - @SuppressWarnings("unused") // Called by internal code - public Lock lockProvisionState(String provisionStateId) { + public Mutex lockProvisionState(String provisionStateId) { return curator.lock(lockPath(provisionStateId), Duration.ofSeconds(1)); } - public Lock lockOsVersions() { + public Mutex lockOsVersions() { return curator.lock(lockRoot.append("osTargetVersion"), defaultLockTimeout); } - public Lock lockOsVersionStatus() { + public Mutex lockOsVersionStatus() { return curator.lock(lockRoot.append("osVersionStatus"), defaultLockTimeout); } - public Lock lockRoutingPolicies() { + public Mutex lockRoutingPolicies() { return curator.lock(lockRoot.append("routingPolicies"), defaultLockTimeout); } - public Lock lockAuditLog() { + public Mutex lockAuditLog() { return curator.lock(lockRoot.append("auditLog"), defaultLockTimeout); } - public Lock lockNameServiceQueue() { + public Mutex lockNameServiceQueue() { return curator.lock(lockRoot.append("nameServiceQueue"), defaultLockTimeout); } - public Lock lockMeteringRefreshTime() throws TimeoutException { + public Mutex lockMeteringRefreshTime() throws TimeoutException { return tryLock(lockRoot.append("meteringRefreshTime")); } - public Lock lockArchiveBuckets(ZoneId zoneId) { + public Mutex lockArchiveBuckets(ZoneId zoneId) { return curator.lock(lockRoot.append("archiveBuckets").append(zoneId.value()), defaultLockTimeout); } - public Lock lockChangeRequests() { + public Mutex lockChangeRequests() { return curator.lock(lockRoot.append("changeRequests"), defaultLockTimeout); } - public Lock lockNotifications(TenantName tenantName) { + public Mutex lockNotifications(TenantName tenantName) { return curator.lock(lockRoot.append("notifications").append(tenantName.value()), defaultLockTimeout); } - public Lock lockSupportAccess(DeploymentId deploymentId) { + public Mutex lockSupportAccess(DeploymentId deploymentId) { return curator.lock(lockRoot.append("supportAccess").append(deploymentId.dottedString()), defaultLockTimeout); } - public Lock lockDeploymentRetriggerQueue() { + public Mutex lockDeploymentRetriggerQueue() { return curator.lock(lockRoot.append("deploymentRetriggerQueue"), defaultLockTimeout); } @@ -276,7 +279,7 @@ public class CuratorDb { * * Useful for maintenance jobs, where there is no point in running the jobs back to back. */ - private Lock tryLock(Path path) throws TimeoutException { + private Mutex tryLock(Path path) throws TimeoutException { try { return curator.lock(path, tryLockTimeout); } @@ -289,9 +292,9 @@ public class CuratorDb { * * Useful for maintenance jobs, where there is no point in running the jobs back to back. */ - private Lock tryLock(Path path, Path path2) throws TimeoutException { + private Mutex tryLock(Path path, Path path2) throws TimeoutException { try { - return new MultiplePathsLock(path, path2, tryLockTimeout, curator); + return new MultiplePathsLock(curator.lock(path, tryLockTimeout), curator.lock(path2, tryLockTimeout)); } catch (UncheckedTimeoutException e) { throw new TimeoutException(e.getMessage()); @@ -462,13 +465,13 @@ public class CuratorDb { // -------------- Job Runs ------------------------------------------------ - public void writeLastRun(Run run, Application application) { - curator.set(lastRunPath(run.id().application(), run.id().type()), asJson(runSerializer.toSlime(run, application))); + public void writeLastRun(Run run) { + curator.set(lastRunPath(run.id().application(), run.id().type()), asJson(runSerializer.toSlime(run))); } - public void writeHistoricRuns(ApplicationId id, JobType type, Iterable<Run> runs, Application application) { + public void writeHistoricRuns(ApplicationId id, JobType type, Iterable<Run> runs) { Path path = runsPath(id, type); - curator.set(path, asJson(runSerializer.toSlime(runs, application))); + curator.set(path, asJson(runSerializer.toSlime(runs))); } public Optional<Run> readLastRun(ApplicationId id, JobType type) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index ea932fd19c8..32ba583321c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -90,17 +90,9 @@ class RunSerializer { private static final String versionsField = "versions"; private static final String isRedeploymentField = "isRedeployment"; private static final String platformVersionField = "platform"; - private static final String repositoryField = "repository"; - private static final String branchField = "branch"; - private static final String commitField = "commit"; - private static final String authorEmailField = "authorEmail"; private static final String deployedDirectlyField = "deployedDirectly"; - private static final String compileVersionField = "compileVersion"; - private static final String buildTimeField = "buildTime"; - private static final String sourceUrlField = "sourceUrl"; private static final String buildField = "build"; private static final String sourceField = "source"; - private static final String bundleHashField = "bundleHash"; private static final String lastTestRecordField = "lastTestRecord"; private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp"; private static final String noNodesDownSinceField = "noNodesDownSince"; @@ -202,24 +194,20 @@ class RunSerializer { summaryArray.entry(12).asLong())); } - Slime toSlime(Iterable<Run> runs, Application application) { + Slime toSlime(Iterable<Run> runs) { Slime slime = new Slime(); Cursor runArray = slime.setArray(); - runs.forEach(run -> toSlime(run, application.revisions()::get, runArray.addObject())); + runs.forEach(run -> toSlime(run, runArray.addObject())); return slime; } - Slime toSlime(Run run, Application application) { - return toSlime(run, application.revisions()::get); - } - - Slime toSlime(Run run, Function<RevisionId, ApplicationVersion> compatilibity) { + Slime toSlime(Run run) { Slime slime = new Slime(); - toSlime(run, compatilibity, slime.setObject()); + toSlime(run, slime.setObject()); return slime; } - private void toSlime(Run run, Function<RevisionId, ApplicationVersion> compatibility, Cursor runObject) { + private void toSlime(Run run, Cursor runObject) { runObject.setString(applicationField, run.id().application().serializedForm()); runObject.setString(jobTypeField, run.id().type().jobName()); runObject.setBool(isRedeploymentField, run.isRedeployment()); @@ -244,10 +232,10 @@ class RunSerializer { stepDetailsObject.setObject(valueOf(step)).setLong(startTimeField, valueOf(startTime)))); Cursor versionsObject = runObject.setObject(versionsField); - toSlime(run.versions().targetPlatform(), compatibility.apply(run.versions().targetRevision()), versionsObject); + toSlime(run.versions().targetPlatform(), run.versions().targetRevision(), versionsObject); run.versions().sourcePlatform().ifPresent(sourcePlatformVersion -> { toSlime(sourcePlatformVersion, - run.versions().sourceRevision().map(compatibility) + run.versions().sourceRevision() .orElseThrow(() -> new IllegalArgumentException("Source versions must be both present or absent.")), versionsObject.setObject(sourceField)); }); @@ -255,18 +243,10 @@ class RunSerializer { run.reason().ifPresent(reason -> runObject.setString(reasonField, reason)); } - private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) { + private void toSlime(Version platformVersion, RevisionId revsion, Cursor versionsObject) { versionsObject.setString(platformVersionField, platformVersion.toString()); - applicationVersion.buildNumber().ifPresent(number -> versionsObject.setLong(buildField, number)); - applicationVersion.source().map(SourceRevision::repository).ifPresent(repository -> versionsObject.setString(repositoryField, repository)); - applicationVersion.source().map(SourceRevision::branch).ifPresent(branch -> versionsObject.setString(branchField, branch)); - applicationVersion.source().map(SourceRevision::commit).ifPresent(commit -> versionsObject.setString(commitField, commit)); - applicationVersion.authorEmail().ifPresent(email -> versionsObject.setString(authorEmailField, email)); - applicationVersion.compileVersion().ifPresent(version -> versionsObject.setString(compileVersionField, version.toString())); - applicationVersion.buildTime().ifPresent(time -> versionsObject.setLong(buildTimeField, time.toEpochMilli())); - applicationVersion.sourceUrl().ifPresent(url -> versionsObject.setString(sourceUrlField, url)); - applicationVersion.commit().ifPresent(commit -> versionsObject.setString(commitField, commit)); - versionsObject.setBool(deployedDirectlyField, applicationVersion.isDeployedDirectly()); + versionsObject.setLong(buildField, revsion.number()); + versionsObject.setBool(deployedDirectlyField, ! revsion.isProduction()); } // Don't change this - introduce a separate array with new values if needed. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index f57bf75de40..6d146c6ba66 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2465,7 +2465,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .max(Comparator.naturalOrder()) .or(() -> applications.stream() .flatMap(application -> application.instances().values().stream()) - .flatMap(instance -> JobType.allIn(controller.system()).stream() + .flatMap(instance -> JobType.allIn(controller.zoneRegistry()).stream() .filter(job -> job.environment() == Environment.dev) .flatMap(jobType -> controller.jobController().last(instance.id(), jobType).stream())) .map(Run::start) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index cf3642144b1..0bbfaea3e6f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -77,18 +77,18 @@ class JobControllerApiHandlerHelper { Cursor responseObject = slime.setObject(); Cursor jobsArray = responseObject.setArray("deployment"); - Arrays.stream(JobType.values()) - .filter(type -> type.environment().isManuallyDeployed()) - .map(devType -> new JobId(id, devType)) - .forEach(job -> { - Collection<Run> runs = controller.jobController().runs(job).descendingMap().values(); - if (runs.isEmpty()) - return; - - Cursor jobObject = jobsArray.addObject(); - jobObject.setString("jobName", job.type().jobName()); - toSlime(jobObject.setArray("runs"), runs, controller.applications().requireApplication(TenantAndApplicationId.from(id)), 10, baseUriForJobs); - }); + JobType.allIn(controller.zoneRegistry()).stream() + .filter(type -> type.environment().isManuallyDeployed()) + .map(devType -> new JobId(id, devType)) + .forEach(job -> { + Collection<Run> runs = controller.jobController().runs(job).descendingMap().values(); + if (runs.isEmpty()) + return; + + Cursor jobObject = jobsArray.addObject(); + jobObject.setString("jobName", job.type().jobName()); + toSlime(jobObject.setArray("runs"), runs, controller.applications().requireApplication(TenantAndApplicationId.from(id)), 10, baseUriForJobs); + }); return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 1a4a42cb521..effa0906b94 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -210,7 +210,7 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { }); }); } - JobType.allIn(controller.system()).stream() + JobType.allIn(controller.zoneRegistry()).stream() .filter(job -> ! job.environment().isManuallyDeployed()) .map(JobType::jobName).forEach(root.setArray("jobs")::addString); return new SlimeJsonResponse(slime); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 850a8cab123..a5a09ab6551 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -152,7 +153,7 @@ public class RoutingPolicies { } /** Update global DNS records for given policies */ - private void updateGlobalDnsOf(RoutingPolicyList instancePolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + private void updateGlobalDnsOf(RoutingPolicyList instancePolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Mutex lock) { Map<RoutingId, List<RoutingPolicy>> routingTable = instancePolicies.asInstanceRoutingTable(); for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { RoutingId routingId = routeEntry.getKey(); @@ -225,7 +226,7 @@ public class RoutingPolicies { } - private void updateApplicationDnsOf(RoutingPolicyList routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + private void updateApplicationDnsOf(RoutingPolicyList routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Mutex lock) { // In the context of single deployment (which this is) there is only one routing policy per routing ID. I.e. // there is no scenario where more than one deployment within an instance can be a member the same // application-level endpoint. However, to allow this in the future the routing table remains @@ -307,7 +308,7 @@ public class RoutingPolicies { * * @return the updated policies */ - private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Lock lock) { + private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) { Map<RoutingPolicyId, RoutingPolicy> policies = new LinkedHashMap<>(instancePolicies.asMap()); for (LoadBalancer loadBalancer : allocation.loadBalancers) { if (loadBalancer.hostname().isEmpty()) continue; @@ -343,7 +344,7 @@ public class RoutingPolicies { * * @return the updated policies */ - private RoutingPolicyList removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Lock lock) { + private RoutingPolicyList removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) { Map<RoutingPolicyId, RoutingPolicy> newPolicies = new LinkedHashMap<>(instancePolicies.asMap()); Set<RoutingPolicyId> activeIds = allocation.asPolicyIds(); RoutingPolicyList removable = instancePolicies.deployment(allocation.deployment) @@ -363,7 +364,7 @@ public class RoutingPolicies { } /** Remove unreferenced instance endpoints from DNS */ - private void removeGlobalDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Lock lock) { + private void removeGlobalDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Mutex lock) { Set<RoutingId> removalCandidates = new HashSet<>(deploymentPolicies.asInstanceRoutingTable().keySet()); Set<RoutingId> activeRoutingIds = instanceRoutingIds(allocation); removalCandidates.removeAll(activeRoutingIds); @@ -380,7 +381,7 @@ public class RoutingPolicies { } /** Remove unreferenced application endpoints in given allocation from DNS */ - private void removeApplicationDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Lock lock) { + private void removeApplicationDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Mutex lock) { Map<RoutingId, List<RoutingPolicy>> routingTable = deploymentPolicies.asApplicationRoutingTable(); Set<RoutingId> removalCandidates = new HashSet<>(routingTable.keySet()); Set<RoutingId> activeRoutingIds = applicationRoutingIds(allocation); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java index 36a43f80e9a..39fc70aac64 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing.rotation; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import java.util.Objects; @@ -12,9 +13,9 @@ import java.util.Objects; */ public class RotationLock implements AutoCloseable { - private final Lock lock; + private final Mutex lock; - RotationLock(Lock lock) { + RotationLock(Mutex lock) { this.lock = Objects.requireNonNull(lock, "lock cannot be null"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java index cd6e3a49c46..27b61a4fd17 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.support.access; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; @@ -35,7 +36,7 @@ public class SupportAccessControl { } public SupportAccess disallow(DeploymentId deployment, String by) { - try (Lock lock = controller.curator().lockSupportAccess(deployment)) { + try (Mutex lock = controller.curator().lockSupportAccess(deployment)) { var now = controller.clock().instant(); SupportAccess supportAccess = forDeployment(deployment); if (supportAccess.currentStatus(now).state() == NOT_ALLOWED) { @@ -49,7 +50,7 @@ public class SupportAccessControl { } public SupportAccess allow(DeploymentId deployment, Instant until, String by) { - try (Lock lock = controller.curator().lockSupportAccess(deployment)) { + try (Mutex lock = controller.curator().lockSupportAccess(deployment)) { var now = controller.clock().instant(); if (until.isAfter(now.plus(MAX_SUPPORT_ACCESS_TIME))) { throw new IllegalArgumentException("Support access cannot be allowed for more than 10 days"); @@ -61,7 +62,7 @@ public class SupportAccessControl { } public SupportAccess registerGrant(DeploymentId deployment, String by, X509Certificate certificate) { - try (Lock lock = controller.curator().lockSupportAccess(deployment)) { + try (Mutex lock = controller.curator().lockSupportAccess(deployment)) { var now = controller.clock().instant(); SupportAccess supportAccess = forDeployment(deployment); if (certificate.getNotAfter().toInstant().isBefore(now)) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 04ee21117a9..5a330b00809 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -24,10 +24,12 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; import java.util.stream.Collectors; import static ai.vespa.validation.Validation.require; @@ -213,9 +215,9 @@ public class DeploymentTriggerTest { app.runJob(systemTest).runJob(stagingTest); tester.triggerJobs(); tester.runner().run(); - assertEquals(EnumSet.of(productionUsCentral1), tester.jobs().active().stream() - .map(run -> run.id().type()) - .collect(Collectors.toCollection(() -> EnumSet.noneOf(JobType.class)))); + assertEquals(Set.of(productionUsCentral1), tester.jobs().active().stream() + .map(run -> run.id().type()) + .collect(Collectors.toCollection(HashSet::new))); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 1fc99b0da1c..5941812d66d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -159,6 +159,7 @@ public class ApplicationSerializerTest { assertEquals(original.id(), serialized.id()); assertEquals(original.createdAt(), serialized.createdAt()); assertEquals(applicationVersion1, serialized.revisions().last().get()); + assertEquals(applicationVersion1, serialized.revisions().get(serialized.instances().get(id1.instance()).deployments().get(zone1).revision())); assertEquals(original.revisions().last(), serialized.revisions().last()); assertEquals(original.revisions().last().get().authorEmail(), serialized.revisions().last().get().authorEmail()); assertEquals(original.revisions().last().get().buildTime(), serialized.revisions().last().get().buildTime()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index f5922fd74aa..c63af87c08c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -90,21 +90,6 @@ public class RunSerializerTest { assertEquals(new Version(1, 2, 3), run.versions().targetPlatform()); RevisionId revision1 = RevisionId.forDevelopment(123, id.job()); RevisionId revision2 = RevisionId.forProduction(122); - ApplicationVersion applicationVersion1 = new ApplicationVersion(revision1, - Optional.of(new SourceRevision("git@github.com:user/repo.git", "master", "f00bad")), - Optional.of("a@b"), - Optional.of(Version.fromString("6.3.1")), - Optional.of(Instant.ofEpochMilli(100)), - Optional.empty(), - Optional.empty(), - Optional.empty(), - true, - false, - Optional.empty(), - 0); - ApplicationVersion applicationVersion2 = ApplicationVersion.from(revision2, new SourceRevision("git@github.com:user/repo.git", - "master", - "badb17")); assertEquals(revision1, run.versions().targetRevision()); assertEquals("because", run.reason().get()); assertEquals(new Version(1, 2, 2), run.versions().sourcePlatform().get()); @@ -138,8 +123,6 @@ public class RunSerializerTest { .build(), run.steps()); - Map<RevisionId, ApplicationVersion> revisions = Map.of(revision1, applicationVersion1, revision2, applicationVersion2); - run = run.with(1L << 50) .with(Instant.now().truncatedTo(MILLIS)) .noNodesDownSince(Instant.now().truncatedTo(MILLIS)) @@ -148,8 +131,7 @@ public class RunSerializerTest { assertEquals(aborted, run.status()); assertTrue(run.hasEnded()); - // Run phoenix = serializer.runsFromSlime(serializer.toSlime(List.of(run))); // TODO jonmv: use runs again, once compatability code is gone. - Run phoenix = serializer.runFromSlime(serializer.toSlime(run, revisions::get)); + Run phoenix = serializer.runsFromSlime(serializer.toSlime(List.of(run))).get(id); assertEquals(run.id(), phoenix.id()); assertEquals(run.start(), phoenix.start()); assertEquals(run.end(), phoenix.end()); @@ -162,11 +144,11 @@ public class RunSerializerTest { assertEquals(run.isDryRun(), phoenix.isDryRun()); assertEquals(run.reason(), phoenix.reason()); - assertEquals(new String(SlimeUtils.toJsonBytes(serializer.toSlime(run, revisions::get).get(), false), UTF_8), - new String(SlimeUtils.toJsonBytes(serializer.toSlime(phoenix, revisions::get).get(), false), UTF_8)); + assertEquals(new String(SlimeUtils.toJsonBytes(serializer.toSlime(run).get(), false), UTF_8), + new String(SlimeUtils.toJsonBytes(serializer.toSlime(phoenix).get(), false), UTF_8)); Run initial = Run.initial(id, run.versions(), run.isRedeployment(), run.start(), JobProfile.production, Optional.empty()); - assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial, revisions::get))); + assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial))); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json index 2e9618c890d..63869ecfba8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json @@ -6,7 +6,7 @@ { "at": 0, "type": "info", - "message": "Deploying platform version 6.1 and application dev build 1 for devUsEast1 of default ..." + "message": "Deploying platform version 6.1 and application dev build 1 for dev-us-east-1 of default ..." }, { "at": 0, |