diff options
Diffstat (limited to 'controller-server/src/main/java')
90 files changed, 1749 insertions, 1661 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 30f416747e0..af8965bdeff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -7,23 +7,22 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.ApplicationActivity; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.security.PublicKey; import java.time.Instant; -import java.util.ArrayDeque; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.Deque; import java.util.List; import java.util.Map; import java.util.Objects; @@ -31,9 +30,7 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.SortedSet; import java.util.TreeMap; -import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Collectors; @@ -50,8 +47,7 @@ public class Application { private final Instant createdAt; private final DeploymentSpec deploymentSpec; private final ValidationOverrides validationOverrides; - private final Optional<ApplicationVersion> latestVersion; - private final SortedSet<ApplicationVersion> versions; + private final RevisionHistory revisions; private final OptionalLong projectId; private final Optional<IssueId> deploymentIssueId; private final Optional<IssueId> ownershipIssueId; @@ -63,16 +59,16 @@ public class Application { /** Creates an empty application. */ public Application(TenantAndApplicationId id, Instant now) { - this(id, now, DeploymentSpec.empty, ValidationOverrides.empty, - Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), - new ApplicationMetrics(0, 0), Set.of(), OptionalLong.empty(), Optional.empty(), new TreeSet<>(), List.of()); + this(id, now, DeploymentSpec.empty, ValidationOverrides.empty, Optional.empty(), + Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(0, 0), + Set.of(), OptionalLong.empty(), RevisionHistory.empty(), List.of()); } // DO NOT USE! For serialization purposes, only. public Application(TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner, OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, OptionalLong projectId, - Optional<ApplicationVersion> latestVersion, SortedSet<ApplicationVersion> versions, Collection<Instance> instances) { + RevisionHistory revisions, Collection<Instance> instances) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null"); this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); @@ -84,13 +80,12 @@ public class Application { this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null"); this.deployKeys = Objects.requireNonNull(deployKeys, "deployKeys cannot be null"); this.projectId = Objects.requireNonNull(projectId, "projectId cannot be null"); - this.latestVersion = requireNotUnknown(latestVersion); - this.versions = versions; + this.revisions = revisions; this.instances = instances.stream().collect( Collectors.collectingAndThen(Collectors.toMap(Instance::name, Function.identity(), (i1, i2) -> { - throw new IllegalArgumentException("Duplicate key " + i1); + throw new IllegalArgumentException("Duplicate instance " + i1.id()); }, TreeMap::new), Collections::unmodifiableMap) @@ -110,29 +105,8 @@ public class Application { /** Returns the project id of this application, if it has any. */ public OptionalLong projectId() { return projectId; } - /** Returns the last submitted version of this application. */ - public Optional<ApplicationVersion> latestVersion() { - return versions.isEmpty() ? Optional.empty() : Optional.of(versions.last()); - } - - /** Returns the currently deployed versions of the application, ordered from oldest to newest. */ - public SortedSet<ApplicationVersion> versions() { - return versions; - } - - /** Returns the currently deployed versions of the application */ - public Collection<ApplicationVersion> deployableVersions(boolean ascending) { - Deque<ApplicationVersion> versions = new ArrayDeque<>(); - String previousHash = ""; - for (ApplicationVersion version : versions()) { - if (version.bundleHash().isEmpty() || ! previousHash.equals(version.bundleHash().get())) { - if (ascending) versions.addLast(version); - else versions.addFirst(version); - } - previousHash = version.bundleHash().orElse(""); - } - return versions; - } + /** Returns the known revisions for this application. */ + public RevisionHistory revisions() { return revisions; } /** * Returns the last deployed validation overrides of this application, @@ -212,10 +186,10 @@ public class Application { /** * Returns the oldest application version this has deployed in a permanent zone (not test or staging). */ - public Optional<ApplicationVersion> oldestDeployedApplication() { + public Optional<RevisionId> oldestDeployedRevision() { return productionDeployments().values().stream().flatMap(List::stream) - .map(Deployment::applicationVersion) - .filter(version -> ! version.isUnknown() && ! version.isDeployedDirectly()) + .map(Deployment::revision) + .filter(RevisionId::isProduction) .min(Comparator.naturalOrder()); } @@ -241,15 +215,6 @@ public class Application { /** Returns the set of deploy keys for this application. */ public Set<PublicKey> deployKeys() { return deployKeys; } - private static Optional<ApplicationVersion> requireNotUnknown(Optional<ApplicationVersion> latestVersion) { - Objects.requireNonNull(latestVersion, "latestVersion cannot be null"); - latestVersion.ifPresent(version -> { - if (version.isUnknown()) - throw new IllegalArgumentException("latestVersion cannot be unknown"); - }); - return latestVersion; - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 48f909df6b6..6907747646e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.log.LogLevel; import com.yahoo.text.Text; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; @@ -27,7 +28,6 @@ import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; -import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; @@ -41,6 +41,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationS import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; @@ -58,6 +60,7 @@ import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates; import com.yahoo.vespa.hosted.controller.concurrent.Once; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; +import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.notification.Notification; @@ -82,6 +85,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -89,6 +93,8 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.logging.Level; @@ -160,12 +166,13 @@ public class ApplicationController { for (InstanceName instance : application.get().deploymentSpec().instanceNames()) if ( ! application.get().instances().containsKey(instance)) application = withNewInstance(application, id.instance(instance)); + store(application); }); count++; } log.log(Level.INFO, Text.format("Wrote %d applications in %s", count, - Duration.between(start, clock.instant()))); + Duration.between(start, clock.instant()))); }); } @@ -175,7 +182,6 @@ public class ApplicationController { } /** Returns the instance with the given id, or null if it is not present */ - // TODO jonmv: remove or inline public Optional<Instance> getInstance(ApplicationId id) { return getApplication(TenantAndApplicationId.from(id)).flatMap(application -> application.get(id.instance())); } @@ -283,7 +289,7 @@ public class ApplicationController { if (oldest == null || version.isBefore(oldest)) oldest = version; - if (run.status() == RunStatus.success) + if (run.hasSucceeded()) return Optional.of(oldest); } // If no successful run was found, ask the node repository in the relevant zone. @@ -292,7 +298,7 @@ public class ApplicationController { /** Reads the oldest installed platform for the given application and zone from the node repo of that zone. */ private Optional<Version> oldestInstalledPlatform(JobId job) { - return configServer.nodeRepository().list(job.type().zone(controller.system()), + return configServer.nodeRepository().list(job.type().zone(), NodeFilter.all() .applications(job.application()) .states(active, reserved)) @@ -399,7 +405,7 @@ public class ApplicationController { * @throws IllegalArgumentException if the application already exists */ public Application createApplication(TenantAndApplicationId id, Credentials credentials) { - try (Lock lock = lock(id)) { + try (Mutex lock = lock(id)) { if (getApplication(id).isPresent()) throw new IllegalArgumentException("Could not create '" + id + "': Application already exists"); if (getApplication(dashToUnderscore(id)).isPresent()) // VESPA-1945 @@ -450,10 +456,10 @@ public class ApplicationController { throw new IllegalArgumentException("'" + job.application() + "' is a tester application!"); TenantAndApplicationId applicationId = TenantAndApplicationId.from(job.application()); - ZoneId zone = job.type().zone(controller.system()); + ZoneId zone = job.type().zone(); DeploymentId deployment = new DeploymentId(job.application(), zone); - try (Lock deploymentLock = lockForDeployment(job.application(), zone)) { + try (Mutex deploymentLock = lockForDeployment(job.application(), zone)) { Set<ContainerEndpoint> containerEndpoints; Optional<EndpointCertificateMetadata> endpointCertificateMetadata; @@ -464,11 +470,13 @@ public class ApplicationController { throw new IllegalStateException("No deployment expected for " + job + " now, as no job is running"); Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform()); - ApplicationVersion revision = run.versions().sourceApplication().filter(__ -> deploySourceVersions).orElse(run.versions().targetApplication()); + RevisionId revision = run.versions().sourceRevision().filter(__ -> deploySourceVersions).orElse(run.versions().targetRevision()); ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(deployment, revision)); - try (Lock lock = lock(applicationId)) { + AtomicReference<RevisionId> lastRevision = new AtomicReference<>(); + try (Mutex lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); + application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set); Instance instance = application.get().require(job.application().instance()); if ( ! applicationPackage.trustedCertificates().isEmpty() @@ -488,22 +496,25 @@ public class ApplicationController { var quotaUsage = deploymentQuotaUsage(zone, job.application()); // For direct deployments use the full deployment ID, but otherwise use just the tenant and application as - // the source since it's the same application, so it should have the same warnings - NotificationSource source = zone.environment().isManuallyDeployed() ? - NotificationSource.from(deployment) : NotificationSource.from(applicationId); - - @SuppressWarnings("deprecation") - List<String> warnings = Optional.ofNullable(result.prepareResponse().log) - .map(logs -> logs.stream() - .filter(log -> log.applicationPackage) - .filter(log -> LogLevel.parse(log.level).intValue() >= Level.WARNING.intValue()) - .map(log -> log.message) - .sorted() - .distinct() - .collect(Collectors.toList())) - .orElseGet(List::of); - if (warnings.isEmpty()) controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage); - else controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings); + // the source since it's the same application, so it should have the same warnings. + // These notifications are only updated when the last submitted revision is deployed here. + NotificationSource source = zone.environment().isManuallyDeployed() + ? NotificationSource.from(deployment) + : revision.equals(lastRevision.get()) ? NotificationSource.from(applicationId) : null; + if (source != null) { + @SuppressWarnings("deprecation") + List<String> warnings = Optional.ofNullable(result.prepareResponse().log) + .map(logs -> logs.stream() + .filter(log -> log.applicationPackage) + .filter(log -> LogLevel.parse(log.level).intValue() >= Level.WARNING.intValue()) + .map(log -> log.message) + .sorted() + .distinct() + .collect(Collectors.toList())) + .orElseGet(List::of); + if (warnings.isEmpty()) controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage); + else controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings); + } lockApplicationOrThrow(applicationId, application -> store(application.with(job.application().instance(), @@ -538,26 +549,13 @@ public class ApplicationController { controller.jobController().deploymentStatus(application.get()); for (Notification notification : controller.notificationsDb().listNotifications(NotificationSource.from(application.get().id()), true)) { - if ( ! notification.source().instance().map(declaredInstances::contains).orElse(false)) + if ( ! notification.source().instance().map(declaredInstances::contains).orElse(true)) controller.notificationsDb().removeNotifications(notification.source()); if (notification.source().instance().isPresent() && ! notification.source().zoneId().map(application.get().require(notification.source().instance().get()).deployments()::containsKey).orElse(false)) controller.notificationsDb().removeNotifications(notification.source()); } - ApplicationVersion oldestDeployedVersion = application.get().oldestDeployedApplication() - .orElse(ApplicationVersion.unknown); - - List<ApplicationVersion> olderVersions = application.get().versions().stream() - .filter(version -> version.compareTo(oldestDeployedVersion) < 0) - .sorted() - .collect(Collectors.toList()); - - // Remove any version not deployed anywhere - but keep one - for (ApplicationVersion version : olderVersions) { - application = application.withoutVersion(version); - } - store(application); } @@ -629,7 +627,7 @@ public class ApplicationController { deploymentQuota, tenantSecretStores, operatorCertificates, dryRun)); - return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), + return new ActivateResult(new com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { // Even if prepare fails, routing configuration may need to be updated @@ -697,7 +695,6 @@ public class ApplicationController { } applicationStore.removeAll(id.tenant(), id.application()); - applicationStore.removeAllTesters(id.tenant(), id.application()); applicationStore.putMetaTombstone(id.tenant(), id.application(), clock.instant()); credentials.ifPresent(creds -> accessControl.deleteApplication(id, creds)); @@ -753,7 +750,7 @@ public class ApplicationController { * @param action Function which acts on the locked application. */ public void lockApplicationIfPresent(TenantAndApplicationId applicationId, Consumer<LockedApplication> action) { - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { getApplication(applicationId).map(application -> new LockedApplication(application, lock)).ifPresent(action); } } @@ -766,7 +763,7 @@ public class ApplicationController { * @throws IllegalArgumentException when application does not exist. */ public void lockApplicationOrThrow(TenantAndApplicationId applicationId, Consumer<LockedApplication> action) { - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { action.accept(new LockedApplication(requireApplication(applicationId), lock)); } } @@ -839,14 +836,14 @@ public class ApplicationController { * Any operation which stores an application need to first acquire this lock, then read, modify * and store the application, and finally release (close) the lock. */ - Lock lock(TenantAndApplicationId application) { + Mutex lock(TenantAndApplicationId application) { return curator.lock(application); } /** * Returns a lock which provides exclusive rights to deploying this application to the given zone. */ - private Lock lockForDeployment(ApplicationId application, ZoneId zone) { + private Mutex lockForDeployment(ApplicationId application, ZoneId zone) { return curator.lockForDeployment(application, zone); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index c35e8c5a7ac..48e663e7feb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -7,12 +7,12 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.Metric; -import com.yahoo.net.HostName; -import com.yahoo.vespa.curator.Lock; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; @@ -28,14 +28,12 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.JobControlFlags; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.support.access.SupportAccessControl; -import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.yolean.concurrent.Sleeper; import java.time.Clock; @@ -49,7 +47,6 @@ import java.util.Set; import java.util.TreeSet; import java.util.function.Function; import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -68,7 +65,6 @@ public class Controller extends AbstractComponent { private static final Logger log = Logger.getLogger(Controller.class.getName()); - private final Supplier<String> hostnameSupplier; private final CuratorDb curator; private final JobControl jobControl; private final ApplicationController applicationController; @@ -100,15 +96,14 @@ public class Controller extends AbstractComponent { public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, FlagSource flagSource, MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, ControllerConfig controllerConfig) { - this(curator, rotationsConfig, accessControl, HostName::getLocalhost, flagSource, + this(curator, rotationsConfig, accessControl, flagSource, mavenRepository, serviceRegistry, metric, secretStore, controllerConfig, Sleeper.DEFAULT); } public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, - Supplier<String> hostnameSupplier, FlagSource flagSource, MavenRepository mavenRepository, + FlagSource flagSource, MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, ControllerConfig controllerConfig, Sleeper sleeper) { - this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null"); this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); this.serviceRegistry = Objects.requireNonNull(serviceRegistry, "ServiceRegistry cannot be null"); this.zoneRegistry = Objects.requireNonNull(serviceRegistry.zoneRegistry(), "ZoneRegistry cannot be null"); @@ -128,12 +123,12 @@ public class Controller extends AbstractComponent { auditLogger = new AuditLogger(curator, clock); jobControl = new JobControl(new JobControlFlags(curator, flagSource)); archiveBucketDb = new CuratorArchiveBucketDb(this); - notifier = new Notifier(curator, serviceRegistry.mailer()); + notifier = new Notifier(curator, serviceRegistry.zoneRegistry(), serviceRegistry.mailer()); notificationsDb = new NotificationsDb(this); supportAccessControl = new SupportAccessControl(this); // Record the version of this controller - curator().writeControllerVersion(this.hostname(), ControllerVersion.CURRENT); + curator().writeControllerVersion(this.hostname(), serviceRegistry.controllerVersion()); jobController.updateStorage(); } @@ -174,11 +169,6 @@ public class Controller extends AbstractComponent { public ControllerConfig controllerConfig() { return controllerConfig; } - public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, - String environment, String region) { - return serviceRegistry.configServer().getApplicationView(tenantName, applicationName, instanceName, environment, region); - } - /** Replace the current version status by a new one */ public void updateVersionStatus(VersionStatus newStatus) { VersionStatus currentStatus = readVersionStatus(); @@ -199,7 +189,7 @@ public class Controller extends AbstractComponent { /** Remove confidence override for versions matching given filter */ public void removeConfidenceOverride(Predicate<Version> filter) { - try (Lock lock = curator.lockConfidenceOverrides()) { + try (Mutex lock = curator.lockConfidenceOverrides()) { Map<Version, VespaVersion.Confidence> overrides = new LinkedHashMap<>(curator.readConfidenceOverrides()); overrides.keySet().removeIf(filter); curator.writeConfidenceOverrides(overrides); @@ -238,7 +228,7 @@ public class Controller extends AbstractComponent { throw new IllegalArgumentException("Cloud '" + cloudName + "' does not exist in this system"); } Instant scheduledAt = clock.instant(); - try (Lock lock = curator.lockOsVersions()) { + try (Mutex lock = curator.lockOsVersions()) { Map<CloudName, OsVersionTarget> targets = curator.readOsVersionTargets().stream() .collect(Collectors.toMap(t -> t.osVersion().cloud(), Function.identity())); @@ -268,7 +258,7 @@ public class Controller extends AbstractComponent { /** Replace the current OS version status with a new one */ public void updateOsVersionStatus(OsVersionStatus newStatus) { - try (Lock lock = curator.lockOsVersionStatus()) { + try (Mutex lock = curator.lockOsVersionStatus()) { OsVersionStatus currentStatus = curator.readOsVersionStatus(); for (CloudName cloud : clouds()) { Set<Version> newVersions = newStatus.versionsIn(cloud); @@ -282,8 +272,8 @@ public class Controller extends AbstractComponent { } /** Returns the hostname of this controller */ - public com.yahoo.config.provision.HostName hostname() { - return com.yahoo.config.provision.HostName.from(hostnameSupplier.get()); + public HostName hostname() { + return serviceRegistry.getHostname(); } public SystemName system() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index e5ab13d2127..402a4bf49a8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -6,8 +6,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -45,17 +45,15 @@ public class Instance { private final RotationStatus rotationStatus; private final Map<JobType, Instant> jobPauses; private final Change change; - private final Optional<ApplicationVersion> latestDeployed; /** Creates an empty instance */ public Instance(ApplicationId id) { - this(id, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty(), Optional.empty()); + this(id, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty()); } /** Creates an empty instance*/ public Instance(ApplicationId id, Collection<Deployment> deployments, Map<JobType, Instant> jobPauses, - List<AssignedRotation> rotations, RotationStatus rotationStatus, Change change, - Optional<ApplicationVersion> latestDeployed) { + List<AssignedRotation> rotations, RotationStatus rotationStatus, Change change) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.deployments = Objects.requireNonNull(deployments, "deployments cannot be null").stream() .collect(Collectors.toUnmodifiableMap(Deployment::zone, Function.identity())); @@ -63,19 +61,18 @@ public class Instance { this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null")); this.rotationStatus = Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null"); this.change = Objects.requireNonNull(change, "change cannot be null"); - this.latestDeployed = Objects.requireNonNull(latestDeployed, "latestDeployed cannot be null"); } - public Instance withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, + public Instance withNewDeployment(ZoneId zone, RevisionId revision, Version version, Instant instant, Map<DeploymentMetrics.Warning, Integer> warnings, QuotaUsage quotaUsage) { // Use info from previous deployment if available, otherwise create a new one. - Deployment previousDeployment = deployments.getOrDefault(zone, new Deployment(zone, applicationVersion, + Deployment previousDeployment = deployments.getOrDefault(zone, new Deployment(zone, revision, version, instant, DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty())); - Deployment newDeployment = new Deployment(zone, applicationVersion, version, instant, + Deployment newDeployment = new Deployment(zone, revision, version, instant, previousDeployment.metrics().with(warnings), previousDeployment.activity(), quotaUsage, @@ -90,7 +87,7 @@ public class Instance { else jobPauses.remove(jobType); - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } public Instance recordActivityAt(Instant instant, ZoneId zone) { @@ -121,19 +118,15 @@ public class Instance { } public Instance with(List<AssignedRotation> assignedRotations) { - return new Instance(id, deployments.values(), jobPauses, assignedRotations, rotationStatus, change, latestDeployed); + return new Instance(id, deployments.values(), jobPauses, assignedRotations, rotationStatus, change); } public Instance with(RotationStatus rotationStatus) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } public Instance withChange(Change change) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed); - } - - public Instance withLatestDeployed(ApplicationVersion latestDeployed) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, Optional.of(latestDeployed)); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } private Instance with(Deployment deployment) { @@ -143,7 +136,7 @@ public class Instance { } private Instance with(Map<ZoneId, Deployment> deployments) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } public ApplicationId id() { return id; } @@ -188,11 +181,6 @@ public class Instance { return change; } - /** Returns the application version that last rolled out to this instance. */ - public Optional<ApplicationVersion> latestDeployed() { - return latestDeployed; - } - /** Returns the total quota usage for this instance, excluding temporary deployments **/ public QuotaUsage quotaUsage() { return deployments.values().stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 06ff381e4dc..3e822415e96 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -4,11 +4,12 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.InstanceName; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import java.security.PublicKey; @@ -21,8 +22,6 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.function.UnaryOperator; /** @@ -32,7 +31,7 @@ import java.util.function.UnaryOperator; */ public class LockedApplication { - private final Lock lock; + private final Mutex lock; private final TenantAndApplicationId id; private final Instant createdAt; private final DeploymentSpec deploymentSpec; @@ -44,8 +43,7 @@ public class LockedApplication { private final ApplicationMetrics metrics; private final Set<PublicKey> deployKeys; private final OptionalLong projectId; - private final Optional<ApplicationVersion> latestVersion; - private final SortedSet<ApplicationVersion> versions; + private final RevisionHistory revisions; private final Map<InstanceName, Instance> instances; /** @@ -54,20 +52,19 @@ public class LockedApplication { * @param application The application to lock. * @param lock The lock for the application. */ - LockedApplication(Application application, Lock lock) { + LockedApplication(Application application, Mutex lock) { this(Objects.requireNonNull(lock, "lock cannot be null"), application.id(), application.createdAt(), application.deploymentSpec(), application.validationOverrides(), application.deploymentIssueId(), application.ownershipIssueId(), application.owner(), application.majorVersion(), application.metrics(), application.deployKeys(), - application.projectId(), application.latestVersion(), application.versions(), application.instances()); + application.projectId(), application.instances(), application.revisions()); } - private LockedApplication(Lock lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, + private LockedApplication(Mutex lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner, OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, - OptionalLong projectId, Optional<ApplicationVersion> latestVersion, SortedSet<ApplicationVersion> versions, - Map<InstanceName, Instance> instances) { + OptionalLong projectId, Map<InstanceName, Instance> instances, RevisionHistory revisions) { this.lock = lock; this.id = id; this.createdAt = createdAt; @@ -80,8 +77,7 @@ public class LockedApplication { this.metrics = metrics; this.deployKeys = deployKeys; this.projectId = projectId; - this.latestVersion = latestVersion; - this.versions = versions; + this.revisions = revisions; this.instances = Map.copyOf(instances); } @@ -89,7 +85,7 @@ public class LockedApplication { public Application get() { return new Application(id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances.values()); + projectId, revisions, instances.values()); } LockedApplication withNewInstance(InstanceName instance) { @@ -97,7 +93,7 @@ public class LockedApplication { instances.put(instance, new Instance(id.instance(instance))); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication with(InstanceName instance, UnaryOperator<Instance> modification) { @@ -105,7 +101,7 @@ public class LockedApplication { instances.put(instance, modification.apply(instances.get(instance))); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication without(InstanceName instance) { @@ -113,51 +109,43 @@ public class LockedApplication { instances.remove(instance); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); - } - - public LockedApplication withNewSubmission(ApplicationVersion latestVersion) { - SortedSet<ApplicationVersion> applicationVersions = new TreeSet<>(versions); - applicationVersions.add(latestVersion); - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, Optional.of(latestVersion), applicationVersions, instances); + projectId, instances, revisions); } public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication withDeploymentIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, Optional.ofNullable(issueId), ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication with(DeploymentSpec deploymentSpec) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication with(ValidationOverrides validationOverrides) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication withOwnershipIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, Optional.of(issueId), owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication withOwner(User owner) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, Optional.of(owner), majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } /** Set a major version for this, or set to null to remove any major version override */ @@ -165,13 +153,13 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion), - metrics, deployKeys, projectId, latestVersion, versions, instances); + metrics, deployKeys, projectId, instances, revisions); } public LockedApplication with(ApplicationMetrics metrics) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication withDeployKey(PublicKey pemDeployKey) { @@ -179,7 +167,7 @@ public class LockedApplication { keys.add(pemDeployKey); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication withoutDeployKey(PublicKey pemDeployKey) { @@ -187,15 +175,13 @@ public class LockedApplication { keys.remove(pemDeployKey); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } - public LockedApplication withoutVersion(ApplicationVersion version) { - SortedSet<ApplicationVersion> applicationVersions = new TreeSet<>(versions); - applicationVersions.remove(version); + public LockedApplication withRevisions(UnaryOperator<RevisionHistory> change) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, applicationVersions, instances); + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, + deployKeys, projectId, instances, change.apply(revisions)); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index bcc3b9b54c7..7a0e60aacb4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -6,6 +6,7 @@ import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; import com.yahoo.security.KeyUtils; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; @@ -46,7 +47,7 @@ public abstract class LockedTenant { this.lastLoginInfo = requireNonNull(lastLoginInfo); } - static LockedTenant of(Tenant tenant, Lock lock) { + static LockedTenant of(Tenant tenant, Mutex lock) { switch (tenant.type()) { case athenz: return new Athenz((AthenzTenant) tenant); case cloud: return new Cloud((CloudTenant) tenant); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 384a5d0f1ac..beba1cdb358 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Text; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -23,7 +24,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; -import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -75,7 +75,7 @@ public class TenantController { /** Locks a tenant for modification and applies the given action. */ public <T extends LockedTenant> void lockIfPresent(TenantName name, Class<T> token, Consumer<T> action) { - try (Lock lock = lock(name)) { + try (Mutex lock = lock(name)) { get(name).map(tenant -> LockedTenant.of(tenant, lock)) .map(token::cast) .ifPresent(action); @@ -84,7 +84,7 @@ public class TenantController { /** Lock a tenant for modification and apply action. Throws if the tenant does not exist */ public <T extends LockedTenant> void lockOrThrow(TenantName name, Class<T> token, Consumer<T> action) { - try (Lock lock = lock(name)) { + try (Mutex lock = lock(name)) { action.accept(token.cast(LockedTenant.of(require(name), lock))); } } @@ -112,7 +112,7 @@ public class TenantController { /** Create a tenant, provided the given credentials are valid. */ public void create(TenantSpec tenantSpec, Credentials credentials) { - try (Lock lock = lock(tenantSpec.tenant())) { + try (Mutex lock = lock(tenantSpec.tenant())) { TenantId.validate(tenantSpec.tenant().value()); requireNonExistent(tenantSpec.tenant()); curator.writeTenant(accessControl.createTenant(tenantSpec, controller.clock().instant(), credentials, asList())); @@ -138,7 +138,7 @@ public class TenantController { /** Updates the tenant contained in the given tenant spec with new data. */ public void update(TenantSpec tenantSpec, Credentials credentials) { - try (Lock lock = lock(tenantSpec.tenant())) { + try (Mutex lock = lock(tenantSpec.tenant())) { curator.writeTenant(accessControl.updateTenant(tenantSpec, credentials, asList(), controller.applications().asList(tenantSpec.tenant()))); } @@ -149,7 +149,7 @@ public class TenantController { * new instant is later */ public void updateLastLogin(TenantName tenantName, List<LastLoginInfo.UserLevel> userLevels, Instant loggedInAt) { - try (Lock lock = lock(tenantName)) { + try (Mutex lock = lock(tenantName)) { Tenant tenant = require(tenantName); LastLoginInfo loginInfo = tenant.lastLoginInfo(); for (LastLoginInfo.UserLevel userLevel : userLevels) @@ -162,7 +162,7 @@ public class TenantController { /** Deletes the given tenant. */ public void delete(TenantName tenant, Optional<Credentials> credentials, boolean forget) { - try (Lock lock = lock(tenant)) { + try (Mutex lock = lock(tenant)) { Tenant oldTenant = get(tenant, true) .orElseThrow(() -> new NotExistsException("Could not delete tenant '" + tenant + "': Tenant not found")); @@ -203,7 +203,7 @@ public class TenantController { * Any operation which stores a tenant need to first acquire this lock, then read, modify * and store the tenant, and finally release (close) the lock. */ - private Lock lock(TenantName tenant) { + private Mutex lock(TenantName tenant) { return curator.lock(tenant); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 5c0669ad543..411c2c133f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -39,13 +39,6 @@ public class ApplicationList extends AbstractFilteringList<Application, Applicat .collect(Collectors.toUnmodifiableList())); } - // ----------------------------------- Accessors - - /** Returns the ids of the applications in this as an immutable list */ - public List<TenantAndApplicationId> idList() { - return mapToList(Application::id); - } - // ----------------------------------- Filters /** Returns the subset of applications which have at least one production deployment */ @@ -54,13 +47,6 @@ public class ApplicationList extends AbstractFilteringList<Application, Applicat .anyMatch(instance -> instance.productionDeployments().size() > 0)); } - /** Returns the subset of applications which have at least one deployment on a lower version than the given one */ - public ApplicationList onLowerVersionThan(Version version) { - return matching(application -> application.instances().values().stream() - .flatMap(instance -> instance.productionDeployments().values().stream()) - .anyMatch(deployment -> deployment.version().isBefore(version))); - } - /** Returns the subset of applications with at least one declared job in deployment spec. */ public ApplicationList withJobs() { return matching(application -> application.deploymentSpec().steps().stream() @@ -72,45 +58,9 @@ public class ApplicationList extends AbstractFilteringList<Application, Applicat return matching(application -> application.projectId().isPresent()); } - /** Returns the subset of applications that are allowed to upgrade at the given time */ - public ApplicationList canUpgradeAt(Instant instant) { - return matching(application -> application.deploymentSpec().instances().stream() - .allMatch(instance -> instance.canUpgradeAt(instant))); - } - - /** Returns the subset of applications that have at least one assigned rotation */ - public ApplicationList hasRotation() { - return matching(application -> application.instances().values().stream() - .anyMatch(instance -> ! instance.rotations().isEmpty())); - } - - /** - * Returns the subset of applications that hasn't pinned to an an earlier major version than the given one. - * - * @param targetMajorVersion the target major version which applications returned allows upgrading to - * @param defaultMajorVersion the default major version to assume for applications not specifying one - */ - public ApplicationList allowMajorVersion(int targetMajorVersion, int defaultMajorVersion) { - return matching(application -> targetMajorVersion <= application.deploymentSpec().majorVersion() - .orElse(application.majorVersion() - .orElse(defaultMajorVersion))); - } - /** Returns the subset of application which have submitted a non-empty deployment spec. */ public ApplicationList withDeploymentSpec() { return matching(application -> ! DeploymentSpec.empty.equals(application.deploymentSpec())); } - // ----------------------------------- Sorting - - /** - * Returns this list sorted by increasing deployed version. - * If multiple versions are deployed the oldest is used. - * Applications without any deployments are ordered first. - */ - public ApplicationList byIncreasingDeployedVersion() { - return sortedBy(Comparator.comparing(application -> application.oldestDeployedPlatform() - .orElse(Version.emptyVersion))); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java index 244fb952b3f..64cad599168 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import java.util.Objects; import java.util.Optional; @@ -28,22 +29,22 @@ public final class Change { private final Optional<Version> platform; /** The application version we are changing to, or empty if none */ - private final Optional<ApplicationVersion> application; + private final Optional<RevisionId> revision; /** Whether this change is a pin to its contained Vespa version, or to the application's current. */ private final boolean pinned; - private Change(Optional<Version> platform, Optional<ApplicationVersion> application, boolean pinned) { + private Change(Optional<Version> platform, Optional<RevisionId> revision, boolean pinned) { this.platform = requireNonNull(platform, "platform cannot be null"); - this.application = requireNonNull(application, "application cannot be null"); - if (application.isPresent() && (application.get().isUnknown() || application.get().isDeployedDirectly())) { + this.revision = requireNonNull(revision, "revision cannot be null"); + if (revision.isPresent() && ( ! revision.get().isProduction())) { throw new IllegalArgumentException("Application version to deploy must be a known version"); } this.pinned = pinned; } public Change withoutPlatform() { - return new Change(Optional.empty(), application, pinned); + return new Change(Optional.empty(), revision, pinned); } public Change withoutApplication() { @@ -52,7 +53,7 @@ public final class Change { /** Returns whether a change should currently be deployed */ public boolean hasTargets() { - return platform.isPresent() || application.isPresent(); + return platform.isPresent() || revision.isPresent(); } /** Returns whether this is the empty change. */ @@ -64,7 +65,7 @@ public final class Change { public Optional<Version> platform() { return platform; } /** Returns the application version carried by this. */ - public Optional<ApplicationVersion> application() { return application; } + public Optional<RevisionId> revision() { return revision; } public boolean isPinned() { return pinned; } @@ -76,30 +77,30 @@ public final class Change { if (pinned) throw new IllegalArgumentException("Not allowed to set a platform version when pinned."); - return new Change(Optional.of(platformVersion), application, pinned); + return new Change(Optional.of(platformVersion), revision, pinned); } - /** Returns a version of this change which replaces or adds this application change */ - public Change with(ApplicationVersion applicationVersion) { - return new Change(platform, Optional.of(applicationVersion), pinned); + /** Returns a version of this change which replaces or adds this revision change */ + public Change with(RevisionId revision) { + return new Change(platform, Optional.of(revision), pinned); } /** Returns a change with the versions of this, and with the platform version pinned. */ public Change withPin() { - return new Change(platform, application, true); + return new Change(platform, revision, true); } /** Returns a change with the versions of this, and with the platform version unpinned. */ public Change withoutPin() { - return new Change(platform, application, false); + return new Change(platform, revision, false); } /** Returns the change obtained when overwriting elements of the given change with any present in this */ public Change onTopOf(Change other) { if (platform.isPresent()) other = other.with(platform.get()); - if (application.isPresent()) - other = other.with(application.get()); + if (revision.isPresent()) + other = other.with(revision.get()); if (pinned) other = other.withPin(); return other; @@ -112,12 +113,12 @@ public final class Change { Change change = (Change) o; return pinned == change.pinned && Objects.equals(platform, change.platform) && - Objects.equals(application, change.application); + Objects.equals(revision, change.revision); } @Override public int hashCode() { - return Objects.hash(platform, application, pinned); + return Objects.hash(platform, revision, pinned); } @Override @@ -126,23 +127,23 @@ public final class Change { if (pinned) changes.add("pin to " + platform.map(Version::toString).orElse("current platform")); else - platform.ifPresent(version -> changes.add("upgrade to " + version.toString())); - application.ifPresent(version -> changes.add("application change to " + version.id())); + platform.ifPresent(version -> changes.add("upgrade to " + version)); + revision.ifPresent(revision -> changes.add("revision change to " + revision)); changes.setEmptyValue("no change"); return changes.toString(); } - public static Change of(ApplicationVersion applicationVersion) { - return new Change(Optional.empty(), Optional.of(applicationVersion), false); + public static Change of(RevisionId revision) { + return new Change(Optional.empty(), Optional.of(revision), false); } public static Change of(Version platformChange) { return new Change(Optional.of(platformChange), Optional.empty(), false); } - /** Returns whether this change carries an application downgrade relative to the given version. */ - public boolean downgrades(ApplicationVersion version) { - return application.map(version::compareTo).orElse(0) > 0; + /** Returns whether this change carries a revision downgrade relative to the given revision. */ + public boolean downgrades(RevisionId revision) { + return this.revision.map(revision::compareTo).orElse(0) > 0; } /** Returns whether this change carries a platform downgrade relative to the given version. */ @@ -150,9 +151,9 @@ public final class Change { return platform.map(version::compareTo).orElse(0) > 0; } - /** Returns whether this change carries an application upgrade relative to the given version. */ - public boolean upgrades(ApplicationVersion version) { - return application.map(version::compareTo).orElse(0) < 0; + /** Returns whether this change carries a revision upgrade relative to the given revision. */ + public boolean upgrades(RevisionId revision) { + return this.revision.map(revision::compareTo).orElse(0) < 0; } /** Returns whether this change carries a platform upgrade relative to the given version. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 6081c3323c8..2e4afb4e004 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -2,8 +2,8 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import java.time.Instant; import java.util.Objects; @@ -18,7 +18,7 @@ import java.util.OptionalDouble; public class Deployment { private final ZoneId zone; - private final ApplicationVersion applicationVersion; + private final RevisionId revision; private final Version version; private final Instant deployTime; private final DeploymentMetrics metrics; @@ -26,10 +26,10 @@ public class Deployment { private final QuotaUsage quota; private final OptionalDouble cost; - public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime, + public Deployment(ZoneId zone, RevisionId revision, Version version, Instant deployTime, DeploymentMetrics metrics, DeploymentActivity activity, QuotaUsage quota, OptionalDouble cost) { this.zone = Objects.requireNonNull(zone, "zone cannot be null"); - this.applicationVersion = Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null"); + this.revision = Objects.requireNonNull(revision, "revision cannot be null"); this.version = Objects.requireNonNull(version, "version cannot be null"); this.deployTime = Objects.requireNonNull(deployTime, "deployTime cannot be null"); this.metrics = Objects.requireNonNull(metrics, "deploymentMetrics cannot be null"); @@ -41,8 +41,8 @@ public class Deployment { /** Returns the zone this was deployed to */ public ZoneId zone() { return zone; } - /** Returns the deployed application version */ - public ApplicationVersion applicationVersion() { return applicationVersion; } + /** Returns the deployed application revision */ + public RevisionId revision() { return revision; } /** Returns the deployed Vespa version */ public Version version() { return version; } @@ -65,26 +65,26 @@ public class Deployment { public OptionalDouble cost() { return cost; } public Deployment recordActivityAt(Instant instant) { - return new Deployment(zone, applicationVersion, version, deployTime, metrics, + return new Deployment(zone, revision, version, deployTime, metrics, activity.recordAt(instant, metrics), quota, cost); } public Deployment withMetrics(DeploymentMetrics metrics) { - return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); + return new Deployment(zone, revision, version, deployTime, metrics, activity, quota, cost); } public Deployment withQuota(QuotaUsage quota) { - return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); + return new Deployment(zone, revision, version, deployTime, metrics, activity, quota, cost); } public Deployment withCost(double cost) { if (this.cost.isPresent() && Double.compare(this.cost.getAsDouble(), cost) == 0) return this; - return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, OptionalDouble.of(cost)); + return new Deployment(zone, revision, version, deployTime, metrics, activity, quota, OptionalDouble.of(cost)); } public Deployment withoutCost() { if (cost.isEmpty()) return this; - return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, OptionalDouble.empty()); + return new Deployment(zone, revision, version, deployTime, metrics, activity, quota, OptionalDouble.empty()); } @Override @@ -93,7 +93,7 @@ public class Deployment { if (o == null || getClass() != o.getClass()) return false; Deployment that = (Deployment) o; return zone.equals(that.zone) && - applicationVersion.equals(that.applicationVersion) && + revision.equals(that.revision) && version.equals(that.version) && deployTime.equals(that.deployTime) && metrics.equals(that.metrics) && @@ -104,12 +104,12 @@ public class Deployment { @Override public int hashCode() { - return Objects.hash(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); + return Objects.hash(zone, revision, version, deployTime, metrics, activity, quota, cost); } @Override public String toString() { - return "deployment to " + zone + " of " + applicationVersion + " on version " + version + " at " + deployTime; + return "deployment to " + zone + " of " + revision + " on version " + version + " at " + deployTime; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 7fc6b927bdd..8de72893a7c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -160,8 +160,7 @@ public class Endpoint { } private static URI createUrl(String name, TenantAndApplicationId application, Optional<InstanceName> instance, - List<Target> targets, Scope scope, SystemName system, Port port, boolean legacy, - RoutingMethod routingMethod) { + List<Target> targets, Scope scope, SystemName system, Port port, boolean legacy) { String separator = "."; String portPart = port.isDefault() ? "" : ":" + port.port; return URI.create("https://" + @@ -591,8 +590,7 @@ public class Endpoint { Objects.requireNonNull(scope, "scope must be non-null"), Objects.requireNonNull(system, "system must be non-null"), Objects.requireNonNull(port, "port must be non-null"), - legacy, - Objects.requireNonNull(routingMethod, "routingMethod must be non-null")); + legacy); return new Endpoint(application, instance, endpointId, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index 99b14bf289a..6178bfbb89e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -42,7 +42,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL */ public InstanceList compatibleWithPlatform(Version platform, Function<ApplicationId, VersionCompatibility> compatibility) { return matching(id -> instance(id).productionDeployments().values().stream() - .flatMap(deployment -> deployment.applicationVersion().compileVersion().stream()) + .flatMap(deployment -> application(id).revisions().get(deployment.revision()).compileVersion().stream()) .noneMatch(version -> compatibility.apply(id).refuse(platform, version))); } @@ -72,8 +72,9 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which contain declared jobs */ public InstanceList withDeclaredJobs() { - return matching(id -> instances.get(id).jobSteps().values().stream() - .anyMatch(job -> job.isDeclared() && job.job().get().application().equals(id))); + return matching(id -> instances.get(id).application().revisions().last().isPresent() + && instances.get(id).jobSteps().values().stream() + .anyMatch(job -> job.isDeclared() && job.job().get().application().equals(id))); } /** Returns the subset of instances which have at least one deployment on a lower version than the given one, or which have no production deployments */ @@ -95,7 +96,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which are currently deploying a new revision */ public InstanceList changingRevision() { - return matching(id -> instance(id).change().application().isPresent()); + return matching(id -> instance(id).change().revision().isPresent()); } /** Returns the subset of instances which currently have failing jobs on the given version */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 258884a4d11..344ed7ec729 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -66,9 +66,9 @@ public class ApplicationPackage { private static final String trustedCertificatesFile = "security/clients.pem"; private static final String buildMetaFile = "build-meta.json"; - private static final String deploymentFile = "deployment.xml"; + static final String deploymentFile = "deployment.xml"; private static final String validationOverridesFile = "validation-overrides.xml"; - private static final String servicesFile = "services.xml"; + static final String servicesFile = "services.xml"; private final String contentHash; private final String bundleHash; @@ -78,6 +78,7 @@ public class ApplicationPackage { private final ZipArchiveCache files; private final Optional<Version> compileVersion; private final Optional<Instant> buildTime; + private final Optional<Version> parentVersion; private final List<X509Certificate> trustedCertificates; /** @@ -110,6 +111,7 @@ public class ApplicationPackage { Optional<Inspector> buildMetaObject = files.get(buildMetaFile).map(SlimeUtils::jsonToSlime).map(Slime::get); this.compileVersion = buildMetaObject.flatMap(object -> parse(object, "compileVersion", field -> Version.fromString(field.asString()))); this.buildTime = buildMetaObject.flatMap(object -> parse(object, "buildTime", field -> Instant.ofEpochMilli(field.asLong()))); + this.parentVersion = buildMetaObject.flatMap(object -> parse(object, "parentVersion", field -> Version.fromString(field.asString()))); this.trustedCertificates = files.get(trustedCertificatesFile).map(bytes -> X509CertificateUtils.certificateListFromPem(new String(bytes, UTF_8))).orElse(List.of()); @@ -159,6 +161,9 @@ public class ApplicationPackage { /** Returns the time this package was built, if known. */ public Optional<Instant> buildTime() { return buildTime; } + /** Returns the parent version used to compile the package, if known. */ + public Optional<Version> parentVersion() { return parentVersion; } + /** Returns the list of certificates trusted by this application, or an empty list if no trust configured. */ public List<X509Certificate> trustedCertificates() { return trustedCertificates; @@ -166,7 +171,7 @@ public class ApplicationPackage { private static <Type> Optional<Type> parse(Inspector buildMetaObject, String fieldName, Function<Inspector, Type> mapper) { if ( ! buildMetaObject.field(fieldName).valid()) - throw new IllegalArgumentException("Missing value '" + fieldName + "' in '" + buildMetaFile + "'"); + return Optional.empty(); try { return Optional.of(mapper.apply(buildMetaObject.field(fieldName))); } @@ -197,11 +202,11 @@ public class ApplicationPackage { RegionName.defaultName()) .run(); // Populates the zip archive cache with files that would be included. } - catch (RuntimeException e) { + catch (IllegalArgumentException e) { throw e; } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalArgumentException(e); } } @@ -212,7 +217,7 @@ public class ApplicationPackage { entry -> entry.getValue().get()))); } - static byte[] filesZip(Map<String, byte[]> files) { + public static byte[] filesZip(Map<String, byte[]> files) { try (ZipBuilder zipBuilder = new ZipBuilder(files.values().stream().mapToInt(bytes -> bytes.length).sum() + 512)) { files.forEach(zipBuilder::add); zipBuilder.close(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java index 9ab5096ec8f..950eaea904a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java @@ -62,7 +62,7 @@ public class ApplicationPackageValidator { /** Verify that each of the production zones listed in the deployment spec exist in this system */ private void validateSteps(DeploymentSpec deploymentSpec) { for (var spec : deploymentSpec.instances()) { - new DeploymentSteps(spec, controller::system).jobs(); + new DeploymentSteps(spec, controller.zoneRegistry()).jobs(); spec.zones().stream() .filter(zone -> zone.environment() == Environment.prod) .forEach(zone -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java new file mode 100644 index 00000000000..fb352848911 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java @@ -0,0 +1,318 @@ +package com.yahoo.vespa.hosted.controller.application.pkg; + +import com.yahoo.config.application.api.DeploymentInstanceSpec; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.DeploymentSpec.Step; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.path.Path; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.text.Text; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; +import com.yahoo.vespa.hosted.controller.config.ControllerConfig; +import com.yahoo.vespa.hosted.controller.config.ControllerConfig.Steprunner.Testerapp; +import com.yahoo.yolean.Exceptions; + +import javax.security.auth.x500.X500Principal; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.jar.JarInputStream; +import java.util.jar.Manifest; +import java.util.regex.Pattern; + +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.production; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging_setup; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.system; +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.deploymentFile; +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.servicesFile; +import static com.yahoo.vespa.hosted.controller.application.pkg.ZipEntries.transferAndWrite; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toList; + +/** + * Validation and manipulation of test package. + * + * @author jonmv + */ +public class TestPackage { + + // Must match exactly the advertised resources of an AWS instance type. Also consider that the container + // will have ~1.8 GB less memory than equivalent resources in AWS (VESPA-16259). + static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); + static final NodeResources DEFAULT_TESTER_RESOURCES = new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any); + + private final ApplicationPackage applicationPackage; + private final X509Certificate certificate; + + public TestPackage(byte[] testPackage, boolean isPublicSystem, RunId id, Testerapp testerApp, + DeploymentSpec spec, Instant certificateValidFrom, Duration certificateValidDuration) { + + // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip. + Map<String, byte[]> entries = new HashMap<>(); + entries.put("artifacts/.ignore-" + UUID.randomUUID(), new byte[0]); + entries.put("tests/.ignore-" + UUID.randomUUID(), new byte[0]); + + entries.put(servicesFile, + servicesXml( ! isPublicSystem, + certificateValidFrom != null, + testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance())), + testerApp)); + + entries.put(deploymentFile, + deploymentXml(id.tester(), + spec.athenzDomain(), + spec.requireInstance(id.application().instance()) + .athenzService(id.type().zone().environment(), id.type().zone().region()))); + + if (certificateValidFrom != null) { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); + X500Principal subject = new X500Principal("CN=" + id.tester().id().toFullString() + "." + id.type() + "." + id.number()); + this.certificate = X509CertificateBuilder.fromKeypair(keyPair, + subject, + certificateValidFrom, + certificateValidFrom.plus(certificateValidDuration), + SignatureAlgorithm.SHA512_WITH_RSA, + BigInteger.valueOf(1)) + .build(); + entries.put("artifacts/key", KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8)); + entries.put("artifacts/cert", X509CertificateUtils.toPem(certificate).getBytes(UTF_8)); + } + else { + this.certificate = null; + } + + ByteArrayOutputStream buffer = new ByteArrayOutputStream(testPackage.length + 10_000); + transferAndWrite(buffer, new ByteArrayInputStream(testPackage), entries); + this.applicationPackage = new ApplicationPackage(buffer.toByteArray()); + } + + public ApplicationPackage asApplicationPackage() { + return applicationPackage; + } + + public X509Certificate certificate() { + return Objects.requireNonNull(certificate); + } + + public static TestSummary validateTests(DeploymentSpec spec, byte[] testPackage) { + return validateTests(expectedSuites(spec.steps()), testPackage); + } + + static TestSummary validateTests(Collection<Suite> expectedSuites, byte[] testPackage) { + List<String> problems = new ArrayList<>(); + Set<Suite> suites = new LinkedHashSet<>(); + ZipEntries.from(testPackage, __ -> true, 0, false).asList().stream() + .map(entry -> Path.fromString(entry.name())) + .collect(groupingBy(path -> path.elements().size() > 1 ? path.elements().get(0) : "", + mapping(path -> (path.elements().size() > 1 ? path.getChildPath() : path).getRelative(), toList()))) + .forEach((directory, paths) -> { + switch (directory) { + case "components": { + for (String path : paths) { + if (path.endsWith("-tests.jar")) { + try { + byte[] testsJar = ZipEntries.readFile(testPackage, "components/" + path, 1 << 30); + Manifest manifest = new JarInputStream(new ByteArrayInputStream(testsJar)).getManifest(); + for (String suite : manifest.getMainAttributes().getValue("X-JDisc-Test-Bundle-Categories").split(",")) + switch (suite.trim()) { + case "SystemTest": suites.add(system); break; + case "StagingSetup": suites.add(staging_setup); break; + case "StagingTest": suites.add(staging); break; + case "ProductionTest": suites.add(production); break; + default: problems.add("unexpected test suite name '" + suite + "' in bundle manifest"); + } + } + catch (Exception e) { + problems.add("failed reading test bundle manifest: " + Exceptions.toMessageString(e)); + } + } + } + } + break; + case "tests": { + if (paths.stream().anyMatch(Pattern.compile("system-test/.+\\.json").asMatchPredicate())) suites.add(system); + if (paths.stream().anyMatch(Pattern.compile("staging-setup/.+\\.json").asMatchPredicate())) suites.add(staging_setup); + if (paths.stream().anyMatch(Pattern.compile("staging-test/.+\\.json").asMatchPredicate())) suites.add(staging); + if (paths.stream().anyMatch(Pattern.compile("production-test/.+\\.json").asMatchPredicate())) suites.add(production); + } + break; + case "artifacts": { + if (paths.stream().anyMatch(Pattern.compile(".+-tests.jar").asMatchPredicate())) + suites.addAll(expectedSuites); // ಠ_ಠ + + for (String forbidden : List.of("key", "cert")) + if (paths.contains(forbidden)) + problems.add("test package contains 'artifacts/" + forbidden + + "'; this conflicts with credentials used to run tests in Vespa Cloud"); + } + break; + } + }); + + if (expectedSuites.contains(system) && ! suites.contains(system)) + problems.add("test package has no system tests, but <test /> is declared in deployment.xml"); + + if (suites.contains(staging) != suites.contains(staging_setup)) + problems.add("test package has " + (suites.contains(staging) ? "staging tests" : "staging setup") + + ", so it should also include " + (suites.contains(staging) ? "staging setup" : "staging tests")); + else if (expectedSuites.contains(staging) && ! suites.contains(staging)) + problems.add("test package has no staging setup and tests, but <staging /> is declared in deployment.xml"); + + if (suites.contains(production) != expectedSuites.contains(production)) + problems.add("test package has " + (suites.contains(production) ? "" : "no ") + "production tests, " + + "but " + (suites.contains(production) ? "no " : "") + "production tests are declared in deployment.xml"); + + if ( ! problems.isEmpty()) + problems.add("see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"); + + return new TestSummary(problems, suites); + } + + public static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) { + NodeResources nodeResources = spec.steps().stream() + .filter(step -> step.concerns(zone.environment())) + .findFirst() + .flatMap(step -> step.zones().get(0).testerFlavor()) + .map(NodeResources::fromLegacyName) + .orElse(zone.region().value().contains("aws-") ? DEFAULT_TESTER_RESOURCES_AWS + : DEFAULT_TESTER_RESOURCES); + return nodeResources.with(NodeResources.DiskSpeed.any); + } + + /** Returns the generated services.xml content for the tester application. */ + public static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, + NodeResources resources, ControllerConfig.Steprunner.Testerapp config) { + int jdiscMemoryGb = 2; // 2Gb memory for tester application (excessive?). + int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb()); + + // Of the remaining memory, split 50/50 between Surefire running the tests and the rest + int testMemoryMb = (int) (1024 * (resources.memoryGb() - jdiscMemoryGb) / 2); + + String resourceString = Text.format("<resources vcpu=\"%.2f\" memory=\"%.2fGb\" disk=\"%.2fGb\" disk-speed=\"%s\" storage-type=\"%s\"/>", + resources.vcpu(), resources.memoryGb(), resources.diskGb(), resources.diskSpeed().name(), resources.storageType().name()); + + String runtimeProviderClass = config.runtimeProviderClass(); + String tenantCdBundle = config.tenantCdBundle(); + + String servicesXml = + "<?xml version='1.0' encoding='UTF-8'?>\n" + + "<services xmlns:deploy='vespa' version='1.0'>\n" + + " <container version='1.0' id='tester'>\n" + + "\n" + + " <component id=\"com.yahoo.vespa.hosted.testrunner.TestRunner\" bundle=\"vespa-testrunner-components\">\n" + + " <config name=\"com.yahoo.vespa.hosted.testrunner.test-runner\">\n" + + " <artifactsPath>artifacts</artifactsPath>\n" + + " <surefireMemoryMb>" + testMemoryMb + "</surefireMemoryMb>\n" + + " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + + " <useTesterCertificate>" + useTesterCertificate + "</useTesterCertificate>\n" + + " </config>\n" + + " </component>\n" + + "\n" + + " <handler id=\"com.yahoo.vespa.testrunner.TestRunnerHandler\" bundle=\"vespa-osgi-testrunner\">\n" + + " <binding>http://*/tester/v1/*</binding>\n" + + " </handler>\n" + + "\n" + + " <component id=\"" + runtimeProviderClass + "\" bundle=\"" + tenantCdBundle + "\" />\n" + + "\n" + + " <component id=\"com.yahoo.vespa.testrunner.JunitRunner\" bundle=\"vespa-osgi-testrunner\">\n" + + " <config name=\"com.yahoo.vespa.testrunner.junit-test-runner\">\n" + + " <artifactsPath>artifacts</artifactsPath>\n" + + " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + + " </config>\n" + + " </component>\n" + + "\n" + + " <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" + + " <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" + + " <artifactsPath>artifacts</artifactsPath>\n" + + " <testsPath>tests</testsPath>\n" + + " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + + " </config>\n" + + " </component>\n" + + "\n" + + " <nodes count=\"1\">\n" + + " <jvm allocated-memory=\"" + jdiscMemoryPct + "%\"/>\n" + + " " + resourceString + "\n" + + " </nodes>\n" + + " </container>\n" + + "</services>\n"; + + return servicesXml.getBytes(UTF_8); + } + + /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */ + public static byte[] deploymentXml(TesterId id, Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) { + String deploymentSpec = + "<?xml version='1.0' encoding='UTF-8'?>\n" + + "<deployment version=\"1.0\" " + + athenzDomain.map(domain -> "athenz-domain=\"" + domain.value() + "\" ").orElse("") + + athenzService.map(service -> "athenz-service=\"" + service.value() + "\" ").orElse("") + ">" + + " <instance id=\"" + id.id().instance().value() + "\" />" + + "</deployment>"; + return deploymentSpec.getBytes(UTF_8); + } + + static Set<Suite> expectedSuites(List<Step> steps) { + Set<Suite> suites = new HashSet<>(); + if (steps.isEmpty()) return suites; + for (Step step : steps) { + if (step.isTest()) { + if (step.concerns(Environment.prod)) suites.add(production); + if (step.concerns(Environment.test)) suites.add(system); + if (step.concerns(Environment.staging)) { suites.add(staging); suites.add(staging_setup); } + } + else + suites.addAll(expectedSuites(step.steps())); + } + return suites; + } + + + public static class TestSummary { + + private final List<String> problems; + private final List<Suite> suites; + + public TestSummary(List<String> problems, Set<Suite> suites) { + this.problems = List.copyOf(problems); + this.suites = List.copyOf(suites); + } + + public List<String> problems() { + return problems; + } + + public List<Suite> suites() { + return suites; + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java index a6cb7f23fc3..8392a77bad5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java @@ -13,6 +13,8 @@ import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; @@ -35,19 +37,28 @@ public class ZipEntries { /** Copies the zipped content from in to out, adding/overwriting an entry with the given name and content. */ public static void transferAndWrite(OutputStream out, InputStream in, String name, byte[] content) { + transferAndWrite(out, in, Map.of(name, content)); + } + + /** Copies the zipped content from in to out, adding/overwriting/removing (on {@code null}) entries as specified. */ + public static void transferAndWrite(OutputStream out, InputStream in, Map<String, byte[]> entries) { try (ZipOutputStream zipOut = new ZipOutputStream(out); ZipInputStream zipIn = new ZipInputStream(in)) { for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) { - if (entry.getName().equals(name)) + if (entries.containsKey(entry.getName())) continue; zipOut.putNextEntry(new ZipEntry(entry.getName())); zipIn.transferTo(zipOut); zipOut.closeEntry(); } - zipOut.putNextEntry(new ZipEntry(name)); - zipOut.write(content); - zipOut.closeEntry(); + for (Entry<String, byte[]> entry : entries.entrySet()) { + if (entry.getValue() != null) { + zipOut.putNextEntry(new ZipEntry(entry.getKey())); + zipOut.write(entry.getValue()); + zipOut.closeEntry(); + } + } } catch (IOException e) { throw new UncheckedIOException(e); @@ -76,6 +87,10 @@ public class ZipEntries { return new ZipEntries(entries); } + public static byte[] readFile(byte[] zip, String name, int maxEntrySizeInBytes) { + return from(zip, name::equals, maxEntrySizeInBytes, true).asList().get(0).contentOrThrow(); + } + public List<ZipEntryWithContent> asList() { return entries; } public static class ZipEntryWithContent { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index f5900604627..2ad28893e18 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -7,7 +7,6 @@ import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; import com.yahoo.vespa.athenz.api.AthenzDomain; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java index 78d65766075..34e7955e02a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.auditlog; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -70,7 +71,7 @@ public class AuditLogger { Instant now = clock.instant(); AuditLog.Entry entry = new AuditLog.Entry(now, principal.getName(), method.get(), pathAndQueryOf(request.getUri()), Optional.of(new String(data, StandardCharsets.UTF_8))); - try (Lock lock = db.lockAuditLog()) { + try (Mutex lock = db.lockAuditLog()) { AuditLog auditLog = db.readAuditLog() .pruneBefore(now.minus(entryTtl)) .with(entry) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index a042215616c..cc7031bab5a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -10,14 +10,16 @@ import com.yahoo.config.application.api.DeploymentSpec.DeclaredTest; import com.yahoo.config.application.api.DeploymentSpec.DeclaredZone; import com.yahoo.config.application.api.DeploymentSpec.UpgradeRollout; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -26,6 +28,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -42,13 +45,12 @@ import static com.yahoo.config.application.api.DeploymentSpec.RevisionTarget.nex import static com.yahoo.config.provision.Environment.prod; import static com.yahoo.config.provision.Environment.staging; import static com.yahoo.config.provision.Environment.test; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; import static java.util.Objects.requireNonNull; import static java.util.function.BinaryOperator.maxBy; import static java.util.stream.Collectors.collectingAndThen; +import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toUnmodifiableList; @@ -59,51 +61,33 @@ import static java.util.stream.Collectors.toUnmodifiableList; */ public class DeploymentStatus { - public static List<JobId> jobsFor(Application application, SystemName system) { - if (DeploymentSpec.empty.equals(application.deploymentSpec())) - return List.of(); - - return application.deploymentSpec().instances().stream() - .flatMap(spec -> Stream.concat(Stream.of(systemTest, stagingTest), - flatten(spec).filter(step -> step.concerns(prod)) - .map(step -> { - if (step instanceof DeclaredZone) - return JobType.from(system, prod, ((DeclaredZone) step).region().get()); - return JobType.testFrom(system, ((DeclaredTest) step).region()); - }) - .flatMap(Optional::stream)) - .map(type -> new JobId(application.id().instance(spec.name()), type))) - .collect(toUnmodifiableList()); - } - - private static Stream<DeploymentSpec.Step> flatten(DeploymentSpec.Step step) { - return step instanceof DeploymentSpec.Steps ? step.steps().stream().flatMap(DeploymentStatus::flatten) : Stream.of(step); - } - private static <T> List<T> union(List<T> first, List<T> second) { return Stream.concat(first.stream(), second.stream()).distinct().collect(toUnmodifiableList()); } private final Application application; private final JobList allJobs; - private final SystemName system; + private final JobType systemTest; + private final JobType stagingTest; private final Version systemVersion; private final Function<InstanceName, VersionCompatibility> versionCompatibility; private final Instant now; private final Map<JobId, StepStatus> jobSteps; private final List<StepStatus> allSteps; - public DeploymentStatus(Application application, Map<JobId, JobStatus> allJobs, SystemName system, + public DeploymentStatus(Application application, Function<JobId, JobStatus> allJobs, ZoneRegistry zones, Version systemVersion, Function<InstanceName, VersionCompatibility> versionCompatibility, Instant now) { this.application = requireNonNull(application); - this.allJobs = JobList.from(allJobs.values()); - this.system = requireNonNull(system); + this.systemTest = JobType.systemTest(zones); + this.stagingTest = JobType.stagingTest(zones); this.systemVersion = requireNonNull(systemVersion); this.versionCompatibility = versionCompatibility; this.now = requireNonNull(now); List<StepStatus> allSteps = new ArrayList<>(); - this.jobSteps = jobDependencies(application.deploymentSpec(), allSteps); + Map<JobId, JobStatus> jobs = new HashMap<>(); + this.jobSteps = jobDependencies(application.deploymentSpec(), allSteps, job -> jobs.computeIfAbsent(job, allJobs)); this.allSteps = Collections.unmodifiableList(allSteps); + this.allJobs = JobList.from(jobSteps.keySet().stream().map(allJobs).collect(toList())); } /** The application this deployment status concerns. */ @@ -143,9 +127,9 @@ public class DeploymentStatus { } /** Whether any job is failing on versions selected by the given filter, with errors other than lack of capacity in a test zone.. */ - public boolean hasFailures(Predicate<ApplicationVersion> versionFilter) { + public boolean hasFailures(Predicate<RevisionId> revisionFilter) { return ! allJobs.failingHard() - .matching(job -> versionFilter.test(job.lastTriggered().get().versions().targetApplication())) + .matching(job -> revisionFilter.test(job.lastTriggered().get().versions().targetRevision())) .isEmpty(); } @@ -172,6 +156,8 @@ public class DeploymentStatus { * and any test jobs for any outstanding change, which will likely be needed to later deploy this change. */ public Map<JobId, List<Job>> jobsToRun() { + if (application.revisions().last().isEmpty()) return Map.of(); + Map<InstanceName, Change> changes = new LinkedHashMap<>(); for (InstanceName instance : application.deploymentSpec().instanceNames()) changes.put(instance, application.require(instance).change()); @@ -196,6 +182,8 @@ public class DeploymentStatus { } private Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes, boolean eagerTests) { + if (application.revisions().last().isEmpty()) return Map.of(); + Map<JobId, List<Job>> productionJobs = new LinkedHashMap<>(); changes.forEach((instance, change) -> productionJobs.putAll(productionJobs(instance, change, eagerTests))); Map<JobId, List<Job>> testJobs = testJobs(productionJobs); @@ -256,7 +244,7 @@ public class DeploymentStatus { public Optional<Deployment> deploymentFor(JobId job) { return Optional.ofNullable(application.require(job.application().instance()) - .deployments().get(job.type().zone(system))); + .deployments().get(job.type().zone())); } /** @@ -269,19 +257,47 @@ public class DeploymentStatus { public Change outstandingChange(InstanceName instance) { StepStatus status = instanceSteps().get(instance); if (status == null) return Change.empty(); - boolean ascending = next == application.deploymentSpec().requireInstance(instance).revisionTarget(); - for (ApplicationVersion version : application.deployableVersions(ascending)) { - if (status.dependenciesCompletedAt(Change.of(version), Optional.empty()).map(now::isBefore).orElse(true)) continue; - Change change = Change.of(version); + DeploymentInstanceSpec spec = application.deploymentSpec().requireInstance(instance); + boolean ascending = next == spec.revisionTarget(); + int cumulativeRisk = 0; + int nextRisk = 0; + int skippedCumulativeRisk = 0; + Instant readySince = now; + Change candidate = Change.empty(); + for (ApplicationVersion version : application.revisions().deployable(ascending)) { + // A revision is only a candidate if it upgrades, and does not downgrade, this instance. + Change change = Change.of(version.id()); if (application.productionDeployments().getOrDefault(instance, List.of()).stream() - .anyMatch(deployment -> change.downgrades(deployment.applicationVersion()))) continue; - if ( ! application.require(instance).change().application().map(change::upgrades).orElse(true)) continue; + .anyMatch(deployment -> change.downgrades(deployment.revision()))) continue; + if ( ! application.require(instance).change().revision().map(change::upgrades).orElse(true)) continue; if (hasCompleted(instance, change)) - if (ascending) continue; - else break; - return change; + if (ascending) continue; // Keep looking for the next revision which is an upgrade, or ... + else return Change.empty(); // ... if the latest is already complete, there's nothing outstanding. + + // This revision contains something new, so start aggregating the risk score. + skippedCumulativeRisk += version.risk(); + nextRisk = nextRisk > 0 ? nextRisk : version.risk(); + // If it's not yet ready to roll out, we keep looking. + Optional<Instant> readyAt = status.dependenciesCompletedAt(Change.of(version.id()), Optional.empty()); + if (readyAt.map(now::isBefore).orElse(true)) continue; + + // It's ready. If looking for the latest, max risk is 0, and we'll return now; otherwise, we _may_ keep on looking for more. + cumulativeRisk += skippedCumulativeRisk; + skippedCumulativeRisk = 0; + nextRisk = 0; + if (cumulativeRisk >= spec.maxRisk()) + return candidate.equals(Change.empty()) ? change : candidate; // If the first candidate exceeds max risk, we have to accept that. + + // Otherwise, we may note this as a candidate, and keep looking for a newer revision, unless that makes us exceed max risk. + if (readyAt.get().isBefore(readySince)) readySince = readyAt.get(); + candidate = change; } - return Change.empty(); + // If min risk is ready, or max idle time has passed, we return the candidate. Otherwise, no outstanding change is ready. + return instanceJobs(instance).values().stream().allMatch(jobs -> jobs.lastTriggered().isEmpty()) + || cumulativeRisk >= spec.minRisk() + || cumulativeRisk + nextRisk > spec.maxRisk() + || ! now.isBefore(readySince.plus(Duration.ofHours(spec.maxIdleHours()))) + ? candidate : Change.empty(); } /** Earliest instant when job was triggered with given versions, or both system and staging tests were successful. */ @@ -305,7 +321,7 @@ public class DeploymentStatus { .type(type).asList().stream() .flatMap(status -> RunList.from(status) .on(versions) - .status(RunStatus.success) + .matching(Run::hasSucceeded) .asList().stream() .map(Run::start)) .min(naturalOrder()); @@ -324,9 +340,9 @@ public class DeploymentStatus { // When computing eager test jobs for outstanding changes, assume current change completes successfully. Optional<Deployment> deployment = deploymentFor(job); Optional<Version> existingPlatform = deployment.map(Deployment::version); - Optional<ApplicationVersion> existingApplication = deployment.map(Deployment::applicationVersion); - boolean deployingCompatibilityChange = areIncompatible(existingPlatform, change.application(), instance) - || areIncompatible(change.platform(), existingApplication, instance); + Optional<RevisionId> existingRevision = deployment.map(Deployment::revision); + boolean deployingCompatibilityChange = areIncompatible(existingPlatform, change.revision(), job) + || areIncompatible(change.platform(), existingRevision, job); if (assumeUpgradesSucceed) { if (deployingCompatibilityChange) // No eager tests for this. return; @@ -334,34 +350,36 @@ public class DeploymentStatus { Change currentChange = application.require(instance).change(); Versions target = Versions.from(currentChange, application, deployment, systemVersion); existingPlatform = Optional.of(target.targetPlatform()); - existingApplication = Optional.of(target.targetApplication()); + existingRevision = Optional.of(target.targetRevision()); } List<Job> toRun = new ArrayList<>(); List<Change> changes = deployingCompatibilityChange ? List.of(change) : changes(job, step, change); for (Change partial : changes) { Job jobToRun = new Job(job.type(), - Versions.from(partial, application, existingPlatform, existingApplication, systemVersion), + Versions.from(partial, application, existingPlatform, existingRevision, systemVersion), step.readyAt(partial, Optional.of(job)), partial); toRun.add(jobToRun); // Assume first partial change is applied before the second. existingPlatform = Optional.of(jobToRun.versions.targetPlatform()); - existingApplication = Optional.of(jobToRun.versions.targetApplication()); + existingRevision = Optional.of(jobToRun.versions.targetRevision()); } jobs.put(job, toRun); }); return jobs; } - private boolean areIncompatible(Optional<Version> platform, Optional<ApplicationVersion> application, InstanceName instance) { + private boolean areIncompatible(Optional<Version> platform, Optional<RevisionId> revision, JobId job) { + Optional<Version> compileVersion = revision.map(application.revisions()::get) + .flatMap(ApplicationVersion::compileVersion); return platform.isPresent() - && application.flatMap(ApplicationVersion::compileVersion).isPresent() - && versionCompatibility.apply(instance).refuse(platform.get(), application.get().compileVersion().get()); + && compileVersion.isPresent() + && versionCompatibility.apply(job.application().instance()).refuse(platform.get(), compileVersion.get()); } /** Changes to deploy with the given job, possibly split in two steps. */ private List<Change> changes(JobId job, StepStatus step, Change change) { - if (change.platform().isEmpty() || change.application().isEmpty() || change.isPinned()) + if (change.platform().isEmpty() || change.revision().isEmpty() || change.isPinned()) return List.of(change); if ( step.completedAt(change.withoutApplication(), Optional.of(job)).isPresent() @@ -370,7 +388,7 @@ public class DeploymentStatus { // For a dual change, where both targets remain, we determine what to run by looking at when the two parts became ready: // for deployments, we look at dependencies; for production tests, this may be overridden by what is already deployed. - JobId deployment = new JobId(job.application(), JobType.from(system, job.type().zone(system)).get()); + JobId deployment = new JobId(job.application(), JobType.deploymentTo(job.type().zone())); UpgradeRollout rollout = application.deploymentSpec().requireInstance(job.application().instance()).upgradeRollout(); if (job.type().isTest()) { Optional<Instant> platformDeployedAt = jobSteps.get(deployment).completedAt(change.withoutApplication(), Optional.of(deployment)); @@ -468,7 +486,7 @@ public class DeploymentStatus { if ( job.type().isProduction() && job.type().isDeployment() && allJobs.successOn(productionJob.versions()).type(testType).isEmpty() && testJobs.keySet().stream() - .noneMatch(test -> test.type() == testType + .noneMatch(test -> test.type().equals(testType) && testJobs.get(test).stream().anyMatch(testJob -> testJob.versions().equals(productionJob.versions())))) { JobId testJob = firstDeclaredOrElseImplicitTest(testType); testJobs.merge(testJob, @@ -486,31 +504,44 @@ public class DeploymentStatus { private JobId firstDeclaredOrElseImplicitTest(JobType testJob) { return application.deploymentSpec().instanceNames().stream() .map(name -> new JobId(application.id().instance(name), testJob)) + .filter(jobSteps::containsKey) .min(comparing(id -> ! jobSteps.get(id).isDeclared())).orElseThrow(); } /** JobId of any declared test of the given type, for the given instance. */ private Optional<JobId> declaredTest(ApplicationId instanceId, JobType testJob) { JobId jobId = new JobId(instanceId, testJob); - return jobSteps.get(jobId).isDeclared() ? Optional.of(jobId) : Optional.empty(); + return jobSteps.containsKey(jobId) && jobSteps.get(jobId).isDeclared() ? Optional.of(jobId) : Optional.empty(); } /** A DAG of the dependencies between the primitive steps in the spec, with iteration order equal to declaration order. */ - private Map<JobId, StepStatus> jobDependencies(DeploymentSpec spec, List<StepStatus> allSteps) { + private Map<JobId, StepStatus> jobDependencies(DeploymentSpec spec, List<StepStatus> allSteps, Function<JobId, JobStatus> jobs) { if (DeploymentSpec.empty.equals(spec)) return Map.of(); Map<JobId, StepStatus> dependencies = new LinkedHashMap<>(); List<StepStatus> previous = List.of(); for (DeploymentSpec.Step step : spec.steps()) - previous = fillStep(dependencies, allSteps, step, previous, null); + previous = fillStep(dependencies, allSteps, step, previous, null, jobs, + instanceWithImplicitTest(test, spec), + instanceWithImplicitTest(staging, spec)); return Collections.unmodifiableMap(dependencies); } + private static InstanceName instanceWithImplicitTest(Environment environment, DeploymentSpec spec) { + InstanceName first = null; + for (DeploymentInstanceSpec step : spec.instances()) { + if (step.concerns(environment)) return null; + first = first != null ? first : step.name(); + } + return first; + } + /** Adds the primitive steps contained in the given step, which depend on the given previous primitives, to the dependency graph. */ - private List<StepStatus> fillStep(Map<JobId, StepStatus> dependencies, List<StepStatus> allSteps, - DeploymentSpec.Step step, List<StepStatus> previous, InstanceName instance) { + private List<StepStatus> fillStep(Map<JobId, StepStatus> dependencies, List<StepStatus> allSteps, DeploymentSpec.Step step, + List<StepStatus> previous, InstanceName instance, Function<JobId, JobStatus> jobs, + InstanceName implicitSystemTest, InstanceName implicitStagingTest) { if (step.steps().isEmpty() && ! (step instanceof DeploymentInstanceSpec)) { if (instance == null) return previous; // Ignore test and staging outside all instances. @@ -522,31 +553,28 @@ public class DeploymentStatus { } JobType jobType; + JobId jobId; StepStatus stepStatus; if (step.concerns(test) || step.concerns(staging)) { - jobType = JobType.from(system, ((DeclaredZone) step).environment(), null) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); - stepStatus = JobStepStatus.ofTestDeployment((DeclaredZone) step, List.of(), this, instance, jobType, true); + jobType = step.concerns(test) ? systemTest : stagingTest; + jobId = new JobId(application.id().instance(instance), jobType); + stepStatus = JobStepStatus.ofTestDeployment((DeclaredZone) step, List.of(), this, jobs.apply(jobId), true); previous = new ArrayList<>(previous); previous.add(stepStatus); } else if (step.isTest()) { - jobType = JobType.testFrom(system, ((DeclaredTest) step).region()) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); - JobType preType = JobType.from(system, prod, ((DeclaredTest) step).region()) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); - stepStatus = JobStepStatus.ofProductionTest((DeclaredTest) step, previous, this, instance, jobType, preType); + jobType = JobType.test(((DeclaredTest) step).region()); + jobId = new JobId(application.id().instance(instance), jobType); + stepStatus = JobStepStatus.ofProductionTest((DeclaredTest) step, previous, this, jobs.apply(jobId)); previous = List.of(stepStatus); } else if (step.concerns(prod)) { - jobType = JobType.from(system, ((DeclaredZone) step).environment(), ((DeclaredZone) step).region().get()) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); - stepStatus = JobStepStatus.ofProductionDeployment((DeclaredZone) step, previous, this, instance, jobType); + jobType = JobType.prod(((DeclaredZone) step).region().get()); + jobId = new JobId(application.id().instance(instance), jobType); + stepStatus = JobStepStatus.ofProductionDeployment((DeclaredZone) step, previous, this, jobs.apply(jobId)); previous = List.of(stepStatus); } else return previous; // Empty container steps end up here, and are simply ignored. - JobId jobId = new JobId(application.id().instance(instance), jobType); - allSteps.removeIf(existing -> existing.job().equals(Optional.of(jobId))); // Replace implicit tests with explicit ones. allSteps.add(stepStatus); dependencies.put(jobId, stepStatus); return previous; @@ -558,27 +586,32 @@ public class DeploymentStatus { instance = spec.name(); allSteps.add(instanceStatus); previous = List.of(instanceStatus); - for (JobType test : List.of(systemTest, stagingTest)) { - JobId job = new JobId(application.id().instance(instance), test); - if ( ! dependencies.containsKey(job)) { - var testStatus = JobStepStatus.ofTestDeployment(new DeclaredZone(test.environment()), List.of(), - this, job.application().instance(), test, false); - dependencies.put(job, testStatus); - allSteps.add(testStatus); - } + if (instance.equals(implicitSystemTest)) { + JobId job = new JobId(application.id().instance(instance), systemTest); + JobStepStatus testStatus = JobStepStatus.ofTestDeployment(new DeclaredZone(test), List.of(), + this, jobs.apply(job), false); + dependencies.put(job, testStatus); + allSteps.add(testStatus); + } + if (instance.equals(implicitStagingTest)) { + JobId job = new JobId(application.id().instance(instance), stagingTest); + JobStepStatus testStatus = JobStepStatus.ofTestDeployment(new DeclaredZone(staging), List.of(), + this, jobs.apply(job), false); + dependencies.put(job, testStatus); + allSteps.add(testStatus); } } if (step.isOrdered()) { for (DeploymentSpec.Step nested : step.steps()) - previous = fillStep(dependencies, allSteps, nested, previous, instance); + previous = fillStep(dependencies, allSteps, nested, previous, instance, jobs, implicitSystemTest, implicitStagingTest); return previous; } List<StepStatus> parallel = new ArrayList<>(); for (DeploymentSpec.Step nested : step.steps()) - parallel.addAll(fillStep(dependencies, allSteps, nested, previous, instance)); + parallel.addAll(fillStep(dependencies, allSteps, nested, previous, instance, jobs, implicitSystemTest, implicitStagingTest)); return List.copyOf(parallel); } @@ -718,7 +751,7 @@ public class DeploymentStatus { @Override Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { return ( (change.platform().isEmpty() || change.platform().equals(instance.change().platform())) - && (change.application().isEmpty() || change.application().equals(instance.change().application())) + && (change.revision().isEmpty() || change.revision().equals(instance.change().revision())) || step().steps().stream().noneMatch(step -> step.concerns(prod))) ? dependenciesCompletedAt(change, dependent).or(() -> Optional.of(Instant.EPOCH).filter(__ -> change.hasTargets())) : Optional.empty(); @@ -732,7 +765,7 @@ public class DeploymentStatus { while ( blocker.window().includes(current) && now.plus(Duration.ofDays(7)).isAfter(current) && ( change.platform().isPresent() && blocker.blocksVersions() - || change.application().isPresent() && blocker.blocksRevisions())) { + || change.revision().isPresent() && blocker.blocksRevisions())) { blocked = true; current = current.plus(Duration.ofHours(1)).truncatedTo(ChronoUnit.HOURS); } @@ -773,7 +806,7 @@ public class DeploymentStatus { if (job.firstFailing().isEmpty() || ! job.firstFailing().get().hasEnded()) return Optional.empty(); Versions lastVersions = job.lastCompleted().get().versions(); if (change.platform().isPresent() && ! change.platform().get().equals(lastVersions.targetPlatform())) return Optional.empty(); - if (change.application().isPresent() && ! change.application().get().equals(lastVersions.targetApplication())) return Optional.empty(); + if (change.revision().isPresent() && ! change.revision().get().equals(lastVersions.targetRevision())) return Optional.empty(); if (job.id().type().environment().isTest() && job.isNodeAllocationFailure()) return Optional.empty(); Instant firstFailing = job.firstFailing().get().end().get(); @@ -787,10 +820,9 @@ public class DeploymentStatus { } private static JobStepStatus ofProductionDeployment(DeclaredZone step, List<StepStatus> dependencies, - DeploymentStatus status, InstanceName instance, JobType jobType) { + DeploymentStatus status, JobStatus job) { ZoneId zone = ZoneId.from(step.environment(), step.region().get()); - JobStatus job = status.instanceJobs(instance).get(jobType); - Optional<Deployment> existingDeployment = Optional.ofNullable(status.application().require(instance) + Optional<Deployment> existingDeployment = Optional.ofNullable(status.application().require(job.id().application().instance()) .deployments().get(zone)); return new JobStepStatus(StepType.deployment, step, dependencies, job, status) { @@ -811,21 +843,21 @@ public class DeploymentStatus { && ! existingDeployment.map(Deployment::version).equals(change.platform())) return Optional.empty(); - if ( change.application().isPresent() - && ! existingDeployment.map(Deployment::applicationVersion).equals(change.application()) + if ( change.revision().isPresent() + && ! existingDeployment.map(Deployment::revision).equals(change.revision()) && dependent.equals(job())) // Job should (re-)run in this case, but other dependents need not wait. return Optional.empty(); - Change fullChange = status.application().require(instance).change(); - if (existingDeployment.map(deployment -> ! (change.upgrades(deployment.version()) || change.upgrades(deployment.applicationVersion())) - && (fullChange.downgrades(deployment.version()) || fullChange.downgrades(deployment.applicationVersion()))) + Change fullChange = status.application().require(job.id().application().instance()).change(); + if (existingDeployment.map(deployment -> ! (change.upgrades(deployment.version()) || change.upgrades(deployment.revision())) + && (fullChange.downgrades(deployment.version()) || fullChange.downgrades(deployment.revision()))) .orElse(false)) return job.lastCompleted().flatMap(Run::end); Optional<Instant> end = Optional.empty(); for (Run run : job.runs().descendingMap().values()) { if (run.versions().targetsMatch(change)) { - if (run.status() == RunStatus.success) end = run.end(); + if (run.hasSucceeded()) end = run.end(); } else if (dependent.equals(job())) // If strict completion, consider only last time this change was deployed. break; @@ -836,11 +868,8 @@ public class DeploymentStatus { } private static JobStepStatus ofProductionTest(DeclaredTest step, List<StepStatus> dependencies, - DeploymentStatus status, InstanceName instance, - JobType testType, JobType prodType) { - JobStatus job = status.instanceJobs(instance).get(testType); - JobId prodId = new JobId(status.application().id().instance(instance), prodType); - + DeploymentStatus status, JobStatus job) { + JobId prodId = new JobId(job.id().application(), JobType.deploymentTo(job.id().type().zone())); return new JobStepStatus(StepType.test, step, dependencies, job, status) { @Override Optional<Instant> readyAt(Change change, Optional<JobId> dependent) { @@ -855,7 +884,7 @@ public class DeploymentStatus { Optional<Instant> deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId)); return (dependent.equals(job()) ? job.lastTriggered().filter(run -> deployedAt.map(at -> ! run.start().isBefore(at)).orElse(false)).stream() : job.runs().values().stream()) - .filter(run -> run.status() == RunStatus.success) + .filter(Run::hasSucceeded) .filter(run -> run.versions().targetsMatch(change)) .flatMap(run -> run.end().stream()).findFirst(); } @@ -863,9 +892,7 @@ public class DeploymentStatus { } private static JobStepStatus ofTestDeployment(DeclaredZone step, List<StepStatus> dependencies, - DeploymentStatus status, InstanceName instance, - JobType jobType, boolean declared) { - JobStatus job = status.instanceJobs(instance).get(jobType); + DeploymentStatus status, JobStatus job, boolean declared) { return new JobStepStatus(StepType.test, step, dependencies, job, status) { @Override Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { @@ -875,9 +902,9 @@ public class DeploymentStatus { status.application, Optional.of(deployment), status.systemVersion))) - .orElseGet(() -> (change.platform().isEmpty() || change.platform().get().equals(run.versions().targetPlatform())) - && (change.application().isEmpty() || change.application().get().equals(run.versions().targetApplication())))) - .status(RunStatus.success) + .orElseGet(() -> (change.platform().isEmpty() || change.platform().get().equals(run.versions().targetPlatform())) + && (change.revision().isEmpty() || change.revision().get().equals(run.versions().targetRevision())))) + .matching(Run::hasSucceeded) .asList().stream() .map(run -> run.end().get()) .max(naturalOrder()); @@ -897,7 +924,7 @@ public class DeploymentStatus { private final Change change; public Job(JobType type, Versions versions, Optional<Instant> readyAt, Change change) { - this.versions = type == systemTest ? versions.withoutSources() : versions; + this.versions = type.isSystemTest() ? versions.withoutSources() : versions; this.readyAt = readyAt; this.change = change; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java index 7ab895654f3..44079a90097 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Deployment; import java.util.Collection; @@ -29,16 +30,16 @@ import static java.util.stream.Collectors.collectingAndThen; public class DeploymentSteps { private final DeploymentInstanceSpec spec; - private final Supplier<SystemName> system; + private final ZoneRegistry zones; - public DeploymentSteps(DeploymentInstanceSpec spec, Supplier<SystemName> system) { + public DeploymentSteps(DeploymentInstanceSpec spec, ZoneRegistry zones) { this.spec = Objects.requireNonNull(spec, "spec cannot be null"); - this.system = Objects.requireNonNull(system, "system cannot be null"); + this.zones = Objects.requireNonNull(zones, "system cannot be null"); } /** Returns jobs for this, in the order they should run */ public List<JobType> jobs() { - return Stream.concat(production().isEmpty() ? Stream.of() : Stream.of(JobType.systemTest, JobType.stagingTest), + return Stream.concat(production().isEmpty() ? Stream.of() : Stream.of(JobType.systemTest(zones), JobType.stagingTest(zones)), spec.steps().stream().flatMap(step -> toJobs(step).stream())) .distinct() .collect(Collectors.toUnmodifiableList()); @@ -67,7 +68,6 @@ public class DeploymentSteps { public List<JobType> toJobs(DeploymentSpec.Step step) { return step.zones().stream() .map(this::toJob) - .flatMap(Optional::stream) .collect(Collectors.toUnmodifiableList()); } @@ -93,8 +93,13 @@ public class DeploymentSteps { } /** Resolve job from deployment zone */ - private Optional<JobType> toJob(DeploymentSpec.DeclaredZone zone) { - return JobType.from(system.get(), zone.environment(), zone.region().orElse(null)); + private JobType toJob(DeploymentSpec.DeclaredZone zone) { + switch (zone.environment()) { + case prod: return JobType.prod(zone.region().get()); + case test: return JobType.systemTest(zones); + case staging: return JobType.stagingTest(zones); + default: throw new IllegalArgumentException("region must be one with automated deployments, but got: " + zone.environment()); + } } /** Resolve jobs from steps */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 7ef8a4b4ae6..be07a2b0cb1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -8,6 +8,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.text.Text; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; @@ -17,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -72,22 +74,7 @@ public class DeploymentTrigger { } public DeploymentSteps steps(DeploymentInstanceSpec spec) { - return new DeploymentSteps(spec, controller::system); - } - - public void notifyOfSubmission(TenantAndApplicationId id, ApplicationVersion version, long projectId) { - if (applications().getApplication(id).isEmpty()) { - log.log(Level.WARNING, "Ignoring submission from project '" + projectId + - "': Unknown application '" + id + "'"); - return; - } - - applications().lockApplicationOrThrow(id, application -> { - application = application.withProjectId(OptionalLong.of(projectId)); - application = application.withNewSubmission(version); - applications().store(application); - }); - triggerNewRevision(id); + return new DeploymentSteps(spec, controller.zoneRegistry()); } /** @@ -100,11 +87,11 @@ public class DeploymentTrigger { DeploymentStatus status = jobs.deploymentStatus(application.get()); for (InstanceName instanceName : application.get().deploymentSpec().instanceNames()) { Change outstanding = outstandingChange(status, instanceName); - if ( outstanding.hasTargets() + if (outstanding.hasTargets() && status.instanceSteps().get(instanceName) .readyAt(outstanding) .map(readyAt -> ! readyAt.isAfter(clock.instant())).orElse(false) - && acceptNewApplicationVersion(status, instanceName, outstanding.application().get())) { + && acceptNewRevision(status, instanceName, outstanding.revision().get())) { application = application.with(instanceName, instance -> withRemainingChange(instance, outstanding.onTopOf(instance.change()), status)); } @@ -116,7 +103,9 @@ public class DeploymentTrigger { /** Returns any outstanding change for the given instance, coupled with any necessary platform upgrade. */ private Change outstandingChange(DeploymentStatus status, InstanceName instance) { Change outstanding = status.outstandingChange(instance); - Optional<Version> compileVersion = outstanding.application().flatMap(ApplicationVersion::compileVersion); + Optional<Version> compileVersion = outstanding.revision() + .map(status.application().revisions()::get) + .flatMap(ApplicationVersion::compileVersion); // If the outstanding revision requires a certain platform for compatibility, add that here. VersionCompatibility compatibility = applications().versionCompatibility(status.application().id().instance(instance)); @@ -246,7 +235,7 @@ public class DeploymentTrigger { DeploymentStatus status = jobs.deploymentStatus(application); Change change = instance.change(); - if ( ! upgradeRevision && change.application().isPresent()) change = change.withoutApplication(); + if ( ! upgradeRevision && change.revision().isPresent()) change = change.withoutApplication(); if ( ! upgradePlatform && change.platform().isPresent()) change = change.withoutPlatform(); Versions versions = Versions.from(change, application, status.deploymentFor(job), controller.readSystemVersion()); DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(job.type(), versions, Optional.of(controller.clock().instant()), instance.change()); @@ -267,24 +256,23 @@ public class DeploymentTrigger { private List<JobId> forceTriggerManualJob(JobId job, String reason) { Run last = jobs.last(job).orElseThrow(() -> new IllegalArgumentException(job + " has never been run")); Versions target = new Versions(controller.readSystemVersion(), - last.versions().targetApplication(), + last.versions().targetRevision(), Optional.of(last.versions().targetPlatform()), - Optional.of(last.versions().targetApplication())); + Optional.of(last.versions().targetRevision())); jobs.start(job.application(), job.type(), target, true, Optional.of(reason)); return List.of(job); } /** Retrigger job. If the job is already running, it will be canceled, and retrigger enqueued. */ public Optional<JobId> reTriggerOrAddToQueue(DeploymentId deployment, String reason) { - JobType jobType = JobType.from(controller.system(), deployment.zoneId()) - .orElseThrow(() -> new IllegalArgumentException(Text.format("No job to trigger for (system/zone): %s/%s", controller.system().value(), deployment.zoneId().value()))); + JobType jobType = JobType.deploymentTo(deployment.zoneId()); Optional<Run> existingRun = controller.jobController().active(deployment.applicationId()).stream() .filter(run -> run.id().type().equals(jobType)) .findFirst(); if (existingRun.isPresent()) { Run run = existingRun.get(); - try (Lock lock = controller.curator().lockDeploymentRetriggerQueue()) { + try (Mutex lock = controller.curator().lockDeploymentRetriggerQueue()) { List<RetriggerEntry> retriggerEntries = controller.curator().readRetriggerEntries(); List<RetriggerEntry> newList = new ArrayList<>(retriggerEntries); RetriggerEntry requiredEntry = new RetriggerEntry(new JobId(deployment.applicationId(), jobType), run.id().number() + 1); @@ -366,7 +354,7 @@ public class DeploymentTrigger { /** Returns the set of all jobs which have changes to propagate from the upstream steps. */ private List<Job> computeReadyJobs() { return jobs.deploymentStatuses(ApplicationList.from(applications().readable()) - .withProjectId() // Need to keep this, as we have applications with deployment spec that shouldn't be orchestrated. + .withProjectId() // Need to keep this, as we have applications with deployment spec that shouldn't be orchestrated. // Maybe not any longer? .withDeploymentSpec()) .withChanges() .asList().stream() @@ -397,7 +385,7 @@ public class DeploymentTrigger { /** Returns whether the application is healthy in all other production zones. */ private boolean isUnhealthyInAnotherZone(Application application, JobId job) { for (Deployment deployment : application.require(job.application().instance()).productionDeployments().values()) { - if ( ! deployment.zone().equals(job.type().zone(controller.system())) + if ( ! deployment.zone().equals(job.type().zone()) && ! controller.applications().isHealthy(new DeploymentId(job.application(), deployment.zone()))) return true; } @@ -426,9 +414,7 @@ public class DeploymentTrigger { boolean blocked = status.jobs().get(job).get().isRunning(); if ( ! job.type().isTest()) { - Optional<JobStatus> productionTest = JobType.testFrom(controller.system(), job.type().zone(controller.system()).region()) - .map(type -> new JobId(job.application(), type)) - .flatMap(status.jobs()::get); + Optional<JobStatus> productionTest = status.jobs().get(new JobId(job.application(), JobType.productionTestOf(job.type().zone()))); if (productionTest.isPresent()) { abortIfOutdated(status, jobs, productionTest.get().id()); // Production deployments are also blocked by their declared tests, if the next versions to run @@ -445,16 +431,16 @@ public class DeploymentTrigger { // ---------- Change management o_O ---------- - private boolean acceptNewApplicationVersion(DeploymentStatus status, InstanceName instance, ApplicationVersion version) { + private boolean acceptNewRevision(DeploymentStatus status, InstanceName instance, RevisionId revision) { if (status.application().deploymentSpec().instance(instance).isEmpty()) return false; // Unknown instance. - boolean isChangingRevision = status.application().require(instance).change().application().isPresent(); + boolean isChangingRevision = status.application().require(instance).change().revision().isPresent(); DeploymentInstanceSpec spec = status.application().deploymentSpec().requireInstance(instance); - Predicate<ApplicationVersion> versionFilter = spec.revisionTarget() == DeploymentSpec.RevisionTarget.next - ? failing -> status.application().require(instance).change().application().get().compareTo(failing) == 0 - : failing -> version.compareTo(failing) > 0; + Predicate<RevisionId> revisionFilter = spec.revisionTarget() == DeploymentSpec.RevisionTarget.next + ? failing -> status.application().require(instance).change().revision().get().compareTo(failing) == 0 + : failing -> revision.compareTo(failing) > 0; switch (spec.revisionChange()) { case whenClear: return ! isChangingRevision; - case whenFailing: return ! isChangingRevision || status.hasFailures(versionFilter); + case whenFailing: return ! isChangingRevision || status.hasFailures(revisionFilter); case always: return true; default: throw new IllegalStateException("Unknown revision upgrade policy"); } @@ -464,18 +450,15 @@ public class DeploymentTrigger { Change remaining = change; if (status.hasCompleted(instance.name(), change.withoutApplication())) remaining = remaining.withoutPlatform(); - if (status.hasCompleted(instance.name(), change.withoutPlatform())) { + if (status.hasCompleted(instance.name(), change.withoutPlatform())) remaining = remaining.withoutApplication(); - if (change.application().isPresent()) - instance = instance.withLatestDeployed(change.application().get()); - } return instance.withChange(remaining); } // ---------- Version and job helpers ---------- private Job deploymentJob(Instance instance, Versions versions, JobType jobType, JobStatus jobStatus, Instant availableSince) { - return new Job(instance, versions, jobType, availableSince, jobStatus.isNodeAllocationFailure(), instance.change().application().isPresent()); + return new Job(instance, versions, jobType, availableSince, jobStatus.isNodeAllocationFailure(), instance.change().revision().isPresent()); } // ---------- Data containers ---------- @@ -510,7 +493,7 @@ public class DeploymentTrigger { public String toString() { return jobType + " for " + instanceId + " on (" + versions.targetPlatform() + versions.sourcePlatform().map(version -> " <-- " + version).orElse("") + - ", " + versions.targetApplication().id() + versions.sourceApplication().map(version -> " <-- " + version.id()).orElse("") + + ", " + versions.targetRevision() + versions.sourceRevision().map(version -> " <-- " + version).orElse("") + "), ready since " + availableSince; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index beef090d214..52e5431b552 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -1,17 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; +import ai.vespa.http.DomainName; import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.Notifications; import com.yahoo.config.application.api.Notifications.When; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.AthenzDomain; -import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; @@ -21,10 +18,6 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.text.Text; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -36,11 +29,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.application.ActivateResult; @@ -48,7 +40,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.config.ControllerConfig; +import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; @@ -89,6 +81,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Nod import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; @@ -125,13 +118,6 @@ public class InternalStepRunner implements StepRunner { private static final Logger logger = Logger.getLogger(InternalStepRunner.class.getName()); - static final NodeResources DEFAULT_TESTER_RESOURCES = - new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any); - // Must match exactly the advertised resources of an AWS instance type. Also consider that the container - // will have ~1.8 GB less memory than equivalent resources in AWS (VESPA-16259). - static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = - new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); - private final Controller controller; private final TestConfigSerializer testConfigSerializer; private final DeploymentFailureMails mails; @@ -184,15 +170,15 @@ public class InternalStepRunner implements StepRunner { Versions versions = controller.jobController().run(id).get().versions(); logger.log("Deploying platform version " + versions.sourcePlatform().orElse(versions.targetPlatform()) + - " and application version " + - versions.sourceApplication().orElse(versions.targetApplication()).id() + " ..."); + " and application " + + versions.sourceRevision().orElse(versions.targetRevision()) + " ..."); return deployReal(id, true, logger); } private Optional<RunStatus> deployReal(RunId id, DualLogger logger) { Versions versions = controller.jobController().run(id).get().versions(); logger.log("Deploying platform version " + versions.targetPlatform() + - " and application version " + versions.targetApplication().id() + " ..."); + " and application " + versions.targetRevision() + " ..."); return deployReal(id, false, logger); } @@ -218,7 +204,7 @@ public class InternalStepRunner implements StepRunner { logger.log("Deploying the tester container on platform " + platform + " ..."); return deploy(() -> controller.applications().deployTester(id.tester(), testerPackage(id), - id.type().zone(controller.system()), + id.type().zone(), platform), controller.jobController().run(id).get() .stepInfo(deployTester).get() @@ -315,19 +301,19 @@ public class InternalStepRunner implements StepRunner { Version platform = setTheStage ? versions.sourcePlatform().orElse(versions.targetPlatform()) : versions.targetPlatform(); Run run = controller.jobController().run(id).get(); - Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone(controller.system())), + Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone()), Optional.of(platform)); if (services.isEmpty()) { logger.log("Config status not currently available -- will retry."); return Optional.empty(); } - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), + List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), NodeFilter.all() .applications(id.application()) .states(active)); Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); - List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), + List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), NodeFilter.all() .hostnames(parentHostnames)); boolean firstTick = run.convergenceSummary().isEmpty(); @@ -358,8 +344,8 @@ public class InternalStepRunner implements StepRunner { } if (summary.converged()) { controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); - if (endpointsAvailable(id.application(), id.type().zone(controller.system()), logger)) { - if (containersAreUp(id.application(), id.type().zone(controller.system()), logger)) { + if (endpointsAvailable(id.application(), id.type().zone(), logger)) { + if (containersAreUp(id.application(), id.type().zone(), logger)) { logger.log("Installation succeeded!"); return Optional.of(running); } @@ -441,7 +427,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> installTester(RunId id, DualLogger logger) { Run run = controller.jobController().run(id).get(); Version platform = testerPlatformVersion(id); - ZoneId zone = id.type().zone(controller.system()); + ZoneId zone = id.type().zone(); ApplicationId testerId = id.tester().id(); Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(testerId, zone), @@ -512,7 +498,7 @@ public class InternalStepRunner implements StepRunner { return false; } for (var endpoint : endpoints.get(zone)) { - HostName endpointName = HostName.from(endpoint.dnsName()); + DomainName endpointName = DomainName.of(endpoint.dnsName()); var ipAddress = controller.jobController().cloud().resolveHostName(endpointName); if (ipAddress.isEmpty()) { logger.log(INFO, "DNS lookup yielded no IP address for '" + endpointName + "'."); @@ -533,7 +519,7 @@ public class InternalStepRunner implements StepRunner { var loadBalancerAddress = controller.jobController().cloud().resolveHostName(policy.canonicalName()); if ( ! loadBalancerAddress.equals(ipAddress)) { logger.log(INFO, "IP address of CNAME '" + endpointName + "' (" + ipAddress.get() + ") and load balancer '" + - policy.canonicalName() + "' (" + loadBalancerAddress.orElse("empty") + ") are not equal"); + policy.canonicalName() + "' (" + loadBalancerAddress.orElse(null) + ") are not equal"); return false; } } @@ -610,7 +596,7 @@ public class InternalStepRunner implements StepRunner { .productionDeployments().keySet().stream() .map(zone -> new DeploymentId(id.application(), zone)) .collect(Collectors.toSet()); - ZoneId zoneId = id.type().zone(controller.system()); + ZoneId zoneId = id.type().zone(); deployments.add(new DeploymentId(id.application(), zoneId)); logger.log("Attempting to find endpoints ..."); @@ -637,6 +623,7 @@ public class InternalStepRunner implements StepRunner { return Optional.of(running); } + @SuppressWarnings("fallthrough") private Optional<RunStatus> endTests(RunId id, boolean isSetup, DualLogger logger) { Optional<Deployment> deployment = deployment(id.application(), id.type()); if (deployment.isEmpty()) { @@ -678,12 +665,14 @@ public class InternalStepRunner implements StepRunner { controller.jobController().updateTestReport(id); return Optional.of(error); case NO_TESTS: - TesterCloud.Suite suite = TesterCloud.Suite.of(id.type(), isSetup); - logger.log(INFO, "No tests were found in the test package, for test suite '" + suite + "'"); - logger.log(INFO, "The test package must either contain basic HTTP tests under 'tests/<suite-name>/', " + - "or a Java test bundle under 'components/' with at least one test with the annotation " + - "for this suite. See docs.vespa.ai/en/testing.html for details."); - return Optional.of(allowNoTests(id.application()) ? running : testFailure); + if ( ! isSetup) { // TODO: consider changing this Later™ + TesterCloud.Suite suite = TesterCloud.Suite.of(id.type(), isSetup); + logger.log(INFO, "No tests were found in the test package, for test suite '" + suite + "'"); + logger.log(INFO, "The test package should either contain basic HTTP tests under 'tests/<suite-name>/', " + + "or a Java test bundle under 'components/' with at least one test with the annotation " + + "for this suite. See docs.vespa.ai/en/testing.html for details."); + return Optional.of(noTests); + } case SUCCESS: logger.log("Tests completed successfully."); controller.jobController().updateTestReport(id); @@ -693,12 +682,6 @@ public class InternalStepRunner implements StepRunner { } } - private boolean allowNoTests(ApplicationId appId) { - return Flags.ALLOW_NO_TESTS.bindTo(controller.flagSource()) - .with(FetchVector.Dimension.TENANT_ID, appId.tenant().value()) - .value(); - } - private Optional<RunStatus> copyVespaLogs(RunId id, DualLogger logger) { if (deployment(id.application(), id.type()).isPresent()) try { @@ -726,8 +709,8 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deactivateReal(RunId id, DualLogger logger) { try { - logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.applications().deactivate(id.application(), id.type().zone(controller.system())); + logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone() + " ..."); + controller.applications().deactivate(id.application(), id.type().zone()); return Optional.of(running); } catch (RuntimeException e) { @@ -741,7 +724,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deactivateTester(RunId id, DualLogger logger) { try { - logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone() + " ..."); controller.jobController().deactivateTester(id.tester(), id.type()); return Optional.of(running); } @@ -786,14 +769,14 @@ public class InternalStepRunner implements StepRunner { Application application = controller.applications().requireApplication(TenantAndApplicationId.from(run.id().application())); Notifications notifications = application.deploymentSpec().requireInstance(run.id().application().instance()).notifications(); - boolean newCommit = application.require(run.id().application().instance()).change().application() - .map(run.versions().targetApplication()::equals) + boolean newCommit = application.require(run.id().application().instance()).change().revision() + .map(run.versions().targetRevision()::equals) .orElse(false); When when = newCommit ? failingCommit : failing; List<String> recipients = new ArrayList<>(notifications.emailAddressesFor(when)); if (notifications.emailRolesFor(when).contains(author)) - run.versions().targetApplication().authorEmail().ifPresent(recipients::add); + application.revisions().get(run.versions().targetRevision()).authorEmail().ifPresent(recipients::add); if (recipients.isEmpty()) return; @@ -834,6 +817,10 @@ public class InternalStepRunner implements StepRunner { case testFailure: updater.accept("one or more verification tests against the deployment failed. Please review test output in the deployment job log."); return; + case noTests: + controller.notificationsDb().setNotification(source, Notification.Type.deployment, Notification.Level.warning, + "no tests were found for this job type. Please review test output in the deployment job log."); + return; case error: case endpointCertificateTimeout: break; @@ -848,6 +835,7 @@ public class InternalStepRunner implements StepRunner { switch (run.status()) { case running: case aborted: + case noTests: case success: return Optional.empty(); case nodeAllocationFailure: @@ -860,16 +848,16 @@ public class InternalStepRunner implements StepRunner { return Optional.of(mails.testFailure(run.id(), recipients)); case error: case endpointCertificateTimeout: - return Optional.of(mails.systemError(run.id(), recipients)); + break; default: logger.log(WARNING, "Don't know what mail to send for run status '" + run.status() + "'"); - return Optional.of(mails.systemError(run.id(), recipients)); } + return Optional.of(mails.systemError(run.id(), recipients)); } /** Returns the deployment of the real application in the zone of the given job, if it exists. */ private Optional<Deployment> deployment(ApplicationId id, JobType type) { - return Optional.ofNullable(application(id).deployments().get(type.zone(controller.system()))); + return Optional.ofNullable(application(id).deployments().get(type.zone())); } /** Returns the real application with the given id. */ @@ -904,141 +892,29 @@ public class InternalStepRunner implements StepRunner { /** Returns the application package for the tester application, assembled from a generated config, fat-jar and services.xml. */ private ApplicationPackage testerPackage(RunId id) { - ApplicationVersion version = controller.jobController().run(id).get().versions().targetApplication(); + RevisionId revision = controller.jobController().run(id).get().versions().targetRevision(); DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); - - ZoneId zone = id.type().zone(controller.system()); + byte[] testZip = controller.applications().applicationStore().getTester(id.application().tenant(), + id.application().application(), revision); boolean useTesterCertificate = useTesterCertificate(id); - byte[] servicesXml = servicesXml( ! controller.system().isPublic(), - useTesterCertificate, - testerResourcesFor(zone, spec.requireInstance(id.application().instance())), - controller.controllerConfig().steprunner().testerapp()); - byte[] testPackage = controller.applications().applicationStore().getTester(id.application().tenant(), id.application().application(), version); - byte[] deploymentXml = deploymentXml(id.tester(), - spec.athenzDomain(), - spec.requireInstance(id.application().instance()).athenzService(zone.environment(), zone.region())); - - try (ZipBuilder zipBuilder = new ZipBuilder(testPackage.length + servicesXml.length + deploymentXml.length + 1000)) { - // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip. - zipBuilder.add(testPackage); - zipBuilder.add("artifacts/.ignore-" + UUID.randomUUID(), new byte[0]); - zipBuilder.add("tests/.ignore-" + UUID.randomUUID(), new byte[0]); - - zipBuilder.add("services.xml", servicesXml); - zipBuilder.add("deployment.xml", deploymentXml); - if (useTesterCertificate) - appendAndStoreCertificate(zipBuilder, id); - - zipBuilder.close(); - return new ApplicationPackage(zipBuilder.toByteArray()); - } - } + TestPackage testPackage = new TestPackage(testZip, + controller.system().isPublic(), + id, + controller.controllerConfig().steprunner().testerapp(), + spec, + useTesterCertificate ? controller.clock().instant() : null, + timeouts.testerCertificate()); + if (useTesterCertificate) controller.jobController().storeTesterCertificate(id, testPackage.certificate()); - private void appendAndStoreCertificate(ZipBuilder zipBuilder, RunId id) { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - X500Principal subject = new X500Principal("CN=" + id.tester().id().toFullString() + "." + id.type() + "." + id.number()); - X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, - subject, - controller.clock().instant(), - controller.clock().instant().plus(timeouts.testerCertificate()), - SignatureAlgorithm.SHA512_WITH_RSA, - BigInteger.valueOf(1)) - .build(); - controller.jobController().storeTesterCertificate(id, certificate); - zipBuilder.add("artifacts/key", KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8)); - zipBuilder.add("artifacts/cert", X509CertificateUtils.toPem(certificate).getBytes(UTF_8)); + return testPackage.asApplicationPackage(); } private DeploymentId getTesterDeploymentId(RunId runId) { - ZoneId zoneId = runId.type().zone(controller.system()); + ZoneId zoneId = runId.type().zone(); return new DeploymentId(runId.tester().id(), zoneId); } - static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) { - NodeResources nodeResources = spec.steps().stream() - .filter(step -> step.concerns(zone.environment())) - .findFirst() - .flatMap(step -> step.zones().get(0).testerFlavor()) - .map(NodeResources::fromLegacyName) - .orElse(zone.region().value().contains("aws-") ? - DEFAULT_TESTER_RESOURCES_AWS : DEFAULT_TESTER_RESOURCES); - return nodeResources.with(NodeResources.DiskSpeed.any); - } - - /** Returns the generated services.xml content for the tester application. */ - static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, - NodeResources resources, ControllerConfig.Steprunner.Testerapp config) { - int jdiscMemoryGb = 2; // 2Gb memory for tester application (excessive?). - int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb()); - - // Of the remaining memory, split 50/50 between Surefire running the tests and the rest - int testMemoryMb = (int) (1024 * (resources.memoryGb() - jdiscMemoryGb) / 2); - - String resourceString = Text.format( - "<resources vcpu=\"%.2f\" memory=\"%.2fGb\" disk=\"%.2fGb\" disk-speed=\"%s\" storage-type=\"%s\"/>", - resources.vcpu(), resources.memoryGb(), resources.diskGb(), resources.diskSpeed().name(), resources.storageType().name()); - - String runtimeProviderClass = config.runtimeProviderClass(); - String tenantCdBundle = config.tenantCdBundle(); - - String servicesXml = - "<?xml version='1.0' encoding='UTF-8'?>\n" + - "<services xmlns:deploy='vespa' version='1.0'>\n" + - " <container version='1.0' id='tester'>\n" + - "\n" + - " <component id=\"com.yahoo.vespa.hosted.testrunner.TestRunner\" bundle=\"vespa-testrunner-components\">\n" + - " <config name=\"com.yahoo.vespa.hosted.testrunner.test-runner\">\n" + - " <artifactsPath>artifacts</artifactsPath>\n" + - " <surefireMemoryMb>" + testMemoryMb + "</surefireMemoryMb>\n" + - " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + - " <useTesterCertificate>" + useTesterCertificate + "</useTesterCertificate>\n" + - " </config>\n" + - " </component>\n" + - "\n" + - " <handler id=\"com.yahoo.vespa.testrunner.TestRunnerHandler\" bundle=\"vespa-osgi-testrunner\">\n" + - " <binding>http://*/tester/v1/*</binding>\n" + - " </handler>\n" + - "\n" + - " <component id=\"" + runtimeProviderClass + "\" bundle=\"" + tenantCdBundle + "\" />\n" + - "\n" + - " <component id=\"com.yahoo.vespa.testrunner.JunitRunner\" bundle=\"vespa-osgi-testrunner\">\n" + - " <config name=\"com.yahoo.vespa.testrunner.junit-test-runner\">\n" + - " <artifactsPath>artifacts</artifactsPath>\n" + - " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + - " </config>\n" + - " </component>\n" + - "\n" + - " <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" + - " <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" + - " <artifactsPath>artifacts</artifactsPath>\n" + - " <testsPath>tests</testsPath>\n" + - " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + - " </config>\n" + - " </component>\n" + - "\n" + - " <nodes count=\"1\">\n" + - " <jvm allocated-memory=\"" + jdiscMemoryPct + "%\"/>\n" + - " " + resourceString + "\n" + - " </nodes>\n" + - " </container>\n" + - "</services>\n"; - - return servicesXml.getBytes(UTF_8); - } - - /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */ - private static byte[] deploymentXml(TesterId id, Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) { - String deploymentSpec = - "<?xml version='1.0' encoding='UTF-8'?>\n" + - "<deployment version=\"1.0\" " + - athenzDomain.map(domain -> "athenz-domain=\"" + domain.value() + "\" ").orElse("") + - athenzService.map(service -> "athenz-service=\"" + service.value() + "\" ").orElse("") + ">" + - " <instance id=\"" + id.id().instance().value() + "\" />" + - "</deployment>"; - return deploymentSpec.getBytes(UTF_8); - } - /** Logger which logs to a {@link JobController}, as well as to the parent class' {@link Logger}. */ private class DualLogger { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index b9f09cc31ea..1d56e2db08b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -6,15 +6,18 @@ import com.yahoo.component.Version; import com.yahoo.component.VersionCompatibility; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.curator.Lock; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; @@ -25,9 +28,13 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff; +import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.TestSummary; +import com.yahoo.vespa.hosted.controller.notification.Notification; +import com.yahoo.vespa.hosted.controller.notification.Notification.Type; +import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.security.cert.X509Certificate; @@ -37,19 +44,21 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.function.UnaryOperator; import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Stream; import static com.yahoo.collections.Iterables.reversed; @@ -63,10 +72,11 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup; import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static java.time.temporal.ChronoUnit.SECONDS; +import static java.util.Comparator.naturalOrder; import static java.util.function.Predicate.not; import static java.util.logging.Level.INFO; +import static java.util.logging.Level.WARNING; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toUnmodifiableList; /** @@ -86,6 +96,8 @@ public class JobController { public static final Duration maxHistoryAge = Duration.ofDays(60); + private static final Logger log = Logger.getLogger(JobController.class.getName()); + private final int historyLength; private final Controller controller; private final CuratorDb curator; @@ -125,7 +137,7 @@ public class JobController { /** Returns the logged entries for the given run, which are after the given id threshold. */ public Optional<RunLog> details(RunId id, long after) { - try (Lock __ = curator.lock(id.application(), id.type())) { + try (Mutex __ = curator.lock(id.application(), id.type())) { Run run = runs(id.application(), id.type()).get(id); if (run == null) return Optional.empty(); @@ -162,7 +174,7 @@ public class JobController { if ( ! run.hasStep(copyVespaLogs)) return run; - ZoneId zone = id.type().zone(controller.system()); + ZoneId zone = id.type().zone(); Optional<Deployment> deployment = Optional.ofNullable(controller.applications().requireInstance(id.application()) .deployments().get(zone)); if (deployment.isEmpty() || deployment.get().at().isBefore(run.start())) @@ -190,7 +202,7 @@ public class JobController { if (step.isEmpty()) return run; - List<LogEntry> entries = cloud.getLog(new DeploymentId(id.tester().id(), id.type().zone(controller.system())), + List<LogEntry> entries = cloud.getLog(new DeploymentId(id.tester().id(), id.type().zone()), run.lastTestLogEntry()); if (entries.isEmpty()) return run; @@ -202,7 +214,7 @@ public class JobController { public void updateTestReport(RunId id) { locked(id, run -> { - Optional<TestReport> report = cloud.getTestReport(new DeploymentId(id.tester().id(), id.type().zone(controller.system()))); + Optional<TestReport> report = cloud.getTestReport(new DeploymentId(id.tester().id(), id.type().zone())); if (report.isEmpty()) { return run; } @@ -229,8 +241,8 @@ public class JobController { } /** Returns all job types which have been run for the given application. */ - public List<JobType> jobs(ApplicationId id) { - return JobType.allIn(controller.system()).stream() + private List<JobType> jobs(ApplicationId id) { + return JobType.allIn(controller.zoneRegistry()).stream() .filter(type -> last(id, type).isPresent()) .collect(toUnmodifiableList()); } @@ -248,6 +260,13 @@ public class JobController { .collect(toUnmodifiableList()); } + /** Returns when given deployment last started deploying, falling back to time of deployment if it cannot be determined from job runs */ + public Instant lastDeploymentStart(ApplicationId instanceId, Deployment deployment) { + return jobStarts(new JobId(instanceId, JobType.deploymentTo(deployment.zone()))).stream() + .findFirst() + .orElseGet(deployment::at); + } + /** Returns an immutable map of all known runs for the given application and job type. */ public NavigableMap<RunId, Run> runs(ApplicationId id, JobType type) { ImmutableSortedMap.Builder<RunId, Run> runs = ImmutableSortedMap.orderedBy(Comparator.comparing(RunId::number)); @@ -309,20 +328,20 @@ public class JobController { /** Returns a list of all active runs for the given application. */ public List<Run> active(TenantAndApplicationId id) { return controller.applications().requireApplication(id).instances().keySet().stream() - .flatMap(name -> Stream.of(JobType.values()) - .map(type -> last(id.instance(name), type)) - .flatMap(Optional::stream) - .filter(run -> !run.hasEnded())) + .flatMap(name -> JobType.allIn(controller.zoneRegistry()).stream() + .map(type -> last(id.instance(name), type)) + .flatMap(Optional::stream) + .filter(run -> ! run.hasEnded())) .collect(toUnmodifiableList()); } /** Returns a list of all active runs for the given instance. */ public List<Run> active(ApplicationId id) { - return Stream.of(JobType.values()) - .map(type -> last(id, type)) - .flatMap(Optional::stream) - .filter(run -> !run.hasEnded()) - .collect(toUnmodifiableList()); + return JobType.allIn(controller.zoneRegistry()).stream() + .map(type -> last(id, type)) + .flatMap(Optional::stream) + .filter(run -> !run.hasEnded()) + .collect(toUnmodifiableList()); } /** Returns the job status of the given job, possibly empty. */ @@ -337,12 +356,8 @@ public class JobController { private DeploymentStatus deploymentStatus(Application application, Version systemVersion) { return new DeploymentStatus(application, - DeploymentStatus.jobsFor(application, controller.system()).stream() - .collect(toMap(job -> job, - job -> jobStatus(job), - (j1, j2) -> { throw new IllegalArgumentException("Duplicate key " + j1.id()); }, - LinkedHashMap::new)), - controller.system(), + this::jobStatus, + controller.zoneRegistry(), systemVersion, instance -> controller.applications().versionCompatibility(application.id().instance(instance)), controller.clock().instant()); @@ -375,7 +390,7 @@ public class JobController { * Throws TimeoutException if some step in this job is still being run. */ public void finish(RunId id) throws TimeoutException { - List<Lock> locks = new ArrayList<>(); + List<Mutex> locks = new ArrayList<>(); try { // Ensure no step is still running before we finish the run — report depends transitively on all the other steps. Run unlockedRun = run(id).get(); @@ -396,7 +411,7 @@ public class JobController { locked(id.application(), id.type(), runs -> { runs.put(run.id(), finishedRun); long last = id.number(); - long successes = runs.values().stream().filter(old -> old.status() == RunStatus.success).count(); + long successes = runs.values().stream().filter(Run::hasSucceeded).count(); var oldEntries = runs.entrySet().iterator(); for (var old = oldEntries.next(); old.getKey().number() <= last - historyLength @@ -405,7 +420,7 @@ public class JobController { // Make sure we keep the last success and the first failing if ( successes == 1 - && old.getValue().status() == RunStatus.success + && old.getValue().hasSucceeded() && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { oldEntries.next(); continue; @@ -417,26 +432,21 @@ public class JobController { }); logs.flush(id); metric.jobFinished(run.id().job(), finishedRun.status()); + pruneRevisions(unlockedRun); - DeploymentId deploymentId = new DeploymentId(unlockedRun.id().application(), unlockedRun.id().job().type().zone(controller.system())); - (unlockedRun.versions().targetApplication().isDeployedDirectly() ? - Stream.of(unlockedRun.id().type()) : - JobType.allIn(controller.system()).stream().filter(jobType -> !jobType.environment().isManuallyDeployed())) - .flatMap(jobType -> controller.jobController().runs(unlockedRun.id().application(), jobType).values().stream()) - .mapToLong(r -> r.versions().targetApplication().buildNumber().orElse(Integer.MAX_VALUE)) - .min() - .ifPresent(oldestBuild -> { - if (unlockedRun.versions().targetApplication().isDeployedDirectly()) - controller.applications().applicationStore().pruneDevDiffs(deploymentId, oldestBuild); - else - controller.applications().applicationStore().pruneDiffs(deploymentId.applicationId().tenant(), deploymentId.applicationId().application(), oldestBuild); - }); return finishedRun; }); } finally { - for (Lock lock : locks) - lock.close(); + for (Mutex lock : locks) { + try { + lock.close(); + } catch (Throwable t) { + log.log(WARNING, "Failed to close the lock " + lock + ": the lock may or may not " + + "have been released in ZooKeeper, and if not this controller " + + "must be restarted to release the lock", t); + } + } } } @@ -451,48 +461,96 @@ public class JobController { } /** Accepts and stores a new application package and test jar pair under a generated application version key. */ - public ApplicationVersion submit(TenantAndApplicationId id, Optional<SourceRevision> revision, Optional<String> authorEmail, - Optional<String> sourceUrl, long projectId, ApplicationPackage applicationPackage, - byte[] testPackageBytes) { + public ApplicationVersion submit(TenantAndApplicationId id, Submission submission, long projectId) { + ApplicationController applications = controller.applications(); AtomicReference<ApplicationVersion> version = new AtomicReference<>(); - controller.applications().lockApplicationOrThrow(id, application -> { - Optional<ApplicationVersion> previousVersion = application.get().latestVersion(); - Optional<ApplicationPackage> previousPackage = previousVersion.flatMap(previous -> controller.applications().applicationStore().find(id.tenant(), id.application(), previous.buildNumber().getAsLong())) + applications.lockApplicationOrThrow(id, application -> { + Optional<ApplicationVersion> previousVersion = application.get().revisions().last(); + Optional<ApplicationPackage> previousPackage = previousVersion.flatMap(previous -> applications.applicationStore().find(id.tenant(), id.application(), previous.buildNumber().getAsLong())) .map(ApplicationPackage::new); long previousBuild = previousVersion.map(latestVersion -> latestVersion.buildNumber().getAsLong()).orElse(0L); - String packageHash = applicationPackage.bundleHash() + ApplicationPackage.calculateHash(testPackageBytes); - version.set(ApplicationVersion.from(revision, 1 + previousBuild, authorEmail, - applicationPackage.compileVersion(), - applicationPackage.buildTime(), - sourceUrl, - revision.map(SourceRevision::commit), - false, - Optional.of(packageHash))); - - byte[] diff = previousPackage.map(previous -> ApplicationPackageDiff.diff(previous, applicationPackage)) - .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage)); - controller.applications().applicationStore().put(id.tenant(), - id.application(), - version.get(), - applicationPackage.zippedContent(), - diff); - controller.applications().applicationStore().putTester(id.tenant(), - id.application(), - version.get(), - testPackageBytes); - controller.applications().applicationStore().putMeta(id.tenant(), - id.application(), - controller.clock().instant(), - applicationPackage.metaDataZip()); - - prunePackages(id); - controller.applications().storeWithUpdatedConfig(application, applicationPackage); - - controller.applications().deploymentTrigger().notifyOfSubmission(id, version.get(), projectId); + version.set(submission.toApplicationVersion(1 + previousBuild)); + + byte[] diff = previousPackage.map(previous -> ApplicationPackageDiff.diff(previous, submission.applicationPackage())) + .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(submission.applicationPackage())); + applications.applicationStore().put(id.tenant(), + id.application(), + version.get().id(), + submission.applicationPackage().zippedContent(), + submission.testPackage(), + diff); + applications.applicationStore().putMeta(id.tenant(), + id.application(), + controller.clock().instant(), + submission.applicationPackage().metaDataZip()); + + application = application.withProjectId(projectId == -1 ? OptionalLong.empty() : OptionalLong.of(projectId)); + application = application.withRevisions(revisions -> revisions.with(version.get())); + application = withPrunedPackages(application); + + TestSummary testSummary = TestPackage.validateTests(submission.applicationPackage().deploymentSpec(), submission.testPackage()); + if (testSummary.problems().isEmpty()) + controller.notificationsDb().removeNotification(NotificationSource.from(id), Type.testPackage); + else + controller.notificationsDb().setNotification(NotificationSource.from(id), + Type.testPackage, + Notification.Level.warning, + testSummary.problems()); + + submission.applicationPackage().parentVersion().ifPresent(parent -> { + if (parent.getMajor() < controller.readSystemVersion().getMajor()) + controller.notificationsDb().setNotification(NotificationSource.from(id), + Type.submission, + Notification.Level.warning, + "Parent version used to compile the application is on a " + + "lower major version than the current Vespa Cloud version"); + else + controller.notificationsDb().removeNotification(NotificationSource.from(id), Type.submission); + }); + + applications.storeWithUpdatedConfig(application, submission.applicationPackage()); + applications.deploymentTrigger().triggerNewRevision(id); }); return version.get(); } + private LockedApplication withPrunedPackages(LockedApplication application){ + TenantAndApplicationId id = application.get().id(); + Optional<RevisionId> oldestDeployed = application.get().oldestDeployedRevision(); + if (oldestDeployed.isPresent()) { + controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed.get()); + + for (ApplicationVersion version : application.get().revisions().withPackage()) + if (version.id().compareTo(oldestDeployed.get()) < 0) + application = application.withRevisions(revisions -> revisions.with(version.withoutPackage())); + } + return application; + } + + /** Forget revisions no longer present in any relevant job history. */ + private void pruneRevisions(Run run) { + TenantAndApplicationId applicationId = TenantAndApplicationId.from(run.id().application()); + boolean isProduction = run.versions().targetRevision().isProduction(); + (isProduction ? deploymentStatus(controller.applications().requireApplication(applicationId)).jobs().asList().stream() + : Stream.of(jobStatus(run.id().job()))) + .flatMap(jobs -> jobs.runs().values().stream()) + .map(r -> r.versions().targetRevision()) + .filter(id -> id.isProduction() == isProduction) + .min(naturalOrder()) + .ifPresent(oldestRevision -> { + controller.applications().lockApplicationOrThrow(applicationId, application -> { + if (isProduction) { + controller.applications().applicationStore().pruneDiffs(run.id().application().tenant(), run.id().application().application(), oldestRevision.number()); + controller.applications().store(application.withRevisions(revisions -> revisions.withoutOlderThan(oldestRevision))); + } + else { + controller.applications().applicationStore().pruneDevDiffs(new DeploymentId(run.id().application(), run.id().job().type().zone()), oldestRevision.number()); + controller.applications().store(application.withRevisions(revisions -> revisions.withoutOlderThan(oldestRevision, run.id().job()))); + } + }); + }); + } + /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, Optional<String> reason) { start(id, type, versions, isRedeployment, JobProfile.of(type), reason); @@ -500,11 +558,12 @@ public class JobController { /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, JobProfile profile, Optional<String> reason) { - if (versions.targetApplication().compileVersion() + ApplicationVersion revision = controller.applications().requireApplication(TenantAndApplicationId.from(id)).revisions().get(versions.targetRevision()); + if (revision.compileVersion() .map(version -> controller.applications().versionCompatibility(id).refuse(versions.targetPlatform(), version)) .orElse(false)) throw new IllegalArgumentException("Will not start a job with incompatible platform version (" + versions.targetPlatform() + ") " + - "and compile versions (" + versions.targetApplication().compileVersion().get() + ")"); + "and compile versions (" + revision.compileVersion().get() + ")"); locked(id, type, __ -> { Optional<Run> last = last(id, type); @@ -525,6 +584,9 @@ public class JobController { /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment.*/ public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage, boolean dryRun) { + if ( ! controller.zoneRegistry().hasZone(type.zone())) + throw new IllegalArgumentException(type.zone() + " is not present in this system"); + controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { if ( ! application.get().instances().containsKey(id.instance())) application = controller.applications().withNewInstance(application, id); @@ -532,24 +594,23 @@ public class JobController { controller.applications().store(application); }); - DeploymentId deploymentId = new DeploymentId(id, type.zone(controller.system())); + DeploymentId deploymentId = new DeploymentId(id, type.zone()); Optional<Run> lastRun = last(id, type); lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); - long build = 1 + lastRun.map(run -> run.versions().targetApplication().buildNumber().orElse(0)).orElse(0L); - ApplicationVersion version = ApplicationVersion.from(Optional.empty(), build, Optional.empty(), - applicationPackage.compileVersion(), - Optional.empty(), Optional.empty(), - Optional.empty(), true, Optional.empty()); + long build = 1 + lastRun.map(run -> run.versions().targetRevision().number()).orElse(0L); + RevisionId revisionId = RevisionId.forDevelopment(build, new JobId(id, type)); + ApplicationVersion version = ApplicationVersion.forDevelopment(revisionId, applicationPackage.compileVersion()); byte[] diff = getDiff(applicationPackage, deploymentId, lastRun); controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - controller.applications().applicationStore().putDev(deploymentId, version, applicationPackage.zippedContent(), diff); - Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, lastRun, id)); + controller.applications().applicationStore().putDev(deploymentId, version.id(), applicationPackage.zippedContent(), diff); + Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, deploymentId, application.get().get(id.instance()))); + controller.applications().store(application.withRevisions(revisions -> revisions.with(version))); start(id, type, - new Versions(targetPlatform, version, lastRun.map(run -> run.versions().targetPlatform()), lastRun.map(run -> run.versions().targetApplication())), + new Versions(targetPlatform, version.id(), lastRun.map(run -> run.versions().targetPlatform()), lastRun.map(run -> run.versions().targetRevision())), false, dryRun ? JobProfile.developmentDryRun : JobProfile.development, Optional.empty()); @@ -562,7 +623,7 @@ public class JobController { /* Application package diff against previous version, or against empty version if previous does not exist or is invalid */ private byte[] getDiff(ApplicationPackage applicationPackage, DeploymentId deploymentId, Optional<Run> lastRun) { - return lastRun.map(run -> run.versions().targetApplication()) + return lastRun.map(run -> run.versions().targetRevision()) .map(prevVersion -> { ApplicationPackage previous; try { @@ -575,24 +636,27 @@ public class JobController { .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage)); } - private Version findTargetPlatform(ApplicationPackage applicationPackage, Optional<Run> lastRun, ApplicationId id) { + private Version findTargetPlatform(ApplicationPackage applicationPackage, DeploymentId id, Optional<Instance> instance) { Optional<Integer> major = applicationPackage.deploymentSpec().majorVersion(); if (major.isPresent()) return controller.applications().lastCompatibleVersion(major.get()) .orElseThrow(() -> new IllegalArgumentException("major " + major.get() + " specified in deployment.xml, " + "but no version on this major was found")); - // Prefer previous platform if possible. - VersionStatus versionStatus = controller.readVersionStatus(); - VersionCompatibility compatibility = controller.applications().versionCompatibility(id); - Optional<Version> target = lastRun.map(run -> run.versions().targetPlatform()).filter(versionStatus::isActive); - if (target.isPresent() && compatibility.accept(target.get(), applicationPackage.compileVersion().orElse(target.get()))) - return target.get(); + VersionCompatibility compatibility = controller.applications().versionCompatibility(id.applicationId()); + + // Prefer previous platform if possible. Candidates are all deployable, ascending, with existing version appended; then reversed. + List<Version> versions = controller.readVersionStatus().deployableVersions().stream() + .map(VespaVersion::versionNumber) + .collect(toList()); + instance.map(Instance::deployments) + .map(deployments -> deployments.get(id.zoneId())) + .map(Deployment::version) + .ifPresent(versions::add); - // Otherwise, use newest, compatible version. - for (VespaVersion platform : reversed(versionStatus.deployableVersions())) - if (compatibility.accept(platform.versionNumber(), applicationPackage.compileVersion().orElse(platform.versionNumber()))) - return platform.versionNumber(); + for (Version target : reversed(versions)) + if (applicationPackage.compileVersion().isEmpty() || compatibility.accept(target, applicationPackage.compileVersion().get())) + return target; throw new IllegalArgumentException("no suitable platform version found" + applicationPackage.compileVersion() @@ -626,7 +690,7 @@ public class JobController { TesterId tester = TesterId.of(id); for (JobType type : jobs(id)) locked(id, type, deactivateTester, __ -> { - try (Lock ___ = curator.lock(id, type)) { + try (Mutex ___ = curator.lock(id, type)) { try { deactivateTester(tester, type); } @@ -634,34 +698,24 @@ public class JobController { // It's probably already deleted, so if we fail, that's OK. } curator.deleteRunData(id, type); - logs.delete(id); } }); + logs.delete(id); + curator.deleteRunData(id); } catch (Exception e) { - return; // Don't remove the data if we couldn't clean up all resources. + log.log(WARNING, "failed cleaning up after deleted application", e); } - curator.deleteRunData(id); }); } public void deactivateTester(TesterId id, JobType type) { - controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone(controller.system()))); - } - - private void prunePackages(TenantAndApplicationId id) { - controller.applications().lockApplicationIfPresent(id, application -> { - application.get().oldestDeployedApplication() - .ifPresent(oldestDeployed -> { - controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed); - controller.applications().applicationStore().pruneTesters(id.tenant(), id.application(), oldestDeployed); - }); - }); + controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone())); } /** Locks all runs and modifies the list of historic runs for the given application and job type. */ private void locked(ApplicationId id, JobType type, Consumer<SortedMap<RunId, Run>> modifications) { - try (Lock __ = curator.lock(id, type)) { + try (Mutex __ = curator.lock(id, type)) { SortedMap<RunId, Run> runs = new TreeMap<>(curator.readHistoricRuns(id, type)); modifications.accept(runs); curator.writeHistoricRuns(id, type, runs.values()); @@ -670,19 +724,18 @@ public class JobController { /** Locks and modifies the run with the given id, provided it is still active. */ public void locked(RunId id, UnaryOperator<Run> modifications) { - try (Lock __ = curator.lock(id.application(), id.type())) { + try (Mutex __ = curator.lock(id.application(), id.type())) { active(id).ifPresent(run -> { - run = modifications.apply(run); - curator.writeLastRun(run); + curator.writeLastRun(modifications.apply(run)); }); } } /** Locks the given step and checks none of its prerequisites are running, then performs the given actions. */ public void locked(ApplicationId id, JobType type, Step step, Consumer<LockedStep> action) throws TimeoutException { - try (Lock lock = curator.lock(id, type, step)) { + try (Mutex lock = curator.lock(id, type, step)) { for (Step prerequisite : step.allPrerequisites(last(id, type).get().steps().keySet())) // Check that no prerequisite is still running. - try (Lock __ = curator.lock(id, type, prerequisite)) { ; } + try (Mutex __ = curator.lock(id, type, prerequisite)) { ; } action.accept(new LockedStep(lock, step)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java index d06bdc45583..387ea755414 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import java.time.Instant; import java.util.Collection; @@ -119,7 +120,7 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { /** Returns the jobs with successful runs matching the given versions — targets only for system test, everything present otherwise. */ public JobList successOn(Versions versions) { - return matching(job -> ! RunList.from(job).status(RunStatus.success).on(versions).isEmpty()); + return matching(job -> ! RunList.from(job).matching(Run::hasSucceeded).on(versions).isEmpty()); } // ----------------------------------- JobRun filtering @@ -174,8 +175,8 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { } /** Returns the subset of jobs where the run of the indicated type was on the given version */ - public JobList on(ApplicationVersion version) { - return matching(run -> run.versions().targetApplication().equals(version)); + public JobList on(RevisionId revision) { + return matching(run -> run.versions().targetRevision().equals(revision)); } /** Returns the subset of jobs where the run of the indicated type was on the given version */ @@ -196,7 +197,7 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { if (job.isSuccess()) return false; if (job.lastSuccess().isEmpty()) return true; // An application which never succeeded is surely bad. if ( ! job.firstFailing().get().versions().targetPlatform().equals(job.lastSuccess().get().versions().targetPlatform())) return false; // Version change may be to blame. - return ! job.firstFailing().get().versions().targetApplication().equals(job.lastSuccess().get().versions().targetApplication()); // Return whether there is an application change. + return ! job.firstFailing().get().versions().targetRevision().equals(job.lastSuccess().get().versions().targetRevision()); // Return whether there is an application change. } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java index 874b1828f5f..14fce806152 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -21,6 +21,7 @@ public class JobMetrics { public static final String deploymentFailure = "deployment.deploymentFailure"; public static final String convergenceFailure = "deployment.convergenceFailure"; public static final String testFailure = "deployment.testFailure"; + public static final String noTests = "deployment.noTests"; public static final String error = "deployment.error"; public static final String abort = "deployment.abort"; public static final String success = "deployment.success"; @@ -46,7 +47,7 @@ public class JobMetrics { "tenantName", id.application().tenant().value(), "app", id.application().application().value() + "." + id.application().instance().value(), "test", Boolean.toString(id.type().isTest()), - "zone", id.type().zone(system.get()).value()); + "zone", id.type().zone().value()); } static String valueOf(RunStatus status) { @@ -56,6 +57,7 @@ public class JobMetrics { case deploymentFailed: return deploymentFailure; case installationFailed: return convergenceFailure; case testFailure: return testFailure; + case noTests: return noTests; case error: return error; case aborted: return abort; case success: return success; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java index aad5d510261..45bf508f026 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java @@ -60,7 +60,7 @@ public class JobStatus { } public boolean isSuccess() { - return lastStatus().isPresent() && lastStatus().get() == RunStatus.success; + return lastCompleted.map(last -> ! last.hasFailed()).orElse(false); } public boolean isRunning() { @@ -90,18 +90,17 @@ public class JobStatus { static Optional<Run> lastSuccess(NavigableMap<RunId, Run> runs) { return runs.descendingMap().values().stream() - .filter(run -> run.status() == RunStatus.success) + .filter(Run::hasSucceeded) .findFirst(); } static Optional<Run> firstFailing(NavigableMap<RunId, Run> runs) { Run failed = null; - loop: for (Run run : runs.descendingMap().values()) - switch (run.status()) { - case running: continue loop; - case success: break loop; - default: failed = run; - } + for (Run run : runs.descendingMap().values()) { + if ( ! run.hasEnded()) continue; + if ( ! run.hasFailed()) break; + failed = run; + } return Optional.ofNullable(failed); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java index d46516582be..8147ccb3180 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; /** @@ -9,7 +10,7 @@ import com.yahoo.vespa.curator.Lock; public class LockedStep { private final Step step; - LockedStep(Lock lock, Step step) { this.step = step; } + LockedStep(Mutex lock, Step step) { this.step = step; } public Step get() { return step; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java index 6f456d2e217..e0c1fef91b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; @@ -23,41 +24,42 @@ public class RetriggerEntrySerializer { private static final String JOB_TYPE_KEY = "jobType"; private static final String MIN_REQUIRED_RUN_ID_KEY = "minimumRunId"; - public static List<RetriggerEntry> fromSlime(Slime slime) { + public List<RetriggerEntry> fromSlime(Slime slime) { return SlimeUtils.entriesStream(slime.get().field("entries")) - .map(RetriggerEntrySerializer::deserializeEntry) + .map(this::deserializeEntry) .collect(Collectors.toList()); } - public static Slime toSlime(List<RetriggerEntry> entryList) { + public Slime toSlime(List<RetriggerEntry> entryList) { Slime slime = new Slime(); Cursor root = slime.setObject(); Cursor entries = root.setArray("entries"); - entryList.forEach(e -> RetriggerEntrySerializer.serializeEntry(entries, e)); + entryList.forEach(e -> serializeEntry(entries, e)); return slime; } - private static void serializeEntry(Cursor array, RetriggerEntry entry) { + private void serializeEntry(Cursor array, RetriggerEntry entry) { Cursor root = array.addObject(); Cursor jobid = root.setObject(JOB_ID_KEY); jobid.setString(APPLICATION_ID_KEY, entry.jobId().application().serializedForm()); - jobid.setString(JOB_TYPE_KEY, entry.jobId().type().jobName()); + jobid.setString(JOB_TYPE_KEY, entry.jobId().type().serialized()); root.setLong(MIN_REQUIRED_RUN_ID_KEY, entry.requiredRun()); } - private static RetriggerEntry deserializeEntry(Inspector inspector) { + private RetriggerEntry deserializeEntry(Inspector inspector) { Inspector jobid = inspector.field(JOB_ID_KEY); ApplicationId applicationId = ApplicationId.fromSerializedForm(require(jobid, APPLICATION_ID_KEY).asString()); - JobType jobType = JobType.fromJobName(require(jobid, JOB_TYPE_KEY).asString()); + JobType jobType = JobType.ofSerialized(require(jobid, JOB_TYPE_KEY).asString()); long minRequiredRunId = require(inspector, MIN_REQUIRED_RUN_ID_KEY).asLong(); return new RetriggerEntry(new JobId(applicationId, jobType), minRequiredRunId); } - private static Inspector require(Inspector inspector, String fieldName) { + private Inspector require(Inspector inspector, String fieldName) { Inspector field = inspector.field(fieldName); if (!field.valid()) { throw new IllegalStateException("Could not deserialize, field not found in json: " + fieldName); } return field; } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java new file mode 100644 index 00000000000..5bdc980f11a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java @@ -0,0 +1,149 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import ai.vespa.validation.Validation; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; + +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Deque; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.TreeMap; + +import static ai.vespa.validation.Validation.require; +import static java.util.Collections.emptyNavigableMap; +import static java.util.stream.Collectors.toList; + +/** + * History of application revisions for an {@link com.yahoo.vespa.hosted.controller.Application}. + * + * @author jonmv + */ +public class RevisionHistory { + + private static final Comparator<JobId> comparator = Comparator.comparing(JobId::application).thenComparing(JobId::type); + + private final NavigableMap<RevisionId, ApplicationVersion> production; + private final NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development; + + private RevisionHistory(NavigableMap<RevisionId, ApplicationVersion> production, + NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development) { + this.production = production; + this.development = development; + } + + public static RevisionHistory empty() { + return ofRevisions(List.of(), Map.of()); + } + + public static RevisionHistory ofRevisions(Collection<ApplicationVersion> productionRevisions, + Map<JobId, ? extends Collection<ApplicationVersion>> developmentRevisions) { + NavigableMap<RevisionId, ApplicationVersion> production = new TreeMap<>(); + for (ApplicationVersion revision : productionRevisions) + production.put(revision.id(), revision); + + // TODO jonmv: remove once it's run once on serialised data + String hash = ""; + for (ApplicationVersion revision : List.copyOf(production.values())) + if (hash.equals(hash = revision.bundleHash().orElse("")) && ! hash.isEmpty()) + production.put(revision.id(), revision.skipped()); + + NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development = new TreeMap<>(comparator); + developmentRevisions.forEach((job, jobRevisions) -> { + NavigableMap<RevisionId, ApplicationVersion> revisions = development.computeIfAbsent(job, __ -> new TreeMap<>()); + for (ApplicationVersion revision : jobRevisions) + revisions.put(revision.id(), revision); + }); + + return new RevisionHistory(production, development); + } + + /** Returns a copy of this without any production revisions older than the given. */ + public RevisionHistory withoutOlderThan(RevisionId id) { + if (production.headMap(id).isEmpty()) return this; + return new RevisionHistory(production.tailMap(id, true), development); + } + + /** Returns a copy of this without any development revisions older than the given. */ + public RevisionHistory withoutOlderThan(RevisionId id, JobId job) { + if ( ! development.containsKey(job) || development.get(job).headMap(id).isEmpty()) return this; + NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development = new TreeMap<>(this.development); + development.compute(job, (__, revisions) -> revisions.tailMap(id, true)); + return new RevisionHistory(production, development); + } + + /** Returns a copy of this with the revision added or updated. */ + public RevisionHistory with(ApplicationVersion revision) { + if (revision.id().isProduction()) { + if ( ! production.isEmpty() && revision.bundleHash().flatMap(hash -> production.lastEntry().getValue().bundleHash().map(hash::equals)).orElse(false)) + revision = revision.skipped(); + + NavigableMap<RevisionId, ApplicationVersion> production = new TreeMap<>(this.production); + production.put(revision.id(), revision); + return new RevisionHistory(production, development); + } + else { + NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development = new TreeMap<>(this.development); + NavigableMap<RevisionId, ApplicationVersion> revisions = development.compute(revision.id().job(), (__, old) -> new TreeMap<>(old != null ? old : emptyNavigableMap())); + if ( ! revisions.isEmpty()) revisions.compute(revisions.lastKey(), (__, last) -> last.withoutPackage()); + revisions.put(revision.id(), revision); + return new RevisionHistory(production, development); + } + } + + // Fallback for when an application version isn't known for the given key. + private static ApplicationVersion revisionOf(RevisionId id) { + return new ApplicationVersion(id, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), false, false, Optional.empty(), 0); + } + + /** Returns the production {@link ApplicationVersion} with this revision ID. */ + public ApplicationVersion get(RevisionId id) { + return id.isProduction() ? production.getOrDefault(id, revisionOf(id)) + : development.getOrDefault(id.job(), emptyNavigableMap()) + .getOrDefault(id, revisionOf(id)); + } + + /** Returns the last submitted production build. */ + public Optional<ApplicationVersion> last() { + return Optional.ofNullable(production.lastEntry()).map(Map.Entry::getValue); + } + + /** Returns all known production revisions we still have the package for, from oldest to newest. */ + public List<ApplicationVersion> withPackage() { + return production.values().stream() + .filter(ApplicationVersion::hasPackage) + .collect(toList()); + } + + /** Returns the currently deployable revisions of the application. */ + public Deque<ApplicationVersion> deployable(boolean ascending) { + Deque<ApplicationVersion> versions = new ArrayDeque<>(); + for (ApplicationVersion version : withPackage()) { + if (version.isDeployable()) { + if (ascending) versions.addLast(version); + else versions.addFirst(version); + } + } + return versions; + } + + /** All known production revisions, in ascending order. */ + public List<ApplicationVersion> production() { + return List.copyOf(production.values()); + } + + /* All known development revisions, in ascending order, per job. */ + public NavigableMap<JobId, List<ApplicationVersion>> development() { + NavigableMap<JobId, List<ApplicationVersion>> copy = new TreeMap<>(comparator); + development.forEach((job, revisions) -> copy.put(job, List.copyOf(revisions.values()))); + return Collections.unmodifiableNavigableMap(copy); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index e73d3f52e1f..03cc6c6ba8d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -13,6 +13,7 @@ import java.util.Optional; import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; @@ -80,8 +81,9 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(Step.Status.of(status))); - return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, this.status == running ? status : this.status, - lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); + RunStatus newStatus = hasFailed() || status == running ? this.status : status; + return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, newStatus, lastTestRecord, + lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); } /** Returns a new Run with a new start time*/ @@ -210,7 +212,7 @@ public class Run { /** Returns whether the run has failed, and should switch to its run-always steps. */ public boolean hasFailed() { - return status != running && status != success; + return status != running && status != success && status != noTests; } /** Returns whether the run has ended, i.e., has become inactive, and can no longer be updated. */ @@ -218,6 +220,8 @@ public class Run { return end.isPresent(); } + public boolean hasSucceeded() { return hasEnded() && ! hasFailed(); } + /** Returns the target, and possibly source, versions for this run. */ public Versions versions() { return versions; @@ -297,7 +301,7 @@ public class Run { return steps.entrySet().stream() .filter(entry -> entry.getValue().status() == unfinished && entry.getKey().prerequisites().stream() - .allMatch(step -> steps.get(step) == null + .allMatch(step -> steps.get(step) == null || steps.get(step).status() == succeeded)) .map(Map.Entry::getKey) .collect(Collectors.toUnmodifiableList()); @@ -310,7 +314,7 @@ public class Run { && entry.getKey().alwaysRun() && entry.getKey().prerequisites().stream() .filter(Step::alwaysRun) - .allMatch(step -> steps.get(step) == null + .allMatch(step -> steps.get(step) == null || steps.get(step).status() != unfinished)) .map(Map.Entry::getKey) .collect(Collectors.toUnmodifiableList()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java index 00cd4bd5c6c..80c6552d3d4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java @@ -38,7 +38,7 @@ public class RunList extends AbstractFilteringList<Run, RunList> { private static boolean matchingVersions(Run run, Versions versions) { return versions.targetsMatch(run.versions()) - && (versions.sourcesMatchIfPresent(run.versions()) || run.id().type() == JobType.systemTest); + && (versions.sourcesMatchIfPresent(run.versions()) || run.id().type().isSystemTest()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 0bb4a30425e..9ca634b19fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -26,6 +26,9 @@ public enum RunStatus { /** The verification tests failed. */ testFailure, + /** No tests, for a test job. */ + noTests, + /** An unexpected error occurred. */ error, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 2e669808c44..82d154dcf03 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -3,11 +3,9 @@ package com.yahoo.vespa.hosted.controller.deployment; import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toUnmodifiableList; /** * Steps that make up a deployment job. See {@link JobProfile} for preset profiles. @@ -115,6 +113,7 @@ public enum Step { case success : throw new AssertionError("Unexpected run status '" + status + "'!"); case reset : case aborted : return unfinished; + case noTests : case running : return succeeded; default : return failed; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Submission.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Submission.java new file mode 100644 index 00000000000..e366920690b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Submission.java @@ -0,0 +1,57 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; + +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.calculateHash; + +/** + * @author jonmv + */ +public class Submission { + + private final ApplicationPackage applicationPackage; + private final byte[] testPackage; + private final Optional<String> sourceUrl; + private final Optional<SourceRevision> source; + private final Optional<String> authorEmail; + private final Optional<String> description; + private final int risk; + + public Submission(ApplicationPackage applicationPackage, byte[] testPackage, Optional<String> sourceUrl, + Optional<SourceRevision> source, Optional<String> authorEmail, Optional<String> description, int risk) { + this.applicationPackage = applicationPackage; + this.testPackage = testPackage; + this.sourceUrl = sourceUrl; + this.source = source; + this.authorEmail = authorEmail; + this.description = description; + this.risk = risk; + } + + public static Submission basic(ApplicationPackage applicationPackage, byte[] testPackage) { + return new Submission(applicationPackage, testPackage, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), 0); + } + + public ApplicationVersion toApplicationVersion(long number) { + return ApplicationVersion.forProduction(RevisionId.forProduction(number), + source, + authorEmail, + applicationPackage.compileVersion(), + applicationPackage.buildTime(), + sourceUrl, + source.map(SourceRevision::commit), + Optional.of(applicationPackage.bundleHash() + calculateHash(testPackage)), + description, + risk); + } + + public ApplicationPackage applicationPackage() { return applicationPackage; } + + public byte[] testPackage() { return testPackage; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java index c9b488026a5..1680e064234 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; @@ -13,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.application.Endpoint; import java.io.IOException; import java.io.UncheckedIOException; -import java.net.URI; import java.util.List; import java.util.Map; @@ -39,7 +37,7 @@ public class TestConfigSerializer { Cursor root = slime.setObject(); root.setString("application", id.serializedForm()); - root.setString("zone", type.zone(system).value()); + root.setString("zone", type.zone().value()); root.setString("system", system.value()); root.setBool("isCI", isCI); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java index d0b8e773cae..f4c4b8bebd4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -22,24 +23,24 @@ import static java.util.Objects.requireNonNull; public class Versions { private final Version targetPlatform; - private final ApplicationVersion targetApplication; + private final RevisionId targetRevision; private final Optional<Version> sourcePlatform; - private final Optional<ApplicationVersion> sourceApplication; + private final Optional<RevisionId> sourceRevision; - public Versions(Version targetPlatform, ApplicationVersion targetApplication, Optional<Version> sourcePlatform, - Optional<ApplicationVersion> sourceApplication) { - if (sourcePlatform.isPresent() ^ sourceApplication.isPresent()) + public Versions(Version targetPlatform, RevisionId targetRevision, Optional<Version> sourcePlatform, + Optional<RevisionId> sourceRevision) { + if (sourcePlatform.isPresent() ^ sourceRevision.isPresent()) throw new IllegalArgumentException("Sources must both be present or absent."); this.targetPlatform = requireNonNull(targetPlatform); - this.targetApplication = requireNonNull(targetApplication); + this.targetRevision = requireNonNull(targetRevision); this.sourcePlatform = requireNonNull(sourcePlatform); - this.sourceApplication = requireNonNull(sourceApplication); + this.sourceRevision = requireNonNull(sourceRevision); } /** A copy of this, without source versions. */ public Versions withoutSources() { - return new Versions(targetPlatform, targetApplication, Optional.empty(), Optional.empty()); + return new Versions(targetPlatform, targetRevision, Optional.empty(), Optional.empty()); } /** Target platform version for this */ @@ -47,9 +48,9 @@ public class Versions { return targetPlatform; } - /** Target application version for this */ - public ApplicationVersion targetApplication() { - return targetApplication; + /** Target revision for this */ + public RevisionId targetRevision() { + return targetRevision; } /** Source platform version for this */ @@ -58,27 +59,27 @@ public class Versions { } /** Source application version for this */ - public Optional<ApplicationVersion> sourceApplication() { - return sourceApplication; + public Optional<RevisionId> sourceRevision() { + return sourceRevision; } /** Returns whether source versions are present and match those of the given job other versions. */ public boolean sourcesMatchIfPresent(Versions versions) { return (sourcePlatform.map(targetPlatform::equals).orElse(true) || sourcePlatform.equals(versions.sourcePlatform())) && - (sourceApplication.map(targetApplication::equals).orElse(true) || - sourceApplication.equals(versions.sourceApplication())); + (sourceRevision.map(targetRevision::equals).orElse(true) || + sourceRevision.equals(versions.sourceRevision())); } public boolean targetsMatch(Versions versions) { return targetPlatform.equals(versions.targetPlatform()) && - targetApplication.equals(versions.targetApplication()); + targetRevision.equals(versions.targetRevision()); } /** Returns wheter this change could result in the given target versions. */ public boolean targetsMatch(Change change) { return change.platform().map(targetPlatform::equals).orElse(true) - && change.application().map(targetApplication::equals).orElse(true); + && change.revision().map(targetRevision::equals).orElse(true); } @Override @@ -87,43 +88,43 @@ public class Versions { if ( ! (o instanceof Versions)) return false; Versions versions = (Versions) o; return Objects.equals(targetPlatform, versions.targetPlatform) && - Objects.equals(targetApplication, versions.targetApplication) && + Objects.equals(targetRevision, versions.targetRevision) && Objects.equals(sourcePlatform, versions.sourcePlatform) && - Objects.equals(sourceApplication, versions.sourceApplication); + Objects.equals(sourceRevision, versions.sourceRevision); } @Override public int hashCode() { - return Objects.hash(targetPlatform, targetApplication, sourcePlatform, sourceApplication); + return Objects.hash(targetPlatform, targetRevision, sourcePlatform, sourceRevision); } @Override public String toString() { - return Text.format("platform %s%s, application %s%s", - sourcePlatform.filter(source -> !source.equals(targetPlatform)) + return Text.format("platform %s%s, revision %s%s", + sourcePlatform.filter(source -> ! source.equals(targetPlatform)) .map(source -> source + " -> ").orElse(""), targetPlatform, - sourceApplication.filter(source -> !source.equals(targetApplication)) - .map(source -> source.id() + " -> ").orElse(""), - targetApplication.id()); + sourceRevision.filter(source -> ! source.equals(targetRevision)) + .map(source -> source + " -> ").orElse(""), + targetRevision); } /** Create versions using given change and application */ public static Versions from(Change change, Application application, Optional<Version> existingPlatform, - Optional<ApplicationVersion> existingApplication, Version defaultPlatformVersion) { + Optional<RevisionId> existingRevision, Version defaultPlatformVersion) { return new Versions(targetPlatform(application, change, existingPlatform, defaultPlatformVersion), - targetApplication(application, change, existingApplication), + targetRevision(application, change, existingRevision), existingPlatform, - existingApplication); + existingRevision); } /** Create versions using given change and application */ public static Versions from(Change change, Application application, Optional<Deployment> deployment, Version defaultPlatformVersion) { return new Versions(targetPlatform(application, change, deployment.map(Deployment::version), defaultPlatformVersion), - targetApplication(application, change, deployment.map(Deployment::applicationVersion)), + targetRevision(application, change, deployment.map(Deployment::revision)), deployment.map(Deployment::version), - deployment.map(Deployment::applicationVersion)); + deployment.map(Deployment::revision)); } private static Version targetPlatform(Application application, Change change, Optional<Version> existing, @@ -135,17 +136,17 @@ public class Versions { .orElseGet(() -> application.oldestDeployedPlatform().orElse(defaultVersion)); } - private static ApplicationVersion targetApplication(Application application, Change change, - Optional<ApplicationVersion> existing) { - return change.application() + private static RevisionId targetRevision(Application application, Change change, + Optional<RevisionId> existing) { + return change.revision() .or(() -> existing) - .orElseGet(() -> defaultApplicationVersion(application)); + .orElseGet(() -> defaultRevision(application)); } - private static ApplicationVersion defaultApplicationVersion(Application application) { - return application.oldestDeployedApplication() - .or(application::latestVersion) - .orElse(ApplicationVersion.unknown); + private static RevisionId defaultRevision(Application application) { + return application.oldestDeployedRevision() + .or(() -> application.revisions().last().map(ApplicationVersion::id)) + .orElseThrow(() -> new IllegalStateException("no known prod revisions, but asked for one, for " + application)); } private static <T extends Comparable<T>> Optional<T> max(Optional<T> o1, Optional<T> o2) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index aecb1e7a2c1..540e8489e6d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; @@ -79,7 +80,7 @@ public class NameServiceForwarder { } protected void forward(NameServiceRequest request, NameServiceQueue.Priority priority) { - try (Lock lock = db.lockNameServiceQueue()) { + try (Mutex lock = db.lockNameServiceQueue()) { NameServiceQueue queue = db.readNameServiceQueue(); var queued = queue.requests().size(); if (queued >= QUEUE_CAPACITY) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 0e880bb627c..02e1818932e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -103,7 +102,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { } } return new ApplicationSummary(app.id().defaultInstance(), app.activity().lastQueried(), app.activity().lastWritten(), - app.latestVersion().flatMap(version -> version.buildTime()), metrics); + app.revisions().last().flatMap(version -> version.buildTime()), metrics); } /** Escalate ownership issues which have not been closed before a defined amount of time has passed. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java index 7ede040773e..8765884e23c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java @@ -7,12 +7,10 @@ import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; -import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; -import java.time.Instant; import java.util.List; import java.util.Optional; import java.util.function.Consumer; @@ -49,8 +47,7 @@ public class CloudTrialExpirer extends ControllerMaintainer { private void moveInactiveTenantsToNonePlan() { var predicate = tenantReadersNotLoggedIn(loginExpiry) - .and(this::tenantHasTrialPlan) - .and(this::tenantHasNoDeployments); + .and(this::tenantHasTrialPlan); forTenant("'none' plan", predicate, this::setPlanNone); } @@ -63,11 +60,12 @@ public class CloudTrialExpirer extends ControllerMaintainer { } private void forTenant(String name, Predicate<Tenant> p, Consumer<List<Tenant>> c) { - var predicate = ((Predicate<Tenant>) this::tenantIsCloudTenant) - .and(this::tenantIsNotExemptFromExpiry); + var predicate = p.and(this::tenantIsCloudTenant) + .and(this::tenantIsNotExemptFromExpiry) + .and(this::tenantHasNoDeployments); var tenants = controller().tenants().asList().stream() - .filter(predicate.and(p)) + .filter(predicate) .collect(Collectors.toList()); if (! tenants.isEmpty()) { @@ -121,7 +119,15 @@ public class CloudTrialExpirer extends ControllerMaintainer { private void tombstoneTenants(List<Tenant> tenants) { tenants.forEach(tenant -> { + deleteApplicationsWithNoDeployments(tenant); controller().tenants().delete(tenant.name(), Optional.empty(), false); }); } + + private void deleteApplicationsWithNoDeployments(Tenant tenant) { + controller().applications().asList(tenant.name()).forEach(application -> { + // this only removes applications with no active deployments + controller().applications().deleteApplication(application.id(), Optional.empty()); + }); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index deafcd35e9b..041d0694ca9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; import java.time.Duration; @@ -36,7 +37,7 @@ public class ControllerMaintenance extends AbstractComponent { @Inject @SuppressWarnings("unused") // instantiated by Dependency Injection - public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement) { + public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement, AthenzClientFactory athenzClientFactory) { Intervals intervals = new Intervals(controller.system()); upgrader = new Upgrader(controller, intervals.defaultInterval); maintainers.add(upgrader); @@ -44,7 +45,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new DeploymentExpirer(controller, intervals.defaultInterval)); maintainers.add(new DeploymentUpgrader(controller, intervals.defaultInterval)); maintainers.add(new DeploymentIssueReporter(controller, controller.serviceRegistry().deploymentIssues(), intervals.defaultInterval)); - maintainers.add(new MetricsReporter(controller, metric)); + maintainers.add(new MetricsReporter(controller, metric, athenzClientFactory.createZmsClient())); maintainers.add(new OutstandingChangeDeployer(controller, intervals.outstandingChangeDeployer)); maintainers.add(new VersionStatusUpdater(controller, intervals.versionStatusUpdater)); maintainers.add(new ReadyJobsTrigger(controller, intervals.readyJobsTrigger)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 9a8ba9afca2..97f3f955a20 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -8,11 +8,9 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.time.Instant; import java.util.Optional; import java.util.logging.Level; @@ -59,13 +57,8 @@ public class DeploymentExpirer extends ControllerMaintainer { Optional<Duration> ttl = controller().zoneRegistry().getDeploymentTimeToLive(deployment.zone()); if (ttl.isEmpty()) return false; - Optional<JobId> jobId = JobType.from(controller().system(), deployment.zone()) - .map(type -> new JobId(instance, type)); - if (jobId.isEmpty()) return false; - - return controller().jobController().jobStarts(jobId.get()).stream().findFirst() - .map(start -> start.plus(ttl.get()).isBefore(controller().clock().instant())) - .orElse(false); + return controller().jobController().lastDeploymentStart(instance, deployment) + .plus(ttl.get()).isBefore(controller().clock().instant()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 3a047d33be5..6b058537c2d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -21,7 +21,6 @@ import java.time.Duration; import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index a2abd6493cb..c86f79ce188 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -19,8 +19,6 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; -import static com.yahoo.collections.Iterables.reversed; - /** * Upgrades instances in manually deployed zones to the system version, at a convenient time. * @@ -49,12 +47,12 @@ public class DeploymentUpgrader extends ControllerMaintainer { for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) try { - JobId job = new JobId(instance.id(), JobType.from(controller().system(), deployment.zone()).get()); + JobId job = new JobId(instance.id(), JobType.deploymentTo(deployment.zone())); if ( ! deployment.zone().environment().isManuallyDeployed()) continue; Run last = controller().jobController().last(job).get(); - Versions target = new Versions(targetPlatform, last.versions().targetApplication(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetApplication())); - if (last.versions().targetApplication().compileVersion() + Versions target = new Versions(targetPlatform, last.versions().targetRevision(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetRevision())); + if (application.revisions().get(last.versions().targetRevision()).compileVersion() .map(version -> controller().applications().versionCompatibility(instance.id()).refuse(version, target.targetPlatform())) .orElse(false)) continue; if ( ! deployment.version().isBefore(target.targetPlatform())) continue; @@ -62,7 +60,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); attempts.incrementAndGet(); - controller().jobController().start(instance.id(), JobType.from(controller().system(), deployment.zone()).get(), target, true, Optional.of("automated upgrade")); + controller().jobController().start(instance.id(), JobType.deploymentTo(deployment.zone()), target, true, Optional.of("automated upgrade")); } catch (Exception e) { failures.incrementAndGet(); log.log(Level.WARNING, "Failed upgrading " + deployment + " of " + instance + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 15f8d6380c0..193fb89eb99 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -6,6 +6,7 @@ import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -82,7 +83,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { var refreshedCertificateMetadata = endpointCertificateMetadata .withVersion(latestAvailableVersion.getAsInt()) .withLastRefreshed(clock.instant().getEpochSecond()); - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { if (Optional.of(endpointCertificateMetadata).equals(curator.readEndpointCertificateMetadata(applicationId))) { curator.writeEndpointCertificateMetadata(applicationId, refreshedCertificateMetadata); // Certificate not validated here, but on deploy. } @@ -103,7 +104,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { controller().applications().getInstance(applicationId) .ifPresent(instance -> instance.productionDeployments().forEach((zone, deployment) -> { if (deployment.at().isBefore(refreshTime)) { - JobType job = JobType.from(controller().system(), zone).orElseThrow(); + JobType job = JobType.deploymentTo(zone); deploymentTrigger.reTrigger(applicationId, job, "re-triggered by EndpointCertificateMaintainer"); log.info("Re-triggering deployment job " + job.jobName() + " for instance " + applicationId.serializedForm() + " to roll out refreshed endpoint certificate"); @@ -128,7 +129,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { curator.readAllEndpointCertificateMetadata().forEach((applicationId, storedMetaData) -> { var lastRequested = Instant.ofEpochSecond(storedMetaData.lastRequested()); if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(applicationId)) { - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { if (Optional.of(storedMetaData).equals(curator.readEndpointCertificateMetadata(applicationId))) { log.log(Level.INFO, "Cert for app " + applicationId.serializedForm() + " has not been requested in a month and app has no deployments, deleting from provider and ZK"); @@ -140,7 +141,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { }); } - private Lock lock(ApplicationId applicationId) { + private Mutex lock(ApplicationId applicationId) { return curator.lock(TenantAndApplicationId.from(applicationId)); } @@ -169,7 +170,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { EndpointCertificateMetadata storedAppMetadata = storedAppEntry.getValue(); if (storedAppMetadata.certName().equals(unknownCertDetails.cert_key_keyname())) { matchFound = true; - try (Lock lock = lock(storedApp)) { + try (Mutex lock = lock(storedApp)) { if (Optional.of(storedAppMetadata).equals(curator.readEndpointCertificateMetadata(storedApp))) { log.log(Level.INFO, "Cert for app " + storedApp.serializedForm() + " has a new leafRequestId " + unknownCertDetails.request_id() + ", updating in ZK"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 6aa43d4db47..294d5bad42d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -62,17 +63,20 @@ public class MetricsReporter extends ControllerMaintainer { public static final String REMAINING_ROTATIONS = "remaining_rotations"; public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; public static final String OPERATION_PREFIX = "operation."; + public static final String ZMS_QUOTA_USAGE = "zms.quota.usage"; private final Metric metric; private final Clock clock; + private final ZmsClient zmsClient; // Keep track of reported node counts for each version private final ConcurrentHashMap<NodeCountKey, Long> nodeCounts = new ConcurrentHashMap<>(); - public MetricsReporter(Controller controller, Metric metric) { + public MetricsReporter(Controller controller, Metric metric, ZmsClient zmsClient) { super(controller, Duration.ofMinutes(1)); // use fixed rate for metrics this.metric = metric; this.clock = controller.clock(); + this.zmsClient = zmsClient; } @Override @@ -85,6 +89,7 @@ public class MetricsReporter extends ControllerMaintainer { reportAuditLog(); reportBrokenSystemVersion(versionStatus); reportTenantMetrics(); + reportZmsQuotaMetrics(); return 1.0; } @@ -167,7 +172,7 @@ public class MetricsReporter extends ControllerMaintainer { }); for (Application application : applications.asList()) - application.latestVersion() + application.revisions().last() .flatMap(ApplicationVersion::buildTime) .ifPresent(buildTime -> metric.set(DEPLOYMENT_BUILD_AGE_SECONDS, controller().clock().instant().getEpochSecond() - buildTime.getEpochSecond(), @@ -252,6 +257,20 @@ public class MetricsReporter extends ControllerMaintainer { }); } + private void reportZmsQuotaMetrics() { + var quota = zmsClient.getQuotaUsage(); + reportZmsQuota("subdomains", quota.getSubdomainUsage()); + reportZmsQuota("services", quota.getServiceUsage()); + reportZmsQuota("policies", quota.getPolicyUsage()); + reportZmsQuota("roles", quota.getRoleUsage()); + reportZmsQuota("groups", quota.getGroupUsage()); + } + + private void reportZmsQuota(String resourceType, double usage) { + var context = metric.createContext(Map.of("resourceType", resourceType)); + metric.set(ZMS_QUOTA_USAGE, usage, context); + } + private Map<NodeVersion, Duration> platformChangeDurations(VersionStatus versionStatus) { return changeDurations(versionStatus.versions(), VespaVersion::nodeVersions); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index e412a5cc9f7..4ed34a91029 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; @@ -66,7 +67,9 @@ public class Upgrader extends ControllerMaintainer { /** Returns a list of all production application instances, except those which are pinned, which we should not manipulate here. */ private InstanceList instances(Version systemVersion) { - return InstanceList.from(controller().jobController().deploymentStatuses(ApplicationList.from(controller().applications().readable()), systemVersion)) + return InstanceList.from(controller().jobController().deploymentStatuses(ApplicationList.from(controller().applications().readable()) + .withProjectId(), + systemVersion)) .withDeclaredJobs() .shuffle(random) .byIncreasingDeployedVersion() @@ -174,7 +177,7 @@ public class Upgrader extends ControllerMaintainer { " for version " + version.toFullString() + ": Version may be in use by applications"); } - try (Lock lock = curator.lockConfidenceOverrides()) { + try (Mutex lock = curator.lockConfidenceOverrides()) { Map<Version, Confidence> overrides = new LinkedHashMap<>(curator.readConfidenceOverrides()); overrides.put(version, confidence); curator.writeConfidenceOverrides(overrides); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java index 05a7e2368d1..163a570768f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index f7e085bd90f..d0b3b9f4c7f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -106,6 +106,10 @@ public class VcmrMaintainer extends ControllerMaintainer { return Status.REQUIRES_OPERATOR_ACTION; } + if (byActionState.getOrDefault(State.OUT_OF_SYNC, 0L) > 0) { + return Status.OUT_OF_SYNC; + } + if (byActionState.getOrDefault(State.RETIRING, 0L) > 0) { return Status.IN_PROGRESS; } @@ -170,6 +174,9 @@ public class VcmrMaintainer extends ControllerMaintainer { addReport(zoneId, changeRequest, node); + if (isOutOfSync(node, hostAction)) + return hostAction.withState(State.OUT_OF_SYNC); + if (isPostponed(changeRequest, hostAction)) { LOG.fine(() -> changeRequest.getChangeRequestSource().getId() + " is postponed, recycling " + node.hostname()); recycleNode(zoneId, node, hostAction); @@ -239,7 +246,7 @@ public class VcmrMaintainer extends ControllerMaintainer { } private boolean hasRetired(Node node, HostAction hostAction) { - return hostAction.getState() == State.RETIRING && + return List.of(State.RETIRING, State.REQUIRES_OPERATOR_ACTION).contains(hostAction.getState()) && node.state() == Node.State.parked; } @@ -248,6 +255,12 @@ public class VcmrMaintainer extends ControllerMaintainer { && node.state() == Node.State.active; } + // Determines if node state is unexpected based on previous action taken + private boolean isOutOfSync(Node node, HostAction action) { + return action.getState() == State.RETIRED && node.state() != Node.State.parked || + action.getState() == State.RETIRING && !node.wantToRetire(); + } + private Map<ZoneId, List<Node>> nodesByZone() { return controller().zoneRegistry() .zones() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index 6597d59027c..71b8f1cd9b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -10,11 +10,11 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.util.logging.Level; +import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.aborted; import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.broken; import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.high; import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.low; import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.normal; -import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.aborted; /** * This maintenance job periodically updates the version status. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java index c39ee031e27..8a363405c41 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java @@ -66,10 +66,16 @@ public class Notification { } public enum Type { - /** Related to contents of application package, e.g. usage of deprecated features/syntax */ + /** Related to contents of application package, e.g., usage of deprecated features/syntax */ applicationPackage, - /** Related to deployment of application, e.g. system test failure, node allocation failure, internal errors, etc. */ + /** Related to contents of application package, e.g., old parent or compile version, or errors detectable on submission */ + submission, + + /** Related to contents of application test package, e.g., mismatch between deployment spec and provided tests */ + testPackage, + + /** Related to deployment of application, e.g., system test failure, node allocation failure, internal errors, etc. */ deployment, /** Application cluster is (near) external feed blocked */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java index aa62028749b..7876099cb21 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java @@ -5,6 +5,7 @@ import com.yahoo.collections.Pair; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Text; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; @@ -65,7 +66,7 @@ public class NotificationsDb { */ public void setNotification(NotificationSource source, Type type, Level level, List<String> messages) { Optional<Notification> changed = Optional.empty(); - try (Lock lock = curatorDb.lockNotifications(source.tenant())) { + try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { var existingNotifications = curatorDb.readNotifications(source.tenant()); List<Notification> notifications = existingNotifications.stream() .filter(notification -> !source.equals(notification.source()) || type != notification.type()) @@ -82,7 +83,7 @@ public class NotificationsDb { /** Remove the notification with the given source and type */ public void removeNotification(NotificationSource source, Type type) { - try (Lock lock = curatorDb.lockNotifications(source.tenant())) { + try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { List<Notification> initial = curatorDb.readNotifications(source.tenant()); List<Notification> filtered = initial.stream() .filter(notification -> !source.equals(notification.source()) || type != notification.type()) @@ -94,7 +95,7 @@ public class NotificationsDb { /** Remove all notifications for this source or sources contained by this source */ public void removeNotifications(NotificationSource source) { - try (Lock lock = curatorDb.lockNotifications(source.tenant())) { + try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { if (source.application().isEmpty()) { // Source is tenant curatorDb.deleteNotifications(source.tenant()); return; @@ -130,7 +131,7 @@ public class NotificationsDb { .collect(Collectors.toUnmodifiableList()); NotificationSource deploymentSource = NotificationSource.from(deploymentId); - try (Lock lock = curatorDb.lockNotifications(deploymentSource.tenant())) { + try (Mutex lock = curatorDb.lockNotifications(deploymentSource.tenant())) { List<Notification> initial = curatorDb.readNotifications(deploymentSource.tenant()); List<Notification> updated = Stream.concat( initial.stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java index 594908a3bc1..6b14872b07d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java @@ -1,16 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notify; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; import com.yahoo.text.Text; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; +import java.net.URI; import java.util.Collection; import java.util.List; import java.util.Objects; @@ -25,12 +30,14 @@ import java.util.stream.Collectors; */ public class Notifier { private final CuratorDb curatorDb; + private final ZoneRegistry zoneRegistry; private final Mailer mailer; private static final Logger log = Logger.getLogger(Notifier.class.getName()); - public Notifier(CuratorDb curatorDb, Mailer mailer) { + public Notifier(CuratorDb curatorDb, ZoneRegistry zoneRegistry, Mailer mailer) { this.curatorDb = Objects.requireNonNull(curatorDb); + this.zoneRegistry = Objects.requireNonNull(zoneRegistry); this.mailer = Objects.requireNonNull(mailer); } @@ -56,7 +63,12 @@ public class Notifier { private boolean skipSource(NotificationSource source) { // Limit sources to production systems only. Dev and test systems cause too much noise at the moment. - return source.jobType().map(t -> !t.isProduction()).orElse(false); + if (source.zoneId().map(z -> z.environment() != Environment.prod).orElse(false)) { + return true; + } else if (source.jobType().map(t -> !t.isProduction()).orElse(false)) { + return true; + } + return false; } public void dispatch(Notification notification) { @@ -82,12 +94,41 @@ public class Notifier { } private Mail mailOf(Notification n, Collection<String> recipients) { - var subject = Text.format("[%s] Vespa Notification for %s", n.level().toString().toUpperCase(), n.type().name()); + var source = n.source(); + var subject = Text.format("[%s] %s Vespa Notification for %s", n.level().toString().toUpperCase(), n.type().name(), applicationIdSource(source)); var body = new StringBuilder(); body.append("Source: ").append(n.source().toString()).append("\n") .append("\n") - .append(String.join("\n", n.messages())); - return new Mail(recipients, subject.toString(), body.toString()); + .append(String.join("\n", n.messages())) + .append("\n") + .append(url(source).toString()); + return new Mail(recipients, subject, body.toString()); + } + + private String applicationIdSource(NotificationSource source) { + StringBuilder sb = new StringBuilder(); + sb.append(source.tenant().value()); + source.application().ifPresent(applicationName -> sb.append(".").append(applicationName.value())); + source.instance().ifPresent(instanceName -> sb.append(".").append(instanceName.value())); + return sb.toString(); + } + + private URI url(NotificationSource source) { + if (source.application().isPresent()) { + if (source.instance().isPresent()) { + if (source.jobType().isPresent() && source.runNumber().isPresent()) { + return zoneRegistry.dashboardUrl( + new RunId(ApplicationId.from(source.tenant(), + source.application().get(), + source.instance().get()), + source.jobType().get(), + source.runNumber().getAsLong())); + } + return zoneRegistry.dashboardUrl(ApplicationId.from(source.tenant(), source.application().get(), source.instance().get())); + } + return zoneRegistry.dashboardUrl(source.tenant(), source.application().get()); + } + return zoneRegistry.dashboardUrl(source.tenant()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index dee3c822465..48d8627f407 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -4,9 +4,11 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyUtils; import com.yahoo.slime.ArrayTraverser; @@ -18,7 +20,9 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; @@ -30,6 +34,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; @@ -49,8 +54,6 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; /** * Serializes {@link Application}s to/from slime. @@ -76,8 +79,9 @@ public class ApplicationSerializer { private static final String instancesField = "instances"; private static final String deployingField = "deployingField"; private static final String projectIdField = "projectId"; - private static final String latestVersionField = "latestVersion"; private static final String versionsField = "versions"; + private static final String prodVersionsField = "prodVersions"; + private static final String devVersionsField = "devVersions"; private static final String pinnedField = "pinned"; private static final String deploymentIssueField = "deploymentIssueId"; private static final String ownershipIssueIdField = "ownershipIssueId"; @@ -97,7 +101,6 @@ public class ApplicationSerializer { private static final String deploymentJobsField = "deploymentJobs"; // TODO jonmv: clean up serialisation format private static final String assignedRotationsField = "assignedRotations"; private static final String assignedRotationEndpointField = "endpointId"; - private static final String latestDeployedField = "latestDeployed"; // Deployment fields private static final String zoneField = "zone"; @@ -110,8 +113,12 @@ public class ApplicationSerializer { private static final String repositoryField = "repositoryField"; private static final String branchField = "branchField"; private static final String commitField = "commitField"; + private static final String descriptionField = "description"; + private static final String riskField = "risk"; private static final String authorEmailField = "authorEmailField"; private static final String deployedDirectlyField = "deployedDirectly"; + private static final String hasPackageField = "hasPackage"; + private static final String shouldSkipField = "shouldSkip"; private static final String compileVersionField = "compileVersion"; private static final String buildTimeField = "buildTime"; private static final String sourceUrlField = "sourceUrl"; @@ -167,8 +174,7 @@ public class ApplicationSerializer { root.setDouble(queryQualityField, application.metrics().queryServiceQuality()); root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); deployKeysToSlime(application.deployKeys(), root.setArray(pemDeployKeysField)); - application.latestVersion().ifPresent(version -> toSlime(version, root.setObject(latestVersionField))); - versionsToSlime(application, root.setArray(versionsField)); + revisionsToSlime(application.revisions(), root.setArray(prodVersionsField), root.setArray(devVersionsField)); instancesToSlime(application, root.setArray(instancesField)); return slime; } @@ -182,7 +188,6 @@ public class ApplicationSerializer { assignedRotationsToSlime(instance.rotations(), instanceObject); toSlime(instance.rotationStatus(), instanceObject.setArray(rotationStatusField)); toSlime(instance.change(), instanceObject, deployingField); - instance.latestDeployed().ifPresent(version -> toSlime(version, instanceObject.setObject(latestDeployedField))); } } @@ -199,7 +204,7 @@ public class ApplicationSerializer { zoneIdToSlime(deployment.zone(), object.setObject(zoneField)); object.setString(versionField, deployment.version().toString()); object.setLong(deployTimeField, deployment.at().toEpochMilli()); - toSlime(deployment.applicationVersion(), object.setObject(applicationPackageRevisionField)); + toSlime(deployment.revision(), object.setObject(applicationPackageRevisionField)); deploymentMetricsToSlime(deployment.metrics(), object); deployment.activity().lastQueried().ifPresent(instant -> object.setLong(lastQueriedField, instant.toEpochMilli())); deployment.activity().lastWritten().ifPresent(instant -> object.setLong(lastWrittenField, instant.toEpochMilli())); @@ -228,8 +233,23 @@ public class ApplicationSerializer { object.setString(regionField, zone.region().value()); } - private void versionsToSlime(Application application, Cursor object) { - application.versions().forEach(version -> toSlime(version, object.addObject())); + private void revisionsToSlime(RevisionHistory revisions, Cursor revisionsArray, Cursor devRevisionsArray) { + revisionsToSlime(revisions.production(), revisionsArray); + revisions.development().forEach((job, devRevisions) -> { + Cursor devRevisionsObject = devRevisionsArray.addObject(); + devRevisionsObject.setString(instanceNameField, job.application().instance().value()); + devRevisionsObject.setString(jobTypeField, job.type().serialized()); + revisionsToSlime(devRevisions, devRevisionsObject.setArray(versionsField)); + }); + } + + private void revisionsToSlime(Iterable<ApplicationVersion> revisions, Cursor revisionsArray) { + revisions.forEach(version -> toSlime(version, revisionsArray.addObject())); + } + + private void toSlime(RevisionId revision, Cursor object) { + object.setLong(applicationBuildNumberField, revision.number()); + object.setBool(deployedDirectlyField, ! revision.isProduction()); } private void toSlime(ApplicationVersion applicationVersion, Cursor object) { @@ -241,6 +261,10 @@ public class ApplicationSerializer { applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url)); applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit)); object.setBool(deployedDirectlyField, applicationVersion.isDeployedDirectly()); + object.setBool(hasPackageField, applicationVersion.hasPackage()); + object.setBool(shouldSkipField, applicationVersion.shouldSkip()); + applicationVersion.description().ifPresent(description -> object.setString(descriptionField, description)); + if (applicationVersion.risk() != 0) object.setLong(riskField, applicationVersion.risk()); applicationVersion.bundleHash().ifPresent(bundleHash -> object.setString(bundleHashField, bundleHash)); } @@ -254,7 +278,7 @@ public class ApplicationSerializer { Cursor jobStatusArray = cursor.setArray(jobStatusField); jobPauses.forEach((type, until) -> { Cursor jobPauseObject = jobStatusArray.addObject(); - jobPauseObject.setString(jobTypeField, type.jobName()); + jobPauseObject.setString(jobTypeField, type.serialized()); jobPauseObject.setLong(pausedUntilField, until.toEpochMilli()); }); } @@ -265,8 +289,8 @@ public class ApplicationSerializer { Cursor object = parentObject.setObject(fieldName); if (deploying.platform().isPresent()) object.setString(versionField, deploying.platform().get().toString()); - if (deploying.application().isPresent()) - toSlime(deploying.application().get(), object); + if (deploying.revision().isPresent()) + toSlime(deploying.revision().get(), object); if (deploying.isPinned()) object.setBool(pinnedField, true); } @@ -321,42 +345,50 @@ public class ApplicationSerializer { Set<PublicKey> deployKeys = deployKeysFromSlime(root.field(pemDeployKeysField)); List<Instance> instances = instancesFromSlime(id, root.field(instancesField)); OptionalLong projectId = SlimeUtils.optionalLong(root.field(projectIdField)); - Optional<ApplicationVersion> latestVersion = latestVersionFromSlime(root.field(latestVersionField)); - SortedSet<ApplicationVersion> versions = versionsFromSlime(root.field(versionsField)); + RevisionHistory revisions = revisionsFromSlime(root.field(prodVersionsField), root.field(devVersionsField), id); return new Application(id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, - deployKeys, projectId, latestVersion, versions, instances); + deployKeys, projectId, revisions, instances); } - private Optional<ApplicationVersion> latestVersionFromSlime(Inspector latestVersionObject) { - return Optional.of(applicationVersionFromSlime(latestVersionObject)) - .filter(version -> ! version.isUnknown()); + private RevisionHistory revisionsFromSlime(Inspector prodVersionsArray, Inspector devVersionsArray, TenantAndApplicationId id) { + List<ApplicationVersion> revisions = revisionsFromSlime(prodVersionsArray, null); + Map<JobId, List<ApplicationVersion>> devRevisions = new HashMap<>(); + devVersionsArray.traverse((ArrayTraverser) (__, devRevisionsObject) -> { + JobId job = jobIdFromSlime(id, devRevisionsObject); + devRevisions.put(job, revisionsFromSlime(devRevisionsObject.field(versionsField), job)); + }); + + return RevisionHistory.ofRevisions(revisions, devRevisions); } - private SortedSet<ApplicationVersion> versionsFromSlime(Inspector versionsObject) { - SortedSet<ApplicationVersion> versions = new TreeSet<>(); - versionsObject.traverse((ArrayTraverser) (name, object) -> versions.add(applicationVersionFromSlime(object))); - return versions; + private JobId jobIdFromSlime(TenantAndApplicationId base, Inspector idObject) { + return new JobId(base.instance(idObject.field(instanceNameField).asString()), + JobType.ofSerialized(idObject.field(jobTypeField).asString())); + } + + private List<ApplicationVersion> revisionsFromSlime(Inspector versionsArray, JobId job) { + List<ApplicationVersion> revisions = new ArrayList<>(); + versionsArray.traverse((ArrayTraverser) (__, revisionObject) -> revisions.add(applicationVersionFromSlime(revisionObject, job))); + return revisions; } private List<Instance> instancesFromSlime(TenantAndApplicationId id, Inspector field) { List<Instance> instances = new ArrayList<>(); field.traverse((ArrayTraverser) (name, object) -> { InstanceName instanceName = InstanceName.from(object.field(instanceNameField).asString()); - List<Deployment> deployments = deploymentsFromSlime(object.field(deploymentsField)); + List<Deployment> deployments = deploymentsFromSlime(object.field(deploymentsField), id.instance(instanceName)); Map<JobType, Instant> jobPauses = jobPausesFromSlime(object.field(deploymentJobsField)); List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(object); RotationStatus rotationStatus = rotationStatusFromSlime(object); Change change = changeFromSlime(object.field(deployingField)); - Optional<ApplicationVersion> latestDeployed = latestVersionFromSlime(object.field(latestDeployedField)); instances.add(new Instance(id.instance(instanceName), deployments, jobPauses, assignedRotations, rotationStatus, - change, - latestDeployed)); + change)); }); return instances; } @@ -367,15 +399,16 @@ public class ApplicationSerializer { return keys; } - private List<Deployment> deploymentsFromSlime(Inspector array) { + private List<Deployment> deploymentsFromSlime(Inspector array, ApplicationId id) { List<Deployment> deployments = new ArrayList<>(); - array.traverse((ArrayTraverser) (int i, Inspector item) -> deployments.add(deploymentFromSlime(item))); + array.traverse((ArrayTraverser) (int i, Inspector item) -> deployments.add(deploymentFromSlime(item, id))); return deployments; } - private Deployment deploymentFromSlime(Inspector deploymentObject) { - return new Deployment(zoneIdFromSlime(deploymentObject.field(zoneField)), - applicationVersionFromSlime(deploymentObject.field(applicationPackageRevisionField)), + private Deployment deploymentFromSlime(Inspector deploymentObject, ApplicationId id) { + ZoneId zone = zoneIdFromSlime(deploymentObject.field(zoneField)); + return new Deployment(zone, + revisionFromSlime(deploymentObject.field(applicationPackageRevisionField), new JobId(id, JobType.deploymentTo(zone))), Version.fromString(deploymentObject.field(versionField).asString()), SlimeUtils.instant(deploymentObject.field(deployTimeField)), deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)), @@ -432,22 +465,30 @@ public class ApplicationSerializer { return ZoneId.from(object.field(environmentField).asString(), object.field(regionField).asString()); } - private ApplicationVersion applicationVersionFromSlime(Inspector object) { - if ( ! object.valid()) return ApplicationVersion.unknown; - OptionalLong applicationBuildNumber = SlimeUtils.optionalLong(object.field(applicationBuildNumberField)); - if (applicationBuildNumber.isEmpty()) - return ApplicationVersion.unknown; + private RevisionId revisionFromSlime(Inspector object, JobId job) { + long build = object.field(applicationBuildNumberField).asLong(); + boolean production = object.field(deployedDirectlyField).valid() // TODO jonmv: remove after migration + && build > 0 + && ! object.field(deployedDirectlyField).asBool(); + return production ? RevisionId.forProduction(build) : RevisionId.forDevelopment(build, job); + } + private ApplicationVersion applicationVersionFromSlime(Inspector object, JobId job) { + RevisionId id = revisionFromSlime(object, job); Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); Optional<String> authorEmail = SlimeUtils.optionalString(object.field(authorEmailField)); Optional<Version> compileVersion = SlimeUtils.optionalString(object.field(compileVersionField)).map(Version::fromString); Optional<Instant> buildTime = SlimeUtils.optionalInstant(object.field(buildTimeField)); Optional<String> sourceUrl = SlimeUtils.optionalString(object.field(sourceUrlField)); Optional<String> commit = SlimeUtils.optionalString(object.field(commitField)); - boolean deployedDirectly = object.field(deployedDirectlyField).asBool(); + boolean hasPackage = object.field(hasPackageField).asBool(); + boolean shouldSkip = object.field(shouldSkipField).asBool(); + Optional<String> description = SlimeUtils.optionalString(object.field(descriptionField)); + int risk = (int) object.field(riskField).asLong(); Optional<String> bundleHash = SlimeUtils.optionalString(object.field(bundleHashField)); - return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly, bundleHash); + return new ApplicationVersion(id, sourceRevision, authorEmail, compileVersion, buildTime, sourceUrl, + commit, bundleHash, hasPackage, shouldSkip, description, risk); } private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) { @@ -460,9 +501,8 @@ public class ApplicationSerializer { private Map<JobType, Instant> jobPausesFromSlime(Inspector object) { Map<JobType, Instant> jobPauses = new HashMap<>(); object.field(jobStatusField).traverse((ArrayTraverser) (__, jobPauseObject) -> - JobType.fromOptionalJobName(jobPauseObject.field(jobTypeField).asString()) - .ifPresent(jobType -> jobPauses.put(jobType, - SlimeUtils.instant(jobPauseObject.field(pausedUntilField))))); + jobPauses.put(JobType.ofSerialized(jobPauseObject.field(jobTypeField).asString()), + SlimeUtils.instant(jobPauseObject.field(pausedUntilField)))); return jobPauses; } @@ -473,7 +513,7 @@ public class ApplicationSerializer { if (versionFieldValue.valid()) change = Change.of(Version.fromString(versionFieldValue.asString())); if (object.field(applicationBuildNumberField).valid()) - change = change.with(applicationVersionFromSlime(object)); + change = change.with(revisionFromSlime(object, null)); if (object.field(pinnedField).asBool()) change = change.withPin(); return change; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java index 059eb37bb59..9721026c628 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java @@ -2,10 +2,10 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Step; @@ -110,10 +110,8 @@ public class BufferedLogStore { store.delete(id); } - /** Deletes all logs for the given application. */ + /** Deletes all logs in permanent storage for the given application. */ public void delete(ApplicationId id) { - for (JobType type : JobType.values()) - buffer.deleteLog(id, type); store.delete(id); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java index 8b599b45558..1ec349b7dab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java @@ -4,10 +4,10 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; /** - * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.ControllerVersion}. + * Serializer for {@link ControllerVersion}. * * @author mpolden */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 84ff3d5d8c3..c3c68f7596f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -7,19 +7,18 @@ import com.yahoo.component.Version; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.curator.MultiplePathsLock; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -38,12 +37,10 @@ import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; - import java.io.IOException; import java.io.UncheckedIOException; import java.nio.ByteBuffer; @@ -103,8 +100,6 @@ public class CuratorDb { private final ControllerVersionSerializer controllerVersionSerializer = new ControllerVersionSerializer(); private final ConfidenceOverrideSerializer confidenceOverrideSerializer = new ConfidenceOverrideSerializer(); private final TenantSerializer tenantSerializer = new TenantSerializer(); - private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); - private final RunSerializer runSerializer = new RunSerializer(); private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); private final OsVersionTargetSerializer osVersionTargetSerializer = new OsVersionTargetSerializer(osVersionSerializer); private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer, nodeVersionSerializer); @@ -112,10 +107,13 @@ public class CuratorDb { private final ZoneRoutingPolicySerializer zoneRoutingPolicySerializer = new ZoneRoutingPolicySerializer(routingPolicySerializer); private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer(); private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer(); + private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); + private final RunSerializer runSerializer = new RunSerializer(); + private final RetriggerEntrySerializer retriggerEntrySerializer = new RetriggerEntrySerializer(); + private final NotificationsSerializer notificationsSerializer = new NotificationsSerializer(); private final Curator curator; private final Duration tryLockTimeout; - private final StringFlag lockScheme; // For each application id (path), store the ZK node version and its deserialised data - update when version changes. // This will grow to keep all applications in memory, but this should be OK @@ -125,14 +123,13 @@ public class CuratorDb { private final Map<Path, Pair<Integer, NavigableMap<RunId, Run>>> cachedHistoricRuns = new ConcurrentHashMap<>(); @Inject - public CuratorDb(Curator curator, FlagSource flagSource) { - this(curator, defaultTryLockTimeout, flagSource); + public CuratorDb(Curator curator, ServiceRegistry services) { + this(curator, defaultTryLockTimeout, services.zoneRegistry().system()); } - CuratorDb(Curator curator, Duration tryLockTimeout, FlagSource flagSource) { + CuratorDb(Curator curator, Duration tryLockTimeout, SystemName system) { this.curator = curator; this.tryLockTimeout = tryLockTimeout; - this.lockScheme = Flags.CONTROLLER_LOCK_SCHEME.bindTo(flagSource); } /** Returns all hostnames configured to be part of this ZooKeeper cluster */ @@ -145,71 +142,35 @@ public class CuratorDb { // -------------- Locks --------------------------------------------------- - public Lock lock(TenantName name) { + public Mutex lock(TenantName name) { return curator.lock(lockPath(name), defaultLockTimeout.multipliedBy(2)); } - public Lock lock(TenantAndApplicationId id) { - switch (lockScheme.value()) { - case "BOTH": - return new MultiplePathsLock(lockPath(id), legacyLockPath(id), defaultLockTimeout.multipliedBy(2), curator); - case "OLD": - return curator.lock(legacyLockPath(id), defaultLockTimeout.multipliedBy(2)); - case "NEW": - return curator.lock(lockPath(id), defaultLockTimeout.multipliedBy(2)); - default: - throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); - } + public Mutex lock(TenantAndApplicationId id) { + return curator.lock(lockPath(id), defaultLockTimeout.multipliedBy(2)); } - public Lock lockForDeployment(ApplicationId id, ZoneId zone) { - switch (lockScheme.value()) { - case "BOTH": - return new MultiplePathsLock(lockPath(id, zone), legacyLockPath(id, zone), deployLockTimeout, curator); - case "OLD": - return curator.lock(legacyLockPath(id, zone), deployLockTimeout); - case "NEW": - return curator.lock(lockPath(id, zone), deployLockTimeout); - default: - throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); - } + public Mutex lockForDeployment(ApplicationId id, ZoneId zone) { + return curator.lock(lockPath(id, zone), deployLockTimeout); } - public Lock lock(ApplicationId id, JobType type) { - switch (lockScheme.value()) { - case "BOTH": - return new MultiplePathsLock(lockPath(id, type), legacyLockPath(id, type), defaultLockTimeout, curator); - case "OLD": - return curator.lock(legacyLockPath(id, type), defaultLockTimeout); - case "NEW": - return curator.lock(lockPath(id, type), defaultLockTimeout); - default: - throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); - } + public Mutex lock(ApplicationId id, JobType type) { + return curator.lock(lockPath(id, type), defaultLockTimeout); } - public Lock lock(ApplicationId id, JobType type, Step step) throws TimeoutException { - switch (lockScheme.value()) { - case "BOTH": - return tryLock(lockPath(id, type, step), legacyLockPath(id, type, step)); - case "OLD": - return tryLock(legacyLockPath(id, type, step)); - case "NEW": - return tryLock(lockPath(id, type, step)); - default: - throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); - } + public Mutex lock(ApplicationId id, JobType type, Step step) throws TimeoutException { + return tryLock(lockPath(id, type, step)); } - public Lock lockRotations() { + public Mutex lockRotations() { return curator.lock(lockRoot.append("rotations"), defaultLockTimeout); } - public Lock lockConfidenceOverrides() { + public Mutex lockConfidenceOverrides() { return curator.lock(lockRoot.append("confidenceOverrides"), defaultLockTimeout); } - public Lock lockMaintenanceJob(String jobName) { + public Mutex lockMaintenanceJob(String jobName) { try { return tryLock(lockRoot.append("maintenanceJobLocks").append(jobName)); } catch (TimeoutException e) { @@ -217,52 +178,51 @@ public class CuratorDb { } } - @SuppressWarnings("unused") // Called by internal code - public Lock lockProvisionState(String provisionStateId) { + public Mutex lockProvisionState(String provisionStateId) { return curator.lock(lockPath(provisionStateId), Duration.ofSeconds(1)); } - public Lock lockOsVersions() { + public Mutex lockOsVersions() { return curator.lock(lockRoot.append("osTargetVersion"), defaultLockTimeout); } - public Lock lockOsVersionStatus() { + public Mutex lockOsVersionStatus() { return curator.lock(lockRoot.append("osVersionStatus"), defaultLockTimeout); } - public Lock lockRoutingPolicies() { + public Mutex lockRoutingPolicies() { return curator.lock(lockRoot.append("routingPolicies"), defaultLockTimeout); } - public Lock lockAuditLog() { + public Mutex lockAuditLog() { return curator.lock(lockRoot.append("auditLog"), defaultLockTimeout); } - public Lock lockNameServiceQueue() { + public Mutex lockNameServiceQueue() { return curator.lock(lockRoot.append("nameServiceQueue"), defaultLockTimeout); } - public Lock lockMeteringRefreshTime() throws TimeoutException { + public Mutex lockMeteringRefreshTime() throws TimeoutException { return tryLock(lockRoot.append("meteringRefreshTime")); } - public Lock lockArchiveBuckets(ZoneId zoneId) { + public Mutex lockArchiveBuckets(ZoneId zoneId) { return curator.lock(lockRoot.append("archiveBuckets").append(zoneId.value()), defaultLockTimeout); } - public Lock lockChangeRequests() { + public Mutex lockChangeRequests() { return curator.lock(lockRoot.append("changeRequests"), defaultLockTimeout); } - public Lock lockNotifications(TenantName tenantName) { + public Mutex lockNotifications(TenantName tenantName) { return curator.lock(lockRoot.append("notifications").append(tenantName.value()), defaultLockTimeout); } - public Lock lockSupportAccess(DeploymentId deploymentId) { + public Mutex lockSupportAccess(DeploymentId deploymentId) { return curator.lock(lockRoot.append("supportAccess").append(deploymentId.dottedString()), defaultLockTimeout); } - public Lock lockDeploymentRetriggerQueue() { + public Mutex lockDeploymentRetriggerQueue() { return curator.lock(lockRoot.append("deploymentRetriggerQueue"), defaultLockTimeout); } @@ -272,7 +232,7 @@ public class CuratorDb { * * Useful for maintenance jobs, where there is no point in running the jobs back to back. */ - private Lock tryLock(Path path) throws TimeoutException { + private Mutex tryLock(Path path) throws TimeoutException { try { return curator.lock(path, tryLockTimeout); } @@ -281,19 +241,6 @@ public class CuratorDb { } } - /** Try locking with a low timeout, meaning it is OK to fail lock acquisition. - * - * Useful for maintenance jobs, where there is no point in running the jobs back to back. - */ - private Lock tryLock(Path path, Path path2) throws TimeoutException { - try { - return new MultiplePathsLock(path, path2, tryLockTimeout, curator); - } - catch (UncheckedTimeoutException e) { - throw new TimeoutException(e.getMessage()); - } - } - private <T> Optional<T> read(Path path, Function<byte[], T> mapper) { return curator.getData(path).filter(data -> data.length > 0).map(mapper); } @@ -383,7 +330,7 @@ public class CuratorDb { } public Optional<Tenant> readTenant(TenantName name) { - return readSlime(tenantPath(name)).map(bytes -> tenantSerializer.tenantFrom(bytes)); + return readSlime(tenantPath(name)).map(tenantSerializer::tenantFrom); } public List<Tenant> readTenants() { @@ -415,7 +362,7 @@ public class CuratorDb { .map(stat -> cachedApplications.compute(path, (__, old) -> old != null && old.getFirst() == stat.getVersion() ? old - : new Pair<>(stat.getVersion(), read(path, applicationSerializer::fromSlime).get())).getSecond()); + : new Pair<>(stat.getVersion(), read(path, bytes -> applicationSerializer.fromSlime(bytes)).get())).getSecond()); } public List<Application> readApplications(boolean canFail) { @@ -676,7 +623,7 @@ public class CuratorDb { public List<Notification> readNotifications(TenantName tenantName) { return readSlime(notificationsPath(tenantName)) - .map(slime -> NotificationsSerializer.fromSlime(tenantName, slime)).orElseGet(List::of); + .map(slime -> notificationsSerializer.fromSlime(tenantName, slime)).orElseGet(List::of); } @@ -687,7 +634,7 @@ public class CuratorDb { } public void writeNotifications(TenantName tenantName, List<Notification> notifications) { - curator.set(notificationsPath(tenantName), asJson(NotificationsSerializer.toSlime(notifications))); + curator.set(notificationsPath(tenantName), asJson(notificationsSerializer.toSlime(notifications))); } public void deleteNotifications(TenantName tenantName) { @@ -708,11 +655,11 @@ public class CuratorDb { // -------------- Job Retrigger entries ----------------------------------- public List<RetriggerEntry> readRetriggerEntries() { - return readSlime(deploymentRetriggerPath()).map(RetriggerEntrySerializer::fromSlime).orElseGet(List::of); + return readSlime(deploymentRetriggerPath()).map(retriggerEntrySerializer::fromSlime).orElseGet(List::of); } public void writeRetriggerEntries(List<RetriggerEntry> retriggerEntries) { - curator.set(deploymentRetriggerPath(), asJson(RetriggerEntrySerializer.toSlime(retriggerEntries))); + curator.set(deploymentRetriggerPath(), asJson(retriggerEntrySerializer.toSlime(retriggerEntries))); } // -------------- Paths --------------------------------------------------- @@ -722,32 +669,6 @@ public class CuratorDb { .append(tenant.value()); } - private Path legacyLockPath(TenantAndApplicationId application) { - return lockPath(application.tenant()) - .append(application.application().value()); - } - - private Path legacyLockPath(ApplicationId instance) { - return legacyLockPath(TenantAndApplicationId.from(instance)) - .append(instance.instance().value()); - } - - private Path legacyLockPath(ApplicationId instance, ZoneId zone) { - return legacyLockPath(instance) - .append(zone.environment().value()) - .append(zone.region().value()); - } - - private Path legacyLockPath(ApplicationId instance, JobType type) { - return legacyLockPath(instance) - .append(type.jobName()); - } - - private Path legacyLockPath(ApplicationId instance, JobType type, Step step) { - return legacyLockPath(instance, type) - .append(step.name()); - } - private Path lockPath(TenantAndApplicationId application) { return lockRoot.append(application.tenant().value() + ":" + application.application().value()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java index 65c9859ad72..21414339a87 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java @@ -2,9 +2,9 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.InMemoryFlagSource; - import java.time.Duration; /** @@ -18,21 +18,21 @@ public class MockCuratorDb extends CuratorDb { private final MockCurator curator; @Inject - public MockCuratorDb() { - this("test-controller:2222"); + public MockCuratorDb(ConfigserverConfig config) { + this("test-controller:2222", SystemName.from(config.system())); + } + + public MockCuratorDb(SystemName system) { + this("test-controller:2222", system); } - public MockCuratorDb(String zooKeeperEnsembleConnectionSpec) { - this(new MockCurator() { - @Override - public String zooKeeperEnsembleConnectionSpec() { - return zooKeeperEnsembleConnectionSpec; - } - }); + public MockCuratorDb(String zooKeeperEnsembleConnectionSpec, SystemName system) { + this(new MockCurator() { @Override public String zooKeeperEnsembleConnectionSpec() { return zooKeeperEnsembleConnectionSpec; } }, + system); } - public MockCuratorDb(MockCurator curator) { - super(curator, Duration.ofMillis(100), new InMemoryFlagSource()); + public MockCuratorDb(MockCurator curator, SystemName system) { + super(curator, Duration.ofMillis(100), system); this.curator = curator; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java index 10ec9dce5f7..0f1f531d589 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java @@ -46,7 +46,7 @@ public class NodeVersionSerializer { public List<NodeVersion> nodeVersionsFromSlime(Inspector array, Version version) { List<NodeVersion> nodeVersions = new ArrayList<>(); array.traverse((ArrayTraverser) (i, entry) -> { - var hostname = HostName.from(entry.field(hostnameField).asString()); + var hostname = HostName.of(entry.field(hostnameField).asString()); var zone = ZoneId.from(entry.field(zoneField).asString()); var wantedVersion = Version.fromString(entry.field(wantedVersionField).asString()); var suspendedAt = SlimeUtils.optionalInstant(entry.field(suspendedAtField)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java index 10763e1f22c..16ec240a116 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; @@ -43,7 +44,7 @@ public class NotificationsSerializer { private static final String jobTypeField = "jobId"; private static final String runNumberField = "runNumber"; - public static Slime toSlime(List<Notification> notifications) { + public Slime toSlime(List<Notification> notifications) { Slime slime = new Slime(); Cursor notificationsArray = slime.setObject().setArray(notificationsFieldName); @@ -59,20 +60,20 @@ public class NotificationsSerializer { notification.source().instance().ifPresent(instance -> notificationObject.setString(instanceField, instance.value())); notification.source().zoneId().ifPresent(zoneId -> notificationObject.setString(zoneField, zoneId.value())); notification.source().clusterId().ifPresent(clusterId -> notificationObject.setString(clusterIdField, clusterId.value())); - notification.source().jobType().ifPresent(jobType -> notificationObject.setString(jobTypeField, jobType.jobName())); + notification.source().jobType().ifPresent(jobType -> notificationObject.setString(jobTypeField, jobType.serialized())); notification.source().runNumber().ifPresent(runNumber -> notificationObject.setLong(runNumberField, runNumber)); } return slime; } - public static List<Notification> fromSlime(TenantName tenantName, Slime slime) { + public List<Notification> fromSlime(TenantName tenantName, Slime slime) { return SlimeUtils.entriesStream(slime.get().field(notificationsFieldName)) .map(inspector -> fromInspector(tenantName, inspector)) .collect(Collectors.toUnmodifiableList()); } - private static Notification fromInspector(TenantName tenantName, Inspector inspector) { + private Notification fromInspector(TenantName tenantName, Inspector inspector) { return new Notification( SlimeUtils.instant(inspector.field(atFieldName)), typeFrom(inspector.field(typeField)), @@ -83,7 +84,7 @@ public class NotificationsSerializer { SlimeUtils.optionalString(inspector.field(instanceField)).map(InstanceName::from), SlimeUtils.optionalString(inspector.field(zoneField)).map(ZoneId::from), SlimeUtils.optionalString(inspector.field(clusterIdField)).map(ClusterSpec.Id::from), - SlimeUtils.optionalString(inspector.field(jobTypeField)).map(JobType::fromJobName), + SlimeUtils.optionalString(inspector.field(jobTypeField)).map(jobName -> JobType.ofSerialized(jobName)), SlimeUtils.optionalLong(inspector.field(runNumberField))), SlimeUtils.entriesStream(inspector.field(messagesField)).map(Inspector::asString).collect(Collectors.toUnmodifiableList())); } @@ -91,6 +92,8 @@ public class NotificationsSerializer { private static String asString(Notification.Type type) { switch (type) { case applicationPackage: return "applicationPackage"; + case submission: return "submission"; + case testPackage: return "testPackage"; case deployment: return "deployment"; case feedBlock: return "feedBlock"; case reindex: return "reindex"; @@ -101,6 +104,8 @@ public class NotificationsSerializer { private static Notification.Type typeFrom(Inspector field) { switch (field.asString()) { case "applicationPackage": return Notification.Type.applicationPackage; + case "submission": return Notification.Type.submission; + case "testPackage": return Notification.Type.testPackage; case "deployment": return Notification.Type.deployment; case "feedBlock": return Notification.Type.feedBlock; case "reindex": return Notification.Type.reindex; @@ -125,4 +130,5 @@ public class NotificationsSerializer { default: throw new IllegalArgumentException("Unknown serialized notification level value '" + field.asString() + "'"); } } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 17337f823c0..72c16ae0110 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import ai.vespa.http.DomainName; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -83,7 +83,7 @@ public class RoutingPolicySerializer { ClusterSpec.Id.from(inspect.field(clusterField).asString()), ZoneId.from(inspect.field(zoneField).asString())); policies.add(new RoutingPolicy(id, - HostName.from(inspect.field(canonicalNameField).asString()), + DomainName.of(inspect.field(canonicalNameField).asString()), SlimeUtils.optionalString(inspect.field(dnsZoneField)), instanceEndpoints, applicationEndpoints, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index c3d81b8dcd5..dd28978d948 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -10,10 +11,9 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; @@ -28,7 +28,6 @@ import java.util.Collections; import java.util.EnumMap; import java.util.NavigableMap; import java.util.Optional; -import java.util.OptionalLong; import java.util.TreeMap; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; @@ -36,6 +35,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentF import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCertificateTimeout; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; @@ -74,7 +74,6 @@ class RunSerializer { // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - // TODO: Remove "steps" when there are no traces of it in the controllers private static final String stepsField = "steps"; private static final String stepDetailsField = "stepDetails"; private static final String startTimeField = "startTime"; @@ -88,17 +87,9 @@ class RunSerializer { private static final String versionsField = "versions"; private static final String isRedeploymentField = "isRedeployment"; private static final String platformVersionField = "platform"; - private static final String repositoryField = "repository"; - private static final String branchField = "branch"; - private static final String commitField = "commit"; - private static final String authorEmailField = "authorEmail"; private static final String deployedDirectlyField = "deployedDirectly"; - private static final String compileVersionField = "compileVersion"; - private static final String buildTimeField = "buildTime"; - private static final String sourceUrlField = "sourceUrl"; private static final String buildField = "build"; private static final String sourceField = "source"; - private static final String bundleHashField = "bundleHash"; private static final String lastTestRecordField = "lastTestRecord"; private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp"; private static final String noNodesDownSinceField = "noNodesDownSince"; @@ -134,11 +125,12 @@ class RunSerializer { steps.put(typedStep, new StepInfo(typedStep, stepStatusOf(status.asString()), startTime)); }); - return new Run(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), - JobType.fromJobName(runObject.field(jobTypeField).asString()), - runObject.field(numberField).asLong()), + RunId id = new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), + JobType.ofSerialized(runObject.field(jobTypeField).asString()), + runObject.field(numberField).asLong()); + return new Run(id, steps, - versionsFromSlime(runObject.field(versionsField)), + versionsFromSlime(runObject.field(versionsField), id), runObject.field(isRedeploymentField).asBool(), SlimeUtils.instant(runObject.field(startField)), SlimeUtils.optionalInstant(runObject.field(endField)), @@ -155,40 +147,26 @@ class RunSerializer { SlimeUtils.optionalString(runObject.field(reasonField))); } - private Versions versionsFromSlime(Inspector versionsObject) { + private Versions versionsFromSlime(Inspector versionsObject, RunId id) { Version targetPlatformVersion = Version.fromString(versionsObject.field(platformVersionField).asString()); - ApplicationVersion targetApplicationVersion = applicationVersionFrom(versionsObject); + RevisionId targetRevision = revisionFrom(versionsObject, id); Optional<Version> sourcePlatformVersion = versionsObject.field(sourceField).valid() ? Optional.of(Version.fromString(versionsObject.field(sourceField).field(platformVersionField).asString())) : Optional.empty(); - Optional<ApplicationVersion> sourceApplicationVersion = versionsObject.field(sourceField).valid() - ? Optional.of(applicationVersionFrom(versionsObject.field(sourceField))) + Optional<RevisionId> sourceRevision = versionsObject.field(sourceField).valid() + ? Optional.of(revisionFrom(versionsObject.field(sourceField), id)) : Optional.empty(); - return new Versions(targetPlatformVersion, targetApplicationVersion, sourcePlatformVersion, sourceApplicationVersion); + return new Versions(targetPlatformVersion, targetRevision, sourcePlatformVersion, sourceRevision); } - private ApplicationVersion applicationVersionFrom(Inspector versionObject) { - if ( ! versionObject.field(buildField).valid()) - return ApplicationVersion.unknown; - + private RevisionId revisionFrom(Inspector versionObject, RunId id) { long buildNumber = versionObject.field(buildField).asLong(); - // TODO jonmv: Remove source revision - Optional<SourceRevision> source = Optional.of(new SourceRevision(versionObject.field(repositoryField).asString(), - versionObject.field(branchField).asString(), - versionObject.field(commitField).asString())) - .filter(revision -> ! revision.commit().isBlank() && ! revision.repository().isBlank() && ! revision.branch().isBlank()); - Optional<String> authorEmail = SlimeUtils.optionalString(versionObject.field(authorEmailField)); - Optional<Version> compileVersion = SlimeUtils.optionalString(versionObject.field(compileVersionField)).map(Version::fromString); - Optional<Instant> buildTime = SlimeUtils.optionalInstant(versionObject.field(buildTimeField)); - Optional<String> sourceUrl = SlimeUtils.optionalString(versionObject.field(sourceUrlField)); - Optional<String> commit = SlimeUtils.optionalString(versionObject.field(commitField)); - boolean deployedDirectly = versionObject.field(deployedDirectlyField).asBool(); - Optional<String> bundleHash = SlimeUtils.optionalString(versionObject.field(bundleHashField)); - - return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, - compileVersion, buildTime, sourceUrl, commit, deployedDirectly, bundleHash); + boolean production = versionObject.field(deployedDirectlyField).valid() // TODO jonmv: remove after migration + && buildNumber > 0 + && ! versionObject.field(deployedDirectlyField).asBool(); + return production ? RevisionId.forProduction(buildNumber) : RevisionId.forDevelopment(buildNumber, id.job()); } // Don't change this — introduce a separate array instead. @@ -228,7 +206,7 @@ class RunSerializer { private void toSlime(Run run, Cursor runObject) { runObject.setString(applicationField, run.id().application().serializedForm()); - runObject.setString(jobTypeField, run.id().type().jobName()); + runObject.setString(jobTypeField, run.id().type().serialized()); runObject.setBool(isRedeploymentField, run.isRedeployment()); runObject.setLong(numberField, run.id().number()); runObject.setLong(startField, run.start().toEpochMilli()); @@ -251,10 +229,10 @@ class RunSerializer { stepDetailsObject.setObject(valueOf(step)).setLong(startTimeField, valueOf(startTime)))); Cursor versionsObject = runObject.setObject(versionsField); - toSlime(run.versions().targetPlatform(), run.versions().targetApplication(), versionsObject); + toSlime(run.versions().targetPlatform(), run.versions().targetRevision(), versionsObject); run.versions().sourcePlatform().ifPresent(sourcePlatformVersion -> { toSlime(sourcePlatformVersion, - run.versions().sourceApplication() + run.versions().sourceRevision() .orElseThrow(() -> new IllegalArgumentException("Source versions must be both present or absent.")), versionsObject.setObject(sourceField)); }); @@ -262,19 +240,10 @@ class RunSerializer { run.reason().ifPresent(reason -> runObject.setString(reasonField, reason)); } - private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) { + private void toSlime(Version platformVersion, RevisionId revsion, Cursor versionsObject) { versionsObject.setString(platformVersionField, platformVersion.toString()); - applicationVersion.buildNumber().ifPresent(number -> versionsObject.setLong(buildField, number)); - // TODO jonmv: Remove source revision. - applicationVersion.source().map(SourceRevision::repository).ifPresent(repository -> versionsObject.setString(repositoryField, repository)); - applicationVersion.source().map(SourceRevision::branch).ifPresent(branch -> versionsObject.setString(branchField, branch)); - applicationVersion.source().map(SourceRevision::commit).ifPresent(commit -> versionsObject.setString(commitField, commit)); - applicationVersion.authorEmail().ifPresent(email -> versionsObject.setString(authorEmailField, email)); - applicationVersion.compileVersion().ifPresent(version -> versionsObject.setString(compileVersionField, version.toString())); - applicationVersion.buildTime().ifPresent(time -> versionsObject.setLong(buildTimeField, time.toEpochMilli())); - applicationVersion.sourceUrl().ifPresent(url -> versionsObject.setString(sourceUrlField, url)); - applicationVersion.commit().ifPresent(commit -> versionsObject.setString(commitField, commit)); - versionsObject.setBool(deployedDirectlyField, applicationVersion.isDeployedDirectly()); + versionsObject.setLong(buildField, revsion.number()); + versionsObject.setBool(deployedDirectlyField, ! revsion.isProduction()); } // Don't change this - introduce a separate array with new values if needed. @@ -368,6 +337,7 @@ class RunSerializer { case deploymentFailed : return "deploymentFailed"; case installationFailed : return "installationFailed"; case testFailure : return "testFailure"; + case noTests : return "noTests"; case error : return "error"; case success : return "success"; case aborted : return "aborted"; @@ -380,11 +350,11 @@ class RunSerializer { static RunStatus runStatusOf(String status) { switch (status) { case "running" : return running; - case "outOfCapacity" : return nodeAllocationFailure; // TODO: Remove after March 2022 case "nodeAllocationFailure" : return nodeAllocationFailure; case "endpointCertificateTimeout" : return endpointCertificateTimeout; case "deploymentFailed" : return deploymentFailed; case "installationFailed" : return installationFailed; + case "noTests" : return noTests; case "testFailure" : return testFailure; case "error" : return error; case "success" : return success; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java index e9e5f8cf032..c200ca467da 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java @@ -13,7 +13,6 @@ import java.time.Instant; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java index 79f0088a214..fdd93eedbff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import com.yahoo.container.jdisc.HttpRequest; - import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Path; import ai.vespa.http.HttpURL.Query; +import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.text.Text; + import java.io.InputStream; import java.net.URI; import java.util.List; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java index 05cdc0d0565..886dc27b404 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import com.yahoo.container.jdisc.HttpResponse; import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Path; +import com.yahoo.container.jdisc.HttpResponse; import java.io.IOException; import java.io.OutputStream; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 924f7c0b0a5..88b319d0051 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2,7 +2,10 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.Signatures; +import ai.vespa.http.DomainName; +import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Query; +import ai.vespa.validation.Validation; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; @@ -26,10 +29,8 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; -import ai.vespa.http.DomainName; import com.yahoo.restapi.ByteArrayResponse; import com.yahoo.restapi.ErrorResponse; -import ai.vespa.http.HttpURL; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; @@ -53,7 +54,6 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbi import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.aws.TenantRoles; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; @@ -67,11 +67,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeReposi import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.vespa.hosted.controller.api.integration.user.User; +import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; @@ -93,6 +94,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer; import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; import com.yahoo.vespa.hosted.controller.notification.Notification; @@ -118,7 +120,6 @@ import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.yolean.Exceptions; import javax.ws.rs.ForbiddenException; @@ -148,7 +149,6 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.Scanner; -import java.util.SortedSet; import java.util.StringJoiner; import java.util.function.Function; import java.util.logging.Level; @@ -260,7 +260,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)).descendingMap(), Optional.ofNullable(request.getProperty("limit")), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.applications().requireApplication(TenantAndApplicationId.from(path.get("tenant"), path.get("application"))), controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)).descendingMap(), Optional.ofNullable(request.getProperty("limit")), request.getUri()); // (((＼（✘෴✘）／))) if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/diff/{number}")) return devApplicationPackageDiff(runIdFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); @@ -268,10 +268,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return getReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/state/v1/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/state/v1/metrics")) return stateV1Metrics(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/state/v1/{*}")) return stateV1(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/orchestrator")) return orchestrator(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/content/{*}")) return content(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.getRest(), request); @@ -284,8 +283,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/state/v1/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); @@ -356,6 +353,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", "all"); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", path.get("choice")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return removeDeployKey(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit/{build}")) return cancelBuild(path.get("tenant"), path.get("application"), path.get("build")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return deleteInstance(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), "all"); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("choice")); @@ -711,7 +709,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { propertyEquals(request, "application", ApplicationName::from, notification.source().application()) && propertyEquals(request, "instance", InstanceName::from, notification.source().instance()) && propertyEquals(request, "zone", ZoneId::from, notification.source().zoneId()) && - propertyEquals(request, "job", JobType::fromJobName, notification.source().jobType()) && + propertyEquals(request, "job", job -> JobType.fromJobName(job, controller.zoneRegistry()), notification.source().jobType()) && propertyEquals(request, "type", Notification.Type::valueOf, Optional.of(notification.type())) && propertyEquals(request, "level", Notification.Level::valueOf, Optional.of(notification.level()))) .forEach(notification -> toSlime(notificationsArray.addObject(), notification, includeTenantFieldInResponse, excludeMessages)); @@ -747,7 +745,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private static String notificationTypeAsString(Notification.Type type) { switch (type) { + case submission: case applicationPackage: return "applicationPackage"; + case testPackage: return "testPackage"; case deployment: return "deployment"; case feedBlock: return "feedBlock"; case reindex: return "reindex"; @@ -800,47 +800,41 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private HttpResponse devApplicationPackage(ApplicationId id, JobType type) { - if ( ! type.environment().isManuallyDeployed()) - throw new IllegalArgumentException("Only manually deployed zones have dev packages"); - - ZoneId zone = type.zone(controller.system()); - ApplicationVersion version = controller.jobController().last(id, type).get().versions().targetApplication(); - byte[] applicationPackage = controller.applications().applicationStore().get(new DeploymentId(id, zone), version); + ZoneId zone = type.zone(); + RevisionId revision = controller.jobController().last(id, type).get().versions().targetRevision(); + byte[] applicationPackage = controller.applications().applicationStore().get(new DeploymentId(id, zone), revision); return new ZipResponse(id.toFullString() + "." + zone.value() + ".zip", applicationPackage); } private HttpResponse devApplicationPackageDiff(RunId runId) { - DeploymentId deploymentId = new DeploymentId(runId.application(), runId.job().type().zone(controller.system())); + DeploymentId deploymentId = new DeploymentId(runId.application(), runId.job().type().zone()); return controller.applications().applicationStore().getDevDiff(deploymentId, runId.number()) .map(ByteArrayResponse::new) .orElseThrow(() -> new NotExistsException("No application package diff found for " + runId)); } private HttpResponse applicationPackage(String tenantName, String applicationName, HttpRequest request) { - var tenantAndApplication = TenantAndApplicationId.from(tenantName, applicationName); - SortedSet<ApplicationVersion> versions = controller.applications().requireApplication(tenantAndApplication).versions(); - if (versions.isEmpty()) - throw new NotExistsException("No application package has been submitted for '" + tenantAndApplication + "'"); - - ApplicationVersion version = Optional.ofNullable(request.getProperty("build")) - .map(build -> { - try { - return Long.parseLong(build); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Invalid build number", e); - } - }) - .map(build -> versions.stream() - .filter(ver -> ver.buildNumber().orElse(-1) == build) - .findFirst() - .orElseThrow(() -> new NotExistsException("No application package found for '" + tenantAndApplication + "' with build number " + build))) - .orElseGet(versions::last); - + TenantAndApplicationId tenantAndApplication = TenantAndApplicationId.from(tenantName, applicationName); + long build; + String parameter = request.getProperty("build"); + if (parameter != null) + try { + build = Validation.requireAtLeast(Long.parseLong(request.getProperty("build")), "build number", 1L); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("invalid value for request parameter 'build'", e); + } + else { + build = controller.applications().requireApplication(tenantAndApplication).revisions().last() + .map(version -> version.id().number()) + .orElseThrow(() -> new NotExistsException("no application package has been submitted for " + tenantAndApplication)); + } + RevisionId revision = RevisionId.forProduction(build); boolean tests = request.getBooleanProperty("tests"); byte[] applicationPackage = tests ? - controller.applications().applicationStore().getTester(tenantAndApplication.tenant(), tenantAndApplication.application(), version) : - controller.applications().applicationStore().get(new DeploymentId(tenantAndApplication.defaultInstance(), ZoneId.defaultId()), version); - String filename = tenantAndApplication + (tests ? "-tests" : "-build") + version.buildNumber().getAsLong() + ".zip"; + controller.applications().applicationStore().getTester(tenantAndApplication.tenant(), tenantAndApplication.application(), revision) : + controller.applications().applicationStore().get(new DeploymentId(tenantAndApplication.defaultInstance(), ZoneId.defaultId()), revision); + String filename = tenantAndApplication + (tests ? "-tests" : "-build") + revision.number() + ".zip"; return new ZipResponse(filename, applicationPackage); } @@ -1121,7 +1115,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { toSlime(node.resources(), nodeObject); nodeObject.setString("clusterId", node.clusterId()); nodeObject.setString("clusterType", valueOf(node.clusterType())); - nodeObject.setBool("down", node.history().stream().anyMatch(event -> "down".equals(event.name()))); + nodeObject.setBool("down", node.down()); nodeObject.setBool("retired", node.retired() || node.wantToRetire()); nodeObject.setBool("restarting", node.wantedRestartGeneration() > node.restartGeneration()); nodeObject.setBool("rebooting", node.wantedRebootGeneration() > node.rebootGeneration()); @@ -1318,7 +1312,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { request.getUri()).toString()); DeploymentStatus status = controller.jobController().deploymentStatus(application); - application.latestVersion().ifPresent(version -> JobControllerApiHandlerHelper.toSlime(object.setObject("latestVersion"), version)); + application.revisions().last().ifPresent(version -> JobControllerApiHandlerHelper.toSlime(object.setObject("latestVersion"), version)); application.projectId().ifPresent(id -> object.setLong("projectId", id)); @@ -1326,11 +1320,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { application.instances().values().stream().findFirst().ifPresent(instance -> { // Currently deploying change if ( ! instance.change().isEmpty()) - toSlime(object.setObject("deploying"), instance.change()); + toSlime(object.setObject("deploying"), instance.change(), application); // Outstanding change if ( ! status.outstandingChange(instance.name()).isEmpty()) - toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name())); + toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name()), application); }); application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion)); @@ -1370,11 +1364,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .sortedJobs(status.instanceJobs(instance.name()).values()); if ( ! instance.change().isEmpty()) - toSlime(object.setObject("deploying"), instance.change()); + toSlime(object.setObject("deploying"), instance.change(), status.application()); // Outstanding change if ( ! status.outstandingChange(instance.name()).isEmpty()) - toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name())); + toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name()), status.application()); // Change blockers Cursor changeBlockers = object.setArray("changeBlockers"); @@ -1395,7 +1389,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { // Deployments sorted according to deployment spec List<Deployment> deployments = deploymentSpec.instance(instance.name()) - .map(spec -> new DeploymentSteps(spec, controller::system)) + .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry())) .map(steps -> steps.sortedDeployments(instance.deployments().values())) .orElse(List.copyOf(instance.deployments().values())); @@ -1441,7 +1435,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { "/instance/" + instance.id().instance().value() + "/job/", request.getUri()).toString()); - application.latestVersion().ifPresent(version -> { + application.revisions().last().ifPresent(version -> { version.sourceUrl().ifPresent(url -> object.setString("sourceUrl", url)); version.commit().ifPresent(commit -> object.setString("commit", commit)); }); @@ -1455,11 +1449,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .sortedJobs(status.instanceJobs(instance.name()).values()); if ( ! instance.change().isEmpty()) - toSlime(object.setObject("deploying"), instance.change()); + toSlime(object.setObject("deploying"), instance.change(), application); // Outstanding change if ( ! status.outstandingChange(instance.name()).isEmpty()) - toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name())); + toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name()), application); // Change blockers Cursor changeBlockers = object.setArray("changeBlockers"); @@ -1483,7 +1477,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { // Deployments sorted according to deployment spec List<Deployment> deployments = application.deploymentSpec().instance(instance.name()) - .map(spec -> new DeploymentSteps(spec, controller::system)) + .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry())) .map(steps -> steps.sortedDeployments(instance.deployments().values())) .orElse(List.copyOf(instance.deployments().values())); Cursor instancesArray = object.setArray("instances"); @@ -1525,7 +1519,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { controller.jobController().active(instance.id()).stream() .map(run -> run.id().job()) .filter(job -> job.type().environment().isManuallyDeployed())) - .map(job -> job.type().zone(controller.system())) + .map(job -> job.type().zone()) .filter(zone -> ! instance.deployments().containsKey(zone)) .forEach(zone -> { Cursor deploymentObject = instancesArray.addObject(); @@ -1574,11 +1568,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(slime); } - private void toSlime(Cursor object, Change change) { + private void toSlime(Cursor object, Change change, Application application) { change.platform().ifPresent(version -> object.setString("version", version.toString())); - change.application() - .filter(version -> !version.isUnknown()) - .ifPresent(version -> JobControllerApiHandlerHelper.toSlime(object.setObject("revision"), version)); + change.revision().ifPresent(revision -> JobControllerApiHandlerHelper.toSlime(object.setObject("revision"), application.revisions().get(revision))); } private void toSlime(Endpoint endpoint, Cursor object) { @@ -1623,8 +1615,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { response.setString("nodes", withPathAndQuery("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/", "recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); response.setString("version", deployment.version().toFullString()); - response.setString("revision", deployment.applicationVersion().id()); - Instant lastDeploymentStart = lastDeploymentStart(deploymentId.applicationId(), deployment); + response.setString("revision", application.revisions().get(deployment.revision()).stringId()); // TODO jonmv or freva: ƪ(`▿▿▿▿´ƪ) + response.setLong("build", deployment.revision().number()); + Instant lastDeploymentStart = controller.jobController().lastDeploymentStart(deploymentId.applicationId(), deployment); response.setLong("deployTimeEpochMs", lastDeploymentStart.toEpochMilli()); controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zoneId()) .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", lastDeploymentStart.plus(deploymentTimeToLive).toEpochMilli())); @@ -1638,22 +1631,18 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (!deployment.zone().environment().isManuallyDeployed()) { DeploymentStatus status = controller.jobController().deploymentStatus(application); - JobType.from(controller.system(), deployment.zone()) - .map(type -> new JobId(instance.id(), type)) - .map(status.jobSteps()::get) + JobId jobId = new JobId(instance.id(), JobType.deploymentTo(deployment.zone())); + Optional.ofNullable(status.jobSteps().get(jobId)) .ifPresent(stepStatus -> { - JobControllerApiHandlerHelper.toSlime( - response.setObject("applicationVersion"), deployment.applicationVersion()); - if (!status.jobsToRun().containsKey(stepStatus.job().get())) + JobControllerApiHandlerHelper.toSlime(response.setObject("applicationVersion"), application.revisions().get(deployment.revision())); + if ( ! status.jobsToRun().containsKey(stepStatus.job().get())) response.setString("status", "complete"); else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true)) response.setString("status", "pending"); else response.setString("status", "running"); }); } else { - var deploymentRun = JobType.from(controller.system(), deploymentId.zoneId()) - .flatMap(jobType -> controller.jobController().last(deploymentId.applicationId(), jobType)); - + var deploymentRun = controller.jobController().last(deploymentId.applicationId(), JobType.deploymentTo(deploymentId.zoneId())); deploymentRun.ifPresent(run -> { response.setString("status", run.hasEnded() ? "complete" : "running"); }); @@ -1685,11 +1674,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { metrics.instant().ifPresent(instant -> metricsObject.setLong("lastUpdated", instant.toEpochMilli())); } - private Instant lastDeploymentStart(ApplicationId instanceId, Deployment deployment) { - return controller.jobController().jobStarts(new JobId(instanceId, JobType.from(controller.system(), deployment.zone()).get())) - .stream().findFirst().orElse(deployment.at()); - } - private void toSlime(RotationState state, Cursor object) { Cursor bcpStatus = object.setObject("bcpStatus"); bcpStatus.setString("rotationStatus", rotationStateString(state)); @@ -1770,7 +1754,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { Cursor root = slime.setObject(); if ( ! instance.change().isEmpty()) { instance.change().platform().ifPresent(version -> root.setString("platform", version.toString())); - instance.change().application().ifPresent(applicationVersion -> root.setString("application", applicationVersion.id())); + instance.change().revision().ifPresent(revision -> root.setString("application", revision.toString())); root.setBool("pinned", instance.change().isPinned()); } return new SlimeJsonResponse(slime); @@ -1786,17 +1770,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse services(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region); - ZoneId zone = requireZone(environment, region); - ServiceApiResponse response = new ServiceApiResponse(zone, - new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), - List.of(controller.zoneRegistry().getConfigServerVipUri(zone)), - request.getUri()); - response.setResponse(applicationView); - return response; - } - private HttpResponse status(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host, HttpURL.Path restPath, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); return controller.serviceRegistry().configServer().getServiceNodePage(deploymentId, @@ -1806,21 +1779,17 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { Query.empty().add(request.getJDiscRequest().parameters())); } - private HttpResponse stateV1Metrics(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host) { + private HttpResponse orchestrator(String tenantName, String applicationName, String instanceName, String environment, String region) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - return controller.serviceRegistry().configServer().getServiceNodePage( - deploymentId, serviceName, DomainName.of(host), HttpURL.Path.parse("/state/v1/metrics"), Query.empty()); + return controller.serviceRegistry().configServer().getServiceNodes(deploymentId); } - private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, HttpURL.Path restPath, HttpRequest request) { + private HttpResponse stateV1(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host, HttpURL.Path rest, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - Map<?,?> result = controller.serviceRegistry().configServer().getServiceApiResponse(deploymentId, serviceName, restPath); - ServiceApiResponse response = new ServiceApiResponse(deploymentId.zoneId(), - deploymentId.applicationId(), - List.of(controller.zoneRegistry().getConfigServerVipUri(deploymentId.zoneId())), - request.getUri()); - response.setResponse(result, serviceName, HttpURL.Path.parse("/state/v1").append(restPath)); - return response; + Query query = Query.empty().add(request.getJDiscRequest().parameters()); + query = query.set("forwarded-url", HttpURL.from(request.getUri()).withQuery(Query.empty()).asURI().toString()); + return controller.serviceRegistry().configServer().getServiceNodePage( + deploymentId, serviceName, DomainName.of(host), HttpURL.Path.parse("/state/v1").append(rest), query); } private HttpResponse content(String tenantName, String applicationName, String instanceName, String environment, String region, HttpURL.Path restPath, HttpRequest request) { @@ -1915,18 +1884,19 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { StringBuilder response = new StringBuilder(); controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - ApplicationVersion version = build == -1 ? application.get().latestVersion().get() - : getApplicationVersion(application.get(), build); - Change change = Change.of(version); + RevisionId revision = build == -1 ? application.get().revisions().last().get().id() + : getRevision(application.get(), build); + Change change = Change.of(revision); controller.applications().deploymentTrigger().forceChange(id, change); response.append("Triggered ").append(change).append(" for ").append(id); }); return new MessageResponse(response.toString()); } - private ApplicationVersion getApplicationVersion(Application application, Long build) { - return application.versions().stream() - .filter(version -> version.buildNumber().stream().anyMatch(build::equals)) + private RevisionId getRevision(Application application, long build) { + return application.revisions().withPackage().stream() + .map(ApplicationVersion::id) + .filter(version -> version.number() == build) .findFirst() .filter(version -> controller.applications().applicationStore().hasBuild(application.id().tenant(), application.id().application(), @@ -1934,6 +1904,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .orElseThrow(() -> new IllegalArgumentException("Build number '" + build + "' was not found")); } + private HttpResponse cancelBuild(String tenantName, String applicationName, String build){ + TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); + RevisionId revision = RevisionId.forProduction(Long.parseLong(build)); + controller.applications().lockApplicationOrThrow(id, application -> { + controller.applications().store(application.withRevisions(revisions -> revisions.with(revisions.get(revision).skipped()))); + }); + return new MessageResponse("Marked build '" + build + "' as non-deployable"); + } + /** Cancel ongoing change for given application, e.g., everything with {"cancel":"all"} */ private HttpResponse cancelDeploy(String tenantName, String applicationName, String instanceName, String choice) { ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); @@ -2053,7 +2032,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); RestartFilter restartFilter = new RestartFilter() - .withHostName(Optional.ofNullable(request.getProperty("hostname")).map(HostName::from)) + .withHostName(Optional.ofNullable(request.getProperty("hostname")).map(HostName::of)) .withClusterType(Optional.ofNullable(request.getProperty("clusterType")).map(ClusterSpec.Type::from)) .withClusterId(Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)); @@ -2080,7 +2059,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP)); controller.applications().verifyApplicationIdentityConfiguration(id.tenant(), Optional.of(id.instance()), - Optional.of(type.zone(controller.system())), + Optional.of(type.zone()), applicationPackage, Optional.of(requireUserPrincipal(request))); @@ -2193,7 +2172,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .flatMap(instance -> instance.productionDeployments().keySet().stream()) .map(zone -> new DeploymentId(prodInstanceId, zone)) .collect(Collectors.toCollection(HashSet::new)); - ZoneId testedZone = type.zone(controller.system()); + ZoneId testedZone = type.zone(); // If a production job is specified, the production deployment of the orchestrated instance is the relevant one, // as user instances should not exist in prod. @@ -2295,7 +2274,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { try { node = nodeRepository.getNode(zone, hostname); } catch (IllegalArgumentException e) { - throw new NotExistsException(new Hostname(hostname)); + throw new NotExistsException(hostname); } ApplicationId app = ApplicationId.from(tenant, application, instance); ApplicationId owner = node.owner().orElseThrow(() -> new IllegalArgumentException("Node has no owner")); @@ -2477,17 +2456,17 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .flatMap(application -> application.instances().values().stream()) .flatMap(instance -> instance.deployments().values().stream() .filter(deployment -> deployment.zone().environment() == Environment.dev) - .map(deployment -> lastDeploymentStart(instance.id(), deployment))) + .map(deployment -> controller.jobController().lastDeploymentStart(instance.id(), deployment))) .max(Comparator.naturalOrder()) .or(() -> applications.stream() .flatMap(application -> application.instances().values().stream()) - .flatMap(instance -> JobType.allIn(controller.system()).stream() + .flatMap(instance -> JobType.allIn(controller.zoneRegistry()).stream() .filter(job -> job.environment() == Environment.dev) .flatMap(jobType -> controller.jobController().last(instance.id(), jobType).stream())) .map(Run::start) .max(Comparator.naturalOrder())); Optional<Instant> lastSubmission = applications.stream() - .flatMap(app -> app.latestVersion().flatMap(ApplicationVersion::buildTime).stream()) + .flatMap(app -> app.revisions().last().flatMap(ApplicationVersion::buildTime).stream()) .max(Comparator.naturalOrder()); object.setLong("createdAtMillis", tenant.createdAt().toEpochMilli()); if (tenant.type() == Tenant.Type.deleted) @@ -2537,15 +2516,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } } - private void toSlime(Run run, Cursor object) { - object.setLong("id", run.id().number()); - object.setString("version", run.versions().targetPlatform().toFullString()); - if ( ! run.versions().targetApplication().isUnknown()) - JobControllerApiHandlerHelper.toSlime(object.setObject("revision"), run.versions().targetApplication()); - object.setString("reason", "unknown reason"); - object.setLong("at", run.end().orElse(run.start()).toEpochMilli()); - } - private Slime toSlime(InputStream jsonStream) { try { byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000); @@ -2713,11 +2683,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return ApplicationId.from(path.get("tenant"), path.get("application"), path.get("instance")); } - private static JobType jobTypeFromPath(Path path) { - return JobType.fromJobName(path.get("jobtype")); + private JobType jobTypeFromPath(Path path) { + return JobType.fromJobName(path.get("jobtype"), controller.zoneRegistry()); } - private static RunId runIdFromPath(Path path) { + private RunId runIdFromPath(Path path) { long number = Long.parseLong(path.get("number")); return new RunId(appIdFromPath(path), jobTypeFromPath(path), number); } @@ -2725,7 +2695,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse submit(String tenant, String application, HttpRequest request) { Map<String, byte[]> dataParts = parseDataParts(request); Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get(); - long projectId = Math.max(1, submitOptions.field("projectId").asLong()); // Absence of this means it's not a prod app :/ + long projectId = submitOptions.field("projectId").asLong(); // Absence of this means it's not a prod app :/ + projectId = projectId == 0 ? 1 : projectId; Optional<String> repository = optional("repository", submitOptions); Optional<String> branch = optional("branch", submitOptions); Optional<String> commit = optional("commit", submitOptions); @@ -2734,6 +2705,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { : Optional.empty(); Optional<String> sourceUrl = optional("sourceUrl", submitOptions); Optional<String> authorEmail = optional("authorEmail", submitOptions); + Optional<String> description = optional("description", submitOptions); + int risk = (int) submitOptions.field("risk").asLong(); sourceUrl.map(URI::create).ifPresent(url -> { if (url.getHost() == null || url.getScheme() == null) @@ -2742,29 +2715,26 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP), true); + byte[] testPackage = dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP); + Submission submission = new Submission(applicationPackage, testPackage, sourceUrl, sourceRevision, authorEmail, description, risk); + controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant), Optional.empty(), Optional.empty(), applicationPackage, Optional.of(requireUserPrincipal(request))); - ensureApplicationExists(TenantAndApplicationId.from(tenant, application), request); - - return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), - tenant, - application, - sourceRevision, - authorEmail, - sourceUrl, - projectId, - applicationPackage, - dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP)); + TenantAndApplicationId id = TenantAndApplicationId.from(tenant, application); + ensureApplicationExists(id, request); + return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), id, submission, projectId); } private HttpResponse removeAllProdDeployments(String tenant, String application) { - JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application, - Optional.empty(), Optional.empty(), Optional.empty(), 1, - ApplicationPackage.deploymentRemoval(), new byte[0]); + JobControllerApiHandlerHelper.submitResponse(controller.jobController(), + TenantAndApplicationId.from(tenant, application), + new Submission(ApplicationPackage.deploymentRemoval(), new byte[0], Optional.empty(), + Optional.empty(), Optional.empty(), Optional.empty(), 0), + 0); return new MessageResponse("All deployments removed"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 2375b5cf049..80425609aa6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.Change; @@ -33,6 +34,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -43,7 +45,6 @@ import java.time.format.TextStyle; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -77,29 +78,29 @@ class JobControllerApiHandlerHelper { Cursor responseObject = slime.setObject(); Cursor jobsArray = responseObject.setArray("deployment"); - Arrays.stream(JobType.values()) - .filter(type -> type.environment().isManuallyDeployed()) - .map(devType -> new JobId(id, devType)) - .forEach(job -> { - Collection<Run> runs = controller.jobController().runs(job).descendingMap().values(); - if (runs.isEmpty()) - return; - - Cursor jobObject = jobsArray.addObject(); - jobObject.setString("jobName", job.type().jobName()); - toSlime(jobObject.setArray("runs"), runs, 10, baseUriForJobs); - }); + JobType.allIn(controller.zoneRegistry()).stream() + .filter(type -> type.environment().isManuallyDeployed()) + .map(devType -> new JobId(id, devType)) + .forEach(job -> { + Collection<Run> runs = controller.jobController().runs(job).descendingMap().values(); + if (runs.isEmpty()) + return; + + Cursor jobObject = jobsArray.addObject(); + jobObject.setString("jobName", job.type().jobName()); + toSlime(jobObject.setArray("runs"), runs, controller.applications().requireApplication(TenantAndApplicationId.from(id)), 10, baseUriForJobs); + }); return new SlimeJsonResponse(slime); } /** Returns a response with the runs for the given job type. */ - static HttpResponse runResponse(Map<RunId, Run> runs, Optional<String> limitStr, URI baseUriForJobType) { + static HttpResponse runResponse(Application application, Map<RunId, Run> runs, Optional<String> limitStr, URI baseUriForJobType) { Slime slime = new Slime(); Cursor cursor = slime.setObject(); int limit = limitStr.map(Integer::parseInt).orElse(Integer.MAX_VALUE); - toSlime(cursor.setArray("runs"), runs.values(), limit, baseUriForJobType); + toSlime(cursor.setArray("runs"), runs.values(), application, limit, baseUriForJobType); return new SlimeJsonResponse(slime); } @@ -194,19 +195,8 @@ class JobControllerApiHandlerHelper { * * @return Response with the new application version */ - static HttpResponse submitResponse(JobController jobController, String tenant, String application, - Optional<SourceRevision> sourceRevision, Optional<String> authorEmail, - Optional<String> sourceUrl, long projectId, - ApplicationPackage applicationPackage, byte[] testPackage) { - ApplicationVersion version = jobController.submit(TenantAndApplicationId.from(tenant, application), - sourceRevision, - authorEmail, - sourceUrl, - projectId, - applicationPackage, - testPackage); - - return new MessageResponse(version.toString()); + static HttpResponse submitResponse(JobController jobController, TenantAndApplicationId id, Submission submission, long projectId) { + return new MessageResponse("application " + jobController.submit(id, submission, projectId)); } /** Aborts any job of the given type. */ @@ -230,6 +220,7 @@ class JobControllerApiHandlerHelper { case aborted: return "aborted"; case error: return "error"; case testFailure: return "testFailure"; + case noTests: return "noTests"; case endpointCertificateTimeout: return "endpointCertificateTimeout"; case nodeAllocationFailure: return "nodeAllocationFailure"; case installationFailed: return "installationFailed"; @@ -276,7 +267,7 @@ class JobControllerApiHandlerHelper { stepStatus.coolingDownUntil(change).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli())); stepStatus.blockedUntil(Change.of(controller.systemVersion(versionStatus))) // Dummy version — just anything with a platform. .ifPresent(until -> stepObject.setLong("platformBlockedUntil", until.toEpochMilli())); - application.latestVersion().map(Change::of).flatMap(stepStatus::blockedUntil) // Dummy version — just anything with an application. + stepStatus.blockedUntil(Change.of(RevisionId.forProduction(1))) // Dummy version — just anything with an application. .ifPresent(until -> stepObject.setLong("applicationBlockedUntil", until.toEpochMilli())); if (stepStatus.type() == DeploymentStatus.StepType.delay) @@ -286,7 +277,7 @@ class JobControllerApiHandlerHelper { Cursor deployingObject = stepObject.setObject("deploying"); if ( ! change.isEmpty()) { change.platform().ifPresent(version -> deployingObject.setString("platform", version.toFullString())); - change.application().ifPresent(version -> toSlime(deployingObject.setObject("application"), version)); + change.revision().ifPresent(revision -> toSlime(deployingObject.setObject("application"), application.revisions().get(revision))); } Cursor latestVersionsObject = stepObject.setObject("latestVersions"); @@ -303,38 +294,33 @@ class JobControllerApiHandlerHelper { || deployments.stream().anyMatch(deployment -> deployment.version().isBefore(latestPlatform.versionNumber()))); Cursor availableArray = latestPlatformObject.setArray("available"); + boolean isUpgrade = true; for (VespaVersion available : availablePlatforms) { if ( deployments.stream().anyMatch(deployment -> deployment.version().isAfter(available.versionNumber())) || deployments.stream().noneMatch(deployment -> deployment.version().isBefore(available.versionNumber())) && ! deployments.isEmpty() || status.hasCompleted(stepStatus.instance(), Change.of(available.versionNumber())) - || change.platform().map(available.versionNumber()::compareTo).orElse(1) <= 0) - break; + || change.platform().map(available.versionNumber()::compareTo).orElse(1) < 0) + isUpgrade = false; - availableArray.addObject().setString("platform", available.versionNumber().toFullString()); + Cursor platformObject = availableArray.addObject(); + platformObject.setString("platform", available.versionNumber().toFullString()); + platformObject.setBool("upgrade", isUpgrade || change.platform().map(available.versionNumber()::equals).orElse(false)); } - change.platform().ifPresent(version -> availableArray.addObject().setString("platform", version.toFullString())); toSlime(latestPlatformObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksVersions)); } - List<ApplicationVersion> availableApplications = new ArrayList<>(application.deployableVersions(false)); + List<ApplicationVersion> availableApplications = new ArrayList<>(application.revisions().deployable(false)); if ( ! availableApplications.isEmpty()) { var latestApplication = availableApplications.get(0); Cursor latestApplicationObject = latestVersionsObject.setObject("application"); toSlime(latestApplicationObject.setObject("application"), latestApplication); latestApplicationObject.setLong("at", latestApplication.buildTime().orElse(Instant.EPOCH).toEpochMilli()); - latestApplicationObject.setBool("upgrade", change.application().map(latestApplication::compareTo).orElse(1) > 0 && deployments.isEmpty() - || deployments.stream().anyMatch(deployment -> deployment.applicationVersion().compareTo(latestApplication) < 0)); + latestApplicationObject.setBool("upgrade", change.revision().map(latestApplication.id()::compareTo).orElse(1) > 0 && deployments.isEmpty() + || deployments.stream().anyMatch(deployment -> deployment.revision().compareTo(latestApplication.id()) < 0)); Cursor availableArray = latestApplicationObject.setArray("available"); - for (ApplicationVersion available : availableApplications) { - if ( deployments.stream().anyMatch(deployment -> deployment.applicationVersion().compareTo(available) > 0) - || deployments.stream().noneMatch(deployment -> deployment.applicationVersion().compareTo(available) < 0) && ! deployments.isEmpty() - || status.hasCompleted(stepStatus.instance(), Change.of(available)) - || change.application().map(available::compareTo).orElse(1) <= 0) - break; - + for (ApplicationVersion available : availableApplications) toSlime(availableArray.addObject().setObject("application"), available); - } - change.application().ifPresent(version -> toSlime(availableArray.addObject().setObject("application"), version)); + toSlime(latestApplicationObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksRevisions)); } } @@ -346,12 +332,12 @@ class JobControllerApiHandlerHelper { "/job/" + job.type().jobName()).normalize(); stepObject.setString("url", baseUriForJob.toString()); stepObject.setString("environment", job.type().environment().value()); - stepObject.setString("region", job.type().zone(controller.system()).value()); + stepObject.setString("region", job.type().zone().value()); if (job.type().isProduction() && job.type().isDeployment()) { status.deploymentFor(job).ifPresent(deployment -> { stepObject.setString("currentPlatform", deployment.version().toFullString()); - toSlime(stepObject.setObject("currentApplication"), deployment.applicationVersion()); + toSlime(stepObject.setObject("currentApplication"), application.revisions().get(deployment.revision())); }); } @@ -367,19 +353,26 @@ class JobControllerApiHandlerHelper { continue; // Run will be contained in the "runs" array. Cursor runObject = toRunArray.addObject(); - toSlime(runObject.setObject("versions"), versions.versions()); + toSlime(runObject.setObject("versions"), versions.versions(), application); } - toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), 10, baseUriForJob); + toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), application, 10, baseUriForJob); }); } Cursor buildsArray = responseObject.setArray("builds"); - application.versions().stream().sorted(reverseOrder()).forEach(version -> toSlime(buildsArray.addObject(), version)); + application.revisions().withPackage().stream().sorted(reverseOrder()).forEach(version -> toRichSlime(buildsArray.addObject(), version)); return new SlimeJsonResponse(slime); } + static void toRichSlime(Cursor versionObject, ApplicationVersion version) { + toSlime(versionObject, version); + version.description().ifPresent(description -> versionObject.setString("description", description)); + if (version.risk() != 0) versionObject.setLong("risk", version.risk()); + versionObject.setBool("deployable", version.isDeployable()); + } + static void toSlime(Cursor versionObject, ApplicationVersion version) { version.buildNumber().ifPresent(id -> versionObject.setLong("build", id)); version.compileVersion().ifPresent(platform -> versionObject.setString("compileVersion", platform.toFullString())); @@ -387,11 +380,11 @@ class JobControllerApiHandlerHelper { version.commit().ifPresent(commit -> versionObject.setString("commit", commit)); } - private static void toSlime(Cursor versionsObject, Versions versions) { + private static void toSlime(Cursor versionsObject, Versions versions, Application application) { versionsObject.setString("targetPlatform", versions.targetPlatform().toFullString()); - toSlime(versionsObject.setObject("targetApplication"), versions.targetApplication()); + toSlime(versionsObject.setObject("targetApplication"), application.revisions().get(versions.targetRevision())); versions.sourcePlatform().ifPresent(platform -> versionsObject.setString("sourcePlatform", platform.toFullString())); - versions.sourceApplication().ifPresent(application -> toSlime(versionsObject.setObject("sourceApplication"), application)); + versions.sourceRevision().ifPresent(revision -> toSlime(versionsObject.setObject("sourceApplication"), application.revisions().get(revision))); } private static void toSlime(Cursor blockersArray, Stream<ChangeBlocker> blockers) { @@ -427,7 +420,7 @@ class JobControllerApiHandlerHelper { return candidates; } - private static void toSlime(Cursor runsArray, Collection<Run> runs, int limit, URI baseUriForJob) { + private static void toSlime(Cursor runsArray, Collection<Run> runs, Application application, int limit, URI baseUriForJob) { runs.stream().limit(limit).forEach(run -> { Cursor runObject = runsArray.addObject(); runObject.setLong("id", run.id().number()); @@ -435,7 +428,7 @@ class JobControllerApiHandlerHelper { runObject.setLong("start", run.start().toEpochMilli()); run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli())); runObject.setString("status", run.status().name()); - toSlime(runObject.setObject("versions"), run.versions()); + toSlime(runObject.setObject("versions"), run.versions(), application); Cursor runStepsArray = runObject.setArray("steps"); run.steps().forEach((step, info) -> { Cursor runStepObject = runStepsArray.addObject(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java deleted file mode 100644 index 67b47aa976a..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.container.jdisc.HttpResponse; -import ai.vespa.http.HttpURL; -import ai.vespa.http.HttpURL.Path; -import ai.vespa.http.HttpURL.Query; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; -import com.yahoo.vespa.serviceview.bindings.ClusterView; -import com.yahoo.vespa.serviceview.bindings.ServiceView; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * A response containing a service view for an application deployment. - * This does not define the API response but merely proxies the API response provided by Vespa, with URLs - * rewritten to include zone and application information allow proxying through the controller - * - * @author Steinar Knutsen - * @author bratseth - */ -class ServiceApiResponse extends HttpResponse { - - private final ZoneId zone; - private final ApplicationId application; - private final List<URI> configServerURIs; - private final Slime slime; - private final HttpURL requestUri; - - // Only set for one of the setResponse calls - private String serviceName = null; - private Path restPath = null; - - public ServiceApiResponse(ZoneId zone, ApplicationId application, List<URI> configServerURIs, URI requestUri) { - super(200); - this.zone = zone; - this.application = application; - this.configServerURIs = configServerURIs; - this.slime = new Slime(); - this.requestUri = HttpURL.from(requestUri).withQuery(Query.empty()); - } - - public void setResponse(ApplicationView applicationView) { - Cursor clustersArray = slime.setObject().setArray("clusters"); - for (ClusterView clusterView : applicationView.clusters) { - Cursor clusterObject = clustersArray.addObject(); - clusterObject.setString("name", clusterView.name); - clusterObject.setString("type", clusterView.type); - setNullableString("url", rewriteIfUrl(clusterView.url, requestUri), clusterObject); - Cursor servicesArray = clusterObject.setArray("services"); - for (ServiceView serviceView : clusterView.services) { - Cursor serviceObject = servicesArray.addObject(); - setNullableString("url", rewriteIfUrl(serviceView.url, requestUri), serviceObject); - serviceObject.setString("serviceType", serviceView.serviceType); - serviceObject.setString("serviceName", serviceView.serviceName); - serviceObject.setString("configId", serviceView.configId); - serviceObject.setString("host", serviceView.host); - } - } - } - - public void setResponse(Map<?,?> responseData, String serviceName, Path restPath) { - this.serviceName = serviceName; - this.restPath = restPath; - mapToSlime(responseData, slime.setObject()); - } - - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { - return "application/json"; - } - - @SuppressWarnings("unchecked") - private void mapToSlime(Map<?,?> data, Cursor object) { - for (Map.Entry<String, Object> entry : ((Map<String, Object>)data).entrySet()) - fieldToSlime(entry.getKey(), entry.getValue(), object); - } - - private void fieldToSlime(String key, Object value, Cursor object) { - if (value instanceof String) { - if (key.equals("url") || key.equals("link")) - value = rewriteIfUrl((String)value, generateLocalLinkPrefix(serviceName, restPath)); - setNullableString(key, (String)value, object); - } - else if (value instanceof Integer) { - object.setLong(key, (int)value); - } - else if (value instanceof Long) { - object.setLong(key, (long)value); - } - else if (value instanceof Float) { - object.setDouble(key, (double)value); - } - else if (value instanceof Double) { - object.setDouble(key, (double)value); - } - else if (value instanceof List) { - listToSlime((List)value, object.setArray(key)); - } - else if (value instanceof Map) { - mapToSlime((Map<?,?>)value, object.setObject(key)); - } - } - - private void listToSlime(List<?> list, Cursor array) { - for (Object entry : list) - entryToSlime(entry, array); - } - - private void entryToSlime(Object entry, Cursor array) { - if (entry instanceof String) - addNullableString(rewriteIfUrl((String)entry, generateLocalLinkPrefix(serviceName, restPath)), array); - else if (entry instanceof Integer) - array.addLong((long)entry); - else if (entry instanceof Long) - array.addLong((long)entry); - else if (entry instanceof Float) - array.addDouble((double)entry); - else if (entry instanceof Double) - array.addDouble((double)entry); - else if (entry instanceof List) - listToSlime((List)entry, array.addArray()); - else if (entry instanceof Map) - mapToSlime((Map)entry, array.addObject()); - } - - private String rewriteIfUrl(String urlOrAnyString, HttpURL requestUri) { - if (urlOrAnyString == null) return null; - - String hostPattern = "(" + - String.join( - "|", configServerURIs.stream() - .map(URI::toString) - .map(s -> s.substring(0, s.length() -1)) - .map(Pattern::quote) - .toArray(String[]::new)) - + ")"; - - String remoteServicePath = "/serviceview/" - + "v1/tenant/" + application.tenant().value() - + "/application/" + application.application().value() - + "/environment/" + zone.environment().value() - + "/region/" + zone.region().value() - + "/instance/" + application.instance() - + "/service/"; - - Pattern remoteServiceResourcePattern = Pattern.compile("^(" + hostPattern + Pattern.quote(remoteServicePath) + ")"); - Matcher matcher = remoteServiceResourcePattern.matcher(urlOrAnyString); - - if (matcher.find()) { - String proxiedPath = urlOrAnyString.substring(matcher.group().length()); - return requestUri.withPath(requestUri.path().append(Path.parse(proxiedPath))).asURI().toString(); - } else { - return urlOrAnyString; // not a service url - } - } - - private HttpURL generateLocalLinkPrefix(String identifier, Path restPath) { - Path proxiedPath = Path.parse(identifier).append(restPath); - if (requestUri.path().tail(proxiedPath.length()).equals(proxiedPath)) { - return requestUri.withPath(requestUri.path().cut(proxiedPath.length())); - } else { - throw new IllegalStateException("Expected the resource " + requestUri.path() + " to end with " + proxiedPath); - } - } - - private void setNullableString(String key, String valueOrNull, Cursor receivingObject) { - if (valueOrNull == null) - receivingObject.setNix(key); - else - receivingObject.setString(key, valueOrNull); - } - - private void addNullableString(String valueOrNull, Cursor receivingArray) { - if (valueOrNull == null) - receivingArray.addNix(); - else - receivingArray.addString(valueOrNull); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index 45c4978ab9f..33307ea1677 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -19,10 +19,10 @@ import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TenantController; import com.yahoo.vespa.hosted.controller.api.integration.billing.Bill; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.CollectionMethod; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PaymentInstrument; import com.yahoo.vespa.hosted.controller.api.integration.billing.InstrumentOwner; -import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PaymentInstrument; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index a5aca4adf10..0f3e5b7f76b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -28,15 +28,12 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import javax.ws.rs.BadRequestException; import java.math.BigDecimal; import java.time.Clock; -import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; import java.util.Comparator; import java.util.List; import java.util.Optional; -import java.util.logging.Level; /** * @author ogronnesby @@ -182,7 +179,6 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler var tenant = tenants.require(tenantName, CloudTenant.class); var untilAt = untilParameter(requestContext); var usage = billing.createUncommittedBill(tenant.name(), untilAt); - var slime = new Slime(); usageToSlime(slime.setObject(), usage); return slime; @@ -322,7 +318,8 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler private LocalDate untilParameter(RestApi.RequestContext ctx) { return ctx.queryParameters().getString("until") - .map(this::parseLocalDate) + .map(LocalDate::parse) + .map(date -> date.plusDays(1)) .orElseGet(this::tomorrow); } @@ -330,12 +327,6 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return LocalDate.now(clock).plusDays(1); } - private LocalDate parseLocalDate(String until) { - if (until.isEmpty() || until.isBlank()) - return tomorrow(); - else return LocalDate.parse(until); - } - private static String getInspectorFieldOrThrow(Inspector inspector, String field) { if (!inspector.field(field).valid()) throw new BadRequestException("Field " + field + " cannot be null"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java index e1fc68974cc..c72d8ceb089 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java @@ -280,7 +280,7 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { private Optional<ZoneId> affectedZone(List<String> hosts) { NodeFilter affectedHosts = NodeFilter.all().hostnames(hosts.stream() - .map(HostName::from) + .map(HostName::of) .collect(Collectors.toSet())); for (var zone : getProdZones()) { var affectedHostsInZone = controller.serviceRegistry().configServer().nodeRepository().list(zone, affectedHosts); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java index 8caa741d737..467b0c094cc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.configserver; +import ai.vespa.http.HttpURL; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.ErrorResponse; -import ai.vespa.http.HttpURL; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java index 7e250ce62ab..c3155406194 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.controller; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; -import com.yahoo.restapi.SlimeJsonResponse; /** * @author mpolden diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java index e68517f7134..122ea94ab6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.controller; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; -import com.yahoo.restapi.SlimeJsonResponse; /** * @author mpolden diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java index eb74f931b2c..9b400fdfb78 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java @@ -81,16 +81,14 @@ public class BadgeApiHandler extends ThreadedHttpRequestHandler { () -> { DeploymentStatus status = controller.jobController().deploymentStatus(controller.applications().requireApplication(TenantAndApplicationId.from(id))); Predicate<JobStatus> isDeclaredJob = job -> status.jobSteps().get(job.id()) != null && status.jobSteps().get(job.id()).isDeclared(); - return Badges.overviewBadge(id, - status.jobs().instance(id.instance()).matching(isDeclaredJob), - controller.system()); + return Badges.overviewBadge(id, status.jobs().instance(id.instance()).matching(isDeclaredJob)); }); } /** Returns a URI which points to a history badge for the given application and job type. */ private HttpResponse historyBadge(String tenant, String application, String instance, String jobName, String historyLength) { ApplicationId id = ApplicationId.from(tenant, application, instance); - JobType type = JobType.fromJobName(jobName); + JobType type = JobType.fromJobName(jobName, controller.zoneRegistry()); int length = historyLength == null ? 5 : Math.min(32, Math.max(0, Integer.parseInt(historyLength))); return cachedResponse(new Key(id, type, length), controller.clock().instant(), @@ -135,7 +133,7 @@ public class BadgeApiHandler extends ThreadedHttpRequestHandler { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Key key = (Key) o; - return historyLength == key.historyLength && id.equals(key.id) && type == key.type; + return historyLength == key.historyLength && id.equals(key.id) && Objects.equals(type, key.type); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java index 1fe5ebfa9a9..26a5da45bdb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import static java.util.stream.Collectors.toList; @@ -51,14 +52,18 @@ public class Badges { return widthOf(text, 11); } - static String colorOf(Run run, Boolean wasOk) { + static String colorOf(Run run, Optional<RunStatus> previous) { switch (run.status()) { - case running: - return wasOk ? "url(#run-on-success)" : "url(#run-on-failure)"; - case success: - return success; - default: - return failure; + case running: switch (previous.orElse(RunStatus.success)) { + case success: return "url(#run-on-success)"; + case aborted: + case noTests: return "url(#run-on-warning)"; + default: return "url(#run-on-failure)"; + } + case success: return success; + case aborted: + case noTests: return warning; + default: return failure; } } @@ -71,9 +76,10 @@ public class Badges { static final double xPad = 6; static final double logoSize = 16; static final String dark = "#404040"; - static final String success = "#00f244"; + static final String success = "#00f844"; static final String running = "#ab83ff"; static final String failure = "#bf103c"; + static final String warning = "#bd890b"; static void addText(List<String> texts, String text, double x, double width) { addText(texts, text, x, width, 11); @@ -116,13 +122,11 @@ public class Badges { .limit(length) .collect(toList()); - boolean isOk = status.lastCompleted().map(run -> run.status() == RunStatus.success).orElse(true); - text = lastTriggered.id().type().jobName(); textWidth = widthOf(text); dx = xPad + textWidth + xPad; addShade(sections, x, dx); - sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(lastTriggered, isOk) + "'/>\n"); + sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(lastTriggered, status.lastStatus()) + "'/>\n"); addShadow(sections, x + dx); addText(texts, text, x + dx / 2, textWidth); x += dx; @@ -130,7 +134,7 @@ public class Badges { dx = xPad * (192.0 / (32 + runs.size())); // Broader sections with shorter history. for (Run run : runs) { addShade(sections, x, dx); - sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(run, null) + "'/>\n"); + sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(run, Optional.empty()) + "'/>\n"); addShadow(sections, x + dx); dx *= Math.pow(0.3, 1.0 / (runs.size() + 8)); // Gradually narrowing sections with age. x += dx; @@ -140,7 +144,7 @@ public class Badges { return badge(sections, texts, x); } - static String overviewBadge(ApplicationId id, JobList jobs, SystemName system) { + static String overviewBadge(ApplicationId id, JobList jobs) { // Put production tests right after their deployments, for a more compact rendering. List<Run> runs = new ArrayList<>(jobs.lastTriggered().asList()); boolean anyTest = false; @@ -149,7 +153,7 @@ public class Badges { if (run.id().type().isProduction() && run.id().type().isTest()) { anyTest = true; int j = i; - while ( ! runs.get(j - 1).id().type().zone(system).equals(run.id().type().zone(system))) + while ( ! runs.get(j - 1).id().type().zone().equals(run.id().type().zone())) runs.set(j, runs.get(--j)); runs.set(j, run); } @@ -179,7 +183,7 @@ public class Badges { text = nameOf(run.id().type()); textWidth = widthOf(text, isTest ? 9 : 11); dx = xPad + textWidth + (isTest ? 0 : xPad); - boolean wasOk = jobs.get(run.id().job()).flatMap(JobStatus::lastStatus).map(RunStatus.success::equals).orElse(true); + Optional<RunStatus> previous = jobs.get(run.id().job()).flatMap(JobStatus::lastStatus); addText(texts, text, x + (dx - (isTest ? xPad : 0)) / 2, textWidth, isTest ? 9 : 11); @@ -197,10 +201,10 @@ public class Badges { // Add colored section for job ... if (test == null) - sections.add(" <rect x='" + (x - 16) + "' rx='3' width='" + (dx + 16) + "' height='20' fill='" + colorOf(run, wasOk) + "'/>\n"); + sections.add(" <rect x='" + (x - 16) + "' rx='3' width='" + (dx + 16) + "' height='20' fill='" + colorOf(run, previous) + "'/>\n"); // ... with a slant if a test is next. else - sections.add(" <polygon points='" + (x - 6) + " 0 " + (x - 6) + " 20 " + (x + dx - 7) + " 20 " + (x + dx + 1) + " 0' fill='" + colorOf(run, wasOk) + "'/>\n"); + sections.add(" <polygon points='" + (x - 6) + " 0 " + (x - 6) + " 20 " + (x + dx - 7) + " 20 " + (x + dx + 1) + " 0' fill='" + colorOf(run, previous) + "'/>\n"); // Cast a shadow onto the next zone ... if (test == null) @@ -255,6 +259,13 @@ public class Badges { " <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />\n" + " <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />\n" + " </linearGradient>\n" + + // Running color sloshing back and forth on top of the warning color. + " <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'>\n" + + " <stop offset='0' stop-color='" + running + "' />\n" + + " <stop offset='1' stop-color='" + warning + "' />\n" + + " <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />\n" + + " <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />\n" + + " </linearGradient>\n" + // Running color sloshing back and forth on top of the success color. " <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'>\n" + " <stop offset='0' stop-color='" + running + "' />\n" + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 1a4a42cb521..effa0906b94 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -210,7 +210,7 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { }); }); } - JobType.allIn(controller.system()).stream() + JobType.allIn(controller.zoneRegistry()).stream() .filter(job -> ! job.environment().isManuallyDeployed()) .map(JobType::jobName).forEach(root.setArray("jobs")::addString); return new SlimeJsonResponse(slime); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index a7472ced09c..01bd02fdc13 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.TenantController; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; @@ -62,6 +63,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { private final TenantController tenants; private final ExecutorService executor; private final SystemName systemName; + private final ZoneRegistry zones; @Inject public AthenzRoleFilter(AthenzClientFactory athenzClientFactory, Controller controller) { @@ -69,6 +71,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { this.tenants = controller.tenants(); this.executor = Executors.newCachedThreadPool(); this.systemName = controller.system(); + this.zones = controller.zoneRegistry(); } @Override @@ -108,8 +111,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/{*}")) { zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobname}")) { - var jobtype= JobType.fromJobName(path.get("jobname")); - zone = Optional.of(jobtype.zone(systemName)); + zone = Optional.of(JobType.fromJobName(path.get("jobname"), zones).zone()); } else { zone = Optional.empty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index 08ec3caa829..7c695ef51d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -6,7 +6,6 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.role.Action; import com.yahoo.vespa.hosted.controller.api.role.Enforcer; @@ -15,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java index 985919581ef..b7c77e7bfb4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import java.util.logging.Level; import com.yahoo.security.KeyUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -24,6 +23,7 @@ import java.security.PublicKey; import java.util.Base64; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import static java.nio.charset.StandardCharsets.UTF_8; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 33e6632b8e1..025e8dff659 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.restapi.user; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -28,7 +27,7 @@ import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; -import com.yahoo.vespa.hosted.controller.api.integration.user.User; +import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; import com.yahoo.vespa.hosted.controller.api.role.Role; @@ -172,7 +171,6 @@ public class UserApiHandler extends ThreadedHttpRequestHandler { toSlime(root.setObject("user"), user); Cursor tenants = root.setObject("tenants"); - InstanceName userInstance = InstanceName.from(user.nickname()); tenantRolesByTenantName.keySet().stream() .sorted() .forEach(tenant -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index 5b165a9ef37..320321080c8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -7,13 +7,13 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.Path; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.yolean.Exceptions; import java.util.Comparator; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 800588fdf8c..e3f1e9b5f94 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.zone.v2; +import ai.vespa.http.HttpURL; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; -import ai.vespa.http.HttpURL; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index b98ef717dd3..a5a09ab6551 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; +import ai.vespa.http.DomainName; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -152,7 +153,7 @@ public class RoutingPolicies { } /** Update global DNS records for given policies */ - private void updateGlobalDnsOf(RoutingPolicyList instancePolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + private void updateGlobalDnsOf(RoutingPolicyList instancePolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Mutex lock) { Map<RoutingId, List<RoutingPolicy>> routingTable = instancePolicies.asInstanceRoutingTable(); for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { RoutingId routingId = routeEntry.getKey(); @@ -215,7 +216,7 @@ public class RoutingPolicies { } var weightedTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone(), weight); - endpoints.computeIfAbsent(regionEndpoint, (k) -> new RegionEndpoint(new LatencyAliasTarget(HostName.from(regionEndpoint.dnsName()), + endpoints.computeIfAbsent(regionEndpoint, (k) -> new RegionEndpoint(new LatencyAliasTarget(DomainName.of(regionEndpoint.dnsName()), policy.dnsZone().get(), policy.id().zone()))) .zoneTargets() @@ -225,7 +226,7 @@ public class RoutingPolicies { } - private void updateApplicationDnsOf(RoutingPolicyList routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + private void updateApplicationDnsOf(RoutingPolicyList routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Mutex lock) { // In the context of single deployment (which this is) there is only one routing policy per routing ID. I.e. // there is no scenario where more than one deployment within an instance can be a member the same // application-level endpoint. However, to allow this in the future the routing table remains @@ -307,7 +308,7 @@ public class RoutingPolicies { * * @return the updated policies */ - private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Lock lock) { + private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) { Map<RoutingPolicyId, RoutingPolicy> policies = new LinkedHashMap<>(instancePolicies.asMap()); for (LoadBalancer loadBalancer : allocation.loadBalancers) { if (loadBalancer.hostname().isEmpty()) continue; @@ -343,7 +344,7 @@ public class RoutingPolicies { * * @return the updated policies */ - private RoutingPolicyList removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Lock lock) { + private RoutingPolicyList removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) { Map<RoutingPolicyId, RoutingPolicy> newPolicies = new LinkedHashMap<>(instancePolicies.asMap()); Set<RoutingPolicyId> activeIds = allocation.asPolicyIds(); RoutingPolicyList removable = instancePolicies.deployment(allocation.deployment) @@ -363,7 +364,7 @@ public class RoutingPolicies { } /** Remove unreferenced instance endpoints from DNS */ - private void removeGlobalDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Lock lock) { + private void removeGlobalDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Mutex lock) { Set<RoutingId> removalCandidates = new HashSet<>(deploymentPolicies.asInstanceRoutingTable().keySet()); Set<RoutingId> activeRoutingIds = instanceRoutingIds(allocation); removalCandidates.removeAll(activeRoutingIds); @@ -380,7 +381,7 @@ public class RoutingPolicies { } /** Remove unreferenced application endpoints in given allocation from DNS */ - private void removeApplicationDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Lock lock) { + private void removeApplicationDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Mutex lock) { Map<RoutingId, List<RoutingPolicy>> routingTable = deploymentPolicies.asApplicationRoutingTable(); Set<RoutingId> removalCandidates = new HashSet<>(routingTable.keySet()); Set<RoutingId> activeRoutingIds = applicationRoutingIds(allocation); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 34736c16a6b..1ccb3205816 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; +import ai.vespa.http.DomainName; import com.google.common.collect.ImmutableSortedSet; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.text.Text; @@ -27,14 +27,14 @@ import java.util.Set; public class RoutingPolicy { private final RoutingPolicyId id; - private final HostName canonicalName; + private final DomainName canonicalName; private final Optional<String> dnsZone; private final Set<EndpointId> instanceEndpoints; private final Set<EndpointId> applicationEndpoints; private final Status status; /** DO NOT USE. Public for serialization purposes */ - public RoutingPolicy(RoutingPolicyId id, HostName canonicalName, Optional<String> dnsZone, + public RoutingPolicy(RoutingPolicyId id, DomainName canonicalName, Optional<String> dnsZone, Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, Status status) { this.id = Objects.requireNonNull(id, "id must be non-null"); this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); @@ -50,7 +50,7 @@ public class RoutingPolicy { } /** The canonical name for the load balancer this applies to (rhs of a CNAME or ALIAS record) */ - public HostName canonicalName() { + public DomainName canonicalName() { return canonicalName; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java index 0cf7101cac0..8eeab8c20e3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.routing.rotation; import com.yahoo.text.Text; + import java.util.Objects; /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java index 36a43f80e9a..39fc70aac64 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing.rotation; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import java.util.Objects; @@ -12,9 +13,9 @@ import java.util.Objects; */ public class RotationLock implements AutoCloseable { - private final Lock lock; + private final Mutex lock; - RotationLock(Lock lock) { + RotationLock(Mutex lock) { this.lock = Objects.requireNonNull(lock, "lock cannot be null"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java index a4af9f8e268..27b61a4fd17 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java @@ -1,13 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.support.access; -import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import java.security.Principal; import java.security.cert.X509Certificate; import java.time.Instant; import java.time.Period; @@ -37,7 +36,7 @@ public class SupportAccessControl { } public SupportAccess disallow(DeploymentId deployment, String by) { - try (Lock lock = controller.curator().lockSupportAccess(deployment)) { + try (Mutex lock = controller.curator().lockSupportAccess(deployment)) { var now = controller.clock().instant(); SupportAccess supportAccess = forDeployment(deployment); if (supportAccess.currentStatus(now).state() == NOT_ALLOWED) { @@ -51,7 +50,7 @@ public class SupportAccessControl { } public SupportAccess allow(DeploymentId deployment, Instant until, String by) { - try (Lock lock = controller.curator().lockSupportAccess(deployment)) { + try (Mutex lock = controller.curator().lockSupportAccess(deployment)) { var now = controller.clock().instant(); if (until.isAfter(now.plus(MAX_SUPPORT_ACCESS_TIME))) { throw new IllegalArgumentException("Support access cannot be allowed for more than 10 days"); @@ -63,7 +62,7 @@ public class SupportAccessControl { } public SupportAccess registerGrant(DeploymentId deployment, String by, X509Certificate certificate) { - try (Lock lock = controller.curator().lockSupportAccess(deployment)) { + try (Mutex lock = controller.curator().lockSupportAccess(deployment)) { var now = controller.clock().instant(); SupportAccess supportAccess = forDeployment(deployment); if (certificate.getNotAfter().toInstant().isBefore(now)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/ControllerVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/ControllerVersion.java deleted file mode 100644 index 5b293081dc2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/ControllerVersion.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.component.Version; -import com.yahoo.component.Vtag; - -import java.time.Instant; -import java.util.Objects; - -/** - * A controller's Vespa version and commit details. - * - * @author mpolden - */ -public class ControllerVersion implements Comparable<ControllerVersion> { - - /** The current version of this controller */ - public static final ControllerVersion CURRENT = new ControllerVersion(Vtag.currentVersion, Vtag.commitSha, - Vtag.commitDate); - - private final Version version; - private final String commitSha; - private final Instant commitDate; - - public ControllerVersion(Version version, String commitSha, Instant commitDate) { - this.version = Objects.requireNonNull(version); - this.commitSha = Objects.requireNonNull(commitSha); - this.commitDate = Objects.requireNonNull(commitDate); - } - - /** Vespa version */ - public Version version() { - return version; - } - - /** Commit SHA of this */ - public String commitSha() { - return commitSha; - } - - /** The time this was committed */ - public Instant commitDate() { - return commitDate; - } - - @Override - public String toString() { - return version + ", commit " + commitSha + " @ " + commitDate; - } - - @Override - public int compareTo(ControllerVersion o) { - return version.compareTo(o.version); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ControllerVersion that = (ControllerVersion) o; - return version.equals(that.version); - } - - @Override - public int hashCode() { - return Objects.hash(version); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java index c1abe38a2a9..179a64d9491 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import com.yahoo.vespa.hosted.controller.deployment.JobList; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import java.util.ArrayList; import java.util.Collection; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 3fa440e694e..117abd52193 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -197,7 +198,7 @@ public class VersionStatus { .add(controller.hostname()); } else { for (String host : controller.curator().cluster()) { - HostName hostname = HostName.from(host); + HostName hostname = HostName.of(host); versions.computeIfAbsent(controller.curator().readControllerVersion(hostname), (k) -> new ArrayList<>()) .add(hostname); } |