diff options
author | Øyvind Grønnesby <oyving@yahooinc.com> | 2022-04-27 15:22:17 +0200 |
---|---|---|
committer | Øyvind Grønnesby <oyving@yahooinc.com> | 2022-04-27 15:22:17 +0200 |
commit | c11fe459990970522286a8fad77e77caf2316ba7 (patch) | |
tree | 6caa5280473758e26384b93e2a0a3de85de67bb7 /controller-server | |
parent | 4994eb64560b266edd052f4852706dde0630ee3c (diff) | |
parent | d1be606eb7b3cc5a77ec171890cf002be8c7ba88 (diff) |
Merge remote-tracking branch 'origin/master' into ogronnesby/billing-v2-fixes
Conflicts:
controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java
Diffstat (limited to 'controller-server')
275 files changed, 4318 insertions, 3505 deletions
diff --git a/controller-server/pom.xml b/controller-server/pom.xml index ca8951124ef..5cf53929a98 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -56,13 +56,6 @@ <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>serviceview</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -246,17 +239,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <!-- Illegal reflective access by LogFileHandler via com.yahoo.io.NativeIO --> - <argLine> - --add-opens=java.base/java.io=ALL-UNNAMED - </argLine> - </configuration> - </plugin> </plugins> </build> </project> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 30f416747e0..af8965bdeff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -7,23 +7,22 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.ApplicationActivity; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.security.PublicKey; import java.time.Instant; -import java.util.ArrayDeque; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.Deque; import java.util.List; import java.util.Map; import java.util.Objects; @@ -31,9 +30,7 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.SortedSet; import java.util.TreeMap; -import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Collectors; @@ -50,8 +47,7 @@ public class Application { private final Instant createdAt; private final DeploymentSpec deploymentSpec; private final ValidationOverrides validationOverrides; - private final Optional<ApplicationVersion> latestVersion; - private final SortedSet<ApplicationVersion> versions; + private final RevisionHistory revisions; private final OptionalLong projectId; private final Optional<IssueId> deploymentIssueId; private final Optional<IssueId> ownershipIssueId; @@ -63,16 +59,16 @@ public class Application { /** Creates an empty application. */ public Application(TenantAndApplicationId id, Instant now) { - this(id, now, DeploymentSpec.empty, ValidationOverrides.empty, - Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), - new ApplicationMetrics(0, 0), Set.of(), OptionalLong.empty(), Optional.empty(), new TreeSet<>(), List.of()); + this(id, now, DeploymentSpec.empty, ValidationOverrides.empty, Optional.empty(), + Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(0, 0), + Set.of(), OptionalLong.empty(), RevisionHistory.empty(), List.of()); } // DO NOT USE! For serialization purposes, only. public Application(TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner, OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, OptionalLong projectId, - Optional<ApplicationVersion> latestVersion, SortedSet<ApplicationVersion> versions, Collection<Instance> instances) { + RevisionHistory revisions, Collection<Instance> instances) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null"); this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); @@ -84,13 +80,12 @@ public class Application { this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null"); this.deployKeys = Objects.requireNonNull(deployKeys, "deployKeys cannot be null"); this.projectId = Objects.requireNonNull(projectId, "projectId cannot be null"); - this.latestVersion = requireNotUnknown(latestVersion); - this.versions = versions; + this.revisions = revisions; this.instances = instances.stream().collect( Collectors.collectingAndThen(Collectors.toMap(Instance::name, Function.identity(), (i1, i2) -> { - throw new IllegalArgumentException("Duplicate key " + i1); + throw new IllegalArgumentException("Duplicate instance " + i1.id()); }, TreeMap::new), Collections::unmodifiableMap) @@ -110,29 +105,8 @@ public class Application { /** Returns the project id of this application, if it has any. */ public OptionalLong projectId() { return projectId; } - /** Returns the last submitted version of this application. */ - public Optional<ApplicationVersion> latestVersion() { - return versions.isEmpty() ? Optional.empty() : Optional.of(versions.last()); - } - - /** Returns the currently deployed versions of the application, ordered from oldest to newest. */ - public SortedSet<ApplicationVersion> versions() { - return versions; - } - - /** Returns the currently deployed versions of the application */ - public Collection<ApplicationVersion> deployableVersions(boolean ascending) { - Deque<ApplicationVersion> versions = new ArrayDeque<>(); - String previousHash = ""; - for (ApplicationVersion version : versions()) { - if (version.bundleHash().isEmpty() || ! previousHash.equals(version.bundleHash().get())) { - if (ascending) versions.addLast(version); - else versions.addFirst(version); - } - previousHash = version.bundleHash().orElse(""); - } - return versions; - } + /** Returns the known revisions for this application. */ + public RevisionHistory revisions() { return revisions; } /** * Returns the last deployed validation overrides of this application, @@ -212,10 +186,10 @@ public class Application { /** * Returns the oldest application version this has deployed in a permanent zone (not test or staging). */ - public Optional<ApplicationVersion> oldestDeployedApplication() { + public Optional<RevisionId> oldestDeployedRevision() { return productionDeployments().values().stream().flatMap(List::stream) - .map(Deployment::applicationVersion) - .filter(version -> ! version.isUnknown() && ! version.isDeployedDirectly()) + .map(Deployment::revision) + .filter(RevisionId::isProduction) .min(Comparator.naturalOrder()); } @@ -241,15 +215,6 @@ public class Application { /** Returns the set of deploy keys for this application. */ public Set<PublicKey> deployKeys() { return deployKeys; } - private static Optional<ApplicationVersion> requireNotUnknown(Optional<ApplicationVersion> latestVersion) { - Objects.requireNonNull(latestVersion, "latestVersion cannot be null"); - latestVersion.ifPresent(version -> { - if (version.isUnknown()) - throw new IllegalArgumentException("latestVersion cannot be unknown"); - }); - return latestVersion; - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 48f909df6b6..6907747646e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.log.LogLevel; import com.yahoo.text.Text; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; @@ -27,7 +28,6 @@ import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; -import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; @@ -41,6 +41,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationS import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; @@ -58,6 +60,7 @@ import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates; import com.yahoo.vespa.hosted.controller.concurrent.Once; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; +import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.notification.Notification; @@ -82,6 +85,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -89,6 +93,8 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.logging.Level; @@ -160,12 +166,13 @@ public class ApplicationController { for (InstanceName instance : application.get().deploymentSpec().instanceNames()) if ( ! application.get().instances().containsKey(instance)) application = withNewInstance(application, id.instance(instance)); + store(application); }); count++; } log.log(Level.INFO, Text.format("Wrote %d applications in %s", count, - Duration.between(start, clock.instant()))); + Duration.between(start, clock.instant()))); }); } @@ -175,7 +182,6 @@ public class ApplicationController { } /** Returns the instance with the given id, or null if it is not present */ - // TODO jonmv: remove or inline public Optional<Instance> getInstance(ApplicationId id) { return getApplication(TenantAndApplicationId.from(id)).flatMap(application -> application.get(id.instance())); } @@ -283,7 +289,7 @@ public class ApplicationController { if (oldest == null || version.isBefore(oldest)) oldest = version; - if (run.status() == RunStatus.success) + if (run.hasSucceeded()) return Optional.of(oldest); } // If no successful run was found, ask the node repository in the relevant zone. @@ -292,7 +298,7 @@ public class ApplicationController { /** Reads the oldest installed platform for the given application and zone from the node repo of that zone. */ private Optional<Version> oldestInstalledPlatform(JobId job) { - return configServer.nodeRepository().list(job.type().zone(controller.system()), + return configServer.nodeRepository().list(job.type().zone(), NodeFilter.all() .applications(job.application()) .states(active, reserved)) @@ -399,7 +405,7 @@ public class ApplicationController { * @throws IllegalArgumentException if the application already exists */ public Application createApplication(TenantAndApplicationId id, Credentials credentials) { - try (Lock lock = lock(id)) { + try (Mutex lock = lock(id)) { if (getApplication(id).isPresent()) throw new IllegalArgumentException("Could not create '" + id + "': Application already exists"); if (getApplication(dashToUnderscore(id)).isPresent()) // VESPA-1945 @@ -450,10 +456,10 @@ public class ApplicationController { throw new IllegalArgumentException("'" + job.application() + "' is a tester application!"); TenantAndApplicationId applicationId = TenantAndApplicationId.from(job.application()); - ZoneId zone = job.type().zone(controller.system()); + ZoneId zone = job.type().zone(); DeploymentId deployment = new DeploymentId(job.application(), zone); - try (Lock deploymentLock = lockForDeployment(job.application(), zone)) { + try (Mutex deploymentLock = lockForDeployment(job.application(), zone)) { Set<ContainerEndpoint> containerEndpoints; Optional<EndpointCertificateMetadata> endpointCertificateMetadata; @@ -464,11 +470,13 @@ public class ApplicationController { throw new IllegalStateException("No deployment expected for " + job + " now, as no job is running"); Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform()); - ApplicationVersion revision = run.versions().sourceApplication().filter(__ -> deploySourceVersions).orElse(run.versions().targetApplication()); + RevisionId revision = run.versions().sourceRevision().filter(__ -> deploySourceVersions).orElse(run.versions().targetRevision()); ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(deployment, revision)); - try (Lock lock = lock(applicationId)) { + AtomicReference<RevisionId> lastRevision = new AtomicReference<>(); + try (Mutex lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); + application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set); Instance instance = application.get().require(job.application().instance()); if ( ! applicationPackage.trustedCertificates().isEmpty() @@ -488,22 +496,25 @@ public class ApplicationController { var quotaUsage = deploymentQuotaUsage(zone, job.application()); // For direct deployments use the full deployment ID, but otherwise use just the tenant and application as - // the source since it's the same application, so it should have the same warnings - NotificationSource source = zone.environment().isManuallyDeployed() ? - NotificationSource.from(deployment) : NotificationSource.from(applicationId); - - @SuppressWarnings("deprecation") - List<String> warnings = Optional.ofNullable(result.prepareResponse().log) - .map(logs -> logs.stream() - .filter(log -> log.applicationPackage) - .filter(log -> LogLevel.parse(log.level).intValue() >= Level.WARNING.intValue()) - .map(log -> log.message) - .sorted() - .distinct() - .collect(Collectors.toList())) - .orElseGet(List::of); - if (warnings.isEmpty()) controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage); - else controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings); + // the source since it's the same application, so it should have the same warnings. + // These notifications are only updated when the last submitted revision is deployed here. + NotificationSource source = zone.environment().isManuallyDeployed() + ? NotificationSource.from(deployment) + : revision.equals(lastRevision.get()) ? NotificationSource.from(applicationId) : null; + if (source != null) { + @SuppressWarnings("deprecation") + List<String> warnings = Optional.ofNullable(result.prepareResponse().log) + .map(logs -> logs.stream() + .filter(log -> log.applicationPackage) + .filter(log -> LogLevel.parse(log.level).intValue() >= Level.WARNING.intValue()) + .map(log -> log.message) + .sorted() + .distinct() + .collect(Collectors.toList())) + .orElseGet(List::of); + if (warnings.isEmpty()) controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage); + else controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings); + } lockApplicationOrThrow(applicationId, application -> store(application.with(job.application().instance(), @@ -538,26 +549,13 @@ public class ApplicationController { controller.jobController().deploymentStatus(application.get()); for (Notification notification : controller.notificationsDb().listNotifications(NotificationSource.from(application.get().id()), true)) { - if ( ! notification.source().instance().map(declaredInstances::contains).orElse(false)) + if ( ! notification.source().instance().map(declaredInstances::contains).orElse(true)) controller.notificationsDb().removeNotifications(notification.source()); if (notification.source().instance().isPresent() && ! notification.source().zoneId().map(application.get().require(notification.source().instance().get()).deployments()::containsKey).orElse(false)) controller.notificationsDb().removeNotifications(notification.source()); } - ApplicationVersion oldestDeployedVersion = application.get().oldestDeployedApplication() - .orElse(ApplicationVersion.unknown); - - List<ApplicationVersion> olderVersions = application.get().versions().stream() - .filter(version -> version.compareTo(oldestDeployedVersion) < 0) - .sorted() - .collect(Collectors.toList()); - - // Remove any version not deployed anywhere - but keep one - for (ApplicationVersion version : olderVersions) { - application = application.withoutVersion(version); - } - store(application); } @@ -629,7 +627,7 @@ public class ApplicationController { deploymentQuota, tenantSecretStores, operatorCertificates, dryRun)); - return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), + return new ActivateResult(new com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { // Even if prepare fails, routing configuration may need to be updated @@ -697,7 +695,6 @@ public class ApplicationController { } applicationStore.removeAll(id.tenant(), id.application()); - applicationStore.removeAllTesters(id.tenant(), id.application()); applicationStore.putMetaTombstone(id.tenant(), id.application(), clock.instant()); credentials.ifPresent(creds -> accessControl.deleteApplication(id, creds)); @@ -753,7 +750,7 @@ public class ApplicationController { * @param action Function which acts on the locked application. */ public void lockApplicationIfPresent(TenantAndApplicationId applicationId, Consumer<LockedApplication> action) { - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { getApplication(applicationId).map(application -> new LockedApplication(application, lock)).ifPresent(action); } } @@ -766,7 +763,7 @@ public class ApplicationController { * @throws IllegalArgumentException when application does not exist. */ public void lockApplicationOrThrow(TenantAndApplicationId applicationId, Consumer<LockedApplication> action) { - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { action.accept(new LockedApplication(requireApplication(applicationId), lock)); } } @@ -839,14 +836,14 @@ public class ApplicationController { * Any operation which stores an application need to first acquire this lock, then read, modify * and store the application, and finally release (close) the lock. */ - Lock lock(TenantAndApplicationId application) { + Mutex lock(TenantAndApplicationId application) { return curator.lock(application); } /** * Returns a lock which provides exclusive rights to deploying this application to the given zone. */ - private Lock lockForDeployment(ApplicationId application, ZoneId zone) { + private Mutex lockForDeployment(ApplicationId application, ZoneId zone) { return curator.lockForDeployment(application, zone); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index c35e8c5a7ac..48e663e7feb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -7,12 +7,12 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.Metric; -import com.yahoo.net.HostName; -import com.yahoo.vespa.curator.Lock; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; @@ -28,14 +28,12 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.JobControlFlags; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.support.access.SupportAccessControl; -import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.yolean.concurrent.Sleeper; import java.time.Clock; @@ -49,7 +47,6 @@ import java.util.Set; import java.util.TreeSet; import java.util.function.Function; import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -68,7 +65,6 @@ public class Controller extends AbstractComponent { private static final Logger log = Logger.getLogger(Controller.class.getName()); - private final Supplier<String> hostnameSupplier; private final CuratorDb curator; private final JobControl jobControl; private final ApplicationController applicationController; @@ -100,15 +96,14 @@ public class Controller extends AbstractComponent { public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, FlagSource flagSource, MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, ControllerConfig controllerConfig) { - this(curator, rotationsConfig, accessControl, HostName::getLocalhost, flagSource, + this(curator, rotationsConfig, accessControl, flagSource, mavenRepository, serviceRegistry, metric, secretStore, controllerConfig, Sleeper.DEFAULT); } public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, - Supplier<String> hostnameSupplier, FlagSource flagSource, MavenRepository mavenRepository, + FlagSource flagSource, MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, ControllerConfig controllerConfig, Sleeper sleeper) { - this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null"); this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); this.serviceRegistry = Objects.requireNonNull(serviceRegistry, "ServiceRegistry cannot be null"); this.zoneRegistry = Objects.requireNonNull(serviceRegistry.zoneRegistry(), "ZoneRegistry cannot be null"); @@ -128,12 +123,12 @@ public class Controller extends AbstractComponent { auditLogger = new AuditLogger(curator, clock); jobControl = new JobControl(new JobControlFlags(curator, flagSource)); archiveBucketDb = new CuratorArchiveBucketDb(this); - notifier = new Notifier(curator, serviceRegistry.mailer()); + notifier = new Notifier(curator, serviceRegistry.zoneRegistry(), serviceRegistry.mailer()); notificationsDb = new NotificationsDb(this); supportAccessControl = new SupportAccessControl(this); // Record the version of this controller - curator().writeControllerVersion(this.hostname(), ControllerVersion.CURRENT); + curator().writeControllerVersion(this.hostname(), serviceRegistry.controllerVersion()); jobController.updateStorage(); } @@ -174,11 +169,6 @@ public class Controller extends AbstractComponent { public ControllerConfig controllerConfig() { return controllerConfig; } - public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, - String environment, String region) { - return serviceRegistry.configServer().getApplicationView(tenantName, applicationName, instanceName, environment, region); - } - /** Replace the current version status by a new one */ public void updateVersionStatus(VersionStatus newStatus) { VersionStatus currentStatus = readVersionStatus(); @@ -199,7 +189,7 @@ public class Controller extends AbstractComponent { /** Remove confidence override for versions matching given filter */ public void removeConfidenceOverride(Predicate<Version> filter) { - try (Lock lock = curator.lockConfidenceOverrides()) { + try (Mutex lock = curator.lockConfidenceOverrides()) { Map<Version, VespaVersion.Confidence> overrides = new LinkedHashMap<>(curator.readConfidenceOverrides()); overrides.keySet().removeIf(filter); curator.writeConfidenceOverrides(overrides); @@ -238,7 +228,7 @@ public class Controller extends AbstractComponent { throw new IllegalArgumentException("Cloud '" + cloudName + "' does not exist in this system"); } Instant scheduledAt = clock.instant(); - try (Lock lock = curator.lockOsVersions()) { + try (Mutex lock = curator.lockOsVersions()) { Map<CloudName, OsVersionTarget> targets = curator.readOsVersionTargets().stream() .collect(Collectors.toMap(t -> t.osVersion().cloud(), Function.identity())); @@ -268,7 +258,7 @@ public class Controller extends AbstractComponent { /** Replace the current OS version status with a new one */ public void updateOsVersionStatus(OsVersionStatus newStatus) { - try (Lock lock = curator.lockOsVersionStatus()) { + try (Mutex lock = curator.lockOsVersionStatus()) { OsVersionStatus currentStatus = curator.readOsVersionStatus(); for (CloudName cloud : clouds()) { Set<Version> newVersions = newStatus.versionsIn(cloud); @@ -282,8 +272,8 @@ public class Controller extends AbstractComponent { } /** Returns the hostname of this controller */ - public com.yahoo.config.provision.HostName hostname() { - return com.yahoo.config.provision.HostName.from(hostnameSupplier.get()); + public HostName hostname() { + return serviceRegistry.getHostname(); } public SystemName system() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index e5ab13d2127..402a4bf49a8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -6,8 +6,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -45,17 +45,15 @@ public class Instance { private final RotationStatus rotationStatus; private final Map<JobType, Instant> jobPauses; private final Change change; - private final Optional<ApplicationVersion> latestDeployed; /** Creates an empty instance */ public Instance(ApplicationId id) { - this(id, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty(), Optional.empty()); + this(id, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty()); } /** Creates an empty instance*/ public Instance(ApplicationId id, Collection<Deployment> deployments, Map<JobType, Instant> jobPauses, - List<AssignedRotation> rotations, RotationStatus rotationStatus, Change change, - Optional<ApplicationVersion> latestDeployed) { + List<AssignedRotation> rotations, RotationStatus rotationStatus, Change change) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.deployments = Objects.requireNonNull(deployments, "deployments cannot be null").stream() .collect(Collectors.toUnmodifiableMap(Deployment::zone, Function.identity())); @@ -63,19 +61,18 @@ public class Instance { this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null")); this.rotationStatus = Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null"); this.change = Objects.requireNonNull(change, "change cannot be null"); - this.latestDeployed = Objects.requireNonNull(latestDeployed, "latestDeployed cannot be null"); } - public Instance withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, + public Instance withNewDeployment(ZoneId zone, RevisionId revision, Version version, Instant instant, Map<DeploymentMetrics.Warning, Integer> warnings, QuotaUsage quotaUsage) { // Use info from previous deployment if available, otherwise create a new one. - Deployment previousDeployment = deployments.getOrDefault(zone, new Deployment(zone, applicationVersion, + Deployment previousDeployment = deployments.getOrDefault(zone, new Deployment(zone, revision, version, instant, DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty())); - Deployment newDeployment = new Deployment(zone, applicationVersion, version, instant, + Deployment newDeployment = new Deployment(zone, revision, version, instant, previousDeployment.metrics().with(warnings), previousDeployment.activity(), quotaUsage, @@ -90,7 +87,7 @@ public class Instance { else jobPauses.remove(jobType); - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } public Instance recordActivityAt(Instant instant, ZoneId zone) { @@ -121,19 +118,15 @@ public class Instance { } public Instance with(List<AssignedRotation> assignedRotations) { - return new Instance(id, deployments.values(), jobPauses, assignedRotations, rotationStatus, change, latestDeployed); + return new Instance(id, deployments.values(), jobPauses, assignedRotations, rotationStatus, change); } public Instance with(RotationStatus rotationStatus) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } public Instance withChange(Change change) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed); - } - - public Instance withLatestDeployed(ApplicationVersion latestDeployed) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, Optional.of(latestDeployed)); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } private Instance with(Deployment deployment) { @@ -143,7 +136,7 @@ public class Instance { } private Instance with(Map<ZoneId, Deployment> deployments) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } public ApplicationId id() { return id; } @@ -188,11 +181,6 @@ public class Instance { return change; } - /** Returns the application version that last rolled out to this instance. */ - public Optional<ApplicationVersion> latestDeployed() { - return latestDeployed; - } - /** Returns the total quota usage for this instance, excluding temporary deployments **/ public QuotaUsage quotaUsage() { return deployments.values().stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 06ff381e4dc..3e822415e96 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -4,11 +4,12 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.InstanceName; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import java.security.PublicKey; @@ -21,8 +22,6 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.function.UnaryOperator; /** @@ -32,7 +31,7 @@ import java.util.function.UnaryOperator; */ public class LockedApplication { - private final Lock lock; + private final Mutex lock; private final TenantAndApplicationId id; private final Instant createdAt; private final DeploymentSpec deploymentSpec; @@ -44,8 +43,7 @@ public class LockedApplication { private final ApplicationMetrics metrics; private final Set<PublicKey> deployKeys; private final OptionalLong projectId; - private final Optional<ApplicationVersion> latestVersion; - private final SortedSet<ApplicationVersion> versions; + private final RevisionHistory revisions; private final Map<InstanceName, Instance> instances; /** @@ -54,20 +52,19 @@ public class LockedApplication { * @param application The application to lock. * @param lock The lock for the application. */ - LockedApplication(Application application, Lock lock) { + LockedApplication(Application application, Mutex lock) { this(Objects.requireNonNull(lock, "lock cannot be null"), application.id(), application.createdAt(), application.deploymentSpec(), application.validationOverrides(), application.deploymentIssueId(), application.ownershipIssueId(), application.owner(), application.majorVersion(), application.metrics(), application.deployKeys(), - application.projectId(), application.latestVersion(), application.versions(), application.instances()); + application.projectId(), application.instances(), application.revisions()); } - private LockedApplication(Lock lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, + private LockedApplication(Mutex lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner, OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, - OptionalLong projectId, Optional<ApplicationVersion> latestVersion, SortedSet<ApplicationVersion> versions, - Map<InstanceName, Instance> instances) { + OptionalLong projectId, Map<InstanceName, Instance> instances, RevisionHistory revisions) { this.lock = lock; this.id = id; this.createdAt = createdAt; @@ -80,8 +77,7 @@ public class LockedApplication { this.metrics = metrics; this.deployKeys = deployKeys; this.projectId = projectId; - this.latestVersion = latestVersion; - this.versions = versions; + this.revisions = revisions; this.instances = Map.copyOf(instances); } @@ -89,7 +85,7 @@ public class LockedApplication { public Application get() { return new Application(id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances.values()); + projectId, revisions, instances.values()); } LockedApplication withNewInstance(InstanceName instance) { @@ -97,7 +93,7 @@ public class LockedApplication { instances.put(instance, new Instance(id.instance(instance))); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication with(InstanceName instance, UnaryOperator<Instance> modification) { @@ -105,7 +101,7 @@ public class LockedApplication { instances.put(instance, modification.apply(instances.get(instance))); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication without(InstanceName instance) { @@ -113,51 +109,43 @@ public class LockedApplication { instances.remove(instance); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); - } - - public LockedApplication withNewSubmission(ApplicationVersion latestVersion) { - SortedSet<ApplicationVersion> applicationVersions = new TreeSet<>(versions); - applicationVersions.add(latestVersion); - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, Optional.of(latestVersion), applicationVersions, instances); + projectId, instances, revisions); } public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication withDeploymentIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, Optional.ofNullable(issueId), ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication with(DeploymentSpec deploymentSpec) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication with(ValidationOverrides validationOverrides) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication withOwnershipIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, Optional.of(issueId), owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication withOwner(User owner) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, Optional.of(owner), majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } /** Set a major version for this, or set to null to remove any major version override */ @@ -165,13 +153,13 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion), - metrics, deployKeys, projectId, latestVersion, versions, instances); + metrics, deployKeys, projectId, instances, revisions); } public LockedApplication with(ApplicationMetrics metrics) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication withDeployKey(PublicKey pemDeployKey) { @@ -179,7 +167,7 @@ public class LockedApplication { keys.add(pemDeployKey); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } public LockedApplication withoutDeployKey(PublicKey pemDeployKey) { @@ -187,15 +175,13 @@ public class LockedApplication { keys.remove(pemDeployKey); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys, - projectId, latestVersion, versions, instances); + projectId, instances, revisions); } - public LockedApplication withoutVersion(ApplicationVersion version) { - SortedSet<ApplicationVersion> applicationVersions = new TreeSet<>(versions); - applicationVersions.remove(version); + public LockedApplication withRevisions(UnaryOperator<RevisionHistory> change) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, latestVersion, applicationVersions, instances); + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, + deployKeys, projectId, instances, change.apply(revisions)); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index bcc3b9b54c7..7a0e60aacb4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -6,6 +6,7 @@ import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; import com.yahoo.security.KeyUtils; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; @@ -46,7 +47,7 @@ public abstract class LockedTenant { this.lastLoginInfo = requireNonNull(lastLoginInfo); } - static LockedTenant of(Tenant tenant, Lock lock) { + static LockedTenant of(Tenant tenant, Mutex lock) { switch (tenant.type()) { case athenz: return new Athenz((AthenzTenant) tenant); case cloud: return new Cloud((CloudTenant) tenant); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 384a5d0f1ac..beba1cdb358 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Text; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -23,7 +24,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; -import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -75,7 +75,7 @@ public class TenantController { /** Locks a tenant for modification and applies the given action. */ public <T extends LockedTenant> void lockIfPresent(TenantName name, Class<T> token, Consumer<T> action) { - try (Lock lock = lock(name)) { + try (Mutex lock = lock(name)) { get(name).map(tenant -> LockedTenant.of(tenant, lock)) .map(token::cast) .ifPresent(action); @@ -84,7 +84,7 @@ public class TenantController { /** Lock a tenant for modification and apply action. Throws if the tenant does not exist */ public <T extends LockedTenant> void lockOrThrow(TenantName name, Class<T> token, Consumer<T> action) { - try (Lock lock = lock(name)) { + try (Mutex lock = lock(name)) { action.accept(token.cast(LockedTenant.of(require(name), lock))); } } @@ -112,7 +112,7 @@ public class TenantController { /** Create a tenant, provided the given credentials are valid. */ public void create(TenantSpec tenantSpec, Credentials credentials) { - try (Lock lock = lock(tenantSpec.tenant())) { + try (Mutex lock = lock(tenantSpec.tenant())) { TenantId.validate(tenantSpec.tenant().value()); requireNonExistent(tenantSpec.tenant()); curator.writeTenant(accessControl.createTenant(tenantSpec, controller.clock().instant(), credentials, asList())); @@ -138,7 +138,7 @@ public class TenantController { /** Updates the tenant contained in the given tenant spec with new data. */ public void update(TenantSpec tenantSpec, Credentials credentials) { - try (Lock lock = lock(tenantSpec.tenant())) { + try (Mutex lock = lock(tenantSpec.tenant())) { curator.writeTenant(accessControl.updateTenant(tenantSpec, credentials, asList(), controller.applications().asList(tenantSpec.tenant()))); } @@ -149,7 +149,7 @@ public class TenantController { * new instant is later */ public void updateLastLogin(TenantName tenantName, List<LastLoginInfo.UserLevel> userLevels, Instant loggedInAt) { - try (Lock lock = lock(tenantName)) { + try (Mutex lock = lock(tenantName)) { Tenant tenant = require(tenantName); LastLoginInfo loginInfo = tenant.lastLoginInfo(); for (LastLoginInfo.UserLevel userLevel : userLevels) @@ -162,7 +162,7 @@ public class TenantController { /** Deletes the given tenant. */ public void delete(TenantName tenant, Optional<Credentials> credentials, boolean forget) { - try (Lock lock = lock(tenant)) { + try (Mutex lock = lock(tenant)) { Tenant oldTenant = get(tenant, true) .orElseThrow(() -> new NotExistsException("Could not delete tenant '" + tenant + "': Tenant not found")); @@ -203,7 +203,7 @@ public class TenantController { * Any operation which stores a tenant need to first acquire this lock, then read, modify * and store the tenant, and finally release (close) the lock. */ - private Lock lock(TenantName tenant) { + private Mutex lock(TenantName tenant) { return curator.lock(tenant); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 5c0669ad543..411c2c133f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -39,13 +39,6 @@ public class ApplicationList extends AbstractFilteringList<Application, Applicat .collect(Collectors.toUnmodifiableList())); } - // ----------------------------------- Accessors - - /** Returns the ids of the applications in this as an immutable list */ - public List<TenantAndApplicationId> idList() { - return mapToList(Application::id); - } - // ----------------------------------- Filters /** Returns the subset of applications which have at least one production deployment */ @@ -54,13 +47,6 @@ public class ApplicationList extends AbstractFilteringList<Application, Applicat .anyMatch(instance -> instance.productionDeployments().size() > 0)); } - /** Returns the subset of applications which have at least one deployment on a lower version than the given one */ - public ApplicationList onLowerVersionThan(Version version) { - return matching(application -> application.instances().values().stream() - .flatMap(instance -> instance.productionDeployments().values().stream()) - .anyMatch(deployment -> deployment.version().isBefore(version))); - } - /** Returns the subset of applications with at least one declared job in deployment spec. */ public ApplicationList withJobs() { return matching(application -> application.deploymentSpec().steps().stream() @@ -72,45 +58,9 @@ public class ApplicationList extends AbstractFilteringList<Application, Applicat return matching(application -> application.projectId().isPresent()); } - /** Returns the subset of applications that are allowed to upgrade at the given time */ - public ApplicationList canUpgradeAt(Instant instant) { - return matching(application -> application.deploymentSpec().instances().stream() - .allMatch(instance -> instance.canUpgradeAt(instant))); - } - - /** Returns the subset of applications that have at least one assigned rotation */ - public ApplicationList hasRotation() { - return matching(application -> application.instances().values().stream() - .anyMatch(instance -> ! instance.rotations().isEmpty())); - } - - /** - * Returns the subset of applications that hasn't pinned to an an earlier major version than the given one. - * - * @param targetMajorVersion the target major version which applications returned allows upgrading to - * @param defaultMajorVersion the default major version to assume for applications not specifying one - */ - public ApplicationList allowMajorVersion(int targetMajorVersion, int defaultMajorVersion) { - return matching(application -> targetMajorVersion <= application.deploymentSpec().majorVersion() - .orElse(application.majorVersion() - .orElse(defaultMajorVersion))); - } - /** Returns the subset of application which have submitted a non-empty deployment spec. */ public ApplicationList withDeploymentSpec() { return matching(application -> ! DeploymentSpec.empty.equals(application.deploymentSpec())); } - // ----------------------------------- Sorting - - /** - * Returns this list sorted by increasing deployed version. - * If multiple versions are deployed the oldest is used. - * Applications without any deployments are ordered first. - */ - public ApplicationList byIncreasingDeployedVersion() { - return sortedBy(Comparator.comparing(application -> application.oldestDeployedPlatform() - .orElse(Version.emptyVersion))); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java index 244fb952b3f..64cad599168 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import java.util.Objects; import java.util.Optional; @@ -28,22 +29,22 @@ public final class Change { private final Optional<Version> platform; /** The application version we are changing to, or empty if none */ - private final Optional<ApplicationVersion> application; + private final Optional<RevisionId> revision; /** Whether this change is a pin to its contained Vespa version, or to the application's current. */ private final boolean pinned; - private Change(Optional<Version> platform, Optional<ApplicationVersion> application, boolean pinned) { + private Change(Optional<Version> platform, Optional<RevisionId> revision, boolean pinned) { this.platform = requireNonNull(platform, "platform cannot be null"); - this.application = requireNonNull(application, "application cannot be null"); - if (application.isPresent() && (application.get().isUnknown() || application.get().isDeployedDirectly())) { + this.revision = requireNonNull(revision, "revision cannot be null"); + if (revision.isPresent() && ( ! revision.get().isProduction())) { throw new IllegalArgumentException("Application version to deploy must be a known version"); } this.pinned = pinned; } public Change withoutPlatform() { - return new Change(Optional.empty(), application, pinned); + return new Change(Optional.empty(), revision, pinned); } public Change withoutApplication() { @@ -52,7 +53,7 @@ public final class Change { /** Returns whether a change should currently be deployed */ public boolean hasTargets() { - return platform.isPresent() || application.isPresent(); + return platform.isPresent() || revision.isPresent(); } /** Returns whether this is the empty change. */ @@ -64,7 +65,7 @@ public final class Change { public Optional<Version> platform() { return platform; } /** Returns the application version carried by this. */ - public Optional<ApplicationVersion> application() { return application; } + public Optional<RevisionId> revision() { return revision; } public boolean isPinned() { return pinned; } @@ -76,30 +77,30 @@ public final class Change { if (pinned) throw new IllegalArgumentException("Not allowed to set a platform version when pinned."); - return new Change(Optional.of(platformVersion), application, pinned); + return new Change(Optional.of(platformVersion), revision, pinned); } - /** Returns a version of this change which replaces or adds this application change */ - public Change with(ApplicationVersion applicationVersion) { - return new Change(platform, Optional.of(applicationVersion), pinned); + /** Returns a version of this change which replaces or adds this revision change */ + public Change with(RevisionId revision) { + return new Change(platform, Optional.of(revision), pinned); } /** Returns a change with the versions of this, and with the platform version pinned. */ public Change withPin() { - return new Change(platform, application, true); + return new Change(platform, revision, true); } /** Returns a change with the versions of this, and with the platform version unpinned. */ public Change withoutPin() { - return new Change(platform, application, false); + return new Change(platform, revision, false); } /** Returns the change obtained when overwriting elements of the given change with any present in this */ public Change onTopOf(Change other) { if (platform.isPresent()) other = other.with(platform.get()); - if (application.isPresent()) - other = other.with(application.get()); + if (revision.isPresent()) + other = other.with(revision.get()); if (pinned) other = other.withPin(); return other; @@ -112,12 +113,12 @@ public final class Change { Change change = (Change) o; return pinned == change.pinned && Objects.equals(platform, change.platform) && - Objects.equals(application, change.application); + Objects.equals(revision, change.revision); } @Override public int hashCode() { - return Objects.hash(platform, application, pinned); + return Objects.hash(platform, revision, pinned); } @Override @@ -126,23 +127,23 @@ public final class Change { if (pinned) changes.add("pin to " + platform.map(Version::toString).orElse("current platform")); else - platform.ifPresent(version -> changes.add("upgrade to " + version.toString())); - application.ifPresent(version -> changes.add("application change to " + version.id())); + platform.ifPresent(version -> changes.add("upgrade to " + version)); + revision.ifPresent(revision -> changes.add("revision change to " + revision)); changes.setEmptyValue("no change"); return changes.toString(); } - public static Change of(ApplicationVersion applicationVersion) { - return new Change(Optional.empty(), Optional.of(applicationVersion), false); + public static Change of(RevisionId revision) { + return new Change(Optional.empty(), Optional.of(revision), false); } public static Change of(Version platformChange) { return new Change(Optional.of(platformChange), Optional.empty(), false); } - /** Returns whether this change carries an application downgrade relative to the given version. */ - public boolean downgrades(ApplicationVersion version) { - return application.map(version::compareTo).orElse(0) > 0; + /** Returns whether this change carries a revision downgrade relative to the given revision. */ + public boolean downgrades(RevisionId revision) { + return this.revision.map(revision::compareTo).orElse(0) > 0; } /** Returns whether this change carries a platform downgrade relative to the given version. */ @@ -150,9 +151,9 @@ public final class Change { return platform.map(version::compareTo).orElse(0) > 0; } - /** Returns whether this change carries an application upgrade relative to the given version. */ - public boolean upgrades(ApplicationVersion version) { - return application.map(version::compareTo).orElse(0) < 0; + /** Returns whether this change carries a revision upgrade relative to the given revision. */ + public boolean upgrades(RevisionId revision) { + return this.revision.map(revision::compareTo).orElse(0) < 0; } /** Returns whether this change carries a platform upgrade relative to the given version. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 6081c3323c8..2e4afb4e004 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -2,8 +2,8 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import java.time.Instant; import java.util.Objects; @@ -18,7 +18,7 @@ import java.util.OptionalDouble; public class Deployment { private final ZoneId zone; - private final ApplicationVersion applicationVersion; + private final RevisionId revision; private final Version version; private final Instant deployTime; private final DeploymentMetrics metrics; @@ -26,10 +26,10 @@ public class Deployment { private final QuotaUsage quota; private final OptionalDouble cost; - public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime, + public Deployment(ZoneId zone, RevisionId revision, Version version, Instant deployTime, DeploymentMetrics metrics, DeploymentActivity activity, QuotaUsage quota, OptionalDouble cost) { this.zone = Objects.requireNonNull(zone, "zone cannot be null"); - this.applicationVersion = Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null"); + this.revision = Objects.requireNonNull(revision, "revision cannot be null"); this.version = Objects.requireNonNull(version, "version cannot be null"); this.deployTime = Objects.requireNonNull(deployTime, "deployTime cannot be null"); this.metrics = Objects.requireNonNull(metrics, "deploymentMetrics cannot be null"); @@ -41,8 +41,8 @@ public class Deployment { /** Returns the zone this was deployed to */ public ZoneId zone() { return zone; } - /** Returns the deployed application version */ - public ApplicationVersion applicationVersion() { return applicationVersion; } + /** Returns the deployed application revision */ + public RevisionId revision() { return revision; } /** Returns the deployed Vespa version */ public Version version() { return version; } @@ -65,26 +65,26 @@ public class Deployment { public OptionalDouble cost() { return cost; } public Deployment recordActivityAt(Instant instant) { - return new Deployment(zone, applicationVersion, version, deployTime, metrics, + return new Deployment(zone, revision, version, deployTime, metrics, activity.recordAt(instant, metrics), quota, cost); } public Deployment withMetrics(DeploymentMetrics metrics) { - return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); + return new Deployment(zone, revision, version, deployTime, metrics, activity, quota, cost); } public Deployment withQuota(QuotaUsage quota) { - return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); + return new Deployment(zone, revision, version, deployTime, metrics, activity, quota, cost); } public Deployment withCost(double cost) { if (this.cost.isPresent() && Double.compare(this.cost.getAsDouble(), cost) == 0) return this; - return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, OptionalDouble.of(cost)); + return new Deployment(zone, revision, version, deployTime, metrics, activity, quota, OptionalDouble.of(cost)); } public Deployment withoutCost() { if (cost.isEmpty()) return this; - return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, OptionalDouble.empty()); + return new Deployment(zone, revision, version, deployTime, metrics, activity, quota, OptionalDouble.empty()); } @Override @@ -93,7 +93,7 @@ public class Deployment { if (o == null || getClass() != o.getClass()) return false; Deployment that = (Deployment) o; return zone.equals(that.zone) && - applicationVersion.equals(that.applicationVersion) && + revision.equals(that.revision) && version.equals(that.version) && deployTime.equals(that.deployTime) && metrics.equals(that.metrics) && @@ -104,12 +104,12 @@ public class Deployment { @Override public int hashCode() { - return Objects.hash(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); + return Objects.hash(zone, revision, version, deployTime, metrics, activity, quota, cost); } @Override public String toString() { - return "deployment to " + zone + " of " + applicationVersion + " on version " + version + " at " + deployTime; + return "deployment to " + zone + " of " + revision + " on version " + version + " at " + deployTime; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 7fc6b927bdd..8de72893a7c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -160,8 +160,7 @@ public class Endpoint { } private static URI createUrl(String name, TenantAndApplicationId application, Optional<InstanceName> instance, - List<Target> targets, Scope scope, SystemName system, Port port, boolean legacy, - RoutingMethod routingMethod) { + List<Target> targets, Scope scope, SystemName system, Port port, boolean legacy) { String separator = "."; String portPart = port.isDefault() ? "" : ":" + port.port; return URI.create("https://" + @@ -591,8 +590,7 @@ public class Endpoint { Objects.requireNonNull(scope, "scope must be non-null"), Objects.requireNonNull(system, "system must be non-null"), Objects.requireNonNull(port, "port must be non-null"), - legacy, - Objects.requireNonNull(routingMethod, "routingMethod must be non-null")); + legacy); return new Endpoint(application, instance, endpointId, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index 99b14bf289a..6178bfbb89e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -42,7 +42,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL */ public InstanceList compatibleWithPlatform(Version platform, Function<ApplicationId, VersionCompatibility> compatibility) { return matching(id -> instance(id).productionDeployments().values().stream() - .flatMap(deployment -> deployment.applicationVersion().compileVersion().stream()) + .flatMap(deployment -> application(id).revisions().get(deployment.revision()).compileVersion().stream()) .noneMatch(version -> compatibility.apply(id).refuse(platform, version))); } @@ -72,8 +72,9 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which contain declared jobs */ public InstanceList withDeclaredJobs() { - return matching(id -> instances.get(id).jobSteps().values().stream() - .anyMatch(job -> job.isDeclared() && job.job().get().application().equals(id))); + return matching(id -> instances.get(id).application().revisions().last().isPresent() + && instances.get(id).jobSteps().values().stream() + .anyMatch(job -> job.isDeclared() && job.job().get().application().equals(id))); } /** Returns the subset of instances which have at least one deployment on a lower version than the given one, or which have no production deployments */ @@ -95,7 +96,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which are currently deploying a new revision */ public InstanceList changingRevision() { - return matching(id -> instance(id).change().application().isPresent()); + return matching(id -> instance(id).change().revision().isPresent()); } /** Returns the subset of instances which currently have failing jobs on the given version */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 258884a4d11..344ed7ec729 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -66,9 +66,9 @@ public class ApplicationPackage { private static final String trustedCertificatesFile = "security/clients.pem"; private static final String buildMetaFile = "build-meta.json"; - private static final String deploymentFile = "deployment.xml"; + static final String deploymentFile = "deployment.xml"; private static final String validationOverridesFile = "validation-overrides.xml"; - private static final String servicesFile = "services.xml"; + static final String servicesFile = "services.xml"; private final String contentHash; private final String bundleHash; @@ -78,6 +78,7 @@ public class ApplicationPackage { private final ZipArchiveCache files; private final Optional<Version> compileVersion; private final Optional<Instant> buildTime; + private final Optional<Version> parentVersion; private final List<X509Certificate> trustedCertificates; /** @@ -110,6 +111,7 @@ public class ApplicationPackage { Optional<Inspector> buildMetaObject = files.get(buildMetaFile).map(SlimeUtils::jsonToSlime).map(Slime::get); this.compileVersion = buildMetaObject.flatMap(object -> parse(object, "compileVersion", field -> Version.fromString(field.asString()))); this.buildTime = buildMetaObject.flatMap(object -> parse(object, "buildTime", field -> Instant.ofEpochMilli(field.asLong()))); + this.parentVersion = buildMetaObject.flatMap(object -> parse(object, "parentVersion", field -> Version.fromString(field.asString()))); this.trustedCertificates = files.get(trustedCertificatesFile).map(bytes -> X509CertificateUtils.certificateListFromPem(new String(bytes, UTF_8))).orElse(List.of()); @@ -159,6 +161,9 @@ public class ApplicationPackage { /** Returns the time this package was built, if known. */ public Optional<Instant> buildTime() { return buildTime; } + /** Returns the parent version used to compile the package, if known. */ + public Optional<Version> parentVersion() { return parentVersion; } + /** Returns the list of certificates trusted by this application, or an empty list if no trust configured. */ public List<X509Certificate> trustedCertificates() { return trustedCertificates; @@ -166,7 +171,7 @@ public class ApplicationPackage { private static <Type> Optional<Type> parse(Inspector buildMetaObject, String fieldName, Function<Inspector, Type> mapper) { if ( ! buildMetaObject.field(fieldName).valid()) - throw new IllegalArgumentException("Missing value '" + fieldName + "' in '" + buildMetaFile + "'"); + return Optional.empty(); try { return Optional.of(mapper.apply(buildMetaObject.field(fieldName))); } @@ -197,11 +202,11 @@ public class ApplicationPackage { RegionName.defaultName()) .run(); // Populates the zip archive cache with files that would be included. } - catch (RuntimeException e) { + catch (IllegalArgumentException e) { throw e; } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalArgumentException(e); } } @@ -212,7 +217,7 @@ public class ApplicationPackage { entry -> entry.getValue().get()))); } - static byte[] filesZip(Map<String, byte[]> files) { + public static byte[] filesZip(Map<String, byte[]> files) { try (ZipBuilder zipBuilder = new ZipBuilder(files.values().stream().mapToInt(bytes -> bytes.length).sum() + 512)) { files.forEach(zipBuilder::add); zipBuilder.close(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java index 9ab5096ec8f..950eaea904a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java @@ -62,7 +62,7 @@ public class ApplicationPackageValidator { /** Verify that each of the production zones listed in the deployment spec exist in this system */ private void validateSteps(DeploymentSpec deploymentSpec) { for (var spec : deploymentSpec.instances()) { - new DeploymentSteps(spec, controller::system).jobs(); + new DeploymentSteps(spec, controller.zoneRegistry()).jobs(); spec.zones().stream() .filter(zone -> zone.environment() == Environment.prod) .forEach(zone -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java new file mode 100644 index 00000000000..fb352848911 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java @@ -0,0 +1,318 @@ +package com.yahoo.vespa.hosted.controller.application.pkg; + +import com.yahoo.config.application.api.DeploymentInstanceSpec; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.DeploymentSpec.Step; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.path.Path; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.text.Text; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; +import com.yahoo.vespa.hosted.controller.config.ControllerConfig; +import com.yahoo.vespa.hosted.controller.config.ControllerConfig.Steprunner.Testerapp; +import com.yahoo.yolean.Exceptions; + +import javax.security.auth.x500.X500Principal; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.jar.JarInputStream; +import java.util.jar.Manifest; +import java.util.regex.Pattern; + +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.production; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging_setup; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.system; +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.deploymentFile; +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.servicesFile; +import static com.yahoo.vespa.hosted.controller.application.pkg.ZipEntries.transferAndWrite; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toList; + +/** + * Validation and manipulation of test package. + * + * @author jonmv + */ +public class TestPackage { + + // Must match exactly the advertised resources of an AWS instance type. Also consider that the container + // will have ~1.8 GB less memory than equivalent resources in AWS (VESPA-16259). + static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); + static final NodeResources DEFAULT_TESTER_RESOURCES = new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any); + + private final ApplicationPackage applicationPackage; + private final X509Certificate certificate; + + public TestPackage(byte[] testPackage, boolean isPublicSystem, RunId id, Testerapp testerApp, + DeploymentSpec spec, Instant certificateValidFrom, Duration certificateValidDuration) { + + // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip. + Map<String, byte[]> entries = new HashMap<>(); + entries.put("artifacts/.ignore-" + UUID.randomUUID(), new byte[0]); + entries.put("tests/.ignore-" + UUID.randomUUID(), new byte[0]); + + entries.put(servicesFile, + servicesXml( ! isPublicSystem, + certificateValidFrom != null, + testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance())), + testerApp)); + + entries.put(deploymentFile, + deploymentXml(id.tester(), + spec.athenzDomain(), + spec.requireInstance(id.application().instance()) + .athenzService(id.type().zone().environment(), id.type().zone().region()))); + + if (certificateValidFrom != null) { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); + X500Principal subject = new X500Principal("CN=" + id.tester().id().toFullString() + "." + id.type() + "." + id.number()); + this.certificate = X509CertificateBuilder.fromKeypair(keyPair, + subject, + certificateValidFrom, + certificateValidFrom.plus(certificateValidDuration), + SignatureAlgorithm.SHA512_WITH_RSA, + BigInteger.valueOf(1)) + .build(); + entries.put("artifacts/key", KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8)); + entries.put("artifacts/cert", X509CertificateUtils.toPem(certificate).getBytes(UTF_8)); + } + else { + this.certificate = null; + } + + ByteArrayOutputStream buffer = new ByteArrayOutputStream(testPackage.length + 10_000); + transferAndWrite(buffer, new ByteArrayInputStream(testPackage), entries); + this.applicationPackage = new ApplicationPackage(buffer.toByteArray()); + } + + public ApplicationPackage asApplicationPackage() { + return applicationPackage; + } + + public X509Certificate certificate() { + return Objects.requireNonNull(certificate); + } + + public static TestSummary validateTests(DeploymentSpec spec, byte[] testPackage) { + return validateTests(expectedSuites(spec.steps()), testPackage); + } + + static TestSummary validateTests(Collection<Suite> expectedSuites, byte[] testPackage) { + List<String> problems = new ArrayList<>(); + Set<Suite> suites = new LinkedHashSet<>(); + ZipEntries.from(testPackage, __ -> true, 0, false).asList().stream() + .map(entry -> Path.fromString(entry.name())) + .collect(groupingBy(path -> path.elements().size() > 1 ? path.elements().get(0) : "", + mapping(path -> (path.elements().size() > 1 ? path.getChildPath() : path).getRelative(), toList()))) + .forEach((directory, paths) -> { + switch (directory) { + case "components": { + for (String path : paths) { + if (path.endsWith("-tests.jar")) { + try { + byte[] testsJar = ZipEntries.readFile(testPackage, "components/" + path, 1 << 30); + Manifest manifest = new JarInputStream(new ByteArrayInputStream(testsJar)).getManifest(); + for (String suite : manifest.getMainAttributes().getValue("X-JDisc-Test-Bundle-Categories").split(",")) + switch (suite.trim()) { + case "SystemTest": suites.add(system); break; + case "StagingSetup": suites.add(staging_setup); break; + case "StagingTest": suites.add(staging); break; + case "ProductionTest": suites.add(production); break; + default: problems.add("unexpected test suite name '" + suite + "' in bundle manifest"); + } + } + catch (Exception e) { + problems.add("failed reading test bundle manifest: " + Exceptions.toMessageString(e)); + } + } + } + } + break; + case "tests": { + if (paths.stream().anyMatch(Pattern.compile("system-test/.+\\.json").asMatchPredicate())) suites.add(system); + if (paths.stream().anyMatch(Pattern.compile("staging-setup/.+\\.json").asMatchPredicate())) suites.add(staging_setup); + if (paths.stream().anyMatch(Pattern.compile("staging-test/.+\\.json").asMatchPredicate())) suites.add(staging); + if (paths.stream().anyMatch(Pattern.compile("production-test/.+\\.json").asMatchPredicate())) suites.add(production); + } + break; + case "artifacts": { + if (paths.stream().anyMatch(Pattern.compile(".+-tests.jar").asMatchPredicate())) + suites.addAll(expectedSuites); // ಠ_ಠ + + for (String forbidden : List.of("key", "cert")) + if (paths.contains(forbidden)) + problems.add("test package contains 'artifacts/" + forbidden + + "'; this conflicts with credentials used to run tests in Vespa Cloud"); + } + break; + } + }); + + if (expectedSuites.contains(system) && ! suites.contains(system)) + problems.add("test package has no system tests, but <test /> is declared in deployment.xml"); + + if (suites.contains(staging) != suites.contains(staging_setup)) + problems.add("test package has " + (suites.contains(staging) ? "staging tests" : "staging setup") + + ", so it should also include " + (suites.contains(staging) ? "staging setup" : "staging tests")); + else if (expectedSuites.contains(staging) && ! suites.contains(staging)) + problems.add("test package has no staging setup and tests, but <staging /> is declared in deployment.xml"); + + if (suites.contains(production) != expectedSuites.contains(production)) + problems.add("test package has " + (suites.contains(production) ? "" : "no ") + "production tests, " + + "but " + (suites.contains(production) ? "no " : "") + "production tests are declared in deployment.xml"); + + if ( ! problems.isEmpty()) + problems.add("see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"); + + return new TestSummary(problems, suites); + } + + public static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) { + NodeResources nodeResources = spec.steps().stream() + .filter(step -> step.concerns(zone.environment())) + .findFirst() + .flatMap(step -> step.zones().get(0).testerFlavor()) + .map(NodeResources::fromLegacyName) + .orElse(zone.region().value().contains("aws-") ? DEFAULT_TESTER_RESOURCES_AWS + : DEFAULT_TESTER_RESOURCES); + return nodeResources.with(NodeResources.DiskSpeed.any); + } + + /** Returns the generated services.xml content for the tester application. */ + public static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, + NodeResources resources, ControllerConfig.Steprunner.Testerapp config) { + int jdiscMemoryGb = 2; // 2Gb memory for tester application (excessive?). + int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb()); + + // Of the remaining memory, split 50/50 between Surefire running the tests and the rest + int testMemoryMb = (int) (1024 * (resources.memoryGb() - jdiscMemoryGb) / 2); + + String resourceString = Text.format("<resources vcpu=\"%.2f\" memory=\"%.2fGb\" disk=\"%.2fGb\" disk-speed=\"%s\" storage-type=\"%s\"/>", + resources.vcpu(), resources.memoryGb(), resources.diskGb(), resources.diskSpeed().name(), resources.storageType().name()); + + String runtimeProviderClass = config.runtimeProviderClass(); + String tenantCdBundle = config.tenantCdBundle(); + + String servicesXml = + "<?xml version='1.0' encoding='UTF-8'?>\n" + + "<services xmlns:deploy='vespa' version='1.0'>\n" + + " <container version='1.0' id='tester'>\n" + + "\n" + + " <component id=\"com.yahoo.vespa.hosted.testrunner.TestRunner\" bundle=\"vespa-testrunner-components\">\n" + + " <config name=\"com.yahoo.vespa.hosted.testrunner.test-runner\">\n" + + " <artifactsPath>artifacts</artifactsPath>\n" + + " <surefireMemoryMb>" + testMemoryMb + "</surefireMemoryMb>\n" + + " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + + " <useTesterCertificate>" + useTesterCertificate + "</useTesterCertificate>\n" + + " </config>\n" + + " </component>\n" + + "\n" + + " <handler id=\"com.yahoo.vespa.testrunner.TestRunnerHandler\" bundle=\"vespa-osgi-testrunner\">\n" + + " <binding>http://*/tester/v1/*</binding>\n" + + " </handler>\n" + + "\n" + + " <component id=\"" + runtimeProviderClass + "\" bundle=\"" + tenantCdBundle + "\" />\n" + + "\n" + + " <component id=\"com.yahoo.vespa.testrunner.JunitRunner\" bundle=\"vespa-osgi-testrunner\">\n" + + " <config name=\"com.yahoo.vespa.testrunner.junit-test-runner\">\n" + + " <artifactsPath>artifacts</artifactsPath>\n" + + " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + + " </config>\n" + + " </component>\n" + + "\n" + + " <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" + + " <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" + + " <artifactsPath>artifacts</artifactsPath>\n" + + " <testsPath>tests</testsPath>\n" + + " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + + " </config>\n" + + " </component>\n" + + "\n" + + " <nodes count=\"1\">\n" + + " <jvm allocated-memory=\"" + jdiscMemoryPct + "%\"/>\n" + + " " + resourceString + "\n" + + " </nodes>\n" + + " </container>\n" + + "</services>\n"; + + return servicesXml.getBytes(UTF_8); + } + + /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */ + public static byte[] deploymentXml(TesterId id, Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) { + String deploymentSpec = + "<?xml version='1.0' encoding='UTF-8'?>\n" + + "<deployment version=\"1.0\" " + + athenzDomain.map(domain -> "athenz-domain=\"" + domain.value() + "\" ").orElse("") + + athenzService.map(service -> "athenz-service=\"" + service.value() + "\" ").orElse("") + ">" + + " <instance id=\"" + id.id().instance().value() + "\" />" + + "</deployment>"; + return deploymentSpec.getBytes(UTF_8); + } + + static Set<Suite> expectedSuites(List<Step> steps) { + Set<Suite> suites = new HashSet<>(); + if (steps.isEmpty()) return suites; + for (Step step : steps) { + if (step.isTest()) { + if (step.concerns(Environment.prod)) suites.add(production); + if (step.concerns(Environment.test)) suites.add(system); + if (step.concerns(Environment.staging)) { suites.add(staging); suites.add(staging_setup); } + } + else + suites.addAll(expectedSuites(step.steps())); + } + return suites; + } + + + public static class TestSummary { + + private final List<String> problems; + private final List<Suite> suites; + + public TestSummary(List<String> problems, Set<Suite> suites) { + this.problems = List.copyOf(problems); + this.suites = List.copyOf(suites); + } + + public List<String> problems() { + return problems; + } + + public List<Suite> suites() { + return suites; + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java index a6cb7f23fc3..8392a77bad5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java @@ -13,6 +13,8 @@ import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; @@ -35,19 +37,28 @@ public class ZipEntries { /** Copies the zipped content from in to out, adding/overwriting an entry with the given name and content. */ public static void transferAndWrite(OutputStream out, InputStream in, String name, byte[] content) { + transferAndWrite(out, in, Map.of(name, content)); + } + + /** Copies the zipped content from in to out, adding/overwriting/removing (on {@code null}) entries as specified. */ + public static void transferAndWrite(OutputStream out, InputStream in, Map<String, byte[]> entries) { try (ZipOutputStream zipOut = new ZipOutputStream(out); ZipInputStream zipIn = new ZipInputStream(in)) { for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) { - if (entry.getName().equals(name)) + if (entries.containsKey(entry.getName())) continue; zipOut.putNextEntry(new ZipEntry(entry.getName())); zipIn.transferTo(zipOut); zipOut.closeEntry(); } - zipOut.putNextEntry(new ZipEntry(name)); - zipOut.write(content); - zipOut.closeEntry(); + for (Entry<String, byte[]> entry : entries.entrySet()) { + if (entry.getValue() != null) { + zipOut.putNextEntry(new ZipEntry(entry.getKey())); + zipOut.write(entry.getValue()); + zipOut.closeEntry(); + } + } } catch (IOException e) { throw new UncheckedIOException(e); @@ -76,6 +87,10 @@ public class ZipEntries { return new ZipEntries(entries); } + public static byte[] readFile(byte[] zip, String name, int maxEntrySizeInBytes) { + return from(zip, name::equals, maxEntrySizeInBytes, true).asList().get(0).contentOrThrow(); + } + public List<ZipEntryWithContent> asList() { return entries; } public static class ZipEntryWithContent { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index f5900604627..2ad28893e18 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -7,7 +7,6 @@ import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; import com.yahoo.vespa.athenz.api.AthenzDomain; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java index 78d65766075..34e7955e02a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.auditlog; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -70,7 +71,7 @@ public class AuditLogger { Instant now = clock.instant(); AuditLog.Entry entry = new AuditLog.Entry(now, principal.getName(), method.get(), pathAndQueryOf(request.getUri()), Optional.of(new String(data, StandardCharsets.UTF_8))); - try (Lock lock = db.lockAuditLog()) { + try (Mutex lock = db.lockAuditLog()) { AuditLog auditLog = db.readAuditLog() .pruneBefore(now.minus(entryTtl)) .with(entry) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index a042215616c..cc7031bab5a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -10,14 +10,16 @@ import com.yahoo.config.application.api.DeploymentSpec.DeclaredTest; import com.yahoo.config.application.api.DeploymentSpec.DeclaredZone; import com.yahoo.config.application.api.DeploymentSpec.UpgradeRollout; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -26,6 +28,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -42,13 +45,12 @@ import static com.yahoo.config.application.api.DeploymentSpec.RevisionTarget.nex import static com.yahoo.config.provision.Environment.prod; import static com.yahoo.config.provision.Environment.staging; import static com.yahoo.config.provision.Environment.test; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; import static java.util.Objects.requireNonNull; import static java.util.function.BinaryOperator.maxBy; import static java.util.stream.Collectors.collectingAndThen; +import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toUnmodifiableList; @@ -59,51 +61,33 @@ import static java.util.stream.Collectors.toUnmodifiableList; */ public class DeploymentStatus { - public static List<JobId> jobsFor(Application application, SystemName system) { - if (DeploymentSpec.empty.equals(application.deploymentSpec())) - return List.of(); - - return application.deploymentSpec().instances().stream() - .flatMap(spec -> Stream.concat(Stream.of(systemTest, stagingTest), - flatten(spec).filter(step -> step.concerns(prod)) - .map(step -> { - if (step instanceof DeclaredZone) - return JobType.from(system, prod, ((DeclaredZone) step).region().get()); - return JobType.testFrom(system, ((DeclaredTest) step).region()); - }) - .flatMap(Optional::stream)) - .map(type -> new JobId(application.id().instance(spec.name()), type))) - .collect(toUnmodifiableList()); - } - - private static Stream<DeploymentSpec.Step> flatten(DeploymentSpec.Step step) { - return step instanceof DeploymentSpec.Steps ? step.steps().stream().flatMap(DeploymentStatus::flatten) : Stream.of(step); - } - private static <T> List<T> union(List<T> first, List<T> second) { return Stream.concat(first.stream(), second.stream()).distinct().collect(toUnmodifiableList()); } private final Application application; private final JobList allJobs; - private final SystemName system; + private final JobType systemTest; + private final JobType stagingTest; private final Version systemVersion; private final Function<InstanceName, VersionCompatibility> versionCompatibility; private final Instant now; private final Map<JobId, StepStatus> jobSteps; private final List<StepStatus> allSteps; - public DeploymentStatus(Application application, Map<JobId, JobStatus> allJobs, SystemName system, + public DeploymentStatus(Application application, Function<JobId, JobStatus> allJobs, ZoneRegistry zones, Version systemVersion, Function<InstanceName, VersionCompatibility> versionCompatibility, Instant now) { this.application = requireNonNull(application); - this.allJobs = JobList.from(allJobs.values()); - this.system = requireNonNull(system); + this.systemTest = JobType.systemTest(zones); + this.stagingTest = JobType.stagingTest(zones); this.systemVersion = requireNonNull(systemVersion); this.versionCompatibility = versionCompatibility; this.now = requireNonNull(now); List<StepStatus> allSteps = new ArrayList<>(); - this.jobSteps = jobDependencies(application.deploymentSpec(), allSteps); + Map<JobId, JobStatus> jobs = new HashMap<>(); + this.jobSteps = jobDependencies(application.deploymentSpec(), allSteps, job -> jobs.computeIfAbsent(job, allJobs)); this.allSteps = Collections.unmodifiableList(allSteps); + this.allJobs = JobList.from(jobSteps.keySet().stream().map(allJobs).collect(toList())); } /** The application this deployment status concerns. */ @@ -143,9 +127,9 @@ public class DeploymentStatus { } /** Whether any job is failing on versions selected by the given filter, with errors other than lack of capacity in a test zone.. */ - public boolean hasFailures(Predicate<ApplicationVersion> versionFilter) { + public boolean hasFailures(Predicate<RevisionId> revisionFilter) { return ! allJobs.failingHard() - .matching(job -> versionFilter.test(job.lastTriggered().get().versions().targetApplication())) + .matching(job -> revisionFilter.test(job.lastTriggered().get().versions().targetRevision())) .isEmpty(); } @@ -172,6 +156,8 @@ public class DeploymentStatus { * and any test jobs for any outstanding change, which will likely be needed to later deploy this change. */ public Map<JobId, List<Job>> jobsToRun() { + if (application.revisions().last().isEmpty()) return Map.of(); + Map<InstanceName, Change> changes = new LinkedHashMap<>(); for (InstanceName instance : application.deploymentSpec().instanceNames()) changes.put(instance, application.require(instance).change()); @@ -196,6 +182,8 @@ public class DeploymentStatus { } private Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes, boolean eagerTests) { + if (application.revisions().last().isEmpty()) return Map.of(); + Map<JobId, List<Job>> productionJobs = new LinkedHashMap<>(); changes.forEach((instance, change) -> productionJobs.putAll(productionJobs(instance, change, eagerTests))); Map<JobId, List<Job>> testJobs = testJobs(productionJobs); @@ -256,7 +244,7 @@ public class DeploymentStatus { public Optional<Deployment> deploymentFor(JobId job) { return Optional.ofNullable(application.require(job.application().instance()) - .deployments().get(job.type().zone(system))); + .deployments().get(job.type().zone())); } /** @@ -269,19 +257,47 @@ public class DeploymentStatus { public Change outstandingChange(InstanceName instance) { StepStatus status = instanceSteps().get(instance); if (status == null) return Change.empty(); - boolean ascending = next == application.deploymentSpec().requireInstance(instance).revisionTarget(); - for (ApplicationVersion version : application.deployableVersions(ascending)) { - if (status.dependenciesCompletedAt(Change.of(version), Optional.empty()).map(now::isBefore).orElse(true)) continue; - Change change = Change.of(version); + DeploymentInstanceSpec spec = application.deploymentSpec().requireInstance(instance); + boolean ascending = next == spec.revisionTarget(); + int cumulativeRisk = 0; + int nextRisk = 0; + int skippedCumulativeRisk = 0; + Instant readySince = now; + Change candidate = Change.empty(); + for (ApplicationVersion version : application.revisions().deployable(ascending)) { + // A revision is only a candidate if it upgrades, and does not downgrade, this instance. + Change change = Change.of(version.id()); if (application.productionDeployments().getOrDefault(instance, List.of()).stream() - .anyMatch(deployment -> change.downgrades(deployment.applicationVersion()))) continue; - if ( ! application.require(instance).change().application().map(change::upgrades).orElse(true)) continue; + .anyMatch(deployment -> change.downgrades(deployment.revision()))) continue; + if ( ! application.require(instance).change().revision().map(change::upgrades).orElse(true)) continue; if (hasCompleted(instance, change)) - if (ascending) continue; - else break; - return change; + if (ascending) continue; // Keep looking for the next revision which is an upgrade, or ... + else return Change.empty(); // ... if the latest is already complete, there's nothing outstanding. + + // This revision contains something new, so start aggregating the risk score. + skippedCumulativeRisk += version.risk(); + nextRisk = nextRisk > 0 ? nextRisk : version.risk(); + // If it's not yet ready to roll out, we keep looking. + Optional<Instant> readyAt = status.dependenciesCompletedAt(Change.of(version.id()), Optional.empty()); + if (readyAt.map(now::isBefore).orElse(true)) continue; + + // It's ready. If looking for the latest, max risk is 0, and we'll return now; otherwise, we _may_ keep on looking for more. + cumulativeRisk += skippedCumulativeRisk; + skippedCumulativeRisk = 0; + nextRisk = 0; + if (cumulativeRisk >= spec.maxRisk()) + return candidate.equals(Change.empty()) ? change : candidate; // If the first candidate exceeds max risk, we have to accept that. + + // Otherwise, we may note this as a candidate, and keep looking for a newer revision, unless that makes us exceed max risk. + if (readyAt.get().isBefore(readySince)) readySince = readyAt.get(); + candidate = change; } - return Change.empty(); + // If min risk is ready, or max idle time has passed, we return the candidate. Otherwise, no outstanding change is ready. + return instanceJobs(instance).values().stream().allMatch(jobs -> jobs.lastTriggered().isEmpty()) + || cumulativeRisk >= spec.minRisk() + || cumulativeRisk + nextRisk > spec.maxRisk() + || ! now.isBefore(readySince.plus(Duration.ofHours(spec.maxIdleHours()))) + ? candidate : Change.empty(); } /** Earliest instant when job was triggered with given versions, or both system and staging tests were successful. */ @@ -305,7 +321,7 @@ public class DeploymentStatus { .type(type).asList().stream() .flatMap(status -> RunList.from(status) .on(versions) - .status(RunStatus.success) + .matching(Run::hasSucceeded) .asList().stream() .map(Run::start)) .min(naturalOrder()); @@ -324,9 +340,9 @@ public class DeploymentStatus { // When computing eager test jobs for outstanding changes, assume current change completes successfully. Optional<Deployment> deployment = deploymentFor(job); Optional<Version> existingPlatform = deployment.map(Deployment::version); - Optional<ApplicationVersion> existingApplication = deployment.map(Deployment::applicationVersion); - boolean deployingCompatibilityChange = areIncompatible(existingPlatform, change.application(), instance) - || areIncompatible(change.platform(), existingApplication, instance); + Optional<RevisionId> existingRevision = deployment.map(Deployment::revision); + boolean deployingCompatibilityChange = areIncompatible(existingPlatform, change.revision(), job) + || areIncompatible(change.platform(), existingRevision, job); if (assumeUpgradesSucceed) { if (deployingCompatibilityChange) // No eager tests for this. return; @@ -334,34 +350,36 @@ public class DeploymentStatus { Change currentChange = application.require(instance).change(); Versions target = Versions.from(currentChange, application, deployment, systemVersion); existingPlatform = Optional.of(target.targetPlatform()); - existingApplication = Optional.of(target.targetApplication()); + existingRevision = Optional.of(target.targetRevision()); } List<Job> toRun = new ArrayList<>(); List<Change> changes = deployingCompatibilityChange ? List.of(change) : changes(job, step, change); for (Change partial : changes) { Job jobToRun = new Job(job.type(), - Versions.from(partial, application, existingPlatform, existingApplication, systemVersion), + Versions.from(partial, application, existingPlatform, existingRevision, systemVersion), step.readyAt(partial, Optional.of(job)), partial); toRun.add(jobToRun); // Assume first partial change is applied before the second. existingPlatform = Optional.of(jobToRun.versions.targetPlatform()); - existingApplication = Optional.of(jobToRun.versions.targetApplication()); + existingRevision = Optional.of(jobToRun.versions.targetRevision()); } jobs.put(job, toRun); }); return jobs; } - private boolean areIncompatible(Optional<Version> platform, Optional<ApplicationVersion> application, InstanceName instance) { + private boolean areIncompatible(Optional<Version> platform, Optional<RevisionId> revision, JobId job) { + Optional<Version> compileVersion = revision.map(application.revisions()::get) + .flatMap(ApplicationVersion::compileVersion); return platform.isPresent() - && application.flatMap(ApplicationVersion::compileVersion).isPresent() - && versionCompatibility.apply(instance).refuse(platform.get(), application.get().compileVersion().get()); + && compileVersion.isPresent() + && versionCompatibility.apply(job.application().instance()).refuse(platform.get(), compileVersion.get()); } /** Changes to deploy with the given job, possibly split in two steps. */ private List<Change> changes(JobId job, StepStatus step, Change change) { - if (change.platform().isEmpty() || change.application().isEmpty() || change.isPinned()) + if (change.platform().isEmpty() || change.revision().isEmpty() || change.isPinned()) return List.of(change); if ( step.completedAt(change.withoutApplication(), Optional.of(job)).isPresent() @@ -370,7 +388,7 @@ public class DeploymentStatus { // For a dual change, where both targets remain, we determine what to run by looking at when the two parts became ready: // for deployments, we look at dependencies; for production tests, this may be overridden by what is already deployed. - JobId deployment = new JobId(job.application(), JobType.from(system, job.type().zone(system)).get()); + JobId deployment = new JobId(job.application(), JobType.deploymentTo(job.type().zone())); UpgradeRollout rollout = application.deploymentSpec().requireInstance(job.application().instance()).upgradeRollout(); if (job.type().isTest()) { Optional<Instant> platformDeployedAt = jobSteps.get(deployment).completedAt(change.withoutApplication(), Optional.of(deployment)); @@ -468,7 +486,7 @@ public class DeploymentStatus { if ( job.type().isProduction() && job.type().isDeployment() && allJobs.successOn(productionJob.versions()).type(testType).isEmpty() && testJobs.keySet().stream() - .noneMatch(test -> test.type() == testType + .noneMatch(test -> test.type().equals(testType) && testJobs.get(test).stream().anyMatch(testJob -> testJob.versions().equals(productionJob.versions())))) { JobId testJob = firstDeclaredOrElseImplicitTest(testType); testJobs.merge(testJob, @@ -486,31 +504,44 @@ public class DeploymentStatus { private JobId firstDeclaredOrElseImplicitTest(JobType testJob) { return application.deploymentSpec().instanceNames().stream() .map(name -> new JobId(application.id().instance(name), testJob)) + .filter(jobSteps::containsKey) .min(comparing(id -> ! jobSteps.get(id).isDeclared())).orElseThrow(); } /** JobId of any declared test of the given type, for the given instance. */ private Optional<JobId> declaredTest(ApplicationId instanceId, JobType testJob) { JobId jobId = new JobId(instanceId, testJob); - return jobSteps.get(jobId).isDeclared() ? Optional.of(jobId) : Optional.empty(); + return jobSteps.containsKey(jobId) && jobSteps.get(jobId).isDeclared() ? Optional.of(jobId) : Optional.empty(); } /** A DAG of the dependencies between the primitive steps in the spec, with iteration order equal to declaration order. */ - private Map<JobId, StepStatus> jobDependencies(DeploymentSpec spec, List<StepStatus> allSteps) { + private Map<JobId, StepStatus> jobDependencies(DeploymentSpec spec, List<StepStatus> allSteps, Function<JobId, JobStatus> jobs) { if (DeploymentSpec.empty.equals(spec)) return Map.of(); Map<JobId, StepStatus> dependencies = new LinkedHashMap<>(); List<StepStatus> previous = List.of(); for (DeploymentSpec.Step step : spec.steps()) - previous = fillStep(dependencies, allSteps, step, previous, null); + previous = fillStep(dependencies, allSteps, step, previous, null, jobs, + instanceWithImplicitTest(test, spec), + instanceWithImplicitTest(staging, spec)); return Collections.unmodifiableMap(dependencies); } + private static InstanceName instanceWithImplicitTest(Environment environment, DeploymentSpec spec) { + InstanceName first = null; + for (DeploymentInstanceSpec step : spec.instances()) { + if (step.concerns(environment)) return null; + first = first != null ? first : step.name(); + } + return first; + } + /** Adds the primitive steps contained in the given step, which depend on the given previous primitives, to the dependency graph. */ - private List<StepStatus> fillStep(Map<JobId, StepStatus> dependencies, List<StepStatus> allSteps, - DeploymentSpec.Step step, List<StepStatus> previous, InstanceName instance) { + private List<StepStatus> fillStep(Map<JobId, StepStatus> dependencies, List<StepStatus> allSteps, DeploymentSpec.Step step, + List<StepStatus> previous, InstanceName instance, Function<JobId, JobStatus> jobs, + InstanceName implicitSystemTest, InstanceName implicitStagingTest) { if (step.steps().isEmpty() && ! (step instanceof DeploymentInstanceSpec)) { if (instance == null) return previous; // Ignore test and staging outside all instances. @@ -522,31 +553,28 @@ public class DeploymentStatus { } JobType jobType; + JobId jobId; StepStatus stepStatus; if (step.concerns(test) || step.concerns(staging)) { - jobType = JobType.from(system, ((DeclaredZone) step).environment(), null) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); - stepStatus = JobStepStatus.ofTestDeployment((DeclaredZone) step, List.of(), this, instance, jobType, true); + jobType = step.concerns(test) ? systemTest : stagingTest; + jobId = new JobId(application.id().instance(instance), jobType); + stepStatus = JobStepStatus.ofTestDeployment((DeclaredZone) step, List.of(), this, jobs.apply(jobId), true); previous = new ArrayList<>(previous); previous.add(stepStatus); } else if (step.isTest()) { - jobType = JobType.testFrom(system, ((DeclaredTest) step).region()) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); - JobType preType = JobType.from(system, prod, ((DeclaredTest) step).region()) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); - stepStatus = JobStepStatus.ofProductionTest((DeclaredTest) step, previous, this, instance, jobType, preType); + jobType = JobType.test(((DeclaredTest) step).region()); + jobId = new JobId(application.id().instance(instance), jobType); + stepStatus = JobStepStatus.ofProductionTest((DeclaredTest) step, previous, this, jobs.apply(jobId)); previous = List.of(stepStatus); } else if (step.concerns(prod)) { - jobType = JobType.from(system, ((DeclaredZone) step).environment(), ((DeclaredZone) step).region().get()) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); - stepStatus = JobStepStatus.ofProductionDeployment((DeclaredZone) step, previous, this, instance, jobType); + jobType = JobType.prod(((DeclaredZone) step).region().get()); + jobId = new JobId(application.id().instance(instance), jobType); + stepStatus = JobStepStatus.ofProductionDeployment((DeclaredZone) step, previous, this, jobs.apply(jobId)); previous = List.of(stepStatus); } else return previous; // Empty container steps end up here, and are simply ignored. - JobId jobId = new JobId(application.id().instance(instance), jobType); - allSteps.removeIf(existing -> existing.job().equals(Optional.of(jobId))); // Replace implicit tests with explicit ones. allSteps.add(stepStatus); dependencies.put(jobId, stepStatus); return previous; @@ -558,27 +586,32 @@ public class DeploymentStatus { instance = spec.name(); allSteps.add(instanceStatus); previous = List.of(instanceStatus); - for (JobType test : List.of(systemTest, stagingTest)) { - JobId job = new JobId(application.id().instance(instance), test); - if ( ! dependencies.containsKey(job)) { - var testStatus = JobStepStatus.ofTestDeployment(new DeclaredZone(test.environment()), List.of(), - this, job.application().instance(), test, false); - dependencies.put(job, testStatus); - allSteps.add(testStatus); - } + if (instance.equals(implicitSystemTest)) { + JobId job = new JobId(application.id().instance(instance), systemTest); + JobStepStatus testStatus = JobStepStatus.ofTestDeployment(new DeclaredZone(test), List.of(), + this, jobs.apply(job), false); + dependencies.put(job, testStatus); + allSteps.add(testStatus); + } + if (instance.equals(implicitStagingTest)) { + JobId job = new JobId(application.id().instance(instance), stagingTest); + JobStepStatus testStatus = JobStepStatus.ofTestDeployment(new DeclaredZone(staging), List.of(), + this, jobs.apply(job), false); + dependencies.put(job, testStatus); + allSteps.add(testStatus); } } if (step.isOrdered()) { for (DeploymentSpec.Step nested : step.steps()) - previous = fillStep(dependencies, allSteps, nested, previous, instance); + previous = fillStep(dependencies, allSteps, nested, previous, instance, jobs, implicitSystemTest, implicitStagingTest); return previous; } List<StepStatus> parallel = new ArrayList<>(); for (DeploymentSpec.Step nested : step.steps()) - parallel.addAll(fillStep(dependencies, allSteps, nested, previous, instance)); + parallel.addAll(fillStep(dependencies, allSteps, nested, previous, instance, jobs, implicitSystemTest, implicitStagingTest)); return List.copyOf(parallel); } @@ -718,7 +751,7 @@ public class DeploymentStatus { @Override Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { return ( (change.platform().isEmpty() || change.platform().equals(instance.change().platform())) - && (change.application().isEmpty() || change.application().equals(instance.change().application())) + && (change.revision().isEmpty() || change.revision().equals(instance.change().revision())) || step().steps().stream().noneMatch(step -> step.concerns(prod))) ? dependenciesCompletedAt(change, dependent).or(() -> Optional.of(Instant.EPOCH).filter(__ -> change.hasTargets())) : Optional.empty(); @@ -732,7 +765,7 @@ public class DeploymentStatus { while ( blocker.window().includes(current) && now.plus(Duration.ofDays(7)).isAfter(current) && ( change.platform().isPresent() && blocker.blocksVersions() - || change.application().isPresent() && blocker.blocksRevisions())) { + || change.revision().isPresent() && blocker.blocksRevisions())) { blocked = true; current = current.plus(Duration.ofHours(1)).truncatedTo(ChronoUnit.HOURS); } @@ -773,7 +806,7 @@ public class DeploymentStatus { if (job.firstFailing().isEmpty() || ! job.firstFailing().get().hasEnded()) return Optional.empty(); Versions lastVersions = job.lastCompleted().get().versions(); if (change.platform().isPresent() && ! change.platform().get().equals(lastVersions.targetPlatform())) return Optional.empty(); - if (change.application().isPresent() && ! change.application().get().equals(lastVersions.targetApplication())) return Optional.empty(); + if (change.revision().isPresent() && ! change.revision().get().equals(lastVersions.targetRevision())) return Optional.empty(); if (job.id().type().environment().isTest() && job.isNodeAllocationFailure()) return Optional.empty(); Instant firstFailing = job.firstFailing().get().end().get(); @@ -787,10 +820,9 @@ public class DeploymentStatus { } private static JobStepStatus ofProductionDeployment(DeclaredZone step, List<StepStatus> dependencies, - DeploymentStatus status, InstanceName instance, JobType jobType) { + DeploymentStatus status, JobStatus job) { ZoneId zone = ZoneId.from(step.environment(), step.region().get()); - JobStatus job = status.instanceJobs(instance).get(jobType); - Optional<Deployment> existingDeployment = Optional.ofNullable(status.application().require(instance) + Optional<Deployment> existingDeployment = Optional.ofNullable(status.application().require(job.id().application().instance()) .deployments().get(zone)); return new JobStepStatus(StepType.deployment, step, dependencies, job, status) { @@ -811,21 +843,21 @@ public class DeploymentStatus { && ! existingDeployment.map(Deployment::version).equals(change.platform())) return Optional.empty(); - if ( change.application().isPresent() - && ! existingDeployment.map(Deployment::applicationVersion).equals(change.application()) + if ( change.revision().isPresent() + && ! existingDeployment.map(Deployment::revision).equals(change.revision()) && dependent.equals(job())) // Job should (re-)run in this case, but other dependents need not wait. return Optional.empty(); - Change fullChange = status.application().require(instance).change(); - if (existingDeployment.map(deployment -> ! (change.upgrades(deployment.version()) || change.upgrades(deployment.applicationVersion())) - && (fullChange.downgrades(deployment.version()) || fullChange.downgrades(deployment.applicationVersion()))) + Change fullChange = status.application().require(job.id().application().instance()).change(); + if (existingDeployment.map(deployment -> ! (change.upgrades(deployment.version()) || change.upgrades(deployment.revision())) + && (fullChange.downgrades(deployment.version()) || fullChange.downgrades(deployment.revision()))) .orElse(false)) return job.lastCompleted().flatMap(Run::end); Optional<Instant> end = Optional.empty(); for (Run run : job.runs().descendingMap().values()) { if (run.versions().targetsMatch(change)) { - if (run.status() == RunStatus.success) end = run.end(); + if (run.hasSucceeded()) end = run.end(); } else if (dependent.equals(job())) // If strict completion, consider only last time this change was deployed. break; @@ -836,11 +868,8 @@ public class DeploymentStatus { } private static JobStepStatus ofProductionTest(DeclaredTest step, List<StepStatus> dependencies, - DeploymentStatus status, InstanceName instance, - JobType testType, JobType prodType) { - JobStatus job = status.instanceJobs(instance).get(testType); - JobId prodId = new JobId(status.application().id().instance(instance), prodType); - + DeploymentStatus status, JobStatus job) { + JobId prodId = new JobId(job.id().application(), JobType.deploymentTo(job.id().type().zone())); return new JobStepStatus(StepType.test, step, dependencies, job, status) { @Override Optional<Instant> readyAt(Change change, Optional<JobId> dependent) { @@ -855,7 +884,7 @@ public class DeploymentStatus { Optional<Instant> deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId)); return (dependent.equals(job()) ? job.lastTriggered().filter(run -> deployedAt.map(at -> ! run.start().isBefore(at)).orElse(false)).stream() : job.runs().values().stream()) - .filter(run -> run.status() == RunStatus.success) + .filter(Run::hasSucceeded) .filter(run -> run.versions().targetsMatch(change)) .flatMap(run -> run.end().stream()).findFirst(); } @@ -863,9 +892,7 @@ public class DeploymentStatus { } private static JobStepStatus ofTestDeployment(DeclaredZone step, List<StepStatus> dependencies, - DeploymentStatus status, InstanceName instance, - JobType jobType, boolean declared) { - JobStatus job = status.instanceJobs(instance).get(jobType); + DeploymentStatus status, JobStatus job, boolean declared) { return new JobStepStatus(StepType.test, step, dependencies, job, status) { @Override Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { @@ -875,9 +902,9 @@ public class DeploymentStatus { status.application, Optional.of(deployment), status.systemVersion))) - .orElseGet(() -> (change.platform().isEmpty() || change.platform().get().equals(run.versions().targetPlatform())) - && (change.application().isEmpty() || change.application().get().equals(run.versions().targetApplication())))) - .status(RunStatus.success) + .orElseGet(() -> (change.platform().isEmpty() || change.platform().get().equals(run.versions().targetPlatform())) + && (change.revision().isEmpty() || change.revision().get().equals(run.versions().targetRevision())))) + .matching(Run::hasSucceeded) .asList().stream() .map(run -> run.end().get()) .max(naturalOrder()); @@ -897,7 +924,7 @@ public class DeploymentStatus { private final Change change; public Job(JobType type, Versions versions, Optional<Instant> readyAt, Change change) { - this.versions = type == systemTest ? versions.withoutSources() : versions; + this.versions = type.isSystemTest() ? versions.withoutSources() : versions; this.readyAt = readyAt; this.change = change; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java index 7ab895654f3..44079a90097 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Deployment; import java.util.Collection; @@ -29,16 +30,16 @@ import static java.util.stream.Collectors.collectingAndThen; public class DeploymentSteps { private final DeploymentInstanceSpec spec; - private final Supplier<SystemName> system; + private final ZoneRegistry zones; - public DeploymentSteps(DeploymentInstanceSpec spec, Supplier<SystemName> system) { + public DeploymentSteps(DeploymentInstanceSpec spec, ZoneRegistry zones) { this.spec = Objects.requireNonNull(spec, "spec cannot be null"); - this.system = Objects.requireNonNull(system, "system cannot be null"); + this.zones = Objects.requireNonNull(zones, "system cannot be null"); } /** Returns jobs for this, in the order they should run */ public List<JobType> jobs() { - return Stream.concat(production().isEmpty() ? Stream.of() : Stream.of(JobType.systemTest, JobType.stagingTest), + return Stream.concat(production().isEmpty() ? Stream.of() : Stream.of(JobType.systemTest(zones), JobType.stagingTest(zones)), spec.steps().stream().flatMap(step -> toJobs(step).stream())) .distinct() .collect(Collectors.toUnmodifiableList()); @@ -67,7 +68,6 @@ public class DeploymentSteps { public List<JobType> toJobs(DeploymentSpec.Step step) { return step.zones().stream() .map(this::toJob) - .flatMap(Optional::stream) .collect(Collectors.toUnmodifiableList()); } @@ -93,8 +93,13 @@ public class DeploymentSteps { } /** Resolve job from deployment zone */ - private Optional<JobType> toJob(DeploymentSpec.DeclaredZone zone) { - return JobType.from(system.get(), zone.environment(), zone.region().orElse(null)); + private JobType toJob(DeploymentSpec.DeclaredZone zone) { + switch (zone.environment()) { + case prod: return JobType.prod(zone.region().get()); + case test: return JobType.systemTest(zones); + case staging: return JobType.stagingTest(zones); + default: throw new IllegalArgumentException("region must be one with automated deployments, but got: " + zone.environment()); + } } /** Resolve jobs from steps */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 7ef8a4b4ae6..be07a2b0cb1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -8,6 +8,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.text.Text; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; @@ -17,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -72,22 +74,7 @@ public class DeploymentTrigger { } public DeploymentSteps steps(DeploymentInstanceSpec spec) { - return new DeploymentSteps(spec, controller::system); - } - - public void notifyOfSubmission(TenantAndApplicationId id, ApplicationVersion version, long projectId) { - if (applications().getApplication(id).isEmpty()) { - log.log(Level.WARNING, "Ignoring submission from project '" + projectId + - "': Unknown application '" + id + "'"); - return; - } - - applications().lockApplicationOrThrow(id, application -> { - application = application.withProjectId(OptionalLong.of(projectId)); - application = application.withNewSubmission(version); - applications().store(application); - }); - triggerNewRevision(id); + return new DeploymentSteps(spec, controller.zoneRegistry()); } /** @@ -100,11 +87,11 @@ public class DeploymentTrigger { DeploymentStatus status = jobs.deploymentStatus(application.get()); for (InstanceName instanceName : application.get().deploymentSpec().instanceNames()) { Change outstanding = outstandingChange(status, instanceName); - if ( outstanding.hasTargets() + if (outstanding.hasTargets() && status.instanceSteps().get(instanceName) .readyAt(outstanding) .map(readyAt -> ! readyAt.isAfter(clock.instant())).orElse(false) - && acceptNewApplicationVersion(status, instanceName, outstanding.application().get())) { + && acceptNewRevision(status, instanceName, outstanding.revision().get())) { application = application.with(instanceName, instance -> withRemainingChange(instance, outstanding.onTopOf(instance.change()), status)); } @@ -116,7 +103,9 @@ public class DeploymentTrigger { /** Returns any outstanding change for the given instance, coupled with any necessary platform upgrade. */ private Change outstandingChange(DeploymentStatus status, InstanceName instance) { Change outstanding = status.outstandingChange(instance); - Optional<Version> compileVersion = outstanding.application().flatMap(ApplicationVersion::compileVersion); + Optional<Version> compileVersion = outstanding.revision() + .map(status.application().revisions()::get) + .flatMap(ApplicationVersion::compileVersion); // If the outstanding revision requires a certain platform for compatibility, add that here. VersionCompatibility compatibility = applications().versionCompatibility(status.application().id().instance(instance)); @@ -246,7 +235,7 @@ public class DeploymentTrigger { DeploymentStatus status = jobs.deploymentStatus(application); Change change = instance.change(); - if ( ! upgradeRevision && change.application().isPresent()) change = change.withoutApplication(); + if ( ! upgradeRevision && change.revision().isPresent()) change = change.withoutApplication(); if ( ! upgradePlatform && change.platform().isPresent()) change = change.withoutPlatform(); Versions versions = Versions.from(change, application, status.deploymentFor(job), controller.readSystemVersion()); DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(job.type(), versions, Optional.of(controller.clock().instant()), instance.change()); @@ -267,24 +256,23 @@ public class DeploymentTrigger { private List<JobId> forceTriggerManualJob(JobId job, String reason) { Run last = jobs.last(job).orElseThrow(() -> new IllegalArgumentException(job + " has never been run")); Versions target = new Versions(controller.readSystemVersion(), - last.versions().targetApplication(), + last.versions().targetRevision(), Optional.of(last.versions().targetPlatform()), - Optional.of(last.versions().targetApplication())); + Optional.of(last.versions().targetRevision())); jobs.start(job.application(), job.type(), target, true, Optional.of(reason)); return List.of(job); } /** Retrigger job. If the job is already running, it will be canceled, and retrigger enqueued. */ public Optional<JobId> reTriggerOrAddToQueue(DeploymentId deployment, String reason) { - JobType jobType = JobType.from(controller.system(), deployment.zoneId()) - .orElseThrow(() -> new IllegalArgumentException(Text.format("No job to trigger for (system/zone): %s/%s", controller.system().value(), deployment.zoneId().value()))); + JobType jobType = JobType.deploymentTo(deployment.zoneId()); Optional<Run> existingRun = controller.jobController().active(deployment.applicationId()).stream() .filter(run -> run.id().type().equals(jobType)) .findFirst(); if (existingRun.isPresent()) { Run run = existingRun.get(); - try (Lock lock = controller.curator().lockDeploymentRetriggerQueue()) { + try (Mutex lock = controller.curator().lockDeploymentRetriggerQueue()) { List<RetriggerEntry> retriggerEntries = controller.curator().readRetriggerEntries(); List<RetriggerEntry> newList = new ArrayList<>(retriggerEntries); RetriggerEntry requiredEntry = new RetriggerEntry(new JobId(deployment.applicationId(), jobType), run.id().number() + 1); @@ -366,7 +354,7 @@ public class DeploymentTrigger { /** Returns the set of all jobs which have changes to propagate from the upstream steps. */ private List<Job> computeReadyJobs() { return jobs.deploymentStatuses(ApplicationList.from(applications().readable()) - .withProjectId() // Need to keep this, as we have applications with deployment spec that shouldn't be orchestrated. + .withProjectId() // Need to keep this, as we have applications with deployment spec that shouldn't be orchestrated. // Maybe not any longer? .withDeploymentSpec()) .withChanges() .asList().stream() @@ -397,7 +385,7 @@ public class DeploymentTrigger { /** Returns whether the application is healthy in all other production zones. */ private boolean isUnhealthyInAnotherZone(Application application, JobId job) { for (Deployment deployment : application.require(job.application().instance()).productionDeployments().values()) { - if ( ! deployment.zone().equals(job.type().zone(controller.system())) + if ( ! deployment.zone().equals(job.type().zone()) && ! controller.applications().isHealthy(new DeploymentId(job.application(), deployment.zone()))) return true; } @@ -426,9 +414,7 @@ public class DeploymentTrigger { boolean blocked = status.jobs().get(job).get().isRunning(); if ( ! job.type().isTest()) { - Optional<JobStatus> productionTest = JobType.testFrom(controller.system(), job.type().zone(controller.system()).region()) - .map(type -> new JobId(job.application(), type)) - .flatMap(status.jobs()::get); + Optional<JobStatus> productionTest = status.jobs().get(new JobId(job.application(), JobType.productionTestOf(job.type().zone()))); if (productionTest.isPresent()) { abortIfOutdated(status, jobs, productionTest.get().id()); // Production deployments are also blocked by their declared tests, if the next versions to run @@ -445,16 +431,16 @@ public class DeploymentTrigger { // ---------- Change management o_O ---------- - private boolean acceptNewApplicationVersion(DeploymentStatus status, InstanceName instance, ApplicationVersion version) { + private boolean acceptNewRevision(DeploymentStatus status, InstanceName instance, RevisionId revision) { if (status.application().deploymentSpec().instance(instance).isEmpty()) return false; // Unknown instance. - boolean isChangingRevision = status.application().require(instance).change().application().isPresent(); + boolean isChangingRevision = status.application().require(instance).change().revision().isPresent(); DeploymentInstanceSpec spec = status.application().deploymentSpec().requireInstance(instance); - Predicate<ApplicationVersion> versionFilter = spec.revisionTarget() == DeploymentSpec.RevisionTarget.next - ? failing -> status.application().require(instance).change().application().get().compareTo(failing) == 0 - : failing -> version.compareTo(failing) > 0; + Predicate<RevisionId> revisionFilter = spec.revisionTarget() == DeploymentSpec.RevisionTarget.next + ? failing -> status.application().require(instance).change().revision().get().compareTo(failing) == 0 + : failing -> revision.compareTo(failing) > 0; switch (spec.revisionChange()) { case whenClear: return ! isChangingRevision; - case whenFailing: return ! isChangingRevision || status.hasFailures(versionFilter); + case whenFailing: return ! isChangingRevision || status.hasFailures(revisionFilter); case always: return true; default: throw new IllegalStateException("Unknown revision upgrade policy"); } @@ -464,18 +450,15 @@ public class DeploymentTrigger { Change remaining = change; if (status.hasCompleted(instance.name(), change.withoutApplication())) remaining = remaining.withoutPlatform(); - if (status.hasCompleted(instance.name(), change.withoutPlatform())) { + if (status.hasCompleted(instance.name(), change.withoutPlatform())) remaining = remaining.withoutApplication(); - if (change.application().isPresent()) - instance = instance.withLatestDeployed(change.application().get()); - } return instance.withChange(remaining); } // ---------- Version and job helpers ---------- private Job deploymentJob(Instance instance, Versions versions, JobType jobType, JobStatus jobStatus, Instant availableSince) { - return new Job(instance, versions, jobType, availableSince, jobStatus.isNodeAllocationFailure(), instance.change().application().isPresent()); + return new Job(instance, versions, jobType, availableSince, jobStatus.isNodeAllocationFailure(), instance.change().revision().isPresent()); } // ---------- Data containers ---------- @@ -510,7 +493,7 @@ public class DeploymentTrigger { public String toString() { return jobType + " for " + instanceId + " on (" + versions.targetPlatform() + versions.sourcePlatform().map(version -> " <-- " + version).orElse("") + - ", " + versions.targetApplication().id() + versions.sourceApplication().map(version -> " <-- " + version.id()).orElse("") + + ", " + versions.targetRevision() + versions.sourceRevision().map(version -> " <-- " + version).orElse("") + "), ready since " + availableSince; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index beef090d214..52e5431b552 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -1,17 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; +import ai.vespa.http.DomainName; import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.Notifications; import com.yahoo.config.application.api.Notifications.When; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.AthenzDomain; -import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; @@ -21,10 +18,6 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.text.Text; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -36,11 +29,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.application.ActivateResult; @@ -48,7 +40,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.config.ControllerConfig; +import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; @@ -89,6 +81,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Nod import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; @@ -125,13 +118,6 @@ public class InternalStepRunner implements StepRunner { private static final Logger logger = Logger.getLogger(InternalStepRunner.class.getName()); - static final NodeResources DEFAULT_TESTER_RESOURCES = - new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any); - // Must match exactly the advertised resources of an AWS instance type. Also consider that the container - // will have ~1.8 GB less memory than equivalent resources in AWS (VESPA-16259). - static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = - new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); - private final Controller controller; private final TestConfigSerializer testConfigSerializer; private final DeploymentFailureMails mails; @@ -184,15 +170,15 @@ public class InternalStepRunner implements StepRunner { Versions versions = controller.jobController().run(id).get().versions(); logger.log("Deploying platform version " + versions.sourcePlatform().orElse(versions.targetPlatform()) + - " and application version " + - versions.sourceApplication().orElse(versions.targetApplication()).id() + " ..."); + " and application " + + versions.sourceRevision().orElse(versions.targetRevision()) + " ..."); return deployReal(id, true, logger); } private Optional<RunStatus> deployReal(RunId id, DualLogger logger) { Versions versions = controller.jobController().run(id).get().versions(); logger.log("Deploying platform version " + versions.targetPlatform() + - " and application version " + versions.targetApplication().id() + " ..."); + " and application " + versions.targetRevision() + " ..."); return deployReal(id, false, logger); } @@ -218,7 +204,7 @@ public class InternalStepRunner implements StepRunner { logger.log("Deploying the tester container on platform " + platform + " ..."); return deploy(() -> controller.applications().deployTester(id.tester(), testerPackage(id), - id.type().zone(controller.system()), + id.type().zone(), platform), controller.jobController().run(id).get() .stepInfo(deployTester).get() @@ -315,19 +301,19 @@ public class InternalStepRunner implements StepRunner { Version platform = setTheStage ? versions.sourcePlatform().orElse(versions.targetPlatform()) : versions.targetPlatform(); Run run = controller.jobController().run(id).get(); - Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone(controller.system())), + Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone()), Optional.of(platform)); if (services.isEmpty()) { logger.log("Config status not currently available -- will retry."); return Optional.empty(); } - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), + List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), NodeFilter.all() .applications(id.application()) .states(active)); Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); - List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), + List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), NodeFilter.all() .hostnames(parentHostnames)); boolean firstTick = run.convergenceSummary().isEmpty(); @@ -358,8 +344,8 @@ public class InternalStepRunner implements StepRunner { } if (summary.converged()) { controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); - if (endpointsAvailable(id.application(), id.type().zone(controller.system()), logger)) { - if (containersAreUp(id.application(), id.type().zone(controller.system()), logger)) { + if (endpointsAvailable(id.application(), id.type().zone(), logger)) { + if (containersAreUp(id.application(), id.type().zone(), logger)) { logger.log("Installation succeeded!"); return Optional.of(running); } @@ -441,7 +427,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> installTester(RunId id, DualLogger logger) { Run run = controller.jobController().run(id).get(); Version platform = testerPlatformVersion(id); - ZoneId zone = id.type().zone(controller.system()); + ZoneId zone = id.type().zone(); ApplicationId testerId = id.tester().id(); Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(testerId, zone), @@ -512,7 +498,7 @@ public class InternalStepRunner implements StepRunner { return false; } for (var endpoint : endpoints.get(zone)) { - HostName endpointName = HostName.from(endpoint.dnsName()); + DomainName endpointName = DomainName.of(endpoint.dnsName()); var ipAddress = controller.jobController().cloud().resolveHostName(endpointName); if (ipAddress.isEmpty()) { logger.log(INFO, "DNS lookup yielded no IP address for '" + endpointName + "'."); @@ -533,7 +519,7 @@ public class InternalStepRunner implements StepRunner { var loadBalancerAddress = controller.jobController().cloud().resolveHostName(policy.canonicalName()); if ( ! loadBalancerAddress.equals(ipAddress)) { logger.log(INFO, "IP address of CNAME '" + endpointName + "' (" + ipAddress.get() + ") and load balancer '" + - policy.canonicalName() + "' (" + loadBalancerAddress.orElse("empty") + ") are not equal"); + policy.canonicalName() + "' (" + loadBalancerAddress.orElse(null) + ") are not equal"); return false; } } @@ -610,7 +596,7 @@ public class InternalStepRunner implements StepRunner { .productionDeployments().keySet().stream() .map(zone -> new DeploymentId(id.application(), zone)) .collect(Collectors.toSet()); - ZoneId zoneId = id.type().zone(controller.system()); + ZoneId zoneId = id.type().zone(); deployments.add(new DeploymentId(id.application(), zoneId)); logger.log("Attempting to find endpoints ..."); @@ -637,6 +623,7 @@ public class InternalStepRunner implements StepRunner { return Optional.of(running); } + @SuppressWarnings("fallthrough") private Optional<RunStatus> endTests(RunId id, boolean isSetup, DualLogger logger) { Optional<Deployment> deployment = deployment(id.application(), id.type()); if (deployment.isEmpty()) { @@ -678,12 +665,14 @@ public class InternalStepRunner implements StepRunner { controller.jobController().updateTestReport(id); return Optional.of(error); case NO_TESTS: - TesterCloud.Suite suite = TesterCloud.Suite.of(id.type(), isSetup); - logger.log(INFO, "No tests were found in the test package, for test suite '" + suite + "'"); - logger.log(INFO, "The test package must either contain basic HTTP tests under 'tests/<suite-name>/', " + - "or a Java test bundle under 'components/' with at least one test with the annotation " + - "for this suite. See docs.vespa.ai/en/testing.html for details."); - return Optional.of(allowNoTests(id.application()) ? running : testFailure); + if ( ! isSetup) { // TODO: consider changing this Later™ + TesterCloud.Suite suite = TesterCloud.Suite.of(id.type(), isSetup); + logger.log(INFO, "No tests were found in the test package, for test suite '" + suite + "'"); + logger.log(INFO, "The test package should either contain basic HTTP tests under 'tests/<suite-name>/', " + + "or a Java test bundle under 'components/' with at least one test with the annotation " + + "for this suite. See docs.vespa.ai/en/testing.html for details."); + return Optional.of(noTests); + } case SUCCESS: logger.log("Tests completed successfully."); controller.jobController().updateTestReport(id); @@ -693,12 +682,6 @@ public class InternalStepRunner implements StepRunner { } } - private boolean allowNoTests(ApplicationId appId) { - return Flags.ALLOW_NO_TESTS.bindTo(controller.flagSource()) - .with(FetchVector.Dimension.TENANT_ID, appId.tenant().value()) - .value(); - } - private Optional<RunStatus> copyVespaLogs(RunId id, DualLogger logger) { if (deployment(id.application(), id.type()).isPresent()) try { @@ -726,8 +709,8 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deactivateReal(RunId id, DualLogger logger) { try { - logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.applications().deactivate(id.application(), id.type().zone(controller.system())); + logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone() + " ..."); + controller.applications().deactivate(id.application(), id.type().zone()); return Optional.of(running); } catch (RuntimeException e) { @@ -741,7 +724,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deactivateTester(RunId id, DualLogger logger) { try { - logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone() + " ..."); controller.jobController().deactivateTester(id.tester(), id.type()); return Optional.of(running); } @@ -786,14 +769,14 @@ public class InternalStepRunner implements StepRunner { Application application = controller.applications().requireApplication(TenantAndApplicationId.from(run.id().application())); Notifications notifications = application.deploymentSpec().requireInstance(run.id().application().instance()).notifications(); - boolean newCommit = application.require(run.id().application().instance()).change().application() - .map(run.versions().targetApplication()::equals) + boolean newCommit = application.require(run.id().application().instance()).change().revision() + .map(run.versions().targetRevision()::equals) .orElse(false); When when = newCommit ? failingCommit : failing; List<String> recipients = new ArrayList<>(notifications.emailAddressesFor(when)); if (notifications.emailRolesFor(when).contains(author)) - run.versions().targetApplication().authorEmail().ifPresent(recipients::add); + application.revisions().get(run.versions().targetRevision()).authorEmail().ifPresent(recipients::add); if (recipients.isEmpty()) return; @@ -834,6 +817,10 @@ public class InternalStepRunner implements StepRunner { case testFailure: updater.accept("one or more verification tests against the deployment failed. Please review test output in the deployment job log."); return; + case noTests: + controller.notificationsDb().setNotification(source, Notification.Type.deployment, Notification.Level.warning, + "no tests were found for this job type. Please review test output in the deployment job log."); + return; case error: case endpointCertificateTimeout: break; @@ -848,6 +835,7 @@ public class InternalStepRunner implements StepRunner { switch (run.status()) { case running: case aborted: + case noTests: case success: return Optional.empty(); case nodeAllocationFailure: @@ -860,16 +848,16 @@ public class InternalStepRunner implements StepRunner { return Optional.of(mails.testFailure(run.id(), recipients)); case error: case endpointCertificateTimeout: - return Optional.of(mails.systemError(run.id(), recipients)); + break; default: logger.log(WARNING, "Don't know what mail to send for run status '" + run.status() + "'"); - return Optional.of(mails.systemError(run.id(), recipients)); } + return Optional.of(mails.systemError(run.id(), recipients)); } /** Returns the deployment of the real application in the zone of the given job, if it exists. */ private Optional<Deployment> deployment(ApplicationId id, JobType type) { - return Optional.ofNullable(application(id).deployments().get(type.zone(controller.system()))); + return Optional.ofNullable(application(id).deployments().get(type.zone())); } /** Returns the real application with the given id. */ @@ -904,141 +892,29 @@ public class InternalStepRunner implements StepRunner { /** Returns the application package for the tester application, assembled from a generated config, fat-jar and services.xml. */ private ApplicationPackage testerPackage(RunId id) { - ApplicationVersion version = controller.jobController().run(id).get().versions().targetApplication(); + RevisionId revision = controller.jobController().run(id).get().versions().targetRevision(); DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); - - ZoneId zone = id.type().zone(controller.system()); + byte[] testZip = controller.applications().applicationStore().getTester(id.application().tenant(), + id.application().application(), revision); boolean useTesterCertificate = useTesterCertificate(id); - byte[] servicesXml = servicesXml( ! controller.system().isPublic(), - useTesterCertificate, - testerResourcesFor(zone, spec.requireInstance(id.application().instance())), - controller.controllerConfig().steprunner().testerapp()); - byte[] testPackage = controller.applications().applicationStore().getTester(id.application().tenant(), id.application().application(), version); - byte[] deploymentXml = deploymentXml(id.tester(), - spec.athenzDomain(), - spec.requireInstance(id.application().instance()).athenzService(zone.environment(), zone.region())); - - try (ZipBuilder zipBuilder = new ZipBuilder(testPackage.length + servicesXml.length + deploymentXml.length + 1000)) { - // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip. - zipBuilder.add(testPackage); - zipBuilder.add("artifacts/.ignore-" + UUID.randomUUID(), new byte[0]); - zipBuilder.add("tests/.ignore-" + UUID.randomUUID(), new byte[0]); - - zipBuilder.add("services.xml", servicesXml); - zipBuilder.add("deployment.xml", deploymentXml); - if (useTesterCertificate) - appendAndStoreCertificate(zipBuilder, id); - - zipBuilder.close(); - return new ApplicationPackage(zipBuilder.toByteArray()); - } - } + TestPackage testPackage = new TestPackage(testZip, + controller.system().isPublic(), + id, + controller.controllerConfig().steprunner().testerapp(), + spec, + useTesterCertificate ? controller.clock().instant() : null, + timeouts.testerCertificate()); + if (useTesterCertificate) controller.jobController().storeTesterCertificate(id, testPackage.certificate()); - private void appendAndStoreCertificate(ZipBuilder zipBuilder, RunId id) { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - X500Principal subject = new X500Principal("CN=" + id.tester().id().toFullString() + "." + id.type() + "." + id.number()); - X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, - subject, - controller.clock().instant(), - controller.clock().instant().plus(timeouts.testerCertificate()), - SignatureAlgorithm.SHA512_WITH_RSA, - BigInteger.valueOf(1)) - .build(); - controller.jobController().storeTesterCertificate(id, certificate); - zipBuilder.add("artifacts/key", KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8)); - zipBuilder.add("artifacts/cert", X509CertificateUtils.toPem(certificate).getBytes(UTF_8)); + return testPackage.asApplicationPackage(); } private DeploymentId getTesterDeploymentId(RunId runId) { - ZoneId zoneId = runId.type().zone(controller.system()); + ZoneId zoneId = runId.type().zone(); return new DeploymentId(runId.tester().id(), zoneId); } - static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) { - NodeResources nodeResources = spec.steps().stream() - .filter(step -> step.concerns(zone.environment())) - .findFirst() - .flatMap(step -> step.zones().get(0).testerFlavor()) - .map(NodeResources::fromLegacyName) - .orElse(zone.region().value().contains("aws-") ? - DEFAULT_TESTER_RESOURCES_AWS : DEFAULT_TESTER_RESOURCES); - return nodeResources.with(NodeResources.DiskSpeed.any); - } - - /** Returns the generated services.xml content for the tester application. */ - static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, - NodeResources resources, ControllerConfig.Steprunner.Testerapp config) { - int jdiscMemoryGb = 2; // 2Gb memory for tester application (excessive?). - int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb()); - - // Of the remaining memory, split 50/50 between Surefire running the tests and the rest - int testMemoryMb = (int) (1024 * (resources.memoryGb() - jdiscMemoryGb) / 2); - - String resourceString = Text.format( - "<resources vcpu=\"%.2f\" memory=\"%.2fGb\" disk=\"%.2fGb\" disk-speed=\"%s\" storage-type=\"%s\"/>", - resources.vcpu(), resources.memoryGb(), resources.diskGb(), resources.diskSpeed().name(), resources.storageType().name()); - - String runtimeProviderClass = config.runtimeProviderClass(); - String tenantCdBundle = config.tenantCdBundle(); - - String servicesXml = - "<?xml version='1.0' encoding='UTF-8'?>\n" + - "<services xmlns:deploy='vespa' version='1.0'>\n" + - " <container version='1.0' id='tester'>\n" + - "\n" + - " <component id=\"com.yahoo.vespa.hosted.testrunner.TestRunner\" bundle=\"vespa-testrunner-components\">\n" + - " <config name=\"com.yahoo.vespa.hosted.testrunner.test-runner\">\n" + - " <artifactsPath>artifacts</artifactsPath>\n" + - " <surefireMemoryMb>" + testMemoryMb + "</surefireMemoryMb>\n" + - " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + - " <useTesterCertificate>" + useTesterCertificate + "</useTesterCertificate>\n" + - " </config>\n" + - " </component>\n" + - "\n" + - " <handler id=\"com.yahoo.vespa.testrunner.TestRunnerHandler\" bundle=\"vespa-osgi-testrunner\">\n" + - " <binding>http://*/tester/v1/*</binding>\n" + - " </handler>\n" + - "\n" + - " <component id=\"" + runtimeProviderClass + "\" bundle=\"" + tenantCdBundle + "\" />\n" + - "\n" + - " <component id=\"com.yahoo.vespa.testrunner.JunitRunner\" bundle=\"vespa-osgi-testrunner\">\n" + - " <config name=\"com.yahoo.vespa.testrunner.junit-test-runner\">\n" + - " <artifactsPath>artifacts</artifactsPath>\n" + - " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + - " </config>\n" + - " </component>\n" + - "\n" + - " <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" + - " <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" + - " <artifactsPath>artifacts</artifactsPath>\n" + - " <testsPath>tests</testsPath>\n" + - " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + - " </config>\n" + - " </component>\n" + - "\n" + - " <nodes count=\"1\">\n" + - " <jvm allocated-memory=\"" + jdiscMemoryPct + "%\"/>\n" + - " " + resourceString + "\n" + - " </nodes>\n" + - " </container>\n" + - "</services>\n"; - - return servicesXml.getBytes(UTF_8); - } - - /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */ - private static byte[] deploymentXml(TesterId id, Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) { - String deploymentSpec = - "<?xml version='1.0' encoding='UTF-8'?>\n" + - "<deployment version=\"1.0\" " + - athenzDomain.map(domain -> "athenz-domain=\"" + domain.value() + "\" ").orElse("") + - athenzService.map(service -> "athenz-service=\"" + service.value() + "\" ").orElse("") + ">" + - " <instance id=\"" + id.id().instance().value() + "\" />" + - "</deployment>"; - return deploymentSpec.getBytes(UTF_8); - } - /** Logger which logs to a {@link JobController}, as well as to the parent class' {@link Logger}. */ private class DualLogger { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index b9f09cc31ea..1d56e2db08b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -6,15 +6,18 @@ import com.yahoo.component.Version; import com.yahoo.component.VersionCompatibility; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.curator.Lock; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; @@ -25,9 +28,13 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff; +import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.TestSummary; +import com.yahoo.vespa.hosted.controller.notification.Notification; +import com.yahoo.vespa.hosted.controller.notification.Notification.Type; +import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.security.cert.X509Certificate; @@ -37,19 +44,21 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.function.UnaryOperator; import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Stream; import static com.yahoo.collections.Iterables.reversed; @@ -63,10 +72,11 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup; import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static java.time.temporal.ChronoUnit.SECONDS; +import static java.util.Comparator.naturalOrder; import static java.util.function.Predicate.not; import static java.util.logging.Level.INFO; +import static java.util.logging.Level.WARNING; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toUnmodifiableList; /** @@ -86,6 +96,8 @@ public class JobController { public static final Duration maxHistoryAge = Duration.ofDays(60); + private static final Logger log = Logger.getLogger(JobController.class.getName()); + private final int historyLength; private final Controller controller; private final CuratorDb curator; @@ -125,7 +137,7 @@ public class JobController { /** Returns the logged entries for the given run, which are after the given id threshold. */ public Optional<RunLog> details(RunId id, long after) { - try (Lock __ = curator.lock(id.application(), id.type())) { + try (Mutex __ = curator.lock(id.application(), id.type())) { Run run = runs(id.application(), id.type()).get(id); if (run == null) return Optional.empty(); @@ -162,7 +174,7 @@ public class JobController { if ( ! run.hasStep(copyVespaLogs)) return run; - ZoneId zone = id.type().zone(controller.system()); + ZoneId zone = id.type().zone(); Optional<Deployment> deployment = Optional.ofNullable(controller.applications().requireInstance(id.application()) .deployments().get(zone)); if (deployment.isEmpty() || deployment.get().at().isBefore(run.start())) @@ -190,7 +202,7 @@ public class JobController { if (step.isEmpty()) return run; - List<LogEntry> entries = cloud.getLog(new DeploymentId(id.tester().id(), id.type().zone(controller.system())), + List<LogEntry> entries = cloud.getLog(new DeploymentId(id.tester().id(), id.type().zone()), run.lastTestLogEntry()); if (entries.isEmpty()) return run; @@ -202,7 +214,7 @@ public class JobController { public void updateTestReport(RunId id) { locked(id, run -> { - Optional<TestReport> report = cloud.getTestReport(new DeploymentId(id.tester().id(), id.type().zone(controller.system()))); + Optional<TestReport> report = cloud.getTestReport(new DeploymentId(id.tester().id(), id.type().zone())); if (report.isEmpty()) { return run; } @@ -229,8 +241,8 @@ public class JobController { } /** Returns all job types which have been run for the given application. */ - public List<JobType> jobs(ApplicationId id) { - return JobType.allIn(controller.system()).stream() + private List<JobType> jobs(ApplicationId id) { + return JobType.allIn(controller.zoneRegistry()).stream() .filter(type -> last(id, type).isPresent()) .collect(toUnmodifiableList()); } @@ -248,6 +260,13 @@ public class JobController { .collect(toUnmodifiableList()); } + /** Returns when given deployment last started deploying, falling back to time of deployment if it cannot be determined from job runs */ + public Instant lastDeploymentStart(ApplicationId instanceId, Deployment deployment) { + return jobStarts(new JobId(instanceId, JobType.deploymentTo(deployment.zone()))).stream() + .findFirst() + .orElseGet(deployment::at); + } + /** Returns an immutable map of all known runs for the given application and job type. */ public NavigableMap<RunId, Run> runs(ApplicationId id, JobType type) { ImmutableSortedMap.Builder<RunId, Run> runs = ImmutableSortedMap.orderedBy(Comparator.comparing(RunId::number)); @@ -309,20 +328,20 @@ public class JobController { /** Returns a list of all active runs for the given application. */ public List<Run> active(TenantAndApplicationId id) { return controller.applications().requireApplication(id).instances().keySet().stream() - .flatMap(name -> Stream.of(JobType.values()) - .map(type -> last(id.instance(name), type)) - .flatMap(Optional::stream) - .filter(run -> !run.hasEnded())) + .flatMap(name -> JobType.allIn(controller.zoneRegistry()).stream() + .map(type -> last(id.instance(name), type)) + .flatMap(Optional::stream) + .filter(run -> ! run.hasEnded())) .collect(toUnmodifiableList()); } /** Returns a list of all active runs for the given instance. */ public List<Run> active(ApplicationId id) { - return Stream.of(JobType.values()) - .map(type -> last(id, type)) - .flatMap(Optional::stream) - .filter(run -> !run.hasEnded()) - .collect(toUnmodifiableList()); + return JobType.allIn(controller.zoneRegistry()).stream() + .map(type -> last(id, type)) + .flatMap(Optional::stream) + .filter(run -> !run.hasEnded()) + .collect(toUnmodifiableList()); } /** Returns the job status of the given job, possibly empty. */ @@ -337,12 +356,8 @@ public class JobController { private DeploymentStatus deploymentStatus(Application application, Version systemVersion) { return new DeploymentStatus(application, - DeploymentStatus.jobsFor(application, controller.system()).stream() - .collect(toMap(job -> job, - job -> jobStatus(job), - (j1, j2) -> { throw new IllegalArgumentException("Duplicate key " + j1.id()); }, - LinkedHashMap::new)), - controller.system(), + this::jobStatus, + controller.zoneRegistry(), systemVersion, instance -> controller.applications().versionCompatibility(application.id().instance(instance)), controller.clock().instant()); @@ -375,7 +390,7 @@ public class JobController { * Throws TimeoutException if some step in this job is still being run. */ public void finish(RunId id) throws TimeoutException { - List<Lock> locks = new ArrayList<>(); + List<Mutex> locks = new ArrayList<>(); try { // Ensure no step is still running before we finish the run — report depends transitively on all the other steps. Run unlockedRun = run(id).get(); @@ -396,7 +411,7 @@ public class JobController { locked(id.application(), id.type(), runs -> { runs.put(run.id(), finishedRun); long last = id.number(); - long successes = runs.values().stream().filter(old -> old.status() == RunStatus.success).count(); + long successes = runs.values().stream().filter(Run::hasSucceeded).count(); var oldEntries = runs.entrySet().iterator(); for (var old = oldEntries.next(); old.getKey().number() <= last - historyLength @@ -405,7 +420,7 @@ public class JobController { // Make sure we keep the last success and the first failing if ( successes == 1 - && old.getValue().status() == RunStatus.success + && old.getValue().hasSucceeded() && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { oldEntries.next(); continue; @@ -417,26 +432,21 @@ public class JobController { }); logs.flush(id); metric.jobFinished(run.id().job(), finishedRun.status()); + pruneRevisions(unlockedRun); - DeploymentId deploymentId = new DeploymentId(unlockedRun.id().application(), unlockedRun.id().job().type().zone(controller.system())); - (unlockedRun.versions().targetApplication().isDeployedDirectly() ? - Stream.of(unlockedRun.id().type()) : - JobType.allIn(controller.system()).stream().filter(jobType -> !jobType.environment().isManuallyDeployed())) - .flatMap(jobType -> controller.jobController().runs(unlockedRun.id().application(), jobType).values().stream()) - .mapToLong(r -> r.versions().targetApplication().buildNumber().orElse(Integer.MAX_VALUE)) - .min() - .ifPresent(oldestBuild -> { - if (unlockedRun.versions().targetApplication().isDeployedDirectly()) - controller.applications().applicationStore().pruneDevDiffs(deploymentId, oldestBuild); - else - controller.applications().applicationStore().pruneDiffs(deploymentId.applicationId().tenant(), deploymentId.applicationId().application(), oldestBuild); - }); return finishedRun; }); } finally { - for (Lock lock : locks) - lock.close(); + for (Mutex lock : locks) { + try { + lock.close(); + } catch (Throwable t) { + log.log(WARNING, "Failed to close the lock " + lock + ": the lock may or may not " + + "have been released in ZooKeeper, and if not this controller " + + "must be restarted to release the lock", t); + } + } } } @@ -451,48 +461,96 @@ public class JobController { } /** Accepts and stores a new application package and test jar pair under a generated application version key. */ - public ApplicationVersion submit(TenantAndApplicationId id, Optional<SourceRevision> revision, Optional<String> authorEmail, - Optional<String> sourceUrl, long projectId, ApplicationPackage applicationPackage, - byte[] testPackageBytes) { + public ApplicationVersion submit(TenantAndApplicationId id, Submission submission, long projectId) { + ApplicationController applications = controller.applications(); AtomicReference<ApplicationVersion> version = new AtomicReference<>(); - controller.applications().lockApplicationOrThrow(id, application -> { - Optional<ApplicationVersion> previousVersion = application.get().latestVersion(); - Optional<ApplicationPackage> previousPackage = previousVersion.flatMap(previous -> controller.applications().applicationStore().find(id.tenant(), id.application(), previous.buildNumber().getAsLong())) + applications.lockApplicationOrThrow(id, application -> { + Optional<ApplicationVersion> previousVersion = application.get().revisions().last(); + Optional<ApplicationPackage> previousPackage = previousVersion.flatMap(previous -> applications.applicationStore().find(id.tenant(), id.application(), previous.buildNumber().getAsLong())) .map(ApplicationPackage::new); long previousBuild = previousVersion.map(latestVersion -> latestVersion.buildNumber().getAsLong()).orElse(0L); - String packageHash = applicationPackage.bundleHash() + ApplicationPackage.calculateHash(testPackageBytes); - version.set(ApplicationVersion.from(revision, 1 + previousBuild, authorEmail, - applicationPackage.compileVersion(), - applicationPackage.buildTime(), - sourceUrl, - revision.map(SourceRevision::commit), - false, - Optional.of(packageHash))); - - byte[] diff = previousPackage.map(previous -> ApplicationPackageDiff.diff(previous, applicationPackage)) - .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage)); - controller.applications().applicationStore().put(id.tenant(), - id.application(), - version.get(), - applicationPackage.zippedContent(), - diff); - controller.applications().applicationStore().putTester(id.tenant(), - id.application(), - version.get(), - testPackageBytes); - controller.applications().applicationStore().putMeta(id.tenant(), - id.application(), - controller.clock().instant(), - applicationPackage.metaDataZip()); - - prunePackages(id); - controller.applications().storeWithUpdatedConfig(application, applicationPackage); - - controller.applications().deploymentTrigger().notifyOfSubmission(id, version.get(), projectId); + version.set(submission.toApplicationVersion(1 + previousBuild)); + + byte[] diff = previousPackage.map(previous -> ApplicationPackageDiff.diff(previous, submission.applicationPackage())) + .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(submission.applicationPackage())); + applications.applicationStore().put(id.tenant(), + id.application(), + version.get().id(), + submission.applicationPackage().zippedContent(), + submission.testPackage(), + diff); + applications.applicationStore().putMeta(id.tenant(), + id.application(), + controller.clock().instant(), + submission.applicationPackage().metaDataZip()); + + application = application.withProjectId(projectId == -1 ? OptionalLong.empty() : OptionalLong.of(projectId)); + application = application.withRevisions(revisions -> revisions.with(version.get())); + application = withPrunedPackages(application); + + TestSummary testSummary = TestPackage.validateTests(submission.applicationPackage().deploymentSpec(), submission.testPackage()); + if (testSummary.problems().isEmpty()) + controller.notificationsDb().removeNotification(NotificationSource.from(id), Type.testPackage); + else + controller.notificationsDb().setNotification(NotificationSource.from(id), + Type.testPackage, + Notification.Level.warning, + testSummary.problems()); + + submission.applicationPackage().parentVersion().ifPresent(parent -> { + if (parent.getMajor() < controller.readSystemVersion().getMajor()) + controller.notificationsDb().setNotification(NotificationSource.from(id), + Type.submission, + Notification.Level.warning, + "Parent version used to compile the application is on a " + + "lower major version than the current Vespa Cloud version"); + else + controller.notificationsDb().removeNotification(NotificationSource.from(id), Type.submission); + }); + + applications.storeWithUpdatedConfig(application, submission.applicationPackage()); + applications.deploymentTrigger().triggerNewRevision(id); }); return version.get(); } + private LockedApplication withPrunedPackages(LockedApplication application){ + TenantAndApplicationId id = application.get().id(); + Optional<RevisionId> oldestDeployed = application.get().oldestDeployedRevision(); + if (oldestDeployed.isPresent()) { + controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed.get()); + + for (ApplicationVersion version : application.get().revisions().withPackage()) + if (version.id().compareTo(oldestDeployed.get()) < 0) + application = application.withRevisions(revisions -> revisions.with(version.withoutPackage())); + } + return application; + } + + /** Forget revisions no longer present in any relevant job history. */ + private void pruneRevisions(Run run) { + TenantAndApplicationId applicationId = TenantAndApplicationId.from(run.id().application()); + boolean isProduction = run.versions().targetRevision().isProduction(); + (isProduction ? deploymentStatus(controller.applications().requireApplication(applicationId)).jobs().asList().stream() + : Stream.of(jobStatus(run.id().job()))) + .flatMap(jobs -> jobs.runs().values().stream()) + .map(r -> r.versions().targetRevision()) + .filter(id -> id.isProduction() == isProduction) + .min(naturalOrder()) + .ifPresent(oldestRevision -> { + controller.applications().lockApplicationOrThrow(applicationId, application -> { + if (isProduction) { + controller.applications().applicationStore().pruneDiffs(run.id().application().tenant(), run.id().application().application(), oldestRevision.number()); + controller.applications().store(application.withRevisions(revisions -> revisions.withoutOlderThan(oldestRevision))); + } + else { + controller.applications().applicationStore().pruneDevDiffs(new DeploymentId(run.id().application(), run.id().job().type().zone()), oldestRevision.number()); + controller.applications().store(application.withRevisions(revisions -> revisions.withoutOlderThan(oldestRevision, run.id().job()))); + } + }); + }); + } + /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, Optional<String> reason) { start(id, type, versions, isRedeployment, JobProfile.of(type), reason); @@ -500,11 +558,12 @@ public class JobController { /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, JobProfile profile, Optional<String> reason) { - if (versions.targetApplication().compileVersion() + ApplicationVersion revision = controller.applications().requireApplication(TenantAndApplicationId.from(id)).revisions().get(versions.targetRevision()); + if (revision.compileVersion() .map(version -> controller.applications().versionCompatibility(id).refuse(versions.targetPlatform(), version)) .orElse(false)) throw new IllegalArgumentException("Will not start a job with incompatible platform version (" + versions.targetPlatform() + ") " + - "and compile versions (" + versions.targetApplication().compileVersion().get() + ")"); + "and compile versions (" + revision.compileVersion().get() + ")"); locked(id, type, __ -> { Optional<Run> last = last(id, type); @@ -525,6 +584,9 @@ public class JobController { /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment.*/ public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage, boolean dryRun) { + if ( ! controller.zoneRegistry().hasZone(type.zone())) + throw new IllegalArgumentException(type.zone() + " is not present in this system"); + controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { if ( ! application.get().instances().containsKey(id.instance())) application = controller.applications().withNewInstance(application, id); @@ -532,24 +594,23 @@ public class JobController { controller.applications().store(application); }); - DeploymentId deploymentId = new DeploymentId(id, type.zone(controller.system())); + DeploymentId deploymentId = new DeploymentId(id, type.zone()); Optional<Run> lastRun = last(id, type); lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); - long build = 1 + lastRun.map(run -> run.versions().targetApplication().buildNumber().orElse(0)).orElse(0L); - ApplicationVersion version = ApplicationVersion.from(Optional.empty(), build, Optional.empty(), - applicationPackage.compileVersion(), - Optional.empty(), Optional.empty(), - Optional.empty(), true, Optional.empty()); + long build = 1 + lastRun.map(run -> run.versions().targetRevision().number()).orElse(0L); + RevisionId revisionId = RevisionId.forDevelopment(build, new JobId(id, type)); + ApplicationVersion version = ApplicationVersion.forDevelopment(revisionId, applicationPackage.compileVersion()); byte[] diff = getDiff(applicationPackage, deploymentId, lastRun); controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - controller.applications().applicationStore().putDev(deploymentId, version, applicationPackage.zippedContent(), diff); - Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, lastRun, id)); + controller.applications().applicationStore().putDev(deploymentId, version.id(), applicationPackage.zippedContent(), diff); + Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, deploymentId, application.get().get(id.instance()))); + controller.applications().store(application.withRevisions(revisions -> revisions.with(version))); start(id, type, - new Versions(targetPlatform, version, lastRun.map(run -> run.versions().targetPlatform()), lastRun.map(run -> run.versions().targetApplication())), + new Versions(targetPlatform, version.id(), lastRun.map(run -> run.versions().targetPlatform()), lastRun.map(run -> run.versions().targetRevision())), false, dryRun ? JobProfile.developmentDryRun : JobProfile.development, Optional.empty()); @@ -562,7 +623,7 @@ public class JobController { /* Application package diff against previous version, or against empty version if previous does not exist or is invalid */ private byte[] getDiff(ApplicationPackage applicationPackage, DeploymentId deploymentId, Optional<Run> lastRun) { - return lastRun.map(run -> run.versions().targetApplication()) + return lastRun.map(run -> run.versions().targetRevision()) .map(prevVersion -> { ApplicationPackage previous; try { @@ -575,24 +636,27 @@ public class JobController { .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage)); } - private Version findTargetPlatform(ApplicationPackage applicationPackage, Optional<Run> lastRun, ApplicationId id) { + private Version findTargetPlatform(ApplicationPackage applicationPackage, DeploymentId id, Optional<Instance> instance) { Optional<Integer> major = applicationPackage.deploymentSpec().majorVersion(); if (major.isPresent()) return controller.applications().lastCompatibleVersion(major.get()) .orElseThrow(() -> new IllegalArgumentException("major " + major.get() + " specified in deployment.xml, " + "but no version on this major was found")); - // Prefer previous platform if possible. - VersionStatus versionStatus = controller.readVersionStatus(); - VersionCompatibility compatibility = controller.applications().versionCompatibility(id); - Optional<Version> target = lastRun.map(run -> run.versions().targetPlatform()).filter(versionStatus::isActive); - if (target.isPresent() && compatibility.accept(target.get(), applicationPackage.compileVersion().orElse(target.get()))) - return target.get(); + VersionCompatibility compatibility = controller.applications().versionCompatibility(id.applicationId()); + + // Prefer previous platform if possible. Candidates are all deployable, ascending, with existing version appended; then reversed. + List<Version> versions = controller.readVersionStatus().deployableVersions().stream() + .map(VespaVersion::versionNumber) + .collect(toList()); + instance.map(Instance::deployments) + .map(deployments -> deployments.get(id.zoneId())) + .map(Deployment::version) + .ifPresent(versions::add); - // Otherwise, use newest, compatible version. - for (VespaVersion platform : reversed(versionStatus.deployableVersions())) - if (compatibility.accept(platform.versionNumber(), applicationPackage.compileVersion().orElse(platform.versionNumber()))) - return platform.versionNumber(); + for (Version target : reversed(versions)) + if (applicationPackage.compileVersion().isEmpty() || compatibility.accept(target, applicationPackage.compileVersion().get())) + return target; throw new IllegalArgumentException("no suitable platform version found" + applicationPackage.compileVersion() @@ -626,7 +690,7 @@ public class JobController { TesterId tester = TesterId.of(id); for (JobType type : jobs(id)) locked(id, type, deactivateTester, __ -> { - try (Lock ___ = curator.lock(id, type)) { + try (Mutex ___ = curator.lock(id, type)) { try { deactivateTester(tester, type); } @@ -634,34 +698,24 @@ public class JobController { // It's probably already deleted, so if we fail, that's OK. } curator.deleteRunData(id, type); - logs.delete(id); } }); + logs.delete(id); + curator.deleteRunData(id); } catch (Exception e) { - return; // Don't remove the data if we couldn't clean up all resources. + log.log(WARNING, "failed cleaning up after deleted application", e); } - curator.deleteRunData(id); }); } public void deactivateTester(TesterId id, JobType type) { - controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone(controller.system()))); - } - - private void prunePackages(TenantAndApplicationId id) { - controller.applications().lockApplicationIfPresent(id, application -> { - application.get().oldestDeployedApplication() - .ifPresent(oldestDeployed -> { - controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed); - controller.applications().applicationStore().pruneTesters(id.tenant(), id.application(), oldestDeployed); - }); - }); + controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone())); } /** Locks all runs and modifies the list of historic runs for the given application and job type. */ private void locked(ApplicationId id, JobType type, Consumer<SortedMap<RunId, Run>> modifications) { - try (Lock __ = curator.lock(id, type)) { + try (Mutex __ = curator.lock(id, type)) { SortedMap<RunId, Run> runs = new TreeMap<>(curator.readHistoricRuns(id, type)); modifications.accept(runs); curator.writeHistoricRuns(id, type, runs.values()); @@ -670,19 +724,18 @@ public class JobController { /** Locks and modifies the run with the given id, provided it is still active. */ public void locked(RunId id, UnaryOperator<Run> modifications) { - try (Lock __ = curator.lock(id.application(), id.type())) { + try (Mutex __ = curator.lock(id.application(), id.type())) { active(id).ifPresent(run -> { - run = modifications.apply(run); - curator.writeLastRun(run); + curator.writeLastRun(modifications.apply(run)); }); } } /** Locks the given step and checks none of its prerequisites are running, then performs the given actions. */ public void locked(ApplicationId id, JobType type, Step step, Consumer<LockedStep> action) throws TimeoutException { - try (Lock lock = curator.lock(id, type, step)) { + try (Mutex lock = curator.lock(id, type, step)) { for (Step prerequisite : step.allPrerequisites(last(id, type).get().steps().keySet())) // Check that no prerequisite is still running. - try (Lock __ = curator.lock(id, type, prerequisite)) { ; } + try (Mutex __ = curator.lock(id, type, prerequisite)) { ; } action.accept(new LockedStep(lock, step)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java index d06bdc45583..387ea755414 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import java.time.Instant; import java.util.Collection; @@ -119,7 +120,7 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { /** Returns the jobs with successful runs matching the given versions — targets only for system test, everything present otherwise. */ public JobList successOn(Versions versions) { - return matching(job -> ! RunList.from(job).status(RunStatus.success).on(versions).isEmpty()); + return matching(job -> ! RunList.from(job).matching(Run::hasSucceeded).on(versions).isEmpty()); } // ----------------------------------- JobRun filtering @@ -174,8 +175,8 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { } /** Returns the subset of jobs where the run of the indicated type was on the given version */ - public JobList on(ApplicationVersion version) { - return matching(run -> run.versions().targetApplication().equals(version)); + public JobList on(RevisionId revision) { + return matching(run -> run.versions().targetRevision().equals(revision)); } /** Returns the subset of jobs where the run of the indicated type was on the given version */ @@ -196,7 +197,7 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { if (job.isSuccess()) return false; if (job.lastSuccess().isEmpty()) return true; // An application which never succeeded is surely bad. if ( ! job.firstFailing().get().versions().targetPlatform().equals(job.lastSuccess().get().versions().targetPlatform())) return false; // Version change may be to blame. - return ! job.firstFailing().get().versions().targetApplication().equals(job.lastSuccess().get().versions().targetApplication()); // Return whether there is an application change. + return ! job.firstFailing().get().versions().targetRevision().equals(job.lastSuccess().get().versions().targetRevision()); // Return whether there is an application change. } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java index 874b1828f5f..14fce806152 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -21,6 +21,7 @@ public class JobMetrics { public static final String deploymentFailure = "deployment.deploymentFailure"; public static final String convergenceFailure = "deployment.convergenceFailure"; public static final String testFailure = "deployment.testFailure"; + public static final String noTests = "deployment.noTests"; public static final String error = "deployment.error"; public static final String abort = "deployment.abort"; public static final String success = "deployment.success"; @@ -46,7 +47,7 @@ public class JobMetrics { "tenantName", id.application().tenant().value(), "app", id.application().application().value() + "." + id.application().instance().value(), "test", Boolean.toString(id.type().isTest()), - "zone", id.type().zone(system.get()).value()); + "zone", id.type().zone().value()); } static String valueOf(RunStatus status) { @@ -56,6 +57,7 @@ public class JobMetrics { case deploymentFailed: return deploymentFailure; case installationFailed: return convergenceFailure; case testFailure: return testFailure; + case noTests: return noTests; case error: return error; case aborted: return abort; case success: return success; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java index aad5d510261..45bf508f026 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java @@ -60,7 +60,7 @@ public class JobStatus { } public boolean isSuccess() { - return lastStatus().isPresent() && lastStatus().get() == RunStatus.success; + return lastCompleted.map(last -> ! last.hasFailed()).orElse(false); } public boolean isRunning() { @@ -90,18 +90,17 @@ public class JobStatus { static Optional<Run> lastSuccess(NavigableMap<RunId, Run> runs) { return runs.descendingMap().values().stream() - .filter(run -> run.status() == RunStatus.success) + .filter(Run::hasSucceeded) .findFirst(); } static Optional<Run> firstFailing(NavigableMap<RunId, Run> runs) { Run failed = null; - loop: for (Run run : runs.descendingMap().values()) - switch (run.status()) { - case running: continue loop; - case success: break loop; - default: failed = run; - } + for (Run run : runs.descendingMap().values()) { + if ( ! run.hasEnded()) continue; + if ( ! run.hasFailed()) break; + failed = run; + } return Optional.ofNullable(failed); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java index d46516582be..8147ccb3180 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; /** @@ -9,7 +10,7 @@ import com.yahoo.vespa.curator.Lock; public class LockedStep { private final Step step; - LockedStep(Lock lock, Step step) { this.step = step; } + LockedStep(Mutex lock, Step step) { this.step = step; } public Step get() { return step; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java index 6f456d2e217..e0c1fef91b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; @@ -23,41 +24,42 @@ public class RetriggerEntrySerializer { private static final String JOB_TYPE_KEY = "jobType"; private static final String MIN_REQUIRED_RUN_ID_KEY = "minimumRunId"; - public static List<RetriggerEntry> fromSlime(Slime slime) { + public List<RetriggerEntry> fromSlime(Slime slime) { return SlimeUtils.entriesStream(slime.get().field("entries")) - .map(RetriggerEntrySerializer::deserializeEntry) + .map(this::deserializeEntry) .collect(Collectors.toList()); } - public static Slime toSlime(List<RetriggerEntry> entryList) { + public Slime toSlime(List<RetriggerEntry> entryList) { Slime slime = new Slime(); Cursor root = slime.setObject(); Cursor entries = root.setArray("entries"); - entryList.forEach(e -> RetriggerEntrySerializer.serializeEntry(entries, e)); + entryList.forEach(e -> serializeEntry(entries, e)); return slime; } - private static void serializeEntry(Cursor array, RetriggerEntry entry) { + private void serializeEntry(Cursor array, RetriggerEntry entry) { Cursor root = array.addObject(); Cursor jobid = root.setObject(JOB_ID_KEY); jobid.setString(APPLICATION_ID_KEY, entry.jobId().application().serializedForm()); - jobid.setString(JOB_TYPE_KEY, entry.jobId().type().jobName()); + jobid.setString(JOB_TYPE_KEY, entry.jobId().type().serialized()); root.setLong(MIN_REQUIRED_RUN_ID_KEY, entry.requiredRun()); } - private static RetriggerEntry deserializeEntry(Inspector inspector) { + private RetriggerEntry deserializeEntry(Inspector inspector) { Inspector jobid = inspector.field(JOB_ID_KEY); ApplicationId applicationId = ApplicationId.fromSerializedForm(require(jobid, APPLICATION_ID_KEY).asString()); - JobType jobType = JobType.fromJobName(require(jobid, JOB_TYPE_KEY).asString()); + JobType jobType = JobType.ofSerialized(require(jobid, JOB_TYPE_KEY).asString()); long minRequiredRunId = require(inspector, MIN_REQUIRED_RUN_ID_KEY).asLong(); return new RetriggerEntry(new JobId(applicationId, jobType), minRequiredRunId); } - private static Inspector require(Inspector inspector, String fieldName) { + private Inspector require(Inspector inspector, String fieldName) { Inspector field = inspector.field(fieldName); if (!field.valid()) { throw new IllegalStateException("Could not deserialize, field not found in json: " + fieldName); } return field; } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java new file mode 100644 index 00000000000..5bdc980f11a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java @@ -0,0 +1,149 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import ai.vespa.validation.Validation; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; + +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Deque; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.TreeMap; + +import static ai.vespa.validation.Validation.require; +import static java.util.Collections.emptyNavigableMap; +import static java.util.stream.Collectors.toList; + +/** + * History of application revisions for an {@link com.yahoo.vespa.hosted.controller.Application}. + * + * @author jonmv + */ +public class RevisionHistory { + + private static final Comparator<JobId> comparator = Comparator.comparing(JobId::application).thenComparing(JobId::type); + + private final NavigableMap<RevisionId, ApplicationVersion> production; + private final NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development; + + private RevisionHistory(NavigableMap<RevisionId, ApplicationVersion> production, + NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development) { + this.production = production; + this.development = development; + } + + public static RevisionHistory empty() { + return ofRevisions(List.of(), Map.of()); + } + + public static RevisionHistory ofRevisions(Collection<ApplicationVersion> productionRevisions, + Map<JobId, ? extends Collection<ApplicationVersion>> developmentRevisions) { + NavigableMap<RevisionId, ApplicationVersion> production = new TreeMap<>(); + for (ApplicationVersion revision : productionRevisions) + production.put(revision.id(), revision); + + // TODO jonmv: remove once it's run once on serialised data + String hash = ""; + for (ApplicationVersion revision : List.copyOf(production.values())) + if (hash.equals(hash = revision.bundleHash().orElse("")) && ! hash.isEmpty()) + production.put(revision.id(), revision.skipped()); + + NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development = new TreeMap<>(comparator); + developmentRevisions.forEach((job, jobRevisions) -> { + NavigableMap<RevisionId, ApplicationVersion> revisions = development.computeIfAbsent(job, __ -> new TreeMap<>()); + for (ApplicationVersion revision : jobRevisions) + revisions.put(revision.id(), revision); + }); + + return new RevisionHistory(production, development); + } + + /** Returns a copy of this without any production revisions older than the given. */ + public RevisionHistory withoutOlderThan(RevisionId id) { + if (production.headMap(id).isEmpty()) return this; + return new RevisionHistory(production.tailMap(id, true), development); + } + + /** Returns a copy of this without any development revisions older than the given. */ + public RevisionHistory withoutOlderThan(RevisionId id, JobId job) { + if ( ! development.containsKey(job) || development.get(job).headMap(id).isEmpty()) return this; + NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development = new TreeMap<>(this.development); + development.compute(job, (__, revisions) -> revisions.tailMap(id, true)); + return new RevisionHistory(production, development); + } + + /** Returns a copy of this with the revision added or updated. */ + public RevisionHistory with(ApplicationVersion revision) { + if (revision.id().isProduction()) { + if ( ! production.isEmpty() && revision.bundleHash().flatMap(hash -> production.lastEntry().getValue().bundleHash().map(hash::equals)).orElse(false)) + revision = revision.skipped(); + + NavigableMap<RevisionId, ApplicationVersion> production = new TreeMap<>(this.production); + production.put(revision.id(), revision); + return new RevisionHistory(production, development); + } + else { + NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development = new TreeMap<>(this.development); + NavigableMap<RevisionId, ApplicationVersion> revisions = development.compute(revision.id().job(), (__, old) -> new TreeMap<>(old != null ? old : emptyNavigableMap())); + if ( ! revisions.isEmpty()) revisions.compute(revisions.lastKey(), (__, last) -> last.withoutPackage()); + revisions.put(revision.id(), revision); + return new RevisionHistory(production, development); + } + } + + // Fallback for when an application version isn't known for the given key. + private static ApplicationVersion revisionOf(RevisionId id) { + return new ApplicationVersion(id, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), false, false, Optional.empty(), 0); + } + + /** Returns the production {@link ApplicationVersion} with this revision ID. */ + public ApplicationVersion get(RevisionId id) { + return id.isProduction() ? production.getOrDefault(id, revisionOf(id)) + : development.getOrDefault(id.job(), emptyNavigableMap()) + .getOrDefault(id, revisionOf(id)); + } + + /** Returns the last submitted production build. */ + public Optional<ApplicationVersion> last() { + return Optional.ofNullable(production.lastEntry()).map(Map.Entry::getValue); + } + + /** Returns all known production revisions we still have the package for, from oldest to newest. */ + public List<ApplicationVersion> withPackage() { + return production.values().stream() + .filter(ApplicationVersion::hasPackage) + .collect(toList()); + } + + /** Returns the currently deployable revisions of the application. */ + public Deque<ApplicationVersion> deployable(boolean ascending) { + Deque<ApplicationVersion> versions = new ArrayDeque<>(); + for (ApplicationVersion version : withPackage()) { + if (version.isDeployable()) { + if (ascending) versions.addLast(version); + else versions.addFirst(version); + } + } + return versions; + } + + /** All known production revisions, in ascending order. */ + public List<ApplicationVersion> production() { + return List.copyOf(production.values()); + } + + /* All known development revisions, in ascending order, per job. */ + public NavigableMap<JobId, List<ApplicationVersion>> development() { + NavigableMap<JobId, List<ApplicationVersion>> copy = new TreeMap<>(comparator); + development.forEach((job, revisions) -> copy.put(job, List.copyOf(revisions.values()))); + return Collections.unmodifiableNavigableMap(copy); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index e73d3f52e1f..03cc6c6ba8d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -13,6 +13,7 @@ import java.util.Optional; import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; @@ -80,8 +81,9 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(Step.Status.of(status))); - return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, this.status == running ? status : this.status, - lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); + RunStatus newStatus = hasFailed() || status == running ? this.status : status; + return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, newStatus, lastTestRecord, + lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); } /** Returns a new Run with a new start time*/ @@ -210,7 +212,7 @@ public class Run { /** Returns whether the run has failed, and should switch to its run-always steps. */ public boolean hasFailed() { - return status != running && status != success; + return status != running && status != success && status != noTests; } /** Returns whether the run has ended, i.e., has become inactive, and can no longer be updated. */ @@ -218,6 +220,8 @@ public class Run { return end.isPresent(); } + public boolean hasSucceeded() { return hasEnded() && ! hasFailed(); } + /** Returns the target, and possibly source, versions for this run. */ public Versions versions() { return versions; @@ -297,7 +301,7 @@ public class Run { return steps.entrySet().stream() .filter(entry -> entry.getValue().status() == unfinished && entry.getKey().prerequisites().stream() - .allMatch(step -> steps.get(step) == null + .allMatch(step -> steps.get(step) == null || steps.get(step).status() == succeeded)) .map(Map.Entry::getKey) .collect(Collectors.toUnmodifiableList()); @@ -310,7 +314,7 @@ public class Run { && entry.getKey().alwaysRun() && entry.getKey().prerequisites().stream() .filter(Step::alwaysRun) - .allMatch(step -> steps.get(step) == null + .allMatch(step -> steps.get(step) == null || steps.get(step).status() != unfinished)) .map(Map.Entry::getKey) .collect(Collectors.toUnmodifiableList()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java index 00cd4bd5c6c..80c6552d3d4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java @@ -38,7 +38,7 @@ public class RunList extends AbstractFilteringList<Run, RunList> { private static boolean matchingVersions(Run run, Versions versions) { return versions.targetsMatch(run.versions()) - && (versions.sourcesMatchIfPresent(run.versions()) || run.id().type() == JobType.systemTest); + && (versions.sourcesMatchIfPresent(run.versions()) || run.id().type().isSystemTest()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 0bb4a30425e..9ca634b19fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -26,6 +26,9 @@ public enum RunStatus { /** The verification tests failed. */ testFailure, + /** No tests, for a test job. */ + noTests, + /** An unexpected error occurred. */ error, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 2e669808c44..82d154dcf03 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -3,11 +3,9 @@ package com.yahoo.vespa.hosted.controller.deployment; import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toUnmodifiableList; /** * Steps that make up a deployment job. See {@link JobProfile} for preset profiles. @@ -115,6 +113,7 @@ public enum Step { case success : throw new AssertionError("Unexpected run status '" + status + "'!"); case reset : case aborted : return unfinished; + case noTests : case running : return succeeded; default : return failed; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Submission.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Submission.java new file mode 100644 index 00000000000..e366920690b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Submission.java @@ -0,0 +1,57 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; + +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.calculateHash; + +/** + * @author jonmv + */ +public class Submission { + + private final ApplicationPackage applicationPackage; + private final byte[] testPackage; + private final Optional<String> sourceUrl; + private final Optional<SourceRevision> source; + private final Optional<String> authorEmail; + private final Optional<String> description; + private final int risk; + + public Submission(ApplicationPackage applicationPackage, byte[] testPackage, Optional<String> sourceUrl, + Optional<SourceRevision> source, Optional<String> authorEmail, Optional<String> description, int risk) { + this.applicationPackage = applicationPackage; + this.testPackage = testPackage; + this.sourceUrl = sourceUrl; + this.source = source; + this.authorEmail = authorEmail; + this.description = description; + this.risk = risk; + } + + public static Submission basic(ApplicationPackage applicationPackage, byte[] testPackage) { + return new Submission(applicationPackage, testPackage, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), 0); + } + + public ApplicationVersion toApplicationVersion(long number) { + return ApplicationVersion.forProduction(RevisionId.forProduction(number), + source, + authorEmail, + applicationPackage.compileVersion(), + applicationPackage.buildTime(), + sourceUrl, + source.map(SourceRevision::commit), + Optional.of(applicationPackage.bundleHash() + calculateHash(testPackage)), + description, + risk); + } + + public ApplicationPackage applicationPackage() { return applicationPackage; } + + public byte[] testPackage() { return testPackage; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java index c9b488026a5..1680e064234 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; @@ -13,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.application.Endpoint; import java.io.IOException; import java.io.UncheckedIOException; -import java.net.URI; import java.util.List; import java.util.Map; @@ -39,7 +37,7 @@ public class TestConfigSerializer { Cursor root = slime.setObject(); root.setString("application", id.serializedForm()); - root.setString("zone", type.zone(system).value()); + root.setString("zone", type.zone().value()); root.setString("system", system.value()); root.setBool("isCI", isCI); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java index d0b8e773cae..f4c4b8bebd4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -22,24 +23,24 @@ import static java.util.Objects.requireNonNull; public class Versions { private final Version targetPlatform; - private final ApplicationVersion targetApplication; + private final RevisionId targetRevision; private final Optional<Version> sourcePlatform; - private final Optional<ApplicationVersion> sourceApplication; + private final Optional<RevisionId> sourceRevision; - public Versions(Version targetPlatform, ApplicationVersion targetApplication, Optional<Version> sourcePlatform, - Optional<ApplicationVersion> sourceApplication) { - if (sourcePlatform.isPresent() ^ sourceApplication.isPresent()) + public Versions(Version targetPlatform, RevisionId targetRevision, Optional<Version> sourcePlatform, + Optional<RevisionId> sourceRevision) { + if (sourcePlatform.isPresent() ^ sourceRevision.isPresent()) throw new IllegalArgumentException("Sources must both be present or absent."); this.targetPlatform = requireNonNull(targetPlatform); - this.targetApplication = requireNonNull(targetApplication); + this.targetRevision = requireNonNull(targetRevision); this.sourcePlatform = requireNonNull(sourcePlatform); - this.sourceApplication = requireNonNull(sourceApplication); + this.sourceRevision = requireNonNull(sourceRevision); } /** A copy of this, without source versions. */ public Versions withoutSources() { - return new Versions(targetPlatform, targetApplication, Optional.empty(), Optional.empty()); + return new Versions(targetPlatform, targetRevision, Optional.empty(), Optional.empty()); } /** Target platform version for this */ @@ -47,9 +48,9 @@ public class Versions { return targetPlatform; } - /** Target application version for this */ - public ApplicationVersion targetApplication() { - return targetApplication; + /** Target revision for this */ + public RevisionId targetRevision() { + return targetRevision; } /** Source platform version for this */ @@ -58,27 +59,27 @@ public class Versions { } /** Source application version for this */ - public Optional<ApplicationVersion> sourceApplication() { - return sourceApplication; + public Optional<RevisionId> sourceRevision() { + return sourceRevision; } /** Returns whether source versions are present and match those of the given job other versions. */ public boolean sourcesMatchIfPresent(Versions versions) { return (sourcePlatform.map(targetPlatform::equals).orElse(true) || sourcePlatform.equals(versions.sourcePlatform())) && - (sourceApplication.map(targetApplication::equals).orElse(true) || - sourceApplication.equals(versions.sourceApplication())); + (sourceRevision.map(targetRevision::equals).orElse(true) || + sourceRevision.equals(versions.sourceRevision())); } public boolean targetsMatch(Versions versions) { return targetPlatform.equals(versions.targetPlatform()) && - targetApplication.equals(versions.targetApplication()); + targetRevision.equals(versions.targetRevision()); } /** Returns wheter this change could result in the given target versions. */ public boolean targetsMatch(Change change) { return change.platform().map(targetPlatform::equals).orElse(true) - && change.application().map(targetApplication::equals).orElse(true); + && change.revision().map(targetRevision::equals).orElse(true); } @Override @@ -87,43 +88,43 @@ public class Versions { if ( ! (o instanceof Versions)) return false; Versions versions = (Versions) o; return Objects.equals(targetPlatform, versions.targetPlatform) && - Objects.equals(targetApplication, versions.targetApplication) && + Objects.equals(targetRevision, versions.targetRevision) && Objects.equals(sourcePlatform, versions.sourcePlatform) && - Objects.equals(sourceApplication, versions.sourceApplication); + Objects.equals(sourceRevision, versions.sourceRevision); } @Override public int hashCode() { - return Objects.hash(targetPlatform, targetApplication, sourcePlatform, sourceApplication); + return Objects.hash(targetPlatform, targetRevision, sourcePlatform, sourceRevision); } @Override public String toString() { - return Text.format("platform %s%s, application %s%s", - sourcePlatform.filter(source -> !source.equals(targetPlatform)) + return Text.format("platform %s%s, revision %s%s", + sourcePlatform.filter(source -> ! source.equals(targetPlatform)) .map(source -> source + " -> ").orElse(""), targetPlatform, - sourceApplication.filter(source -> !source.equals(targetApplication)) - .map(source -> source.id() + " -> ").orElse(""), - targetApplication.id()); + sourceRevision.filter(source -> ! source.equals(targetRevision)) + .map(source -> source + " -> ").orElse(""), + targetRevision); } /** Create versions using given change and application */ public static Versions from(Change change, Application application, Optional<Version> existingPlatform, - Optional<ApplicationVersion> existingApplication, Version defaultPlatformVersion) { + Optional<RevisionId> existingRevision, Version defaultPlatformVersion) { return new Versions(targetPlatform(application, change, existingPlatform, defaultPlatformVersion), - targetApplication(application, change, existingApplication), + targetRevision(application, change, existingRevision), existingPlatform, - existingApplication); + existingRevision); } /** Create versions using given change and application */ public static Versions from(Change change, Application application, Optional<Deployment> deployment, Version defaultPlatformVersion) { return new Versions(targetPlatform(application, change, deployment.map(Deployment::version), defaultPlatformVersion), - targetApplication(application, change, deployment.map(Deployment::applicationVersion)), + targetRevision(application, change, deployment.map(Deployment::revision)), deployment.map(Deployment::version), - deployment.map(Deployment::applicationVersion)); + deployment.map(Deployment::revision)); } private static Version targetPlatform(Application application, Change change, Optional<Version> existing, @@ -135,17 +136,17 @@ public class Versions { .orElseGet(() -> application.oldestDeployedPlatform().orElse(defaultVersion)); } - private static ApplicationVersion targetApplication(Application application, Change change, - Optional<ApplicationVersion> existing) { - return change.application() + private static RevisionId targetRevision(Application application, Change change, + Optional<RevisionId> existing) { + return change.revision() .or(() -> existing) - .orElseGet(() -> defaultApplicationVersion(application)); + .orElseGet(() -> defaultRevision(application)); } - private static ApplicationVersion defaultApplicationVersion(Application application) { - return application.oldestDeployedApplication() - .or(application::latestVersion) - .orElse(ApplicationVersion.unknown); + private static RevisionId defaultRevision(Application application) { + return application.oldestDeployedRevision() + .or(() -> application.revisions().last().map(ApplicationVersion::id)) + .orElseThrow(() -> new IllegalStateException("no known prod revisions, but asked for one, for " + application)); } private static <T extends Comparable<T>> Optional<T> max(Optional<T> o1, Optional<T> o2) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index aecb1e7a2c1..540e8489e6d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; @@ -79,7 +80,7 @@ public class NameServiceForwarder { } protected void forward(NameServiceRequest request, NameServiceQueue.Priority priority) { - try (Lock lock = db.lockNameServiceQueue()) { + try (Mutex lock = db.lockNameServiceQueue()) { NameServiceQueue queue = db.readNameServiceQueue(); var queued = queue.requests().size(); if (queued >= QUEUE_CAPACITY) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 0e880bb627c..02e1818932e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -103,7 +102,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { } } return new ApplicationSummary(app.id().defaultInstance(), app.activity().lastQueried(), app.activity().lastWritten(), - app.latestVersion().flatMap(version -> version.buildTime()), metrics); + app.revisions().last().flatMap(version -> version.buildTime()), metrics); } /** Escalate ownership issues which have not been closed before a defined amount of time has passed. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java index 7ede040773e..8765884e23c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java @@ -7,12 +7,10 @@ import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; -import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; -import java.time.Instant; import java.util.List; import java.util.Optional; import java.util.function.Consumer; @@ -49,8 +47,7 @@ public class CloudTrialExpirer extends ControllerMaintainer { private void moveInactiveTenantsToNonePlan() { var predicate = tenantReadersNotLoggedIn(loginExpiry) - .and(this::tenantHasTrialPlan) - .and(this::tenantHasNoDeployments); + .and(this::tenantHasTrialPlan); forTenant("'none' plan", predicate, this::setPlanNone); } @@ -63,11 +60,12 @@ public class CloudTrialExpirer extends ControllerMaintainer { } private void forTenant(String name, Predicate<Tenant> p, Consumer<List<Tenant>> c) { - var predicate = ((Predicate<Tenant>) this::tenantIsCloudTenant) - .and(this::tenantIsNotExemptFromExpiry); + var predicate = p.and(this::tenantIsCloudTenant) + .and(this::tenantIsNotExemptFromExpiry) + .and(this::tenantHasNoDeployments); var tenants = controller().tenants().asList().stream() - .filter(predicate.and(p)) + .filter(predicate) .collect(Collectors.toList()); if (! tenants.isEmpty()) { @@ -121,7 +119,15 @@ public class CloudTrialExpirer extends ControllerMaintainer { private void tombstoneTenants(List<Tenant> tenants) { tenants.forEach(tenant -> { + deleteApplicationsWithNoDeployments(tenant); controller().tenants().delete(tenant.name(), Optional.empty(), false); }); } + + private void deleteApplicationsWithNoDeployments(Tenant tenant) { + controller().applications().asList(tenant.name()).forEach(application -> { + // this only removes applications with no active deployments + controller().applications().deleteApplication(application.id(), Optional.empty()); + }); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index deafcd35e9b..041d0694ca9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; import java.time.Duration; @@ -36,7 +37,7 @@ public class ControllerMaintenance extends AbstractComponent { @Inject @SuppressWarnings("unused") // instantiated by Dependency Injection - public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement) { + public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement, AthenzClientFactory athenzClientFactory) { Intervals intervals = new Intervals(controller.system()); upgrader = new Upgrader(controller, intervals.defaultInterval); maintainers.add(upgrader); @@ -44,7 +45,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new DeploymentExpirer(controller, intervals.defaultInterval)); maintainers.add(new DeploymentUpgrader(controller, intervals.defaultInterval)); maintainers.add(new DeploymentIssueReporter(controller, controller.serviceRegistry().deploymentIssues(), intervals.defaultInterval)); - maintainers.add(new MetricsReporter(controller, metric)); + maintainers.add(new MetricsReporter(controller, metric, athenzClientFactory.createZmsClient())); maintainers.add(new OutstandingChangeDeployer(controller, intervals.outstandingChangeDeployer)); maintainers.add(new VersionStatusUpdater(controller, intervals.versionStatusUpdater)); maintainers.add(new ReadyJobsTrigger(controller, intervals.readyJobsTrigger)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 9a8ba9afca2..97f3f955a20 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -8,11 +8,9 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.time.Instant; import java.util.Optional; import java.util.logging.Level; @@ -59,13 +57,8 @@ public class DeploymentExpirer extends ControllerMaintainer { Optional<Duration> ttl = controller().zoneRegistry().getDeploymentTimeToLive(deployment.zone()); if (ttl.isEmpty()) return false; - Optional<JobId> jobId = JobType.from(controller().system(), deployment.zone()) - .map(type -> new JobId(instance, type)); - if (jobId.isEmpty()) return false; - - return controller().jobController().jobStarts(jobId.get()).stream().findFirst() - .map(start -> start.plus(ttl.get()).isBefore(controller().clock().instant())) - .orElse(false); + return controller().jobController().lastDeploymentStart(instance, deployment) + .plus(ttl.get()).isBefore(controller().clock().instant()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 3a047d33be5..6b058537c2d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -21,7 +21,6 @@ import java.time.Duration; import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index a2abd6493cb..c86f79ce188 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -19,8 +19,6 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; -import static com.yahoo.collections.Iterables.reversed; - /** * Upgrades instances in manually deployed zones to the system version, at a convenient time. * @@ -49,12 +47,12 @@ public class DeploymentUpgrader extends ControllerMaintainer { for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) try { - JobId job = new JobId(instance.id(), JobType.from(controller().system(), deployment.zone()).get()); + JobId job = new JobId(instance.id(), JobType.deploymentTo(deployment.zone())); if ( ! deployment.zone().environment().isManuallyDeployed()) continue; Run last = controller().jobController().last(job).get(); - Versions target = new Versions(targetPlatform, last.versions().targetApplication(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetApplication())); - if (last.versions().targetApplication().compileVersion() + Versions target = new Versions(targetPlatform, last.versions().targetRevision(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetRevision())); + if (application.revisions().get(last.versions().targetRevision()).compileVersion() .map(version -> controller().applications().versionCompatibility(instance.id()).refuse(version, target.targetPlatform())) .orElse(false)) continue; if ( ! deployment.version().isBefore(target.targetPlatform())) continue; @@ -62,7 +60,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); attempts.incrementAndGet(); - controller().jobController().start(instance.id(), JobType.from(controller().system(), deployment.zone()).get(), target, true, Optional.of("automated upgrade")); + controller().jobController().start(instance.id(), JobType.deploymentTo(deployment.zone()), target, true, Optional.of("automated upgrade")); } catch (Exception e) { failures.incrementAndGet(); log.log(Level.WARNING, "Failed upgrading " + deployment + " of " + instance + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 15f8d6380c0..193fb89eb99 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -6,6 +6,7 @@ import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -82,7 +83,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { var refreshedCertificateMetadata = endpointCertificateMetadata .withVersion(latestAvailableVersion.getAsInt()) .withLastRefreshed(clock.instant().getEpochSecond()); - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { if (Optional.of(endpointCertificateMetadata).equals(curator.readEndpointCertificateMetadata(applicationId))) { curator.writeEndpointCertificateMetadata(applicationId, refreshedCertificateMetadata); // Certificate not validated here, but on deploy. } @@ -103,7 +104,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { controller().applications().getInstance(applicationId) .ifPresent(instance -> instance.productionDeployments().forEach((zone, deployment) -> { if (deployment.at().isBefore(refreshTime)) { - JobType job = JobType.from(controller().system(), zone).orElseThrow(); + JobType job = JobType.deploymentTo(zone); deploymentTrigger.reTrigger(applicationId, job, "re-triggered by EndpointCertificateMaintainer"); log.info("Re-triggering deployment job " + job.jobName() + " for instance " + applicationId.serializedForm() + " to roll out refreshed endpoint certificate"); @@ -128,7 +129,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { curator.readAllEndpointCertificateMetadata().forEach((applicationId, storedMetaData) -> { var lastRequested = Instant.ofEpochSecond(storedMetaData.lastRequested()); if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(applicationId)) { - try (Lock lock = lock(applicationId)) { + try (Mutex lock = lock(applicationId)) { if (Optional.of(storedMetaData).equals(curator.readEndpointCertificateMetadata(applicationId))) { log.log(Level.INFO, "Cert for app " + applicationId.serializedForm() + " has not been requested in a month and app has no deployments, deleting from provider and ZK"); @@ -140,7 +141,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { }); } - private Lock lock(ApplicationId applicationId) { + private Mutex lock(ApplicationId applicationId) { return curator.lock(TenantAndApplicationId.from(applicationId)); } @@ -169,7 +170,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { EndpointCertificateMetadata storedAppMetadata = storedAppEntry.getValue(); if (storedAppMetadata.certName().equals(unknownCertDetails.cert_key_keyname())) { matchFound = true; - try (Lock lock = lock(storedApp)) { + try (Mutex lock = lock(storedApp)) { if (Optional.of(storedAppMetadata).equals(curator.readEndpointCertificateMetadata(storedApp))) { log.log(Level.INFO, "Cert for app " + storedApp.serializedForm() + " has a new leafRequestId " + unknownCertDetails.request_id() + ", updating in ZK"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 6aa43d4db47..294d5bad42d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -62,17 +63,20 @@ public class MetricsReporter extends ControllerMaintainer { public static final String REMAINING_ROTATIONS = "remaining_rotations"; public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; public static final String OPERATION_PREFIX = "operation."; + public static final String ZMS_QUOTA_USAGE = "zms.quota.usage"; private final Metric metric; private final Clock clock; + private final ZmsClient zmsClient; // Keep track of reported node counts for each version private final ConcurrentHashMap<NodeCountKey, Long> nodeCounts = new ConcurrentHashMap<>(); - public MetricsReporter(Controller controller, Metric metric) { + public MetricsReporter(Controller controller, Metric metric, ZmsClient zmsClient) { super(controller, Duration.ofMinutes(1)); // use fixed rate for metrics this.metric = metric; this.clock = controller.clock(); + this.zmsClient = zmsClient; } @Override @@ -85,6 +89,7 @@ public class MetricsReporter extends ControllerMaintainer { reportAuditLog(); reportBrokenSystemVersion(versionStatus); reportTenantMetrics(); + reportZmsQuotaMetrics(); return 1.0; } @@ -167,7 +172,7 @@ public class MetricsReporter extends ControllerMaintainer { }); for (Application application : applications.asList()) - application.latestVersion() + application.revisions().last() .flatMap(ApplicationVersion::buildTime) .ifPresent(buildTime -> metric.set(DEPLOYMENT_BUILD_AGE_SECONDS, controller().clock().instant().getEpochSecond() - buildTime.getEpochSecond(), @@ -252,6 +257,20 @@ public class MetricsReporter extends ControllerMaintainer { }); } + private void reportZmsQuotaMetrics() { + var quota = zmsClient.getQuotaUsage(); + reportZmsQuota("subdomains", quota.getSubdomainUsage()); + reportZmsQuota("services", quota.getServiceUsage()); + reportZmsQuota("policies", quota.getPolicyUsage()); + reportZmsQuota("roles", quota.getRoleUsage()); + reportZmsQuota("groups", quota.getGroupUsage()); + } + + private void reportZmsQuota(String resourceType, double usage) { + var context = metric.createContext(Map.of("resourceType", resourceType)); + metric.set(ZMS_QUOTA_USAGE, usage, context); + } + private Map<NodeVersion, Duration> platformChangeDurations(VersionStatus versionStatus) { return changeDurations(versionStatus.versions(), VespaVersion::nodeVersions); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index e412a5cc9f7..4ed34a91029 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; @@ -66,7 +67,9 @@ public class Upgrader extends ControllerMaintainer { /** Returns a list of all production application instances, except those which are pinned, which we should not manipulate here. */ private InstanceList instances(Version systemVersion) { - return InstanceList.from(controller().jobController().deploymentStatuses(ApplicationList.from(controller().applications().readable()), systemVersion)) + return InstanceList.from(controller().jobController().deploymentStatuses(ApplicationList.from(controller().applications().readable()) + .withProjectId(), + systemVersion)) .withDeclaredJobs() .shuffle(random) .byIncreasingDeployedVersion() @@ -174,7 +177,7 @@ public class Upgrader extends ControllerMaintainer { " for version " + version.toFullString() + ": Version may be in use by applications"); } - try (Lock lock = curator.lockConfidenceOverrides()) { + try (Mutex lock = curator.lockConfidenceOverrides()) { Map<Version, Confidence> overrides = new LinkedHashMap<>(curator.readConfidenceOverrides()); overrides.put(version, confidence); curator.writeConfidenceOverrides(overrides); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java index 05a7e2368d1..163a570768f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index f7e085bd90f..d0b3b9f4c7f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -106,6 +106,10 @@ public class VcmrMaintainer extends ControllerMaintainer { return Status.REQUIRES_OPERATOR_ACTION; } + if (byActionState.getOrDefault(State.OUT_OF_SYNC, 0L) > 0) { + return Status.OUT_OF_SYNC; + } + if (byActionState.getOrDefault(State.RETIRING, 0L) > 0) { return Status.IN_PROGRESS; } @@ -170,6 +174,9 @@ public class VcmrMaintainer extends ControllerMaintainer { addReport(zoneId, changeRequest, node); + if (isOutOfSync(node, hostAction)) + return hostAction.withState(State.OUT_OF_SYNC); + if (isPostponed(changeRequest, hostAction)) { LOG.fine(() -> changeRequest.getChangeRequestSource().getId() + " is postponed, recycling " + node.hostname()); recycleNode(zoneId, node, hostAction); @@ -239,7 +246,7 @@ public class VcmrMaintainer extends ControllerMaintainer { } private boolean hasRetired(Node node, HostAction hostAction) { - return hostAction.getState() == State.RETIRING && + return List.of(State.RETIRING, State.REQUIRES_OPERATOR_ACTION).contains(hostAction.getState()) && node.state() == Node.State.parked; } @@ -248,6 +255,12 @@ public class VcmrMaintainer extends ControllerMaintainer { && node.state() == Node.State.active; } + // Determines if node state is unexpected based on previous action taken + private boolean isOutOfSync(Node node, HostAction action) { + return action.getState() == State.RETIRED && node.state() != Node.State.parked || + action.getState() == State.RETIRING && !node.wantToRetire(); + } + private Map<ZoneId, List<Node>> nodesByZone() { return controller().zoneRegistry() .zones() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index 6597d59027c..71b8f1cd9b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -10,11 +10,11 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.util.logging.Level; +import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.aborted; import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.broken; import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.high; import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.low; import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.normal; -import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.aborted; /** * This maintenance job periodically updates the version status. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java index c39ee031e27..8a363405c41 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java @@ -66,10 +66,16 @@ public class Notification { } public enum Type { - /** Related to contents of application package, e.g. usage of deprecated features/syntax */ + /** Related to contents of application package, e.g., usage of deprecated features/syntax */ applicationPackage, - /** Related to deployment of application, e.g. system test failure, node allocation failure, internal errors, etc. */ + /** Related to contents of application package, e.g., old parent or compile version, or errors detectable on submission */ + submission, + + /** Related to contents of application test package, e.g., mismatch between deployment spec and provided tests */ + testPackage, + + /** Related to deployment of application, e.g., system test failure, node allocation failure, internal errors, etc. */ deployment, /** Application cluster is (near) external feed blocked */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java index aa62028749b..7876099cb21 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java @@ -5,6 +5,7 @@ import com.yahoo.collections.Pair; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Text; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; @@ -65,7 +66,7 @@ public class NotificationsDb { */ public void setNotification(NotificationSource source, Type type, Level level, List<String> messages) { Optional<Notification> changed = Optional.empty(); - try (Lock lock = curatorDb.lockNotifications(source.tenant())) { + try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { var existingNotifications = curatorDb.readNotifications(source.tenant()); List<Notification> notifications = existingNotifications.stream() .filter(notification -> !source.equals(notification.source()) || type != notification.type()) @@ -82,7 +83,7 @@ public class NotificationsDb { /** Remove the notification with the given source and type */ public void removeNotification(NotificationSource source, Type type) { - try (Lock lock = curatorDb.lockNotifications(source.tenant())) { + try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { List<Notification> initial = curatorDb.readNotifications(source.tenant()); List<Notification> filtered = initial.stream() .filter(notification -> !source.equals(notification.source()) || type != notification.type()) @@ -94,7 +95,7 @@ public class NotificationsDb { /** Remove all notifications for this source or sources contained by this source */ public void removeNotifications(NotificationSource source) { - try (Lock lock = curatorDb.lockNotifications(source.tenant())) { + try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { if (source.application().isEmpty()) { // Source is tenant curatorDb.deleteNotifications(source.tenant()); return; @@ -130,7 +131,7 @@ public class NotificationsDb { .collect(Collectors.toUnmodifiableList()); NotificationSource deploymentSource = NotificationSource.from(deploymentId); - try (Lock lock = curatorDb.lockNotifications(deploymentSource.tenant())) { + try (Mutex lock = curatorDb.lockNotifications(deploymentSource.tenant())) { List<Notification> initial = curatorDb.readNotifications(deploymentSource.tenant()); List<Notification> updated = Stream.concat( initial.stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java index 594908a3bc1..6b14872b07d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java @@ -1,16 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notify; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; import com.yahoo.text.Text; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; +import java.net.URI; import java.util.Collection; import java.util.List; import java.util.Objects; @@ -25,12 +30,14 @@ import java.util.stream.Collectors; */ public class Notifier { private final CuratorDb curatorDb; + private final ZoneRegistry zoneRegistry; private final Mailer mailer; private static final Logger log = Logger.getLogger(Notifier.class.getName()); - public Notifier(CuratorDb curatorDb, Mailer mailer) { + public Notifier(CuratorDb curatorDb, ZoneRegistry zoneRegistry, Mailer mailer) { this.curatorDb = Objects.requireNonNull(curatorDb); + this.zoneRegistry = Objects.requireNonNull(zoneRegistry); this.mailer = Objects.requireNonNull(mailer); } @@ -56,7 +63,12 @@ public class Notifier { private boolean skipSource(NotificationSource source) { // Limit sources to production systems only. Dev and test systems cause too much noise at the moment. - return source.jobType().map(t -> !t.isProduction()).orElse(false); + if (source.zoneId().map(z -> z.environment() != Environment.prod).orElse(false)) { + return true; + } else if (source.jobType().map(t -> !t.isProduction()).orElse(false)) { + return true; + } + return false; } public void dispatch(Notification notification) { @@ -82,12 +94,41 @@ public class Notifier { } private Mail mailOf(Notification n, Collection<String> recipients) { - var subject = Text.format("[%s] Vespa Notification for %s", n.level().toString().toUpperCase(), n.type().name()); + var source = n.source(); + var subject = Text.format("[%s] %s Vespa Notification for %s", n.level().toString().toUpperCase(), n.type().name(), applicationIdSource(source)); var body = new StringBuilder(); body.append("Source: ").append(n.source().toString()).append("\n") .append("\n") - .append(String.join("\n", n.messages())); - return new Mail(recipients, subject.toString(), body.toString()); + .append(String.join("\n", n.messages())) + .append("\n") + .append(url(source).toString()); + return new Mail(recipients, subject, body.toString()); + } + + private String applicationIdSource(NotificationSource source) { + StringBuilder sb = new StringBuilder(); + sb.append(source.tenant().value()); + source.application().ifPresent(applicationName -> sb.append(".").append(applicationName.value())); + source.instance().ifPresent(instanceName -> sb.append(".").append(instanceName.value())); + return sb.toString(); + } + + private URI url(NotificationSource source) { + if (source.application().isPresent()) { + if (source.instance().isPresent()) { + if (source.jobType().isPresent() && source.runNumber().isPresent()) { + return zoneRegistry.dashboardUrl( + new RunId(ApplicationId.from(source.tenant(), + source.application().get(), + source.instance().get()), + source.jobType().get(), + source.runNumber().getAsLong())); + } + return zoneRegistry.dashboardUrl(ApplicationId.from(source.tenant(), source.application().get(), source.instance().get())); + } + return zoneRegistry.dashboardUrl(source.tenant(), source.application().get()); + } + return zoneRegistry.dashboardUrl(source.tenant()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index dee3c822465..48d8627f407 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -4,9 +4,11 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyUtils; import com.yahoo.slime.ArrayTraverser; @@ -18,7 +20,9 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; @@ -30,6 +34,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; @@ -49,8 +54,6 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; /** * Serializes {@link Application}s to/from slime. @@ -76,8 +79,9 @@ public class ApplicationSerializer { private static final String instancesField = "instances"; private static final String deployingField = "deployingField"; private static final String projectIdField = "projectId"; - private static final String latestVersionField = "latestVersion"; private static final String versionsField = "versions"; + private static final String prodVersionsField = "prodVersions"; + private static final String devVersionsField = "devVersions"; private static final String pinnedField = "pinned"; private static final String deploymentIssueField = "deploymentIssueId"; private static final String ownershipIssueIdField = "ownershipIssueId"; @@ -97,7 +101,6 @@ public class ApplicationSerializer { private static final String deploymentJobsField = "deploymentJobs"; // TODO jonmv: clean up serialisation format private static final String assignedRotationsField = "assignedRotations"; private static final String assignedRotationEndpointField = "endpointId"; - private static final String latestDeployedField = "latestDeployed"; // Deployment fields private static final String zoneField = "zone"; @@ -110,8 +113,12 @@ public class ApplicationSerializer { private static final String repositoryField = "repositoryField"; private static final String branchField = "branchField"; private static final String commitField = "commitField"; + private static final String descriptionField = "description"; + private static final String riskField = "risk"; private static final String authorEmailField = "authorEmailField"; private static final String deployedDirectlyField = "deployedDirectly"; + private static final String hasPackageField = "hasPackage"; + private static final String shouldSkipField = "shouldSkip"; private static final String compileVersionField = "compileVersion"; private static final String buildTimeField = "buildTime"; private static final String sourceUrlField = "sourceUrl"; @@ -167,8 +174,7 @@ public class ApplicationSerializer { root.setDouble(queryQualityField, application.metrics().queryServiceQuality()); root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); deployKeysToSlime(application.deployKeys(), root.setArray(pemDeployKeysField)); - application.latestVersion().ifPresent(version -> toSlime(version, root.setObject(latestVersionField))); - versionsToSlime(application, root.setArray(versionsField)); + revisionsToSlime(application.revisions(), root.setArray(prodVersionsField), root.setArray(devVersionsField)); instancesToSlime(application, root.setArray(instancesField)); return slime; } @@ -182,7 +188,6 @@ public class ApplicationSerializer { assignedRotationsToSlime(instance.rotations(), instanceObject); toSlime(instance.rotationStatus(), instanceObject.setArray(rotationStatusField)); toSlime(instance.change(), instanceObject, deployingField); - instance.latestDeployed().ifPresent(version -> toSlime(version, instanceObject.setObject(latestDeployedField))); } } @@ -199,7 +204,7 @@ public class ApplicationSerializer { zoneIdToSlime(deployment.zone(), object.setObject(zoneField)); object.setString(versionField, deployment.version().toString()); object.setLong(deployTimeField, deployment.at().toEpochMilli()); - toSlime(deployment.applicationVersion(), object.setObject(applicationPackageRevisionField)); + toSlime(deployment.revision(), object.setObject(applicationPackageRevisionField)); deploymentMetricsToSlime(deployment.metrics(), object); deployment.activity().lastQueried().ifPresent(instant -> object.setLong(lastQueriedField, instant.toEpochMilli())); deployment.activity().lastWritten().ifPresent(instant -> object.setLong(lastWrittenField, instant.toEpochMilli())); @@ -228,8 +233,23 @@ public class ApplicationSerializer { object.setString(regionField, zone.region().value()); } - private void versionsToSlime(Application application, Cursor object) { - application.versions().forEach(version -> toSlime(version, object.addObject())); + private void revisionsToSlime(RevisionHistory revisions, Cursor revisionsArray, Cursor devRevisionsArray) { + revisionsToSlime(revisions.production(), revisionsArray); + revisions.development().forEach((job, devRevisions) -> { + Cursor devRevisionsObject = devRevisionsArray.addObject(); + devRevisionsObject.setString(instanceNameField, job.application().instance().value()); + devRevisionsObject.setString(jobTypeField, job.type().serialized()); + revisionsToSlime(devRevisions, devRevisionsObject.setArray(versionsField)); + }); + } + + private void revisionsToSlime(Iterable<ApplicationVersion> revisions, Cursor revisionsArray) { + revisions.forEach(version -> toSlime(version, revisionsArray.addObject())); + } + + private void toSlime(RevisionId revision, Cursor object) { + object.setLong(applicationBuildNumberField, revision.number()); + object.setBool(deployedDirectlyField, ! revision.isProduction()); } private void toSlime(ApplicationVersion applicationVersion, Cursor object) { @@ -241,6 +261,10 @@ public class ApplicationSerializer { applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url)); applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit)); object.setBool(deployedDirectlyField, applicationVersion.isDeployedDirectly()); + object.setBool(hasPackageField, applicationVersion.hasPackage()); + object.setBool(shouldSkipField, applicationVersion.shouldSkip()); + applicationVersion.description().ifPresent(description -> object.setString(descriptionField, description)); + if (applicationVersion.risk() != 0) object.setLong(riskField, applicationVersion.risk()); applicationVersion.bundleHash().ifPresent(bundleHash -> object.setString(bundleHashField, bundleHash)); } @@ -254,7 +278,7 @@ public class ApplicationSerializer { Cursor jobStatusArray = cursor.setArray(jobStatusField); jobPauses.forEach((type, until) -> { Cursor jobPauseObject = jobStatusArray.addObject(); - jobPauseObject.setString(jobTypeField, type.jobName()); + jobPauseObject.setString(jobTypeField, type.serialized()); jobPauseObject.setLong(pausedUntilField, until.toEpochMilli()); }); } @@ -265,8 +289,8 @@ public class ApplicationSerializer { Cursor object = parentObject.setObject(fieldName); if (deploying.platform().isPresent()) object.setString(versionField, deploying.platform().get().toString()); - if (deploying.application().isPresent()) - toSlime(deploying.application().get(), object); + if (deploying.revision().isPresent()) + toSlime(deploying.revision().get(), object); if (deploying.isPinned()) object.setBool(pinnedField, true); } @@ -321,42 +345,50 @@ public class ApplicationSerializer { Set<PublicKey> deployKeys = deployKeysFromSlime(root.field(pemDeployKeysField)); List<Instance> instances = instancesFromSlime(id, root.field(instancesField)); OptionalLong projectId = SlimeUtils.optionalLong(root.field(projectIdField)); - Optional<ApplicationVersion> latestVersion = latestVersionFromSlime(root.field(latestVersionField)); - SortedSet<ApplicationVersion> versions = versionsFromSlime(root.field(versionsField)); + RevisionHistory revisions = revisionsFromSlime(root.field(prodVersionsField), root.field(devVersionsField), id); return new Application(id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, - deployKeys, projectId, latestVersion, versions, instances); + deployKeys, projectId, revisions, instances); } - private Optional<ApplicationVersion> latestVersionFromSlime(Inspector latestVersionObject) { - return Optional.of(applicationVersionFromSlime(latestVersionObject)) - .filter(version -> ! version.isUnknown()); + private RevisionHistory revisionsFromSlime(Inspector prodVersionsArray, Inspector devVersionsArray, TenantAndApplicationId id) { + List<ApplicationVersion> revisions = revisionsFromSlime(prodVersionsArray, null); + Map<JobId, List<ApplicationVersion>> devRevisions = new HashMap<>(); + devVersionsArray.traverse((ArrayTraverser) (__, devRevisionsObject) -> { + JobId job = jobIdFromSlime(id, devRevisionsObject); + devRevisions.put(job, revisionsFromSlime(devRevisionsObject.field(versionsField), job)); + }); + + return RevisionHistory.ofRevisions(revisions, devRevisions); } - private SortedSet<ApplicationVersion> versionsFromSlime(Inspector versionsObject) { - SortedSet<ApplicationVersion> versions = new TreeSet<>(); - versionsObject.traverse((ArrayTraverser) (name, object) -> versions.add(applicationVersionFromSlime(object))); - return versions; + private JobId jobIdFromSlime(TenantAndApplicationId base, Inspector idObject) { + return new JobId(base.instance(idObject.field(instanceNameField).asString()), + JobType.ofSerialized(idObject.field(jobTypeField).asString())); + } + + private List<ApplicationVersion> revisionsFromSlime(Inspector versionsArray, JobId job) { + List<ApplicationVersion> revisions = new ArrayList<>(); + versionsArray.traverse((ArrayTraverser) (__, revisionObject) -> revisions.add(applicationVersionFromSlime(revisionObject, job))); + return revisions; } private List<Instance> instancesFromSlime(TenantAndApplicationId id, Inspector field) { List<Instance> instances = new ArrayList<>(); field.traverse((ArrayTraverser) (name, object) -> { InstanceName instanceName = InstanceName.from(object.field(instanceNameField).asString()); - List<Deployment> deployments = deploymentsFromSlime(object.field(deploymentsField)); + List<Deployment> deployments = deploymentsFromSlime(object.field(deploymentsField), id.instance(instanceName)); Map<JobType, Instant> jobPauses = jobPausesFromSlime(object.field(deploymentJobsField)); List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(object); RotationStatus rotationStatus = rotationStatusFromSlime(object); Change change = changeFromSlime(object.field(deployingField)); - Optional<ApplicationVersion> latestDeployed = latestVersionFromSlime(object.field(latestDeployedField)); instances.add(new Instance(id.instance(instanceName), deployments, jobPauses, assignedRotations, rotationStatus, - change, - latestDeployed)); + change)); }); return instances; } @@ -367,15 +399,16 @@ public class ApplicationSerializer { return keys; } - private List<Deployment> deploymentsFromSlime(Inspector array) { + private List<Deployment> deploymentsFromSlime(Inspector array, ApplicationId id) { List<Deployment> deployments = new ArrayList<>(); - array.traverse((ArrayTraverser) (int i, Inspector item) -> deployments.add(deploymentFromSlime(item))); + array.traverse((ArrayTraverser) (int i, Inspector item) -> deployments.add(deploymentFromSlime(item, id))); return deployments; } - private Deployment deploymentFromSlime(Inspector deploymentObject) { - return new Deployment(zoneIdFromSlime(deploymentObject.field(zoneField)), - applicationVersionFromSlime(deploymentObject.field(applicationPackageRevisionField)), + private Deployment deploymentFromSlime(Inspector deploymentObject, ApplicationId id) { + ZoneId zone = zoneIdFromSlime(deploymentObject.field(zoneField)); + return new Deployment(zone, + revisionFromSlime(deploymentObject.field(applicationPackageRevisionField), new JobId(id, JobType.deploymentTo(zone))), Version.fromString(deploymentObject.field(versionField).asString()), SlimeUtils.instant(deploymentObject.field(deployTimeField)), deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)), @@ -432,22 +465,30 @@ public class ApplicationSerializer { return ZoneId.from(object.field(environmentField).asString(), object.field(regionField).asString()); } - private ApplicationVersion applicationVersionFromSlime(Inspector object) { - if ( ! object.valid()) return ApplicationVersion.unknown; - OptionalLong applicationBuildNumber = SlimeUtils.optionalLong(object.field(applicationBuildNumberField)); - if (applicationBuildNumber.isEmpty()) - return ApplicationVersion.unknown; + private RevisionId revisionFromSlime(Inspector object, JobId job) { + long build = object.field(applicationBuildNumberField).asLong(); + boolean production = object.field(deployedDirectlyField).valid() // TODO jonmv: remove after migration + && build > 0 + && ! object.field(deployedDirectlyField).asBool(); + return production ? RevisionId.forProduction(build) : RevisionId.forDevelopment(build, job); + } + private ApplicationVersion applicationVersionFromSlime(Inspector object, JobId job) { + RevisionId id = revisionFromSlime(object, job); Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); Optional<String> authorEmail = SlimeUtils.optionalString(object.field(authorEmailField)); Optional<Version> compileVersion = SlimeUtils.optionalString(object.field(compileVersionField)).map(Version::fromString); Optional<Instant> buildTime = SlimeUtils.optionalInstant(object.field(buildTimeField)); Optional<String> sourceUrl = SlimeUtils.optionalString(object.field(sourceUrlField)); Optional<String> commit = SlimeUtils.optionalString(object.field(commitField)); - boolean deployedDirectly = object.field(deployedDirectlyField).asBool(); + boolean hasPackage = object.field(hasPackageField).asBool(); + boolean shouldSkip = object.field(shouldSkipField).asBool(); + Optional<String> description = SlimeUtils.optionalString(object.field(descriptionField)); + int risk = (int) object.field(riskField).asLong(); Optional<String> bundleHash = SlimeUtils.optionalString(object.field(bundleHashField)); - return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly, bundleHash); + return new ApplicationVersion(id, sourceRevision, authorEmail, compileVersion, buildTime, sourceUrl, + commit, bundleHash, hasPackage, shouldSkip, description, risk); } private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) { @@ -460,9 +501,8 @@ public class ApplicationSerializer { private Map<JobType, Instant> jobPausesFromSlime(Inspector object) { Map<JobType, Instant> jobPauses = new HashMap<>(); object.field(jobStatusField).traverse((ArrayTraverser) (__, jobPauseObject) -> - JobType.fromOptionalJobName(jobPauseObject.field(jobTypeField).asString()) - .ifPresent(jobType -> jobPauses.put(jobType, - SlimeUtils.instant(jobPauseObject.field(pausedUntilField))))); + jobPauses.put(JobType.ofSerialized(jobPauseObject.field(jobTypeField).asString()), + SlimeUtils.instant(jobPauseObject.field(pausedUntilField)))); return jobPauses; } @@ -473,7 +513,7 @@ public class ApplicationSerializer { if (versionFieldValue.valid()) change = Change.of(Version.fromString(versionFieldValue.asString())); if (object.field(applicationBuildNumberField).valid()) - change = change.with(applicationVersionFromSlime(object)); + change = change.with(revisionFromSlime(object, null)); if (object.field(pinnedField).asBool()) change = change.withPin(); return change; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java index 059eb37bb59..9721026c628 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java @@ -2,10 +2,10 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Step; @@ -110,10 +110,8 @@ public class BufferedLogStore { store.delete(id); } - /** Deletes all logs for the given application. */ + /** Deletes all logs in permanent storage for the given application. */ public void delete(ApplicationId id) { - for (JobType type : JobType.values()) - buffer.deleteLog(id, type); store.delete(id); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java index 8b599b45558..1ec349b7dab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java @@ -4,10 +4,10 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; /** - * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.ControllerVersion}. + * Serializer for {@link ControllerVersion}. * * @author mpolden */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 84ff3d5d8c3..c3c68f7596f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -7,19 +7,18 @@ import com.yahoo.component.Version; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.curator.MultiplePathsLock; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -38,12 +37,10 @@ import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; - import java.io.IOException; import java.io.UncheckedIOException; import java.nio.ByteBuffer; @@ -103,8 +100,6 @@ public class CuratorDb { private final ControllerVersionSerializer controllerVersionSerializer = new ControllerVersionSerializer(); private final ConfidenceOverrideSerializer confidenceOverrideSerializer = new ConfidenceOverrideSerializer(); private final TenantSerializer tenantSerializer = new TenantSerializer(); - private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); - private final RunSerializer runSerializer = new RunSerializer(); private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); private final OsVersionTargetSerializer osVersionTargetSerializer = new OsVersionTargetSerializer(osVersionSerializer); private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer, nodeVersionSerializer); @@ -112,10 +107,13 @@ public class CuratorDb { private final ZoneRoutingPolicySerializer zoneRoutingPolicySerializer = new ZoneRoutingPolicySerializer(routingPolicySerializer); private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer(); private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer(); + private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); + private final RunSerializer runSerializer = new RunSerializer(); + private final RetriggerEntrySerializer retriggerEntrySerializer = new RetriggerEntrySerializer(); + private final NotificationsSerializer notificationsSerializer = new NotificationsSerializer(); private final Curator curator; private final Duration tryLockTimeout; - private final StringFlag lockScheme; // For each application id (path), store the ZK node version and its deserialised data - update when version changes. // This will grow to keep all applications in memory, but this should be OK @@ -125,14 +123,13 @@ public class CuratorDb { private final Map<Path, Pair<Integer, NavigableMap<RunId, Run>>> cachedHistoricRuns = new ConcurrentHashMap<>(); @Inject - public CuratorDb(Curator curator, FlagSource flagSource) { - this(curator, defaultTryLockTimeout, flagSource); + public CuratorDb(Curator curator, ServiceRegistry services) { + this(curator, defaultTryLockTimeout, services.zoneRegistry().system()); } - CuratorDb(Curator curator, Duration tryLockTimeout, FlagSource flagSource) { + CuratorDb(Curator curator, Duration tryLockTimeout, SystemName system) { this.curator = curator; this.tryLockTimeout = tryLockTimeout; - this.lockScheme = Flags.CONTROLLER_LOCK_SCHEME.bindTo(flagSource); } /** Returns all hostnames configured to be part of this ZooKeeper cluster */ @@ -145,71 +142,35 @@ public class CuratorDb { // -------------- Locks --------------------------------------------------- - public Lock lock(TenantName name) { + public Mutex lock(TenantName name) { return curator.lock(lockPath(name), defaultLockTimeout.multipliedBy(2)); } - public Lock lock(TenantAndApplicationId id) { - switch (lockScheme.value()) { - case "BOTH": - return new MultiplePathsLock(lockPath(id), legacyLockPath(id), defaultLockTimeout.multipliedBy(2), curator); - case "OLD": - return curator.lock(legacyLockPath(id), defaultLockTimeout.multipliedBy(2)); - case "NEW": - return curator.lock(lockPath(id), defaultLockTimeout.multipliedBy(2)); - default: - throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); - } + public Mutex lock(TenantAndApplicationId id) { + return curator.lock(lockPath(id), defaultLockTimeout.multipliedBy(2)); } - public Lock lockForDeployment(ApplicationId id, ZoneId zone) { - switch (lockScheme.value()) { - case "BOTH": - return new MultiplePathsLock(lockPath(id, zone), legacyLockPath(id, zone), deployLockTimeout, curator); - case "OLD": - return curator.lock(legacyLockPath(id, zone), deployLockTimeout); - case "NEW": - return curator.lock(lockPath(id, zone), deployLockTimeout); - default: - throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); - } + public Mutex lockForDeployment(ApplicationId id, ZoneId zone) { + return curator.lock(lockPath(id, zone), deployLockTimeout); } - public Lock lock(ApplicationId id, JobType type) { - switch (lockScheme.value()) { - case "BOTH": - return new MultiplePathsLock(lockPath(id, type), legacyLockPath(id, type), defaultLockTimeout, curator); - case "OLD": - return curator.lock(legacyLockPath(id, type), defaultLockTimeout); - case "NEW": - return curator.lock(lockPath(id, type), defaultLockTimeout); - default: - throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); - } + public Mutex lock(ApplicationId id, JobType type) { + return curator.lock(lockPath(id, type), defaultLockTimeout); } - public Lock lock(ApplicationId id, JobType type, Step step) throws TimeoutException { - switch (lockScheme.value()) { - case "BOTH": - return tryLock(lockPath(id, type, step), legacyLockPath(id, type, step)); - case "OLD": - return tryLock(legacyLockPath(id, type, step)); - case "NEW": - return tryLock(lockPath(id, type, step)); - default: - throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); - } + public Mutex lock(ApplicationId id, JobType type, Step step) throws TimeoutException { + return tryLock(lockPath(id, type, step)); } - public Lock lockRotations() { + public Mutex lockRotations() { return curator.lock(lockRoot.append("rotations"), defaultLockTimeout); } - public Lock lockConfidenceOverrides() { + public Mutex lockConfidenceOverrides() { return curator.lock(lockRoot.append("confidenceOverrides"), defaultLockTimeout); } - public Lock lockMaintenanceJob(String jobName) { + public Mutex lockMaintenanceJob(String jobName) { try { return tryLock(lockRoot.append("maintenanceJobLocks").append(jobName)); } catch (TimeoutException e) { @@ -217,52 +178,51 @@ public class CuratorDb { } } - @SuppressWarnings("unused") // Called by internal code - public Lock lockProvisionState(String provisionStateId) { + public Mutex lockProvisionState(String provisionStateId) { return curator.lock(lockPath(provisionStateId), Duration.ofSeconds(1)); } - public Lock lockOsVersions() { + public Mutex lockOsVersions() { return curator.lock(lockRoot.append("osTargetVersion"), defaultLockTimeout); } - public Lock lockOsVersionStatus() { + public Mutex lockOsVersionStatus() { return curator.lock(lockRoot.append("osVersionStatus"), defaultLockTimeout); } - public Lock lockRoutingPolicies() { + public Mutex lockRoutingPolicies() { return curator.lock(lockRoot.append("routingPolicies"), defaultLockTimeout); } - public Lock lockAuditLog() { + public Mutex lockAuditLog() { return curator.lock(lockRoot.append("auditLog"), defaultLockTimeout); } - public Lock lockNameServiceQueue() { + public Mutex lockNameServiceQueue() { return curator.lock(lockRoot.append("nameServiceQueue"), defaultLockTimeout); } - public Lock lockMeteringRefreshTime() throws TimeoutException { + public Mutex lockMeteringRefreshTime() throws TimeoutException { return tryLock(lockRoot.append("meteringRefreshTime")); } - public Lock lockArchiveBuckets(ZoneId zoneId) { + public Mutex lockArchiveBuckets(ZoneId zoneId) { return curator.lock(lockRoot.append("archiveBuckets").append(zoneId.value()), defaultLockTimeout); } - public Lock lockChangeRequests() { + public Mutex lockChangeRequests() { return curator.lock(lockRoot.append("changeRequests"), defaultLockTimeout); } - public Lock lockNotifications(TenantName tenantName) { + public Mutex lockNotifications(TenantName tenantName) { return curator.lock(lockRoot.append("notifications").append(tenantName.value()), defaultLockTimeout); } - public Lock lockSupportAccess(DeploymentId deploymentId) { + public Mutex lockSupportAccess(DeploymentId deploymentId) { return curator.lock(lockRoot.append("supportAccess").append(deploymentId.dottedString()), defaultLockTimeout); } - public Lock lockDeploymentRetriggerQueue() { + public Mutex lockDeploymentRetriggerQueue() { return curator.lock(lockRoot.append("deploymentRetriggerQueue"), defaultLockTimeout); } @@ -272,7 +232,7 @@ public class CuratorDb { * * Useful for maintenance jobs, where there is no point in running the jobs back to back. */ - private Lock tryLock(Path path) throws TimeoutException { + private Mutex tryLock(Path path) throws TimeoutException { try { return curator.lock(path, tryLockTimeout); } @@ -281,19 +241,6 @@ public class CuratorDb { } } - /** Try locking with a low timeout, meaning it is OK to fail lock acquisition. - * - * Useful for maintenance jobs, where there is no point in running the jobs back to back. - */ - private Lock tryLock(Path path, Path path2) throws TimeoutException { - try { - return new MultiplePathsLock(path, path2, tryLockTimeout, curator); - } - catch (UncheckedTimeoutException e) { - throw new TimeoutException(e.getMessage()); - } - } - private <T> Optional<T> read(Path path, Function<byte[], T> mapper) { return curator.getData(path).filter(data -> data.length > 0).map(mapper); } @@ -383,7 +330,7 @@ public class CuratorDb { } public Optional<Tenant> readTenant(TenantName name) { - return readSlime(tenantPath(name)).map(bytes -> tenantSerializer.tenantFrom(bytes)); + return readSlime(tenantPath(name)).map(tenantSerializer::tenantFrom); } public List<Tenant> readTenants() { @@ -415,7 +362,7 @@ public class CuratorDb { .map(stat -> cachedApplications.compute(path, (__, old) -> old != null && old.getFirst() == stat.getVersion() ? old - : new Pair<>(stat.getVersion(), read(path, applicationSerializer::fromSlime).get())).getSecond()); + : new Pair<>(stat.getVersion(), read(path, bytes -> applicationSerializer.fromSlime(bytes)).get())).getSecond()); } public List<Application> readApplications(boolean canFail) { @@ -676,7 +623,7 @@ public class CuratorDb { public List<Notification> readNotifications(TenantName tenantName) { return readSlime(notificationsPath(tenantName)) - .map(slime -> NotificationsSerializer.fromSlime(tenantName, slime)).orElseGet(List::of); + .map(slime -> notificationsSerializer.fromSlime(tenantName, slime)).orElseGet(List::of); } @@ -687,7 +634,7 @@ public class CuratorDb { } public void writeNotifications(TenantName tenantName, List<Notification> notifications) { - curator.set(notificationsPath(tenantName), asJson(NotificationsSerializer.toSlime(notifications))); + curator.set(notificationsPath(tenantName), asJson(notificationsSerializer.toSlime(notifications))); } public void deleteNotifications(TenantName tenantName) { @@ -708,11 +655,11 @@ public class CuratorDb { // -------------- Job Retrigger entries ----------------------------------- public List<RetriggerEntry> readRetriggerEntries() { - return readSlime(deploymentRetriggerPath()).map(RetriggerEntrySerializer::fromSlime).orElseGet(List::of); + return readSlime(deploymentRetriggerPath()).map(retriggerEntrySerializer::fromSlime).orElseGet(List::of); } public void writeRetriggerEntries(List<RetriggerEntry> retriggerEntries) { - curator.set(deploymentRetriggerPath(), asJson(RetriggerEntrySerializer.toSlime(retriggerEntries))); + curator.set(deploymentRetriggerPath(), asJson(retriggerEntrySerializer.toSlime(retriggerEntries))); } // -------------- Paths --------------------------------------------------- @@ -722,32 +669,6 @@ public class CuratorDb { .append(tenant.value()); } - private Path legacyLockPath(TenantAndApplicationId application) { - return lockPath(application.tenant()) - .append(application.application().value()); - } - - private Path legacyLockPath(ApplicationId instance) { - return legacyLockPath(TenantAndApplicationId.from(instance)) - .append(instance.instance().value()); - } - - private Path legacyLockPath(ApplicationId instance, ZoneId zone) { - return legacyLockPath(instance) - .append(zone.environment().value()) - .append(zone.region().value()); - } - - private Path legacyLockPath(ApplicationId instance, JobType type) { - return legacyLockPath(instance) - .append(type.jobName()); - } - - private Path legacyLockPath(ApplicationId instance, JobType type, Step step) { - return legacyLockPath(instance, type) - .append(step.name()); - } - private Path lockPath(TenantAndApplicationId application) { return lockRoot.append(application.tenant().value() + ":" + application.application().value()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java index 65c9859ad72..21414339a87 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java @@ -2,9 +2,9 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.InMemoryFlagSource; - import java.time.Duration; /** @@ -18,21 +18,21 @@ public class MockCuratorDb extends CuratorDb { private final MockCurator curator; @Inject - public MockCuratorDb() { - this("test-controller:2222"); + public MockCuratorDb(ConfigserverConfig config) { + this("test-controller:2222", SystemName.from(config.system())); + } + + public MockCuratorDb(SystemName system) { + this("test-controller:2222", system); } - public MockCuratorDb(String zooKeeperEnsembleConnectionSpec) { - this(new MockCurator() { - @Override - public String zooKeeperEnsembleConnectionSpec() { - return zooKeeperEnsembleConnectionSpec; - } - }); + public MockCuratorDb(String zooKeeperEnsembleConnectionSpec, SystemName system) { + this(new MockCurator() { @Override public String zooKeeperEnsembleConnectionSpec() { return zooKeeperEnsembleConnectionSpec; } }, + system); } - public MockCuratorDb(MockCurator curator) { - super(curator, Duration.ofMillis(100), new InMemoryFlagSource()); + public MockCuratorDb(MockCurator curator, SystemName system) { + super(curator, Duration.ofMillis(100), system); this.curator = curator; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java index 10ec9dce5f7..0f1f531d589 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java @@ -46,7 +46,7 @@ public class NodeVersionSerializer { public List<NodeVersion> nodeVersionsFromSlime(Inspector array, Version version) { List<NodeVersion> nodeVersions = new ArrayList<>(); array.traverse((ArrayTraverser) (i, entry) -> { - var hostname = HostName.from(entry.field(hostnameField).asString()); + var hostname = HostName.of(entry.field(hostnameField).asString()); var zone = ZoneId.from(entry.field(zoneField).asString()); var wantedVersion = Version.fromString(entry.field(wantedVersionField).asString()); var suspendedAt = SlimeUtils.optionalInstant(entry.field(suspendedAtField)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java index 10763e1f22c..16ec240a116 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; @@ -43,7 +44,7 @@ public class NotificationsSerializer { private static final String jobTypeField = "jobId"; private static final String runNumberField = "runNumber"; - public static Slime toSlime(List<Notification> notifications) { + public Slime toSlime(List<Notification> notifications) { Slime slime = new Slime(); Cursor notificationsArray = slime.setObject().setArray(notificationsFieldName); @@ -59,20 +60,20 @@ public class NotificationsSerializer { notification.source().instance().ifPresent(instance -> notificationObject.setString(instanceField, instance.value())); notification.source().zoneId().ifPresent(zoneId -> notificationObject.setString(zoneField, zoneId.value())); notification.source().clusterId().ifPresent(clusterId -> notificationObject.setString(clusterIdField, clusterId.value())); - notification.source().jobType().ifPresent(jobType -> notificationObject.setString(jobTypeField, jobType.jobName())); + notification.source().jobType().ifPresent(jobType -> notificationObject.setString(jobTypeField, jobType.serialized())); notification.source().runNumber().ifPresent(runNumber -> notificationObject.setLong(runNumberField, runNumber)); } return slime; } - public static List<Notification> fromSlime(TenantName tenantName, Slime slime) { + public List<Notification> fromSlime(TenantName tenantName, Slime slime) { return SlimeUtils.entriesStream(slime.get().field(notificationsFieldName)) .map(inspector -> fromInspector(tenantName, inspector)) .collect(Collectors.toUnmodifiableList()); } - private static Notification fromInspector(TenantName tenantName, Inspector inspector) { + private Notification fromInspector(TenantName tenantName, Inspector inspector) { return new Notification( SlimeUtils.instant(inspector.field(atFieldName)), typeFrom(inspector.field(typeField)), @@ -83,7 +84,7 @@ public class NotificationsSerializer { SlimeUtils.optionalString(inspector.field(instanceField)).map(InstanceName::from), SlimeUtils.optionalString(inspector.field(zoneField)).map(ZoneId::from), SlimeUtils.optionalString(inspector.field(clusterIdField)).map(ClusterSpec.Id::from), - SlimeUtils.optionalString(inspector.field(jobTypeField)).map(JobType::fromJobName), + SlimeUtils.optionalString(inspector.field(jobTypeField)).map(jobName -> JobType.ofSerialized(jobName)), SlimeUtils.optionalLong(inspector.field(runNumberField))), SlimeUtils.entriesStream(inspector.field(messagesField)).map(Inspector::asString).collect(Collectors.toUnmodifiableList())); } @@ -91,6 +92,8 @@ public class NotificationsSerializer { private static String asString(Notification.Type type) { switch (type) { case applicationPackage: return "applicationPackage"; + case submission: return "submission"; + case testPackage: return "testPackage"; case deployment: return "deployment"; case feedBlock: return "feedBlock"; case reindex: return "reindex"; @@ -101,6 +104,8 @@ public class NotificationsSerializer { private static Notification.Type typeFrom(Inspector field) { switch (field.asString()) { case "applicationPackage": return Notification.Type.applicationPackage; + case "submission": return Notification.Type.submission; + case "testPackage": return Notification.Type.testPackage; case "deployment": return Notification.Type.deployment; case "feedBlock": return Notification.Type.feedBlock; case "reindex": return Notification.Type.reindex; @@ -125,4 +130,5 @@ public class NotificationsSerializer { default: throw new IllegalArgumentException("Unknown serialized notification level value '" + field.asString() + "'"); } } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 17337f823c0..72c16ae0110 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import ai.vespa.http.DomainName; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -83,7 +83,7 @@ public class RoutingPolicySerializer { ClusterSpec.Id.from(inspect.field(clusterField).asString()), ZoneId.from(inspect.field(zoneField).asString())); policies.add(new RoutingPolicy(id, - HostName.from(inspect.field(canonicalNameField).asString()), + DomainName.of(inspect.field(canonicalNameField).asString()), SlimeUtils.optionalString(inspect.field(dnsZoneField)), instanceEndpoints, applicationEndpoints, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index c3d81b8dcd5..dd28978d948 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -10,10 +11,9 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; @@ -28,7 +28,6 @@ import java.util.Collections; import java.util.EnumMap; import java.util.NavigableMap; import java.util.Optional; -import java.util.OptionalLong; import java.util.TreeMap; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; @@ -36,6 +35,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentF import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCertificateTimeout; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; @@ -74,7 +74,6 @@ class RunSerializer { // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - // TODO: Remove "steps" when there are no traces of it in the controllers private static final String stepsField = "steps"; private static final String stepDetailsField = "stepDetails"; private static final String startTimeField = "startTime"; @@ -88,17 +87,9 @@ class RunSerializer { private static final String versionsField = "versions"; private static final String isRedeploymentField = "isRedeployment"; private static final String platformVersionField = "platform"; - private static final String repositoryField = "repository"; - private static final String branchField = "branch"; - private static final String commitField = "commit"; - private static final String authorEmailField = "authorEmail"; private static final String deployedDirectlyField = "deployedDirectly"; - private static final String compileVersionField = "compileVersion"; - private static final String buildTimeField = "buildTime"; - private static final String sourceUrlField = "sourceUrl"; private static final String buildField = "build"; private static final String sourceField = "source"; - private static final String bundleHashField = "bundleHash"; private static final String lastTestRecordField = "lastTestRecord"; private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp"; private static final String noNodesDownSinceField = "noNodesDownSince"; @@ -134,11 +125,12 @@ class RunSerializer { steps.put(typedStep, new StepInfo(typedStep, stepStatusOf(status.asString()), startTime)); }); - return new Run(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), - JobType.fromJobName(runObject.field(jobTypeField).asString()), - runObject.field(numberField).asLong()), + RunId id = new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), + JobType.ofSerialized(runObject.field(jobTypeField).asString()), + runObject.field(numberField).asLong()); + return new Run(id, steps, - versionsFromSlime(runObject.field(versionsField)), + versionsFromSlime(runObject.field(versionsField), id), runObject.field(isRedeploymentField).asBool(), SlimeUtils.instant(runObject.field(startField)), SlimeUtils.optionalInstant(runObject.field(endField)), @@ -155,40 +147,26 @@ class RunSerializer { SlimeUtils.optionalString(runObject.field(reasonField))); } - private Versions versionsFromSlime(Inspector versionsObject) { + private Versions versionsFromSlime(Inspector versionsObject, RunId id) { Version targetPlatformVersion = Version.fromString(versionsObject.field(platformVersionField).asString()); - ApplicationVersion targetApplicationVersion = applicationVersionFrom(versionsObject); + RevisionId targetRevision = revisionFrom(versionsObject, id); Optional<Version> sourcePlatformVersion = versionsObject.field(sourceField).valid() ? Optional.of(Version.fromString(versionsObject.field(sourceField).field(platformVersionField).asString())) : Optional.empty(); - Optional<ApplicationVersion> sourceApplicationVersion = versionsObject.field(sourceField).valid() - ? Optional.of(applicationVersionFrom(versionsObject.field(sourceField))) + Optional<RevisionId> sourceRevision = versionsObject.field(sourceField).valid() + ? Optional.of(revisionFrom(versionsObject.field(sourceField), id)) : Optional.empty(); - return new Versions(targetPlatformVersion, targetApplicationVersion, sourcePlatformVersion, sourceApplicationVersion); + return new Versions(targetPlatformVersion, targetRevision, sourcePlatformVersion, sourceRevision); } - private ApplicationVersion applicationVersionFrom(Inspector versionObject) { - if ( ! versionObject.field(buildField).valid()) - return ApplicationVersion.unknown; - + private RevisionId revisionFrom(Inspector versionObject, RunId id) { long buildNumber = versionObject.field(buildField).asLong(); - // TODO jonmv: Remove source revision - Optional<SourceRevision> source = Optional.of(new SourceRevision(versionObject.field(repositoryField).asString(), - versionObject.field(branchField).asString(), - versionObject.field(commitField).asString())) - .filter(revision -> ! revision.commit().isBlank() && ! revision.repository().isBlank() && ! revision.branch().isBlank()); - Optional<String> authorEmail = SlimeUtils.optionalString(versionObject.field(authorEmailField)); - Optional<Version> compileVersion = SlimeUtils.optionalString(versionObject.field(compileVersionField)).map(Version::fromString); - Optional<Instant> buildTime = SlimeUtils.optionalInstant(versionObject.field(buildTimeField)); - Optional<String> sourceUrl = SlimeUtils.optionalString(versionObject.field(sourceUrlField)); - Optional<String> commit = SlimeUtils.optionalString(versionObject.field(commitField)); - boolean deployedDirectly = versionObject.field(deployedDirectlyField).asBool(); - Optional<String> bundleHash = SlimeUtils.optionalString(versionObject.field(bundleHashField)); - - return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, - compileVersion, buildTime, sourceUrl, commit, deployedDirectly, bundleHash); + boolean production = versionObject.field(deployedDirectlyField).valid() // TODO jonmv: remove after migration + && buildNumber > 0 + && ! versionObject.field(deployedDirectlyField).asBool(); + return production ? RevisionId.forProduction(buildNumber) : RevisionId.forDevelopment(buildNumber, id.job()); } // Don't change this — introduce a separate array instead. @@ -228,7 +206,7 @@ class RunSerializer { private void toSlime(Run run, Cursor runObject) { runObject.setString(applicationField, run.id().application().serializedForm()); - runObject.setString(jobTypeField, run.id().type().jobName()); + runObject.setString(jobTypeField, run.id().type().serialized()); runObject.setBool(isRedeploymentField, run.isRedeployment()); runObject.setLong(numberField, run.id().number()); runObject.setLong(startField, run.start().toEpochMilli()); @@ -251,10 +229,10 @@ class RunSerializer { stepDetailsObject.setObject(valueOf(step)).setLong(startTimeField, valueOf(startTime)))); Cursor versionsObject = runObject.setObject(versionsField); - toSlime(run.versions().targetPlatform(), run.versions().targetApplication(), versionsObject); + toSlime(run.versions().targetPlatform(), run.versions().targetRevision(), versionsObject); run.versions().sourcePlatform().ifPresent(sourcePlatformVersion -> { toSlime(sourcePlatformVersion, - run.versions().sourceApplication() + run.versions().sourceRevision() .orElseThrow(() -> new IllegalArgumentException("Source versions must be both present or absent.")), versionsObject.setObject(sourceField)); }); @@ -262,19 +240,10 @@ class RunSerializer { run.reason().ifPresent(reason -> runObject.setString(reasonField, reason)); } - private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) { + private void toSlime(Version platformVersion, RevisionId revsion, Cursor versionsObject) { versionsObject.setString(platformVersionField, platformVersion.toString()); - applicationVersion.buildNumber().ifPresent(number -> versionsObject.setLong(buildField, number)); - // TODO jonmv: Remove source revision. - applicationVersion.source().map(SourceRevision::repository).ifPresent(repository -> versionsObject.setString(repositoryField, repository)); - applicationVersion.source().map(SourceRevision::branch).ifPresent(branch -> versionsObject.setString(branchField, branch)); - applicationVersion.source().map(SourceRevision::commit).ifPresent(commit -> versionsObject.setString(commitField, commit)); - applicationVersion.authorEmail().ifPresent(email -> versionsObject.setString(authorEmailField, email)); - applicationVersion.compileVersion().ifPresent(version -> versionsObject.setString(compileVersionField, version.toString())); - applicationVersion.buildTime().ifPresent(time -> versionsObject.setLong(buildTimeField, time.toEpochMilli())); - applicationVersion.sourceUrl().ifPresent(url -> versionsObject.setString(sourceUrlField, url)); - applicationVersion.commit().ifPresent(commit -> versionsObject.setString(commitField, commit)); - versionsObject.setBool(deployedDirectlyField, applicationVersion.isDeployedDirectly()); + versionsObject.setLong(buildField, revsion.number()); + versionsObject.setBool(deployedDirectlyField, ! revsion.isProduction()); } // Don't change this - introduce a separate array with new values if needed. @@ -368,6 +337,7 @@ class RunSerializer { case deploymentFailed : return "deploymentFailed"; case installationFailed : return "installationFailed"; case testFailure : return "testFailure"; + case noTests : return "noTests"; case error : return "error"; case success : return "success"; case aborted : return "aborted"; @@ -380,11 +350,11 @@ class RunSerializer { static RunStatus runStatusOf(String status) { switch (status) { case "running" : return running; - case "outOfCapacity" : return nodeAllocationFailure; // TODO: Remove after March 2022 case "nodeAllocationFailure" : return nodeAllocationFailure; case "endpointCertificateTimeout" : return endpointCertificateTimeout; case "deploymentFailed" : return deploymentFailed; case "installationFailed" : return installationFailed; + case "noTests" : return noTests; case "testFailure" : return testFailure; case "error" : return error; case "success" : return success; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java index e9e5f8cf032..c200ca467da 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java @@ -13,7 +13,6 @@ import java.time.Instant; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java index 79f0088a214..fdd93eedbff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import com.yahoo.container.jdisc.HttpRequest; - import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Path; import ai.vespa.http.HttpURL.Query; +import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.text.Text; + import java.io.InputStream; import java.net.URI; import java.util.List; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java index 05cdc0d0565..886dc27b404 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import com.yahoo.container.jdisc.HttpResponse; import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Path; +import com.yahoo.container.jdisc.HttpResponse; import java.io.IOException; import java.io.OutputStream; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 924f7c0b0a5..88b319d0051 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2,7 +2,10 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.Signatures; +import ai.vespa.http.DomainName; +import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Query; +import ai.vespa.validation.Validation; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; @@ -26,10 +29,8 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; -import ai.vespa.http.DomainName; import com.yahoo.restapi.ByteArrayResponse; import com.yahoo.restapi.ErrorResponse; -import ai.vespa.http.HttpURL; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; @@ -53,7 +54,6 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbi import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.aws.TenantRoles; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; @@ -67,11 +67,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeReposi import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.vespa.hosted.controller.api.integration.user.User; +import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; @@ -93,6 +94,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer; import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; import com.yahoo.vespa.hosted.controller.notification.Notification; @@ -118,7 +120,6 @@ import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.yolean.Exceptions; import javax.ws.rs.ForbiddenException; @@ -148,7 +149,6 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.Scanner; -import java.util.SortedSet; import java.util.StringJoiner; import java.util.function.Function; import java.util.logging.Level; @@ -260,7 +260,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)).descendingMap(), Optional.ofNullable(request.getProperty("limit")), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.applications().requireApplication(TenantAndApplicationId.from(path.get("tenant"), path.get("application"))), controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)).descendingMap(), Optional.ofNullable(request.getProperty("limit")), request.getUri()); // (((\(✘෴✘)/))) if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/diff/{number}")) return devApplicationPackageDiff(runIdFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); @@ -268,10 +268,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return getReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/state/v1/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/state/v1/metrics")) return stateV1Metrics(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/state/v1/{*}")) return stateV1(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/orchestrator")) return orchestrator(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/content/{*}")) return content(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.getRest(), request); @@ -284,8 +283,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/state/v1/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); @@ -356,6 +353,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", "all"); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", path.get("choice")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return removeDeployKey(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit/{build}")) return cancelBuild(path.get("tenant"), path.get("application"), path.get("build")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return deleteInstance(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), "all"); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("choice")); @@ -711,7 +709,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { propertyEquals(request, "application", ApplicationName::from, notification.source().application()) && propertyEquals(request, "instance", InstanceName::from, notification.source().instance()) && propertyEquals(request, "zone", ZoneId::from, notification.source().zoneId()) && - propertyEquals(request, "job", JobType::fromJobName, notification.source().jobType()) && + propertyEquals(request, "job", job -> JobType.fromJobName(job, controller.zoneRegistry()), notification.source().jobType()) && propertyEquals(request, "type", Notification.Type::valueOf, Optional.of(notification.type())) && propertyEquals(request, "level", Notification.Level::valueOf, Optional.of(notification.level()))) .forEach(notification -> toSlime(notificationsArray.addObject(), notification, includeTenantFieldInResponse, excludeMessages)); @@ -747,7 +745,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private static String notificationTypeAsString(Notification.Type type) { switch (type) { + case submission: case applicationPackage: return "applicationPackage"; + case testPackage: return "testPackage"; case deployment: return "deployment"; case feedBlock: return "feedBlock"; case reindex: return "reindex"; @@ -800,47 +800,41 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private HttpResponse devApplicationPackage(ApplicationId id, JobType type) { - if ( ! type.environment().isManuallyDeployed()) - throw new IllegalArgumentException("Only manually deployed zones have dev packages"); - - ZoneId zone = type.zone(controller.system()); - ApplicationVersion version = controller.jobController().last(id, type).get().versions().targetApplication(); - byte[] applicationPackage = controller.applications().applicationStore().get(new DeploymentId(id, zone), version); + ZoneId zone = type.zone(); + RevisionId revision = controller.jobController().last(id, type).get().versions().targetRevision(); + byte[] applicationPackage = controller.applications().applicationStore().get(new DeploymentId(id, zone), revision); return new ZipResponse(id.toFullString() + "." + zone.value() + ".zip", applicationPackage); } private HttpResponse devApplicationPackageDiff(RunId runId) { - DeploymentId deploymentId = new DeploymentId(runId.application(), runId.job().type().zone(controller.system())); + DeploymentId deploymentId = new DeploymentId(runId.application(), runId.job().type().zone()); return controller.applications().applicationStore().getDevDiff(deploymentId, runId.number()) .map(ByteArrayResponse::new) .orElseThrow(() -> new NotExistsException("No application package diff found for " + runId)); } private HttpResponse applicationPackage(String tenantName, String applicationName, HttpRequest request) { - var tenantAndApplication = TenantAndApplicationId.from(tenantName, applicationName); - SortedSet<ApplicationVersion> versions = controller.applications().requireApplication(tenantAndApplication).versions(); - if (versions.isEmpty()) - throw new NotExistsException("No application package has been submitted for '" + tenantAndApplication + "'"); - - ApplicationVersion version = Optional.ofNullable(request.getProperty("build")) - .map(build -> { - try { - return Long.parseLong(build); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Invalid build number", e); - } - }) - .map(build -> versions.stream() - .filter(ver -> ver.buildNumber().orElse(-1) == build) - .findFirst() - .orElseThrow(() -> new NotExistsException("No application package found for '" + tenantAndApplication + "' with build number " + build))) - .orElseGet(versions::last); - + TenantAndApplicationId tenantAndApplication = TenantAndApplicationId.from(tenantName, applicationName); + long build; + String parameter = request.getProperty("build"); + if (parameter != null) + try { + build = Validation.requireAtLeast(Long.parseLong(request.getProperty("build")), "build number", 1L); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("invalid value for request parameter 'build'", e); + } + else { + build = controller.applications().requireApplication(tenantAndApplication).revisions().last() + .map(version -> version.id().number()) + .orElseThrow(() -> new NotExistsException("no application package has been submitted for " + tenantAndApplication)); + } + RevisionId revision = RevisionId.forProduction(build); boolean tests = request.getBooleanProperty("tests"); byte[] applicationPackage = tests ? - controller.applications().applicationStore().getTester(tenantAndApplication.tenant(), tenantAndApplication.application(), version) : - controller.applications().applicationStore().get(new DeploymentId(tenantAndApplication.defaultInstance(), ZoneId.defaultId()), version); - String filename = tenantAndApplication + (tests ? "-tests" : "-build") + version.buildNumber().getAsLong() + ".zip"; + controller.applications().applicationStore().getTester(tenantAndApplication.tenant(), tenantAndApplication.application(), revision) : + controller.applications().applicationStore().get(new DeploymentId(tenantAndApplication.defaultInstance(), ZoneId.defaultId()), revision); + String filename = tenantAndApplication + (tests ? "-tests" : "-build") + revision.number() + ".zip"; return new ZipResponse(filename, applicationPackage); } @@ -1121,7 +1115,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { toSlime(node.resources(), nodeObject); nodeObject.setString("clusterId", node.clusterId()); nodeObject.setString("clusterType", valueOf(node.clusterType())); - nodeObject.setBool("down", node.history().stream().anyMatch(event -> "down".equals(event.name()))); + nodeObject.setBool("down", node.down()); nodeObject.setBool("retired", node.retired() || node.wantToRetire()); nodeObject.setBool("restarting", node.wantedRestartGeneration() > node.restartGeneration()); nodeObject.setBool("rebooting", node.wantedRebootGeneration() > node.rebootGeneration()); @@ -1318,7 +1312,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { request.getUri()).toString()); DeploymentStatus status = controller.jobController().deploymentStatus(application); - application.latestVersion().ifPresent(version -> JobControllerApiHandlerHelper.toSlime(object.setObject("latestVersion"), version)); + application.revisions().last().ifPresent(version -> JobControllerApiHandlerHelper.toSlime(object.setObject("latestVersion"), version)); application.projectId().ifPresent(id -> object.setLong("projectId", id)); @@ -1326,11 +1320,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { application.instances().values().stream().findFirst().ifPresent(instance -> { // Currently deploying change if ( ! instance.change().isEmpty()) - toSlime(object.setObject("deploying"), instance.change()); + toSlime(object.setObject("deploying"), instance.change(), application); // Outstanding change if ( ! status.outstandingChange(instance.name()).isEmpty()) - toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name())); + toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name()), application); }); application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion)); @@ -1370,11 +1364,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .sortedJobs(status.instanceJobs(instance.name()).values()); if ( ! instance.change().isEmpty()) - toSlime(object.setObject("deploying"), instance.change()); + toSlime(object.setObject("deploying"), instance.change(), status.application()); // Outstanding change if ( ! status.outstandingChange(instance.name()).isEmpty()) - toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name())); + toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name()), status.application()); // Change blockers Cursor changeBlockers = object.setArray("changeBlockers"); @@ -1395,7 +1389,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { // Deployments sorted according to deployment spec List<Deployment> deployments = deploymentSpec.instance(instance.name()) - .map(spec -> new DeploymentSteps(spec, controller::system)) + .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry())) .map(steps -> steps.sortedDeployments(instance.deployments().values())) .orElse(List.copyOf(instance.deployments().values())); @@ -1441,7 +1435,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { "/instance/" + instance.id().instance().value() + "/job/", request.getUri()).toString()); - application.latestVersion().ifPresent(version -> { + application.revisions().last().ifPresent(version -> { version.sourceUrl().ifPresent(url -> object.setString("sourceUrl", url)); version.commit().ifPresent(commit -> object.setString("commit", commit)); }); @@ -1455,11 +1449,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .sortedJobs(status.instanceJobs(instance.name()).values()); if ( ! instance.change().isEmpty()) - toSlime(object.setObject("deploying"), instance.change()); + toSlime(object.setObject("deploying"), instance.change(), application); // Outstanding change if ( ! status.outstandingChange(instance.name()).isEmpty()) - toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name())); + toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name()), application); // Change blockers Cursor changeBlockers = object.setArray("changeBlockers"); @@ -1483,7 +1477,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { // Deployments sorted according to deployment spec List<Deployment> deployments = application.deploymentSpec().instance(instance.name()) - .map(spec -> new DeploymentSteps(spec, controller::system)) + .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry())) .map(steps -> steps.sortedDeployments(instance.deployments().values())) .orElse(List.copyOf(instance.deployments().values())); Cursor instancesArray = object.setArray("instances"); @@ -1525,7 +1519,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { controller.jobController().active(instance.id()).stream() .map(run -> run.id().job()) .filter(job -> job.type().environment().isManuallyDeployed())) - .map(job -> job.type().zone(controller.system())) + .map(job -> job.type().zone()) .filter(zone -> ! instance.deployments().containsKey(zone)) .forEach(zone -> { Cursor deploymentObject = instancesArray.addObject(); @@ -1574,11 +1568,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(slime); } - private void toSlime(Cursor object, Change change) { + private void toSlime(Cursor object, Change change, Application application) { change.platform().ifPresent(version -> object.setString("version", version.toString())); - change.application() - .filter(version -> !version.isUnknown()) - .ifPresent(version -> JobControllerApiHandlerHelper.toSlime(object.setObject("revision"), version)); + change.revision().ifPresent(revision -> JobControllerApiHandlerHelper.toSlime(object.setObject("revision"), application.revisions().get(revision))); } private void toSlime(Endpoint endpoint, Cursor object) { @@ -1623,8 +1615,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { response.setString("nodes", withPathAndQuery("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/", "recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); response.setString("version", deployment.version().toFullString()); - response.setString("revision", deployment.applicationVersion().id()); - Instant lastDeploymentStart = lastDeploymentStart(deploymentId.applicationId(), deployment); + response.setString("revision", application.revisions().get(deployment.revision()).stringId()); // TODO jonmv or freva: ƪ(`▿▿▿▿´ƪ) + response.setLong("build", deployment.revision().number()); + Instant lastDeploymentStart = controller.jobController().lastDeploymentStart(deploymentId.applicationId(), deployment); response.setLong("deployTimeEpochMs", lastDeploymentStart.toEpochMilli()); controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zoneId()) .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", lastDeploymentStart.plus(deploymentTimeToLive).toEpochMilli())); @@ -1638,22 +1631,18 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (!deployment.zone().environment().isManuallyDeployed()) { DeploymentStatus status = controller.jobController().deploymentStatus(application); - JobType.from(controller.system(), deployment.zone()) - .map(type -> new JobId(instance.id(), type)) - .map(status.jobSteps()::get) + JobId jobId = new JobId(instance.id(), JobType.deploymentTo(deployment.zone())); + Optional.ofNullable(status.jobSteps().get(jobId)) .ifPresent(stepStatus -> { - JobControllerApiHandlerHelper.toSlime( - response.setObject("applicationVersion"), deployment.applicationVersion()); - if (!status.jobsToRun().containsKey(stepStatus.job().get())) + JobControllerApiHandlerHelper.toSlime(response.setObject("applicationVersion"), application.revisions().get(deployment.revision())); + if ( ! status.jobsToRun().containsKey(stepStatus.job().get())) response.setString("status", "complete"); else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true)) response.setString("status", "pending"); else response.setString("status", "running"); }); } else { - var deploymentRun = JobType.from(controller.system(), deploymentId.zoneId()) - .flatMap(jobType -> controller.jobController().last(deploymentId.applicationId(), jobType)); - + var deploymentRun = controller.jobController().last(deploymentId.applicationId(), JobType.deploymentTo(deploymentId.zoneId())); deploymentRun.ifPresent(run -> { response.setString("status", run.hasEnded() ? "complete" : "running"); }); @@ -1685,11 +1674,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { metrics.instant().ifPresent(instant -> metricsObject.setLong("lastUpdated", instant.toEpochMilli())); } - private Instant lastDeploymentStart(ApplicationId instanceId, Deployment deployment) { - return controller.jobController().jobStarts(new JobId(instanceId, JobType.from(controller.system(), deployment.zone()).get())) - .stream().findFirst().orElse(deployment.at()); - } - private void toSlime(RotationState state, Cursor object) { Cursor bcpStatus = object.setObject("bcpStatus"); bcpStatus.setString("rotationStatus", rotationStateString(state)); @@ -1770,7 +1754,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { Cursor root = slime.setObject(); if ( ! instance.change().isEmpty()) { instance.change().platform().ifPresent(version -> root.setString("platform", version.toString())); - instance.change().application().ifPresent(applicationVersion -> root.setString("application", applicationVersion.id())); + instance.change().revision().ifPresent(revision -> root.setString("application", revision.toString())); root.setBool("pinned", instance.change().isPinned()); } return new SlimeJsonResponse(slime); @@ -1786,17 +1770,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse services(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region); - ZoneId zone = requireZone(environment, region); - ServiceApiResponse response = new ServiceApiResponse(zone, - new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), - List.of(controller.zoneRegistry().getConfigServerVipUri(zone)), - request.getUri()); - response.setResponse(applicationView); - return response; - } - private HttpResponse status(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host, HttpURL.Path restPath, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); return controller.serviceRegistry().configServer().getServiceNodePage(deploymentId, @@ -1806,21 +1779,17 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { Query.empty().add(request.getJDiscRequest().parameters())); } - private HttpResponse stateV1Metrics(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host) { + private HttpResponse orchestrator(String tenantName, String applicationName, String instanceName, String environment, String region) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - return controller.serviceRegistry().configServer().getServiceNodePage( - deploymentId, serviceName, DomainName.of(host), HttpURL.Path.parse("/state/v1/metrics"), Query.empty()); + return controller.serviceRegistry().configServer().getServiceNodes(deploymentId); } - private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, HttpURL.Path restPath, HttpRequest request) { + private HttpResponse stateV1(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host, HttpURL.Path rest, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - Map<?,?> result = controller.serviceRegistry().configServer().getServiceApiResponse(deploymentId, serviceName, restPath); - ServiceApiResponse response = new ServiceApiResponse(deploymentId.zoneId(), - deploymentId.applicationId(), - List.of(controller.zoneRegistry().getConfigServerVipUri(deploymentId.zoneId())), - request.getUri()); - response.setResponse(result, serviceName, HttpURL.Path.parse("/state/v1").append(restPath)); - return response; + Query query = Query.empty().add(request.getJDiscRequest().parameters()); + query = query.set("forwarded-url", HttpURL.from(request.getUri()).withQuery(Query.empty()).asURI().toString()); + return controller.serviceRegistry().configServer().getServiceNodePage( + deploymentId, serviceName, DomainName.of(host), HttpURL.Path.parse("/state/v1").append(rest), query); } private HttpResponse content(String tenantName, String applicationName, String instanceName, String environment, String region, HttpURL.Path restPath, HttpRequest request) { @@ -1915,18 +1884,19 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { StringBuilder response = new StringBuilder(); controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - ApplicationVersion version = build == -1 ? application.get().latestVersion().get() - : getApplicationVersion(application.get(), build); - Change change = Change.of(version); + RevisionId revision = build == -1 ? application.get().revisions().last().get().id() + : getRevision(application.get(), build); + Change change = Change.of(revision); controller.applications().deploymentTrigger().forceChange(id, change); response.append("Triggered ").append(change).append(" for ").append(id); }); return new MessageResponse(response.toString()); } - private ApplicationVersion getApplicationVersion(Application application, Long build) { - return application.versions().stream() - .filter(version -> version.buildNumber().stream().anyMatch(build::equals)) + private RevisionId getRevision(Application application, long build) { + return application.revisions().withPackage().stream() + .map(ApplicationVersion::id) + .filter(version -> version.number() == build) .findFirst() .filter(version -> controller.applications().applicationStore().hasBuild(application.id().tenant(), application.id().application(), @@ -1934,6 +1904,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .orElseThrow(() -> new IllegalArgumentException("Build number '" + build + "' was not found")); } + private HttpResponse cancelBuild(String tenantName, String applicationName, String build){ + TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); + RevisionId revision = RevisionId.forProduction(Long.parseLong(build)); + controller.applications().lockApplicationOrThrow(id, application -> { + controller.applications().store(application.withRevisions(revisions -> revisions.with(revisions.get(revision).skipped()))); + }); + return new MessageResponse("Marked build '" + build + "' as non-deployable"); + } + /** Cancel ongoing change for given application, e.g., everything with {"cancel":"all"} */ private HttpResponse cancelDeploy(String tenantName, String applicationName, String instanceName, String choice) { ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); @@ -2053,7 +2032,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); RestartFilter restartFilter = new RestartFilter() - .withHostName(Optional.ofNullable(request.getProperty("hostname")).map(HostName::from)) + .withHostName(Optional.ofNullable(request.getProperty("hostname")).map(HostName::of)) .withClusterType(Optional.ofNullable(request.getProperty("clusterType")).map(ClusterSpec.Type::from)) .withClusterId(Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)); @@ -2080,7 +2059,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP)); controller.applications().verifyApplicationIdentityConfiguration(id.tenant(), Optional.of(id.instance()), - Optional.of(type.zone(controller.system())), + Optional.of(type.zone()), applicationPackage, Optional.of(requireUserPrincipal(request))); @@ -2193,7 +2172,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .flatMap(instance -> instance.productionDeployments().keySet().stream()) .map(zone -> new DeploymentId(prodInstanceId, zone)) .collect(Collectors.toCollection(HashSet::new)); - ZoneId testedZone = type.zone(controller.system()); + ZoneId testedZone = type.zone(); // If a production job is specified, the production deployment of the orchestrated instance is the relevant one, // as user instances should not exist in prod. @@ -2295,7 +2274,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { try { node = nodeRepository.getNode(zone, hostname); } catch (IllegalArgumentException e) { - throw new NotExistsException(new Hostname(hostname)); + throw new NotExistsException(hostname); } ApplicationId app = ApplicationId.from(tenant, application, instance); ApplicationId owner = node.owner().orElseThrow(() -> new IllegalArgumentException("Node has no owner")); @@ -2477,17 +2456,17 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .flatMap(application -> application.instances().values().stream()) .flatMap(instance -> instance.deployments().values().stream() .filter(deployment -> deployment.zone().environment() == Environment.dev) - .map(deployment -> lastDeploymentStart(instance.id(), deployment))) + .map(deployment -> controller.jobController().lastDeploymentStart(instance.id(), deployment))) .max(Comparator.naturalOrder()) .or(() -> applications.stream() .flatMap(application -> application.instances().values().stream()) - .flatMap(instance -> JobType.allIn(controller.system()).stream() + .flatMap(instance -> JobType.allIn(controller.zoneRegistry()).stream() .filter(job -> job.environment() == Environment.dev) .flatMap(jobType -> controller.jobController().last(instance.id(), jobType).stream())) .map(Run::start) .max(Comparator.naturalOrder())); Optional<Instant> lastSubmission = applications.stream() - .flatMap(app -> app.latestVersion().flatMap(ApplicationVersion::buildTime).stream()) + .flatMap(app -> app.revisions().last().flatMap(ApplicationVersion::buildTime).stream()) .max(Comparator.naturalOrder()); object.setLong("createdAtMillis", tenant.createdAt().toEpochMilli()); if (tenant.type() == Tenant.Type.deleted) @@ -2537,15 +2516,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } } - private void toSlime(Run run, Cursor object) { - object.setLong("id", run.id().number()); - object.setString("version", run.versions().targetPlatform().toFullString()); - if ( ! run.versions().targetApplication().isUnknown()) - JobControllerApiHandlerHelper.toSlime(object.setObject("revision"), run.versions().targetApplication()); - object.setString("reason", "unknown reason"); - object.setLong("at", run.end().orElse(run.start()).toEpochMilli()); - } - private Slime toSlime(InputStream jsonStream) { try { byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000); @@ -2713,11 +2683,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return ApplicationId.from(path.get("tenant"), path.get("application"), path.get("instance")); } - private static JobType jobTypeFromPath(Path path) { - return JobType.fromJobName(path.get("jobtype")); + private JobType jobTypeFromPath(Path path) { + return JobType.fromJobName(path.get("jobtype"), controller.zoneRegistry()); } - private static RunId runIdFromPath(Path path) { + private RunId runIdFromPath(Path path) { long number = Long.parseLong(path.get("number")); return new RunId(appIdFromPath(path), jobTypeFromPath(path), number); } @@ -2725,7 +2695,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse submit(String tenant, String application, HttpRequest request) { Map<String, byte[]> dataParts = parseDataParts(request); Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get(); - long projectId = Math.max(1, submitOptions.field("projectId").asLong()); // Absence of this means it's not a prod app :/ + long projectId = submitOptions.field("projectId").asLong(); // Absence of this means it's not a prod app :/ + projectId = projectId == 0 ? 1 : projectId; Optional<String> repository = optional("repository", submitOptions); Optional<String> branch = optional("branch", submitOptions); Optional<String> commit = optional("commit", submitOptions); @@ -2734,6 +2705,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { : Optional.empty(); Optional<String> sourceUrl = optional("sourceUrl", submitOptions); Optional<String> authorEmail = optional("authorEmail", submitOptions); + Optional<String> description = optional("description", submitOptions); + int risk = (int) submitOptions.field("risk").asLong(); sourceUrl.map(URI::create).ifPresent(url -> { if (url.getHost() == null || url.getScheme() == null) @@ -2742,29 +2715,26 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP), true); + byte[] testPackage = dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP); + Submission submission = new Submission(applicationPackage, testPackage, sourceUrl, sourceRevision, authorEmail, description, risk); + controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant), Optional.empty(), Optional.empty(), applicationPackage, Optional.of(requireUserPrincipal(request))); - ensureApplicationExists(TenantAndApplicationId.from(tenant, application), request); - - return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), - tenant, - application, - sourceRevision, - authorEmail, - sourceUrl, - projectId, - applicationPackage, - dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP)); + TenantAndApplicationId id = TenantAndApplicationId.from(tenant, application); + ensureApplicationExists(id, request); + return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), id, submission, projectId); } private HttpResponse removeAllProdDeployments(String tenant, String application) { - JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application, - Optional.empty(), Optional.empty(), Optional.empty(), 1, - ApplicationPackage.deploymentRemoval(), new byte[0]); + JobControllerApiHandlerHelper.submitResponse(controller.jobController(), + TenantAndApplicationId.from(tenant, application), + new Submission(ApplicationPackage.deploymentRemoval(), new byte[0], Optional.empty(), + Optional.empty(), Optional.empty(), Optional.empty(), 0), + 0); return new MessageResponse("All deployments removed"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 2375b5cf049..80425609aa6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.Change; @@ -33,6 +34,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -43,7 +45,6 @@ import java.time.format.TextStyle; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -77,29 +78,29 @@ class JobControllerApiHandlerHelper { Cursor responseObject = slime.setObject(); Cursor jobsArray = responseObject.setArray("deployment"); - Arrays.stream(JobType.values()) - .filter(type -> type.environment().isManuallyDeployed()) - .map(devType -> new JobId(id, devType)) - .forEach(job -> { - Collection<Run> runs = controller.jobController().runs(job).descendingMap().values(); - if (runs.isEmpty()) - return; - - Cursor jobObject = jobsArray.addObject(); - jobObject.setString("jobName", job.type().jobName()); - toSlime(jobObject.setArray("runs"), runs, 10, baseUriForJobs); - }); + JobType.allIn(controller.zoneRegistry()).stream() + .filter(type -> type.environment().isManuallyDeployed()) + .map(devType -> new JobId(id, devType)) + .forEach(job -> { + Collection<Run> runs = controller.jobController().runs(job).descendingMap().values(); + if (runs.isEmpty()) + return; + + Cursor jobObject = jobsArray.addObject(); + jobObject.setString("jobName", job.type().jobName()); + toSlime(jobObject.setArray("runs"), runs, controller.applications().requireApplication(TenantAndApplicationId.from(id)), 10, baseUriForJobs); + }); return new SlimeJsonResponse(slime); } /** Returns a response with the runs for the given job type. */ - static HttpResponse runResponse(Map<RunId, Run> runs, Optional<String> limitStr, URI baseUriForJobType) { + static HttpResponse runResponse(Application application, Map<RunId, Run> runs, Optional<String> limitStr, URI baseUriForJobType) { Slime slime = new Slime(); Cursor cursor = slime.setObject(); int limit = limitStr.map(Integer::parseInt).orElse(Integer.MAX_VALUE); - toSlime(cursor.setArray("runs"), runs.values(), limit, baseUriForJobType); + toSlime(cursor.setArray("runs"), runs.values(), application, limit, baseUriForJobType); return new SlimeJsonResponse(slime); } @@ -194,19 +195,8 @@ class JobControllerApiHandlerHelper { * * @return Response with the new application version */ - static HttpResponse submitResponse(JobController jobController, String tenant, String application, - Optional<SourceRevision> sourceRevision, Optional<String> authorEmail, - Optional<String> sourceUrl, long projectId, - ApplicationPackage applicationPackage, byte[] testPackage) { - ApplicationVersion version = jobController.submit(TenantAndApplicationId.from(tenant, application), - sourceRevision, - authorEmail, - sourceUrl, - projectId, - applicationPackage, - testPackage); - - return new MessageResponse(version.toString()); + static HttpResponse submitResponse(JobController jobController, TenantAndApplicationId id, Submission submission, long projectId) { + return new MessageResponse("application " + jobController.submit(id, submission, projectId)); } /** Aborts any job of the given type. */ @@ -230,6 +220,7 @@ class JobControllerApiHandlerHelper { case aborted: return "aborted"; case error: return "error"; case testFailure: return "testFailure"; + case noTests: return "noTests"; case endpointCertificateTimeout: return "endpointCertificateTimeout"; case nodeAllocationFailure: return "nodeAllocationFailure"; case installationFailed: return "installationFailed"; @@ -276,7 +267,7 @@ class JobControllerApiHandlerHelper { stepStatus.coolingDownUntil(change).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli())); stepStatus.blockedUntil(Change.of(controller.systemVersion(versionStatus))) // Dummy version — just anything with a platform. .ifPresent(until -> stepObject.setLong("platformBlockedUntil", until.toEpochMilli())); - application.latestVersion().map(Change::of).flatMap(stepStatus::blockedUntil) // Dummy version — just anything with an application. + stepStatus.blockedUntil(Change.of(RevisionId.forProduction(1))) // Dummy version — just anything with an application. .ifPresent(until -> stepObject.setLong("applicationBlockedUntil", until.toEpochMilli())); if (stepStatus.type() == DeploymentStatus.StepType.delay) @@ -286,7 +277,7 @@ class JobControllerApiHandlerHelper { Cursor deployingObject = stepObject.setObject("deploying"); if ( ! change.isEmpty()) { change.platform().ifPresent(version -> deployingObject.setString("platform", version.toFullString())); - change.application().ifPresent(version -> toSlime(deployingObject.setObject("application"), version)); + change.revision().ifPresent(revision -> toSlime(deployingObject.setObject("application"), application.revisions().get(revision))); } Cursor latestVersionsObject = stepObject.setObject("latestVersions"); @@ -303,38 +294,33 @@ class JobControllerApiHandlerHelper { || deployments.stream().anyMatch(deployment -> deployment.version().isBefore(latestPlatform.versionNumber()))); Cursor availableArray = latestPlatformObject.setArray("available"); + boolean isUpgrade = true; for (VespaVersion available : availablePlatforms) { if ( deployments.stream().anyMatch(deployment -> deployment.version().isAfter(available.versionNumber())) || deployments.stream().noneMatch(deployment -> deployment.version().isBefore(available.versionNumber())) && ! deployments.isEmpty() || status.hasCompleted(stepStatus.instance(), Change.of(available.versionNumber())) - || change.platform().map(available.versionNumber()::compareTo).orElse(1) <= 0) - break; + || change.platform().map(available.versionNumber()::compareTo).orElse(1) < 0) + isUpgrade = false; - availableArray.addObject().setString("platform", available.versionNumber().toFullString()); + Cursor platformObject = availableArray.addObject(); + platformObject.setString("platform", available.versionNumber().toFullString()); + platformObject.setBool("upgrade", isUpgrade || change.platform().map(available.versionNumber()::equals).orElse(false)); } - change.platform().ifPresent(version -> availableArray.addObject().setString("platform", version.toFullString())); toSlime(latestPlatformObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksVersions)); } - List<ApplicationVersion> availableApplications = new ArrayList<>(application.deployableVersions(false)); + List<ApplicationVersion> availableApplications = new ArrayList<>(application.revisions().deployable(false)); if ( ! availableApplications.isEmpty()) { var latestApplication = availableApplications.get(0); Cursor latestApplicationObject = latestVersionsObject.setObject("application"); toSlime(latestApplicationObject.setObject("application"), latestApplication); latestApplicationObject.setLong("at", latestApplication.buildTime().orElse(Instant.EPOCH).toEpochMilli()); - latestApplicationObject.setBool("upgrade", change.application().map(latestApplication::compareTo).orElse(1) > 0 && deployments.isEmpty() - || deployments.stream().anyMatch(deployment -> deployment.applicationVersion().compareTo(latestApplication) < 0)); + latestApplicationObject.setBool("upgrade", change.revision().map(latestApplication.id()::compareTo).orElse(1) > 0 && deployments.isEmpty() + || deployments.stream().anyMatch(deployment -> deployment.revision().compareTo(latestApplication.id()) < 0)); Cursor availableArray = latestApplicationObject.setArray("available"); - for (ApplicationVersion available : availableApplications) { - if ( deployments.stream().anyMatch(deployment -> deployment.applicationVersion().compareTo(available) > 0) - || deployments.stream().noneMatch(deployment -> deployment.applicationVersion().compareTo(available) < 0) && ! deployments.isEmpty() - || status.hasCompleted(stepStatus.instance(), Change.of(available)) - || change.application().map(available::compareTo).orElse(1) <= 0) - break; - + for (ApplicationVersion available : availableApplications) toSlime(availableArray.addObject().setObject("application"), available); - } - change.application().ifPresent(version -> toSlime(availableArray.addObject().setObject("application"), version)); + toSlime(latestApplicationObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksRevisions)); } } @@ -346,12 +332,12 @@ class JobControllerApiHandlerHelper { "/job/" + job.type().jobName()).normalize(); stepObject.setString("url", baseUriForJob.toString()); stepObject.setString("environment", job.type().environment().value()); - stepObject.setString("region", job.type().zone(controller.system()).value()); + stepObject.setString("region", job.type().zone().value()); if (job.type().isProduction() && job.type().isDeployment()) { status.deploymentFor(job).ifPresent(deployment -> { stepObject.setString("currentPlatform", deployment.version().toFullString()); - toSlime(stepObject.setObject("currentApplication"), deployment.applicationVersion()); + toSlime(stepObject.setObject("currentApplication"), application.revisions().get(deployment.revision())); }); } @@ -367,19 +353,26 @@ class JobControllerApiHandlerHelper { continue; // Run will be contained in the "runs" array. Cursor runObject = toRunArray.addObject(); - toSlime(runObject.setObject("versions"), versions.versions()); + toSlime(runObject.setObject("versions"), versions.versions(), application); } - toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), 10, baseUriForJob); + toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), application, 10, baseUriForJob); }); } Cursor buildsArray = responseObject.setArray("builds"); - application.versions().stream().sorted(reverseOrder()).forEach(version -> toSlime(buildsArray.addObject(), version)); + application.revisions().withPackage().stream().sorted(reverseOrder()).forEach(version -> toRichSlime(buildsArray.addObject(), version)); return new SlimeJsonResponse(slime); } + static void toRichSlime(Cursor versionObject, ApplicationVersion version) { + toSlime(versionObject, version); + version.description().ifPresent(description -> versionObject.setString("description", description)); + if (version.risk() != 0) versionObject.setLong("risk", version.risk()); + versionObject.setBool("deployable", version.isDeployable()); + } + static void toSlime(Cursor versionObject, ApplicationVersion version) { version.buildNumber().ifPresent(id -> versionObject.setLong("build", id)); version.compileVersion().ifPresent(platform -> versionObject.setString("compileVersion", platform.toFullString())); @@ -387,11 +380,11 @@ class JobControllerApiHandlerHelper { version.commit().ifPresent(commit -> versionObject.setString("commit", commit)); } - private static void toSlime(Cursor versionsObject, Versions versions) { + private static void toSlime(Cursor versionsObject, Versions versions, Application application) { versionsObject.setString("targetPlatform", versions.targetPlatform().toFullString()); - toSlime(versionsObject.setObject("targetApplication"), versions.targetApplication()); + toSlime(versionsObject.setObject("targetApplication"), application.revisions().get(versions.targetRevision())); versions.sourcePlatform().ifPresent(platform -> versionsObject.setString("sourcePlatform", platform.toFullString())); - versions.sourceApplication().ifPresent(application -> toSlime(versionsObject.setObject("sourceApplication"), application)); + versions.sourceRevision().ifPresent(revision -> toSlime(versionsObject.setObject("sourceApplication"), application.revisions().get(revision))); } private static void toSlime(Cursor blockersArray, Stream<ChangeBlocker> blockers) { @@ -427,7 +420,7 @@ class JobControllerApiHandlerHelper { return candidates; } - private static void toSlime(Cursor runsArray, Collection<Run> runs, int limit, URI baseUriForJob) { + private static void toSlime(Cursor runsArray, Collection<Run> runs, Application application, int limit, URI baseUriForJob) { runs.stream().limit(limit).forEach(run -> { Cursor runObject = runsArray.addObject(); runObject.setLong("id", run.id().number()); @@ -435,7 +428,7 @@ class JobControllerApiHandlerHelper { runObject.setLong("start", run.start().toEpochMilli()); run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli())); runObject.setString("status", run.status().name()); - toSlime(runObject.setObject("versions"), run.versions()); + toSlime(runObject.setObject("versions"), run.versions(), application); Cursor runStepsArray = runObject.setArray("steps"); run.steps().forEach((step, info) -> { Cursor runStepObject = runStepsArray.addObject(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java deleted file mode 100644 index 67b47aa976a..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.container.jdisc.HttpResponse; -import ai.vespa.http.HttpURL; -import ai.vespa.http.HttpURL.Path; -import ai.vespa.http.HttpURL.Query; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; -import com.yahoo.vespa.serviceview.bindings.ClusterView; -import com.yahoo.vespa.serviceview.bindings.ServiceView; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * A response containing a service view for an application deployment. - * This does not define the API response but merely proxies the API response provided by Vespa, with URLs - * rewritten to include zone and application information allow proxying through the controller - * - * @author Steinar Knutsen - * @author bratseth - */ -class ServiceApiResponse extends HttpResponse { - - private final ZoneId zone; - private final ApplicationId application; - private final List<URI> configServerURIs; - private final Slime slime; - private final HttpURL requestUri; - - // Only set for one of the setResponse calls - private String serviceName = null; - private Path restPath = null; - - public ServiceApiResponse(ZoneId zone, ApplicationId application, List<URI> configServerURIs, URI requestUri) { - super(200); - this.zone = zone; - this.application = application; - this.configServerURIs = configServerURIs; - this.slime = new Slime(); - this.requestUri = HttpURL.from(requestUri).withQuery(Query.empty()); - } - - public void setResponse(ApplicationView applicationView) { - Cursor clustersArray = slime.setObject().setArray("clusters"); - for (ClusterView clusterView : applicationView.clusters) { - Cursor clusterObject = clustersArray.addObject(); - clusterObject.setString("name", clusterView.name); - clusterObject.setString("type", clusterView.type); - setNullableString("url", rewriteIfUrl(clusterView.url, requestUri), clusterObject); - Cursor servicesArray = clusterObject.setArray("services"); - for (ServiceView serviceView : clusterView.services) { - Cursor serviceObject = servicesArray.addObject(); - setNullableString("url", rewriteIfUrl(serviceView.url, requestUri), serviceObject); - serviceObject.setString("serviceType", serviceView.serviceType); - serviceObject.setString("serviceName", serviceView.serviceName); - serviceObject.setString("configId", serviceView.configId); - serviceObject.setString("host", serviceView.host); - } - } - } - - public void setResponse(Map<?,?> responseData, String serviceName, Path restPath) { - this.serviceName = serviceName; - this.restPath = restPath; - mapToSlime(responseData, slime.setObject()); - } - - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { - return "application/json"; - } - - @SuppressWarnings("unchecked") - private void mapToSlime(Map<?,?> data, Cursor object) { - for (Map.Entry<String, Object> entry : ((Map<String, Object>)data).entrySet()) - fieldToSlime(entry.getKey(), entry.getValue(), object); - } - - private void fieldToSlime(String key, Object value, Cursor object) { - if (value instanceof String) { - if (key.equals("url") || key.equals("link")) - value = rewriteIfUrl((String)value, generateLocalLinkPrefix(serviceName, restPath)); - setNullableString(key, (String)value, object); - } - else if (value instanceof Integer) { - object.setLong(key, (int)value); - } - else if (value instanceof Long) { - object.setLong(key, (long)value); - } - else if (value instanceof Float) { - object.setDouble(key, (double)value); - } - else if (value instanceof Double) { - object.setDouble(key, (double)value); - } - else if (value instanceof List) { - listToSlime((List)value, object.setArray(key)); - } - else if (value instanceof Map) { - mapToSlime((Map<?,?>)value, object.setObject(key)); - } - } - - private void listToSlime(List<?> list, Cursor array) { - for (Object entry : list) - entryToSlime(entry, array); - } - - private void entryToSlime(Object entry, Cursor array) { - if (entry instanceof String) - addNullableString(rewriteIfUrl((String)entry, generateLocalLinkPrefix(serviceName, restPath)), array); - else if (entry instanceof Integer) - array.addLong((long)entry); - else if (entry instanceof Long) - array.addLong((long)entry); - else if (entry instanceof Float) - array.addDouble((double)entry); - else if (entry instanceof Double) - array.addDouble((double)entry); - else if (entry instanceof List) - listToSlime((List)entry, array.addArray()); - else if (entry instanceof Map) - mapToSlime((Map)entry, array.addObject()); - } - - private String rewriteIfUrl(String urlOrAnyString, HttpURL requestUri) { - if (urlOrAnyString == null) return null; - - String hostPattern = "(" + - String.join( - "|", configServerURIs.stream() - .map(URI::toString) - .map(s -> s.substring(0, s.length() -1)) - .map(Pattern::quote) - .toArray(String[]::new)) - + ")"; - - String remoteServicePath = "/serviceview/" - + "v1/tenant/" + application.tenant().value() - + "/application/" + application.application().value() - + "/environment/" + zone.environment().value() - + "/region/" + zone.region().value() - + "/instance/" + application.instance() - + "/service/"; - - Pattern remoteServiceResourcePattern = Pattern.compile("^(" + hostPattern + Pattern.quote(remoteServicePath) + ")"); - Matcher matcher = remoteServiceResourcePattern.matcher(urlOrAnyString); - - if (matcher.find()) { - String proxiedPath = urlOrAnyString.substring(matcher.group().length()); - return requestUri.withPath(requestUri.path().append(Path.parse(proxiedPath))).asURI().toString(); - } else { - return urlOrAnyString; // not a service url - } - } - - private HttpURL generateLocalLinkPrefix(String identifier, Path restPath) { - Path proxiedPath = Path.parse(identifier).append(restPath); - if (requestUri.path().tail(proxiedPath.length()).equals(proxiedPath)) { - return requestUri.withPath(requestUri.path().cut(proxiedPath.length())); - } else { - throw new IllegalStateException("Expected the resource " + requestUri.path() + " to end with " + proxiedPath); - } - } - - private void setNullableString(String key, String valueOrNull, Cursor receivingObject) { - if (valueOrNull == null) - receivingObject.setNix(key); - else - receivingObject.setString(key, valueOrNull); - } - - private void addNullableString(String valueOrNull, Cursor receivingArray) { - if (valueOrNull == null) - receivingArray.addNix(); - else - receivingArray.addString(valueOrNull); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index 45c4978ab9f..33307ea1677 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -19,10 +19,10 @@ import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TenantController; import com.yahoo.vespa.hosted.controller.api.integration.billing.Bill; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.CollectionMethod; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PaymentInstrument; import com.yahoo.vespa.hosted.controller.api.integration.billing.InstrumentOwner; -import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PaymentInstrument; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index a5aca4adf10..0f3e5b7f76b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -28,15 +28,12 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import javax.ws.rs.BadRequestException; import java.math.BigDecimal; import java.time.Clock; -import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; import java.util.Comparator; import java.util.List; import java.util.Optional; -import java.util.logging.Level; /** * @author ogronnesby @@ -182,7 +179,6 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler var tenant = tenants.require(tenantName, CloudTenant.class); var untilAt = untilParameter(requestContext); var usage = billing.createUncommittedBill(tenant.name(), untilAt); - var slime = new Slime(); usageToSlime(slime.setObject(), usage); return slime; @@ -322,7 +318,8 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler private LocalDate untilParameter(RestApi.RequestContext ctx) { return ctx.queryParameters().getString("until") - .map(this::parseLocalDate) + .map(LocalDate::parse) + .map(date -> date.plusDays(1)) .orElseGet(this::tomorrow); } @@ -330,12 +327,6 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return LocalDate.now(clock).plusDays(1); } - private LocalDate parseLocalDate(String until) { - if (until.isEmpty() || until.isBlank()) - return tomorrow(); - else return LocalDate.parse(until); - } - private static String getInspectorFieldOrThrow(Inspector inspector, String field) { if (!inspector.field(field).valid()) throw new BadRequestException("Field " + field + " cannot be null"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java index e1fc68974cc..c72d8ceb089 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java @@ -280,7 +280,7 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { private Optional<ZoneId> affectedZone(List<String> hosts) { NodeFilter affectedHosts = NodeFilter.all().hostnames(hosts.stream() - .map(HostName::from) + .map(HostName::of) .collect(Collectors.toSet())); for (var zone : getProdZones()) { var affectedHostsInZone = controller.serviceRegistry().configServer().nodeRepository().list(zone, affectedHosts); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java index 8caa741d737..467b0c094cc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.configserver; +import ai.vespa.http.HttpURL; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.ErrorResponse; -import ai.vespa.http.HttpURL; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java index 7e250ce62ab..c3155406194 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.controller; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; -import com.yahoo.restapi.SlimeJsonResponse; /** * @author mpolden diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java index e68517f7134..122ea94ab6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.controller; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; -import com.yahoo.restapi.SlimeJsonResponse; /** * @author mpolden diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java index eb74f931b2c..9b400fdfb78 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java @@ -81,16 +81,14 @@ public class BadgeApiHandler extends ThreadedHttpRequestHandler { () -> { DeploymentStatus status = controller.jobController().deploymentStatus(controller.applications().requireApplication(TenantAndApplicationId.from(id))); Predicate<JobStatus> isDeclaredJob = job -> status.jobSteps().get(job.id()) != null && status.jobSteps().get(job.id()).isDeclared(); - return Badges.overviewBadge(id, - status.jobs().instance(id.instance()).matching(isDeclaredJob), - controller.system()); + return Badges.overviewBadge(id, status.jobs().instance(id.instance()).matching(isDeclaredJob)); }); } /** Returns a URI which points to a history badge for the given application and job type. */ private HttpResponse historyBadge(String tenant, String application, String instance, String jobName, String historyLength) { ApplicationId id = ApplicationId.from(tenant, application, instance); - JobType type = JobType.fromJobName(jobName); + JobType type = JobType.fromJobName(jobName, controller.zoneRegistry()); int length = historyLength == null ? 5 : Math.min(32, Math.max(0, Integer.parseInt(historyLength))); return cachedResponse(new Key(id, type, length), controller.clock().instant(), @@ -135,7 +133,7 @@ public class BadgeApiHandler extends ThreadedHttpRequestHandler { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Key key = (Key) o; - return historyLength == key.historyLength && id.equals(key.id) && type == key.type; + return historyLength == key.historyLength && id.equals(key.id) && Objects.equals(type, key.type); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java index 1fe5ebfa9a9..26a5da45bdb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import static java.util.stream.Collectors.toList; @@ -51,14 +52,18 @@ public class Badges { return widthOf(text, 11); } - static String colorOf(Run run, Boolean wasOk) { + static String colorOf(Run run, Optional<RunStatus> previous) { switch (run.status()) { - case running: - return wasOk ? "url(#run-on-success)" : "url(#run-on-failure)"; - case success: - return success; - default: - return failure; + case running: switch (previous.orElse(RunStatus.success)) { + case success: return "url(#run-on-success)"; + case aborted: + case noTests: return "url(#run-on-warning)"; + default: return "url(#run-on-failure)"; + } + case success: return success; + case aborted: + case noTests: return warning; + default: return failure; } } @@ -71,9 +76,10 @@ public class Badges { static final double xPad = 6; static final double logoSize = 16; static final String dark = "#404040"; - static final String success = "#00f244"; + static final String success = "#00f844"; static final String running = "#ab83ff"; static final String failure = "#bf103c"; + static final String warning = "#bd890b"; static void addText(List<String> texts, String text, double x, double width) { addText(texts, text, x, width, 11); @@ -116,13 +122,11 @@ public class Badges { .limit(length) .collect(toList()); - boolean isOk = status.lastCompleted().map(run -> run.status() == RunStatus.success).orElse(true); - text = lastTriggered.id().type().jobName(); textWidth = widthOf(text); dx = xPad + textWidth + xPad; addShade(sections, x, dx); - sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(lastTriggered, isOk) + "'/>\n"); + sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(lastTriggered, status.lastStatus()) + "'/>\n"); addShadow(sections, x + dx); addText(texts, text, x + dx / 2, textWidth); x += dx; @@ -130,7 +134,7 @@ public class Badges { dx = xPad * (192.0 / (32 + runs.size())); // Broader sections with shorter history. for (Run run : runs) { addShade(sections, x, dx); - sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(run, null) + "'/>\n"); + sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(run, Optional.empty()) + "'/>\n"); addShadow(sections, x + dx); dx *= Math.pow(0.3, 1.0 / (runs.size() + 8)); // Gradually narrowing sections with age. x += dx; @@ -140,7 +144,7 @@ public class Badges { return badge(sections, texts, x); } - static String overviewBadge(ApplicationId id, JobList jobs, SystemName system) { + static String overviewBadge(ApplicationId id, JobList jobs) { // Put production tests right after their deployments, for a more compact rendering. List<Run> runs = new ArrayList<>(jobs.lastTriggered().asList()); boolean anyTest = false; @@ -149,7 +153,7 @@ public class Badges { if (run.id().type().isProduction() && run.id().type().isTest()) { anyTest = true; int j = i; - while ( ! runs.get(j - 1).id().type().zone(system).equals(run.id().type().zone(system))) + while ( ! runs.get(j - 1).id().type().zone().equals(run.id().type().zone())) runs.set(j, runs.get(--j)); runs.set(j, run); } @@ -179,7 +183,7 @@ public class Badges { text = nameOf(run.id().type()); textWidth = widthOf(text, isTest ? 9 : 11); dx = xPad + textWidth + (isTest ? 0 : xPad); - boolean wasOk = jobs.get(run.id().job()).flatMap(JobStatus::lastStatus).map(RunStatus.success::equals).orElse(true); + Optional<RunStatus> previous = jobs.get(run.id().job()).flatMap(JobStatus::lastStatus); addText(texts, text, x + (dx - (isTest ? xPad : 0)) / 2, textWidth, isTest ? 9 : 11); @@ -197,10 +201,10 @@ public class Badges { // Add colored section for job ... if (test == null) - sections.add(" <rect x='" + (x - 16) + "' rx='3' width='" + (dx + 16) + "' height='20' fill='" + colorOf(run, wasOk) + "'/>\n"); + sections.add(" <rect x='" + (x - 16) + "' rx='3' width='" + (dx + 16) + "' height='20' fill='" + colorOf(run, previous) + "'/>\n"); // ... with a slant if a test is next. else - sections.add(" <polygon points='" + (x - 6) + " 0 " + (x - 6) + " 20 " + (x + dx - 7) + " 20 " + (x + dx + 1) + " 0' fill='" + colorOf(run, wasOk) + "'/>\n"); + sections.add(" <polygon points='" + (x - 6) + " 0 " + (x - 6) + " 20 " + (x + dx - 7) + " 20 " + (x + dx + 1) + " 0' fill='" + colorOf(run, previous) + "'/>\n"); // Cast a shadow onto the next zone ... if (test == null) @@ -255,6 +259,13 @@ public class Badges { " <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />\n" + " <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />\n" + " </linearGradient>\n" + + // Running color sloshing back and forth on top of the warning color. + " <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'>\n" + + " <stop offset='0' stop-color='" + running + "' />\n" + + " <stop offset='1' stop-color='" + warning + "' />\n" + + " <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />\n" + + " <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />\n" + + " </linearGradient>\n" + // Running color sloshing back and forth on top of the success color. " <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'>\n" + " <stop offset='0' stop-color='" + running + "' />\n" + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 1a4a42cb521..effa0906b94 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -210,7 +210,7 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { }); }); } - JobType.allIn(controller.system()).stream() + JobType.allIn(controller.zoneRegistry()).stream() .filter(job -> ! job.environment().isManuallyDeployed()) .map(JobType::jobName).forEach(root.setArray("jobs")::addString); return new SlimeJsonResponse(slime); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index a7472ced09c..01bd02fdc13 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.TenantController; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; @@ -62,6 +63,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { private final TenantController tenants; private final ExecutorService executor; private final SystemName systemName; + private final ZoneRegistry zones; @Inject public AthenzRoleFilter(AthenzClientFactory athenzClientFactory, Controller controller) { @@ -69,6 +71,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { this.tenants = controller.tenants(); this.executor = Executors.newCachedThreadPool(); this.systemName = controller.system(); + this.zones = controller.zoneRegistry(); } @Override @@ -108,8 +111,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/{*}")) { zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobname}")) { - var jobtype= JobType.fromJobName(path.get("jobname")); - zone = Optional.of(jobtype.zone(systemName)); + zone = Optional.of(JobType.fromJobName(path.get("jobname"), zones).zone()); } else { zone = Optional.empty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index 08ec3caa829..7c695ef51d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -6,7 +6,6 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.role.Action; import com.yahoo.vespa.hosted.controller.api.role.Enforcer; @@ -15,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java index 985919581ef..b7c77e7bfb4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import java.util.logging.Level; import com.yahoo.security.KeyUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -24,6 +23,7 @@ import java.security.PublicKey; import java.util.Base64; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import static java.nio.charset.StandardCharsets.UTF_8; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 33e6632b8e1..025e8dff659 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.restapi.user; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -28,7 +27,7 @@ import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; -import com.yahoo.vespa.hosted.controller.api.integration.user.User; +import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; import com.yahoo.vespa.hosted.controller.api.role.Role; @@ -172,7 +171,6 @@ public class UserApiHandler extends ThreadedHttpRequestHandler { toSlime(root.setObject("user"), user); Cursor tenants = root.setObject("tenants"); - InstanceName userInstance = InstanceName.from(user.nickname()); tenantRolesByTenantName.keySet().stream() .sorted() .forEach(tenant -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index 5b165a9ef37..320321080c8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -7,13 +7,13 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.Path; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.yolean.Exceptions; import java.util.Comparator; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 800588fdf8c..e3f1e9b5f94 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.zone.v2; +import ai.vespa.http.HttpURL; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; -import ai.vespa.http.HttpURL; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index b98ef717dd3..a5a09ab6551 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; +import ai.vespa.http.DomainName; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -152,7 +153,7 @@ public class RoutingPolicies { } /** Update global DNS records for given policies */ - private void updateGlobalDnsOf(RoutingPolicyList instancePolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + private void updateGlobalDnsOf(RoutingPolicyList instancePolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Mutex lock) { Map<RoutingId, List<RoutingPolicy>> routingTable = instancePolicies.asInstanceRoutingTable(); for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { RoutingId routingId = routeEntry.getKey(); @@ -215,7 +216,7 @@ public class RoutingPolicies { } var weightedTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone(), weight); - endpoints.computeIfAbsent(regionEndpoint, (k) -> new RegionEndpoint(new LatencyAliasTarget(HostName.from(regionEndpoint.dnsName()), + endpoints.computeIfAbsent(regionEndpoint, (k) -> new RegionEndpoint(new LatencyAliasTarget(DomainName.of(regionEndpoint.dnsName()), policy.dnsZone().get(), policy.id().zone()))) .zoneTargets() @@ -225,7 +226,7 @@ public class RoutingPolicies { } - private void updateApplicationDnsOf(RoutingPolicyList routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + private void updateApplicationDnsOf(RoutingPolicyList routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Mutex lock) { // In the context of single deployment (which this is) there is only one routing policy per routing ID. I.e. // there is no scenario where more than one deployment within an instance can be a member the same // application-level endpoint. However, to allow this in the future the routing table remains @@ -307,7 +308,7 @@ public class RoutingPolicies { * * @return the updated policies */ - private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Lock lock) { + private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) { Map<RoutingPolicyId, RoutingPolicy> policies = new LinkedHashMap<>(instancePolicies.asMap()); for (LoadBalancer loadBalancer : allocation.loadBalancers) { if (loadBalancer.hostname().isEmpty()) continue; @@ -343,7 +344,7 @@ public class RoutingPolicies { * * @return the updated policies */ - private RoutingPolicyList removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Lock lock) { + private RoutingPolicyList removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) { Map<RoutingPolicyId, RoutingPolicy> newPolicies = new LinkedHashMap<>(instancePolicies.asMap()); Set<RoutingPolicyId> activeIds = allocation.asPolicyIds(); RoutingPolicyList removable = instancePolicies.deployment(allocation.deployment) @@ -363,7 +364,7 @@ public class RoutingPolicies { } /** Remove unreferenced instance endpoints from DNS */ - private void removeGlobalDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Lock lock) { + private void removeGlobalDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Mutex lock) { Set<RoutingId> removalCandidates = new HashSet<>(deploymentPolicies.asInstanceRoutingTable().keySet()); Set<RoutingId> activeRoutingIds = instanceRoutingIds(allocation); removalCandidates.removeAll(activeRoutingIds); @@ -380,7 +381,7 @@ public class RoutingPolicies { } /** Remove unreferenced application endpoints in given allocation from DNS */ - private void removeApplicationDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Lock lock) { + private void removeApplicationDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Mutex lock) { Map<RoutingId, List<RoutingPolicy>> routingTable = deploymentPolicies.asApplicationRoutingTable(); Set<RoutingId> removalCandidates = new HashSet<>(routingTable.keySet()); Set<RoutingId> activeRoutingIds = applicationRoutingIds(allocation); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 34736c16a6b..1ccb3205816 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; +import ai.vespa.http.DomainName; import com.google.common.collect.ImmutableSortedSet; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.text.Text; @@ -27,14 +27,14 @@ import java.util.Set; public class RoutingPolicy { private final RoutingPolicyId id; - private final HostName canonicalName; + private final DomainName canonicalName; private final Optional<String> dnsZone; private final Set<EndpointId> instanceEndpoints; private final Set<EndpointId> applicationEndpoints; private final Status status; /** DO NOT USE. Public for serialization purposes */ - public RoutingPolicy(RoutingPolicyId id, HostName canonicalName, Optional<String> dnsZone, + public RoutingPolicy(RoutingPolicyId id, DomainName canonicalName, Optional<String> dnsZone, Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, Status status) { this.id = Objects.requireNonNull(id, "id must be non-null"); this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); @@ -50,7 +50,7 @@ public class RoutingPolicy { } /** The canonical name for the load balancer this applies to (rhs of a CNAME or ALIAS record) */ - public HostName canonicalName() { + public DomainName canonicalName() { return canonicalName; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java index 0cf7101cac0..8eeab8c20e3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.routing.rotation; import com.yahoo.text.Text; + import java.util.Objects; /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java index 36a43f80e9a..39fc70aac64 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing.rotation; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Lock; import java.util.Objects; @@ -12,9 +13,9 @@ import java.util.Objects; */ public class RotationLock implements AutoCloseable { - private final Lock lock; + private final Mutex lock; - RotationLock(Lock lock) { + RotationLock(Mutex lock) { this.lock = Objects.requireNonNull(lock, "lock cannot be null"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java index a4af9f8e268..27b61a4fd17 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java @@ -1,13 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.support.access; -import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import java.security.Principal; import java.security.cert.X509Certificate; import java.time.Instant; import java.time.Period; @@ -37,7 +36,7 @@ public class SupportAccessControl { } public SupportAccess disallow(DeploymentId deployment, String by) { - try (Lock lock = controller.curator().lockSupportAccess(deployment)) { + try (Mutex lock = controller.curator().lockSupportAccess(deployment)) { var now = controller.clock().instant(); SupportAccess supportAccess = forDeployment(deployment); if (supportAccess.currentStatus(now).state() == NOT_ALLOWED) { @@ -51,7 +50,7 @@ public class SupportAccessControl { } public SupportAccess allow(DeploymentId deployment, Instant until, String by) { - try (Lock lock = controller.curator().lockSupportAccess(deployment)) { + try (Mutex lock = controller.curator().lockSupportAccess(deployment)) { var now = controller.clock().instant(); if (until.isAfter(now.plus(MAX_SUPPORT_ACCESS_TIME))) { throw new IllegalArgumentException("Support access cannot be allowed for more than 10 days"); @@ -63,7 +62,7 @@ public class SupportAccessControl { } public SupportAccess registerGrant(DeploymentId deployment, String by, X509Certificate certificate) { - try (Lock lock = controller.curator().lockSupportAccess(deployment)) { + try (Mutex lock = controller.curator().lockSupportAccess(deployment)) { var now = controller.clock().instant(); SupportAccess supportAccess = forDeployment(deployment); if (certificate.getNotAfter().toInstant().isBefore(now)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/ControllerVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/ControllerVersion.java deleted file mode 100644 index 5b293081dc2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/ControllerVersion.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.component.Version; -import com.yahoo.component.Vtag; - -import java.time.Instant; -import java.util.Objects; - -/** - * A controller's Vespa version and commit details. - * - * @author mpolden - */ -public class ControllerVersion implements Comparable<ControllerVersion> { - - /** The current version of this controller */ - public static final ControllerVersion CURRENT = new ControllerVersion(Vtag.currentVersion, Vtag.commitSha, - Vtag.commitDate); - - private final Version version; - private final String commitSha; - private final Instant commitDate; - - public ControllerVersion(Version version, String commitSha, Instant commitDate) { - this.version = Objects.requireNonNull(version); - this.commitSha = Objects.requireNonNull(commitSha); - this.commitDate = Objects.requireNonNull(commitDate); - } - - /** Vespa version */ - public Version version() { - return version; - } - - /** Commit SHA of this */ - public String commitSha() { - return commitSha; - } - - /** The time this was committed */ - public Instant commitDate() { - return commitDate; - } - - @Override - public String toString() { - return version + ", commit " + commitSha + " @ " + commitDate; - } - - @Override - public int compareTo(ControllerVersion o) { - return version.compareTo(o.version); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ControllerVersion that = (ControllerVersion) o; - return version.equals(that.version); - } - - @Override - public int hashCode() { - return Objects.hash(version); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java index c1abe38a2a9..179a64d9491 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import com.yahoo.vespa.hosted.controller.deployment.JobList; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import java.util.ArrayList; import java.util.Collection; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 3fa440e694e..117abd52193 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -197,7 +198,7 @@ public class VersionStatus { .add(controller.hostname()); } else { for (String host : controller.curator().cluster()) { - HostName hostname = HostName.from(host); + HostName hostname = HostName.of(host); versions.computeIfAbsent(controller.curator().readControllerVersion(hostname), (k) -> new ArrayList<>()) .add(hostname); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 5ababdd8250..0ecac036913 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller; import com.google.common.collect.Sets; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.Notifications; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; @@ -23,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.dns.LatencyAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; @@ -36,7 +38,12 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.controller.notification.Notification; +import com.yahoo.vespa.hosted.controller.notification.Notification.Level; +import com.yahoo.vespa.hosted.controller.notification.Notification.Type; +import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; @@ -59,10 +66,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static com.yahoo.config.provision.SystemName.main; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -92,13 +99,13 @@ public class ControllerTest { var context = tester.newDeploymentContext(); context.submit(applicationPackage); assertEquals("Application version is known from completion of initial job", - ApplicationVersion.from(DeploymentContext.defaultSourceRevision, 1, "a@b", new Version("6.1"), Instant.ofEpochSecond(1)), - context.instance().change().application().get()); + ApplicationVersion.from(RevisionId.forProduction(1), DeploymentContext.defaultSourceRevision, "a@b", new Version("6.1"), Instant.ofEpochSecond(1)), + context.application().revisions().get(context.instance().change().revision().get())); context.runJob(systemTest); context.runJob(stagingTest); - ApplicationVersion applicationVersion = context.instance().change().application().get(); - assertFalse("Application version has been set during deployment", applicationVersion.isUnknown()); + RevisionId applicationVersion = context.instance().change().revision().get(); + assertTrue("Application version has been set during deployment", applicationVersion.isProduction()); tester.triggerJobs(); // Causes first deployment job to be triggered @@ -169,7 +176,7 @@ public class ControllerTest { e.getMessage()); } assertNotNull("Zone was not removed", - context.instance().deployments().get(productionUsWest1.zone(main))); + context.instance().deployments().get(productionUsWest1.zone())); // prod zone removal is allowed with override applicationPackage = new ApplicationPackageBuilder() @@ -179,7 +186,7 @@ public class ControllerTest { .build(); context.submit(applicationPackage); assertNull("Zone was removed", - context.instance().deployments().get(productionUsWest1.zone(main))); + context.instance().deployments().get(productionUsWest1.zone())); assertNull("Deployment job was removed", context.instanceJobs().get(productionUsWest1)); // Submission has stored application meta. @@ -200,7 +207,7 @@ public class ControllerTest { .get(tester.clock().instant())); assertNull(tester.controllerTester().serviceRegistry().applicationStore() - .getMeta(context.deploymentIdIn(productionUsWest1.zone(main)))); + .getMeta(context.deploymentIdIn(productionUsWest1.zone()))); } @Test @@ -735,7 +742,7 @@ public class ControllerTest { context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version1).build()); assertEquals(version2, context.deployment(zone).version()); - assertEquals(Optional.of(version1), context.deployment(zone).applicationVersion().compileVersion()); + assertEquals(Optional.of(version1), context.application().revisions().get(context.deployment(zone).revision()).compileVersion()); try { context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version1).majorVersion(8).build()); @@ -764,11 +771,11 @@ public class ControllerTest { context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version3).majorVersion(8).build()); assertEquals(version3, context.deployment(zone).version()); - assertEquals(Optional.of(version3), context.deployment(zone).applicationVersion().compileVersion()); + assertEquals(Optional.of(version3), context.application().revisions().get(context.deployment(zone).revision()).compileVersion()); context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version3).build()); assertEquals(version3, context.deployment(zone).version()); - assertEquals(Optional.of(version3), context.deployment(zone).applicationVersion().compileVersion()); + assertEquals(Optional.of(version3), context.application().revisions().get(context.deployment(zone).revision()).compileVersion()); } @Test @@ -872,6 +879,8 @@ public class ControllerTest { @Test public void testDeployWithGlobalEndpointsInMultipleClouds() { tester.controllerTester().zoneRegistry().setZones( + ZoneApiMock.fromId("test.us-west-1"), + ZoneApiMock.fromId("staging.us-west-1"), ZoneApiMock.fromId("prod.us-west-1"), ZoneApiMock.newBuilder().with(CloudName.from("aws")).withId("prod.aws-us-east-1").build() ); @@ -940,19 +949,19 @@ public class ControllerTest { // The weighted record for zone 2's region new Record(Record.Type.ALIAS, RecordName.from("application.tenant.us-east-3-w.vespa.oath.cloud"), - new WeightedAliasTarget(HostName.from("lb-0--tenant:application:default--prod.us-east-3"), + new WeightedAliasTarget(HostName.of("lb-0--tenant.application.default--prod.us-east-3"), "dns-zone-1", ZoneId.from("prod.us-east-3"), 1).pack()), // The 'east' global endpoint, pointing to the weighted record for zone 2's region new Record(Record.Type.ALIAS, RecordName.from("east.application.tenant.global.vespa.oath.cloud"), - new LatencyAliasTarget(HostName.from("application.tenant.us-east-3-w.vespa.oath.cloud"), + new LatencyAliasTarget(HostName.of("application.tenant.us-east-3-w.vespa.oath.cloud"), "dns-zone-1", ZoneId.from("prod.us-east-3")).pack()), // The zone-scoped endpoint pointing to zone 2 with exclusive routing new Record(Record.Type.CNAME, RecordName.from("application.tenant.us-east-3.vespa.oath.cloud"), - RecordData.from("lb-0--tenant:application:default--prod.us-east-3."))); + RecordData.from("lb-0--tenant.application.default--prod.us-east-3."))); assertEquals(expectedRecords, List.copyOf(tester.controllerTester().nameService().records())); } @@ -1040,7 +1049,7 @@ public class ControllerTest { @Test public void testReadableApplications() { - var db = new MockCuratorDb(); + var db = new MockCuratorDb(tester.controller().system()); var tester = new DeploymentTester(new ControllerTester(db)); // Create and deploy two applications @@ -1101,6 +1110,26 @@ public class ControllerTest { } @Test + public void testTestPackageWarnings() { + String deploymentXml = "<deployment version='1.0'>\n" + + " <prod>\n" + + " <region>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>\n"; + ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(deploymentXml); + byte[] testPackage = ApplicationPackage.filesZip(Map.of("tests/staging-test/foo.json", new byte[0])); + var app = tester.newDeploymentContext(); + tester.jobs().submit(app.application().id(), Submission.basic(applicationPackage, testPackage), 1); + assertEquals(List.of(new Notification(tester.clock().instant(), + Type.testPackage, + Level.warning, + NotificationSource.from(app.application().id()), + List.of("test package has staging tests, so it should also include staging setup", + "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"))), + tester.controller().notificationsDb().listNotifications(NotificationSource.from(app.application().id()), true)); + } + + @Test public void testCompileVersion() { DeploymentContext context = tester.newDeploymentContext(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder().region("us-west-1").build(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 75c33952754..a2eefd47b56 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -54,7 +54,7 @@ import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import com.yahoo.yolean.concurrent.Sleeper; @@ -74,6 +74,7 @@ import java.util.logging.Handler; import java.util.logging.Logger; import java.util.stream.Collectors; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -109,11 +110,11 @@ public final class ControllerTester { } public ControllerTester(ServiceRegistryMock serviceRegistryMock) { - this(new AthenzDbMock(), new MockCuratorDb(), defaultRotationsConfig(), serviceRegistryMock); + this(new AthenzDbMock(), new MockCuratorDb(serviceRegistryMock.zoneRegistry().system()), defaultRotationsConfig(), serviceRegistryMock); } public ControllerTester(RotationsConfig rotationsConfig, SystemName system) { - this(new AthenzDbMock(), new MockCuratorDb(), rotationsConfig, new ServiceRegistryMock(system)); + this(new AthenzDbMock(), new MockCuratorDb(system), rotationsConfig, new ServiceRegistryMock(system)); } public ControllerTester(MockCuratorDb curatorDb) { @@ -121,11 +122,11 @@ public final class ControllerTester { } public ControllerTester() { - this(defaultRotationsConfig(), new MockCuratorDb()); + this(defaultRotationsConfig(), new MockCuratorDb(new ServiceRegistryMock().zoneRegistry().system())); } public ControllerTester(SystemName system) { - this(new AthenzDbMock(), new MockCuratorDb(), defaultRotationsConfig(), new ServiceRegistryMock(system)); + this(new AthenzDbMock(), new MockCuratorDb(system), defaultRotationsConfig(), new ServiceRegistryMock(system)); } private ControllerTester(AthenzDbMock athenzDb, boolean inContainer, CuratorDb curator, @@ -215,9 +216,9 @@ public final class ControllerTester { } /** Set the zones and system for this and bootstrap infrastructure nodes */ - public ControllerTester setZones(List<ZoneId> zones, SystemName system) { - zoneRegistry().setZones(zones.stream().map(ZoneApiMock::from).collect(Collectors.toList())) - .setSystemName(system); + public ControllerTester setZones(List<ZoneId> zones) { + ZoneApiMock.Builder builder = ZoneApiMock.newBuilder().withSystem(zoneRegistry().system()); + zoneRegistry().setZones(zones.stream().map(zone -> builder.with(zone).build()).collect(Collectors.toList())); configServer().bootstrap(zones, SystemApplication.notController()); return this; } @@ -239,7 +240,7 @@ public final class ControllerTester { /** Upgrade controller to given version */ public void upgradeController(Version version, String commitSha, Instant commitDate) { for (var hostname : controller().curator().cluster()) { - upgradeController(HostName.from(hostname), version, commitSha, commitDate); + upgradeController(HostName.of(hostname), version, commitSha, commitDate); } } @@ -394,7 +395,6 @@ public final class ControllerTester { serviceRegistry.zoneRegistry().system().isPublic() ? new CloudAccessControl(new MockUserManagement(), flagSource, serviceRegistry) : new AthenzFacade(new AthenzClientFactoryMock(athensDb)), - () -> "test-controller", flagSource, new MockMavenRepository(), serviceRegistry, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java index 974df695d07..96b0e3137cb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java @@ -8,16 +8,17 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.ApplicationData; +import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import org.junit.Test; - import java.nio.file.Files; import java.nio.file.Paths; import java.time.Instant; @@ -27,7 +28,6 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.TreeSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -61,11 +61,10 @@ public class DeploymentQuotaCalculatorTest { @Test public void quota_is_divided_among_prod_and_manual_instances() { - var existing_dev_deployment = new Application(TenantAndApplicationId.from(ApplicationId.defaultId()), Instant.EPOCH, DeploymentSpec.empty, ValidationOverrides.empty, - Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(1, 1), Set.of(), OptionalLong.empty(), - Optional.empty(), new TreeSet<>(), List.of(new Instance(ApplicationId.defaultId()).withNewDeployment( - ZoneId.from(Environment.dev, RegionName.defaultName()), ApplicationVersion.unknown, Version.emptyVersion, Instant.EPOCH, Map.of(), - QuotaUsage.create(0.53d)))); + var existing_dev_deployment = new Application(TenantAndApplicationId.from(ApplicationId.defaultId()), Instant.EPOCH, DeploymentSpec.empty, ValidationOverrides.empty, Optional.empty(), + Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(1, 1), Set.of(), OptionalLong.empty(), RevisionHistory.empty(), + List.of(new Instance(ApplicationId.defaultId()).withNewDeployment(ZoneId.from(Environment.dev, RegionName.defaultName()), + RevisionId.forProduction(1), Version.emptyVersion, Instant.EPOCH, Map.of(), QuotaUsage.create(0.53d)))); Quota calculated = DeploymentQuotaCalculator.calculate(Quota.unlimited().withBudget(2), List.of(existing_dev_deployment), ApplicationId.defaultId(), ZoneId.defaultId(), DeploymentSpec.fromXml( diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiffTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiffTest.java index b2aba721a6f..977304db3fb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiffTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiffTest.java @@ -12,7 +12,6 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff.diff; - import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff.diffAgainstEmpty; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index 4e155e937b9..99e22302c73 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -24,24 +24,24 @@ import static org.junit.Assert.fail; */ public class ApplicationPackageTest { - private static final String deploymentXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<deployment version=\"1.0\">\n" + - " <test />\n" + - " <prod>\n" + - " <parallel>\n" + - " <region active=\"true\">us-central-1</region>\n" + - " </parallel>\n" + - " </prod>\n" + - "</deployment>\n"; - - private static final String servicesXml = "<services version='1.0' xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\">\n" + - " <preprocess:include file='jdisc.xml' />\n" + - " <content version='1.0' if='foo' />\n" + - " <content version='1.0' id='foo' deploy:environment='staging prod' deploy:region='us-east-3 us-central-1'>\n" + - " <preprocess:include file='content/content.xml' />\n" + - " </content>\n" + - " <preprocess:include file='not_found.xml' required='false' />\n" + - "</services>\n"; + static final String deploymentXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<deployment version=\"1.0\">\n" + + " <test />\n" + + " <prod>\n" + + " <parallel>\n" + + " <region active=\"true\">us-central-1</region>\n" + + " </parallel>\n" + + " </prod>\n" + + "</deployment>\n"; + + static final String servicesXml = "<services version='1.0' xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\">\n" + + " <preprocess:include file='jdisc.xml' />\n" + + " <content version='1.0' if='foo' />\n" + + " <content version='1.0' id='foo' deploy:environment='staging prod' deploy:region='us-east-3 us-central-1'>\n" + + " <preprocess:include file='content/content.xml' />\n" + + " </content>\n" + + " <preprocess:include file='not_found.xml' required='false' />\n" + + "</services>\n"; private static final String jdiscXml = "<container id='stateless' version='1.0' />\n"; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java new file mode 100644 index 00000000000..8588bb9ea16 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java @@ -0,0 +1,157 @@ +package com.yahoo.vespa.hosted.controller.application.pkg; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.TestSummary; +import com.yahoo.vespa.hosted.controller.config.ControllerConfig; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.jar.JarOutputStream; +import java.util.zip.ZipEntry; + +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.production; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging_setup; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.system; +import static com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.validateTests; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class TestPackageTest { + + static byte[] testsJar(String... suites) throws IOException { + String manifest = "Manifest-Version: 1.0\n" + + "Created-By: vespa container maven plugin\n" + + "Build-Jdk-Spec: 17\n" + + "Bundle-ManifestVersion: 2\n" + + "Bundle-SymbolicName: canary-application-test\n" + + "Bundle-Version: 1.0.1\n" + + "Bundle-Name: Test & verification application for Vespa\n" + + "X-JDisc-Test-Bundle-Version: 1.0\n" + + "Bundle-Vendor: Yahoo!\n" + + "Bundle-ClassPath: .,dependencies/fest-assert-1.4.jar,dependencies/fest-u\n" + + " til-1.1.6.jar\n" + + "Import-Package: ai.vespa.feed.client;version=\"[1.0.0,2)\",ai.vespa.hosted\n" + + " .cd;version=\"[1.0.0,2)\",com.yahoo.config;version=\"[1.0.0,2)\",com.yahoo.\n" + + " container.jdisc;version=\"[1.0.0,2)\",com.yahoo.jdisc.http;version=\"[1.0.\n" + + " 0,2)\",com.yahoo.slime;version=\"[1.0.0,2)\",java.awt.image;version=\"[0.0.\n" + + " 0,1)\",java.awt;version=\"[0.0.0,1)\",java.beans;version=\"[0.0.0,1)\",java.\n" + + " io;version=\"[0.0.0,1)\",java.lang.annotation;version=\"[0.0.0,1)\",java.la\n" + + " ng.reflect;version=\"[0.0.0,1)\",java.lang;version=\"[0.0.0,1)\",java.math;\n" + + " version=\"[0.0.0,1)\",java.net.http;version=\"[0.0.0,1)\",java.net;version=\n" + + " \"[0.0.0,1)\",java.nio.file;version=\"[0.0.0,1)\",java.security;version=\"[0\n" + + " .0.0,1)\",java.text;version=\"[0.0.0,1)\",java.time.temporal;version=\"[0.0\n" + + " .0,1)\",java.time;version=\"[0.0.0,1)\",java.util.concurrent;version=\"[0.0\n" + + " .0,1)\",java.util.function;version=\"[0.0.0,1)\",java.util.stream;version=\n" + + " \"[0.0.0,1)\",java.util;version=\"[0.0.0,1)\",javax.imageio;version=\"[0.0.0\n" + + " ,1)\",org.junit.jupiter.api;version=\"[5.8.1,6)\"\n" + + "X-JDisc-Test-Bundle-Categories: " + String.join(",", suites) + "\n" + + "\n"; + + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + try (JarOutputStream out = new JarOutputStream(buffer)) { + write("META-INF/MANIFEST.MF", manifest, out); + write("dependencies/foo.jar", "bar", out); + write("META-INF/maven/ai.vespa.test/app/pom.xml", "<project />", out); + write("ai/vespa/test/Test.class", "baz", out); + } + return buffer.toByteArray(); + } + + static void write(String name, String content, JarOutputStream out) throws IOException { + out.putNextEntry(new ZipEntry(name)); + out.write(content.getBytes(UTF_8)); + out.closeEntry(); + } + + @Test + public void testBundleValidation() throws IOException { + byte[] testZip = ApplicationPackage.filesZip(Map.of("components/foo-tests.jar", testsJar("SystemTest", "StagingSetup", "ProductionTest"), + "artifacts/key", new byte[0])); + TestSummary summary = validateTests(List.of(system), testZip); + + assertEquals(List.of(system, staging_setup, production), summary.suites()); + assertEquals(List.of("test package contains 'artifacts/key'; this conflicts with credentials used to run tests in Vespa Cloud", + "test package has staging setup, so it should also include staging tests", + "test package has production tests, but no production tests are declared in deployment.xml", + "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), + summary.problems()); + } + + @Test + public void testFatTestsValidation() { + byte[] testZip = ApplicationPackage.filesZip(Map.of("artifacts/foo-tests.jar", new byte[0])); + TestSummary summary = validateTests(List.of(staging, production), testZip); + + assertEquals(List.of(staging, production), summary.suites()); + assertEquals(List.of("test package has staging tests, so it should also include staging setup", + "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), + summary.problems()); + } + + @Test + public void testBasicTestsValidation() { + byte[] testZip = ApplicationPackage.filesZip(Map.of("tests/staging-test/foo.json", new byte[0], + "tests/staging-setup/foo.json", new byte[0])); + TestSummary summary = validateTests(List.of(system, production), testZip); + assertEquals(List.of(staging_setup, staging), summary.suites()); + assertEquals(List.of("test package has no system tests, but <test /> is declared in deployment.xml", + "test package has no production tests, but production tests are declared in deployment.xml", + "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), + summary.problems()); + } + + @Test + public void generates_correct_tester_flavor() { + DeploymentSpec spec = DeploymentSpec.fromXml("<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" + + " <instance id='first'>\n" + + " <test tester-flavor=\"d-6-16-100\" />\n" + + " <prod>\n" + + " <region active=\"true\">us-west-1</region>\n" + + " <test>us-west-1</test>\n" + + " </prod>\n" + + " </instance>\n" + + " <instance id='second'>\n" + + " <test />\n" + + " <staging />\n" + + " <prod tester-flavor=\"d-6-16-100\">\n" + + " <parallel>\n" + + " <region active=\"true\">us-east-3</region>\n" + + " <region active=\"true\">us-central-1</region>\n" + + " </parallel>\n" + + " <region active=\"true\">us-west-1</region>\n" + + " <test>us-west-1</test>\n" + + " </prod>\n" + + " </instance>\n" + + "</deployment>\n"); + + NodeResources firstResources = TestPackage.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("first")); + assertEquals(TestPackage.DEFAULT_TESTER_RESOURCES, firstResources); + + NodeResources secondResources = TestPackage.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("second")); + assertEquals(6, secondResources.vcpu(), 1e-9); + assertEquals(16, secondResources.memoryGb(), 1e-9); + assertEquals(100, secondResources.diskGb(), 1e-9); + } + + @Test + public void generates_correct_services_xml() throws IOException { + assertEquals(Files.readString(Paths.get("src/test/resources/test_runner_services.xml-cd")), + new String(TestPackage.servicesXml(true, + false, + new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local), + new ControllerConfig.Steprunner.Testerapp.Builder().build()), + UTF_8)); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index 61bcb119d00..df31883b1d5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -52,7 +52,7 @@ public class EndpointCertificatesTest { private final SecretStoreMock secretStore = new SecretStoreMock(); private final CuratorDb mockCuratorDb = tester.curator(); private final ManualClock clock = tester.clock(); - private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(); + private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(new ManualClock()); private final EndpointCertificateValidatorImpl endpointCertificateValidator = new EndpointCertificateValidatorImpl(secretStore, clock); private final EndpointCertificates endpointCertificates = new EndpointCertificates(tester.controller(), endpointCertificateMock, endpointCertificateValidator); private final KeyPair testKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 192); @@ -146,10 +146,10 @@ public class EndpointCertificatesTest { "*.default.default.g.vespa-app.cloud", "default.default.aws-us-east-1a.z.vespa-app.cloud", "*.default.default.aws-us-east-1a.z.vespa-app.cloud", - "default.default.aws-us-east-1c.test.z.vespa-app.cloud", - "*.default.default.aws-us-east-1c.test.z.vespa-app.cloud", - "default.default.aws-us-east-1c.staging.z.vespa-app.cloud", - "*.default.default.aws-us-east-1c.staging.z.vespa-app.cloud" + "default.default.us-east-1.test.z.vespa-app.cloud", + "*.default.default.us-east-1.test.z.vespa-app.cloud", + "default.default.us-east-3.staging.z.vespa-app.cloud", + "*.default.default.us-east-3.staging.z.vespa-app.cloud" ); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificates.getMetadata(testInstance, testZone, DeploymentSpec.empty); assertTrue(endpointCertificateMetadata.isPresent()); @@ -261,10 +261,10 @@ public class EndpointCertificatesTest { "*.a1.t1.aws-us-east-1c.r.vespa-app.cloud", "a1.t1.aws-us-east-1c.z.vespa-app.cloud", "*.a1.t1.aws-us-east-1c.z.vespa-app.cloud", - "a1.t1.aws-us-east-1c.test.z.vespa-app.cloud", - "*.a1.t1.aws-us-east-1c.test.z.vespa-app.cloud", - "a1.t1.aws-us-east-1c.staging.z.vespa-app.cloud", - "*.a1.t1.aws-us-east-1c.staging.z.vespa-app.cloud" + "a1.t1.us-east-1.test.z.vespa-app.cloud", + "*.a1.t1.us-east-1.test.z.vespa-app.cloud", + "a1.t1.us-east-3.staging.z.vespa-app.cloud", + "*.a1.t1.us-east-3.staging.z.vespa-app.cloud" ); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificates.getMetadata(instance, zone1, applicationPackage.deploymentSpec()); assertTrue(endpointCertificateMetadata.isPresent()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index a464e3d7e9b..27cf1554b4d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -317,7 +317,9 @@ public class ApplicationPackageBuilder { } private static byte[] buildMeta(Version compileVersion) { - return ("{\"compileVersion\":\"" + compileVersion.toFullString() + "\",\"buildTime\":1000}").getBytes(UTF_8); + return ("{\"compileVersion\":\"" + compileVersion.toFullString() + + "\",\"buildTime\":1000,\"parentVersion\":\"" + + compileVersion.toFullString() + "\"}").getBytes(UTF_8); } public ApplicationPackage build() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 86c21839c96..fd294f9cf9f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -21,23 +21,24 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; -import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import javax.security.auth.x500.X500Principal; import java.math.BigInteger; @@ -76,6 +77,28 @@ import static org.junit.Assert.assertTrue; */ public class DeploymentContext { + public static final JobType systemTest = JobType.deploymentTo(ZoneId.from("test", "us-east-1")); + public static final JobType stagingTest = JobType.deploymentTo(ZoneId.from("staging", "us-east-3")); + public static final JobType productionUsEast3 = JobType.prod("us-east-3"); + public static final JobType testUsEast3 = JobType.test("us-east-3"); + public static final JobType productionUsWest1 = JobType.prod("us-west-1"); + public static final JobType testUsWest1 = JobType.test("us-west-1"); + public static final JobType productionUsCentral1 = JobType.prod("us-central-1"); + public static final JobType testUsCentral1 = JobType.test("us-central-1"); + public static final JobType productionApNortheast1 = JobType.prod("ap-northeast-1"); + public static final JobType testApNortheast1 = JobType.test("ap-northeast-1"); + public static final JobType productionApNortheast2 = JobType.prod("ap-northeast-2"); + public static final JobType testApNortheast2 = JobType.test("ap-northeast-2"); + public static final JobType productionApSoutheast1 = JobType.prod("ap-southeast-1"); + public static final JobType testApSoutheast1 = JobType.test("ap-southeast-1"); + public static final JobType productionEuWest1 = JobType.prod("eu-west-1"); + public static final JobType testEuWest1 = JobType.test("eu-west-1"); + public static final JobType productionAwsUsEast1a = JobType.prod("aws-us-east-1a"); + public static final JobType testAwsUsEast1a = JobType.test("aws-us-east-1a"); + public static final JobType devUsEast1 = JobType.dev("us-east-1"); + public static final JobType devAwsUsEast2a = JobType.dev("aws-us-east-2a"); + public static final JobType perfUsEast3 = JobType.perf("us-east-3"); + private final AtomicLong salt = new AtomicLong(); // Application packages are expensive to construct, and a given test typically only needs to the test in the context @@ -107,7 +130,7 @@ public class DeploymentContext { private final JobRunner runner; private final DeploymentTester tester; - private ApplicationVersion lastSubmission = null; + private RevisionId lastSubmission = null; private boolean deferDnsUpdates = false; public DeploymentContext(ApplicationId instanceId, DeploymentTester tester) { @@ -169,10 +192,10 @@ public class DeploymentContext { /** Completely deploy the current change */ public DeploymentContext deploy() { Application application = application(); - assertTrue("Application package submitted", application.latestVersion().isPresent()); + assertTrue("Application package submitted", application.revisions().last().isPresent()); assertFalse("Submission is not already deployed", application.instances().values().stream() .anyMatch(instance -> instance.deployments().values().stream() - .anyMatch(deployment -> deployment.applicationVersion().equals(lastSubmission)))); + .anyMatch(deployment -> deployment.revision().equals(lastSubmission)))); completeRollout(application.deploymentSpec().instances().size() > 1); for (var instance : application().instances().values()) { assertFalse(instance.change().hasTargets()); @@ -187,7 +210,7 @@ public class DeploymentContext { .anyMatch(instance -> instance.deployments().values().stream() .anyMatch(deployment -> deployment.version().equals(version)))); assertEquals(version, instance().change().platform().get()); - assertFalse(instance().change().application().isPresent()); + assertFalse(instance().change().revision().isPresent()); completeRollout(); @@ -196,10 +219,10 @@ public class DeploymentContext { .allMatch(deployment -> deployment.version().equals(version)))); for (var spec : application().deploymentSpec().instances()) - for (JobType type : new DeploymentSteps(spec, tester.controller()::system).productionJobs()) + for (JobType type : new DeploymentSteps(spec, tester.controller().zoneRegistry()).productionJobs()) assertTrue(tester.configServer().nodeRepository() - .list(type.zone(tester.controller().system()), - NodeFilter.all().applications(applicationId.defaultInstance())).stream() // TODO jonmv: support more + .list(type.zone(), + NodeFilter.all().applications(applicationId.defaultInstance())).stream() .allMatch(node -> node.currentVersion().equals(version))); assertFalse(instance().change().hasTargets()); @@ -247,7 +270,7 @@ public class DeploymentContext { var clusterId = "default-inactive"; var id = new RoutingPolicyId(instanceId, ClusterSpec.Id.from(clusterId), zone); var policies = new LinkedHashMap<>(tester.controller().routing().policies().read(instanceId).asMap()); - policies.put(id, new RoutingPolicy(id, HostName.from("lb-host"), + policies.put(id, new RoutingPolicy(id, HostName.of("lb-host"), Optional.empty(), Set.of(EndpointId.of("default")), Set.of(), @@ -258,7 +281,12 @@ public class DeploymentContext { /** Submit given application package for deployment */ public DeploymentContext resubmit(ApplicationPackage applicationPackage) { - return submit(applicationPackage, Optional.of(defaultSourceRevision), salt.get()); + return submit(applicationPackage, Optional.of(defaultSourceRevision), salt.get(), 0); + } + + /** Submit given application package for deployment */ + public DeploymentContext submit(ApplicationPackage applicationPackage, int risk) { + return submit(applicationPackage, Optional.of(defaultSourceRevision), salt.incrementAndGet(), risk); } /** Submit given application package for deployment */ @@ -267,24 +295,23 @@ public class DeploymentContext { } /** Submit given application package for deployment */ - public DeploymentContext submit(ApplicationPackage applicationPackage, long salt) { - return submit(applicationPackage, Optional.of(defaultSourceRevision), salt); + public DeploymentContext submit(ApplicationPackage applicationPackage, long salt, int risk) { + return submit(applicationPackage, Optional.of(defaultSourceRevision), salt, risk); } /** Submit given application package for deployment */ public DeploymentContext submit(ApplicationPackage applicationPackage, Optional<SourceRevision> sourceRevision) { - return submit(applicationPackage, sourceRevision, salt.incrementAndGet()); + return submit(applicationPackage, sourceRevision, salt.incrementAndGet(), 0); } /** Submit given application package for deployment */ - public DeploymentContext submit(ApplicationPackage applicationPackage, Optional<SourceRevision> sourceRevision, long salt) { + public DeploymentContext submit(ApplicationPackage applicationPackage, Optional<SourceRevision> sourceRevision, long salt, int risk) { var projectId = tester.controller().applications() .requireApplication(applicationId) .projectId() .orElse(1000); // These are really set through submission, so just pick one if it hasn't been set. var testerpackage = new byte[]{ (byte) (salt >> 56), (byte) (salt >> 48), (byte) (salt >> 40), (byte) (salt >> 32), (byte) (salt >> 24), (byte) (salt >> 16), (byte) (salt >> 8), (byte) salt }; - lastSubmission = jobs.submit(applicationId, sourceRevision, Optional.of("a@b"), Optional.empty(), - projectId, applicationPackage, testerpackage); + lastSubmission = jobs.submit(applicationId, new Submission(applicationPackage, testerpackage, Optional.empty(), sourceRevision, Optional.of("a@b"), Optional.empty(), risk), projectId).id(); return this; } @@ -345,7 +372,7 @@ public class DeploymentContext { } /** Returns the last submitted application version */ - public Optional<ApplicationVersion> lastSubmission() { + public Optional<RevisionId> lastSubmission() { return Optional.ofNullable(lastSubmission); } @@ -389,7 +416,7 @@ public class DeploymentContext { /** Runs a deployment of the given package to the given manually deployable zone. */ public DeploymentContext runJob(ZoneId zone, ApplicationPackage applicationPackage) { - return runJob(JobType.from(tester.controller().system(), zone).get(), applicationPackage, null); + return runJob(JobType.deploymentTo(zone), applicationPackage, null); } /** Pulls the ready job trigger, and then runs the whole of the given job in the instance of this, successfully. */ @@ -397,8 +424,33 @@ public class DeploymentContext { return runJob(type, instanceId); } + /** Runs the job, failing tests with noTests status, or with regular testFailure. */ + public DeploymentContext failTests(JobType type, boolean noTests) { + if ( ! type.isTest()) throw new IllegalArgumentException(type + " does not run tests"); + var job = new JobId(instanceId, type); + triggerJobs(); + doDeploy(job); + if (job.type().isDeployment()) { + doUpgrade(job); + doConverge(job); + if (job.type().environment().isManuallyDeployed()) + return this; + } + + RunId id = currentRun(job).id(); + + assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.endTests)); + tester.cloud().set(noTests ? Status.NO_TESTS : Status.FAILURE); + runner.advance(currentRun(job)); + assertTrue(jobs.run(id).get().hasEnded()); + assertEquals(noTests, jobs.run(id).get().hasSucceeded()); + assertTrue(configServer().nodeRepository().list(job.type().zone(), NodeFilter.all().applications(TesterId.of(instanceId).id())).isEmpty()); + + return this; + } + /** Pulls the ready job trigger, and then runs the whole of job for the given instance, successfully. */ - public DeploymentContext runJob(JobType type, ApplicationId instance) { + private DeploymentContext runJob(JobType type, ApplicationId instance) { var job = new JobId(instance, type); triggerJobs(); doDeploy(job); @@ -465,13 +517,13 @@ public class DeploymentContext { tester.readyJobsTrigger().maintain(); if (type.isProduction()) { - runJob(JobType.systemTest); - runJob(JobType.stagingTest); + runJob(systemTest); + runJob(stagingTest); tester.readyJobsTrigger().maintain(); } Run run = jobs.active().stream() - .filter(r -> r.id().type() == type) + .filter(r -> r.id().type().equals(type)) .findAny() .orElseThrow(() -> new AssertionError(type + " is not among the active: " + jobs.active())); return run.id(); @@ -479,8 +531,8 @@ public class DeploymentContext { /** Start tests in system test stage */ public RunId startSystemTestTests() { - var id = newRun(JobType.systemTest); - var testZone = JobType.systemTest.zone(tester.controller().system()); + var id = newRun(systemTest); + var testZone = systemTest.zone(); runner.run(); if ( ! deferDnsUpdates) flushDnsUpdates(); @@ -494,18 +546,18 @@ public class DeploymentContext { public void assertRunning(JobType type) { assertTrue(jobId(type) + " should be among the active: " + jobs.active(), - jobs.active().stream().anyMatch(run -> run.id().application().equals(instanceId) && run.id().type() == type)); + jobs.active().stream().anyMatch(run -> run.id().application().equals(instanceId) && run.id().type().equals(type))); } public void assertNotRunning(JobType type) { assertFalse(jobId(type) + " should not be among the active: " + jobs.active(), - jobs.active().stream().anyMatch(run -> run.id().application().equals(instanceId) && run.id().type() == type)); + jobs.active().stream().anyMatch(run -> run.id().application().equals(instanceId) && run.id().type().equals(type))); } /** Deploys tester and real app, and completes tester and initial staging installation first if needed. */ private void doDeploy(JobId job) { RunId id = currentRun(job).id(); - ZoneId zone = zone(job); + ZoneId zone = job.type().zone(); DeploymentId deployment = new DeploymentId(job.application(), zone); // First step is always a deployment. @@ -517,7 +569,7 @@ public class DeploymentContext { if (job.type().isTest()) doInstallTester(job); - if (job.type() == JobType.stagingTest) { // Do the initial deployment and installation of the real application. + if (job.type().equals(stagingTest)) { // Do the initial deployment and installation of the real application. assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), tester.configServer().application(job.application(), zone).get().version().get()); configServer().convergeServices(id.application(), zone); @@ -539,7 +591,7 @@ public class DeploymentContext { /** Upgrades nodes to target version. */ private void doUpgrade(JobId job) { RunId id = currentRun(job).id(); - ZoneId zone = zone(job); + ZoneId zone = job.type().zone(); DeploymentId deployment = new DeploymentId(job.application(), zone); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installReal)); @@ -550,7 +602,7 @@ public class DeploymentContext { /** Returns the current run for the given job type, and verifies it is still running normally. */ private Run currentRun(JobId job) { Run run = jobs.last(job) - .filter(r -> r.id().type() == job.type()) + .filter(r -> r.id().type().equals(job.type())) .orElseThrow(() -> new AssertionError(job.type() + " is not among the active: " + jobs.active())); assertFalse(run.id() + " should not have failed yet: " + run, run.hasFailed()); assertFalse(run.id() + " should not have ended yet: " + run, run.hasEnded()); @@ -560,7 +612,7 @@ public class DeploymentContext { /** Lets nodes converge on new application version. */ private void doConverge(JobId job) { RunId id = currentRun(job).id(); - ZoneId zone = zone(job); + ZoneId zone = job.type().zone(); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installReal)); configServer().convergeServices(id.application(), zone); @@ -576,7 +628,7 @@ public class DeploymentContext { /** Installs tester and starts tests. */ private void doInstallTester(JobId job) { RunId id = currentRun(job).id(); - ZoneId zone = zone(job); + ZoneId zone = job.type().zone(); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); configServer().nodeRepository().doUpgrade(new DeploymentId(TesterId.of(job.application()).id(), zone), Optional.empty(), tester.configServer().application(id.tester().id(), zone).get().version().get()); @@ -591,7 +643,7 @@ public class DeploymentContext { /** Completes tests with success. */ private void doTests(JobId job) { RunId id = currentRun(job).id(); - ZoneId zone = zone(job); + ZoneId zone = job.type().zone(); // All installation is complete and endpoints are ready, so tests may begin. if (job.type().isDeployment()) @@ -613,10 +665,6 @@ public class DeploymentContext { return new JobId(instanceId, type); } - private ZoneId zone(JobId job) { - return job.type().zone(tester.controller().system()); - } - private ConfigServerMock configServer() { return tester.configServer(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index f9de291ff0d..4d4d94f9e1f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -3,15 +3,15 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Assert; import org.junit.Test; @@ -19,33 +19,33 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.util.Collection; -import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.config.provision.SystemName.main; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApNortheast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApNortheast2; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApSoutheast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionAwsUsEast1a; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdAwsUsEast1a; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdUsEast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionEuWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testApNortheast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testApNortheast2; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testAwsUsEast1a; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testEuWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsCentral1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsEast3; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsWest1; +import static ai.vespa.validation.Validation.require; +import static com.yahoo.config.provision.SystemName.cd; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionApNortheast1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionApNortheast2; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionApSoutheast1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionAwsUsEast1a; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionEuWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsCentral1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testApNortheast1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testApNortheast2; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testAwsUsEast1a; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testEuWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsCentral1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsWest1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; import static java.util.Collections.emptyList; @@ -118,28 +118,28 @@ public class DeploymentTriggerTest { .build(); app.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3); - Optional<ApplicationVersion> v0 = app.lastSubmission(); + Optional<RevisionId> v0 = app.lastSubmission(); app.submit(applicationPackage); - Optional<ApplicationVersion> v1 = app.lastSubmission(); - assertEquals(v0, app.instance().change().application()); + Optional<RevisionId> v1 = app.lastSubmission(); + assertEquals(v0, app.instance().change().revision()); // Eager tests still run before new revision rolls out. app.runJob(systemTest).runJob(stagingTest); // v0 rolls out completely. app.runJob(testUsEast3); - assertEquals(Optional.empty(), app.instance().change().application()); + assertEquals(Optional.empty(), app.instance().change().revision()); // v1 starts rolling when v0 is done. tester.outstandingChangeDeployer().run(); - assertEquals(v1, app.instance().change().application()); + assertEquals(v1, app.instance().change().revision()); // v1 fails, so v2 starts immediately. app.runJob(productionUsEast3).failDeployment(testUsEast3); app.submit(applicationPackage); - Optional<ApplicationVersion> v2 = app.lastSubmission(); - assertEquals(v2, app.instance().change().application()); + Optional<RevisionId> v2 = app.lastSubmission(); + assertEquals(v2, app.instance().change().revision()); } @Test @@ -207,9 +207,9 @@ public class DeploymentTriggerTest { app.runJob(systemTest).runJob(stagingTest); tester.triggerJobs(); tester.runner().run(); - assertEquals(EnumSet.of(productionUsCentral1), tester.jobs().active().stream() - .map(run -> run.id().type()) - .collect(Collectors.toCollection(() -> EnumSet.noneOf(JobType.class)))); + assertEquals(Set.of(productionUsCentral1), tester.jobs().active().stream() + .map(run -> run.id().type()) + .collect(Collectors.toCollection(HashSet::new))); } @Test @@ -218,9 +218,9 @@ public class DeploymentTriggerTest { .region("us-east-3") .build(); - DeploymentContext app = tester.newDeploymentContext().submit(firstPackage, 5417); + DeploymentContext app = tester.newDeploymentContext().submit(firstPackage, 5417, 0); var version = app.lastSubmission(); - assertEquals(version, app.instance().change().application()); + assertEquals(version, app.instance().change().revision()); app.runJob(systemTest) .runJob(stagingTest) .runJob(productionUsEast3); @@ -235,10 +235,10 @@ public class DeploymentTriggerTest { .test("us-east-3") .build(); - app.submit(secondPackage, 5417); + app.submit(secondPackage, 5417, 0); app.triggerJobs(); assertEquals(List.of(), tester.jobs().active()); - assertEquals(version, app.instance().change().application()); + assertEquals(version, app.instance().change().revision()); tester.clock().advance(Duration.ofHours(1)); app.runJob(testUsEast3); @@ -246,7 +246,7 @@ public class DeploymentTriggerTest { assertEquals(Change.empty(), app.instance().change()); // The original application package is submitted again. No new jobs are added, so no change needs to roll out now. - app.submit(firstPackage, 5417); + app.submit(firstPackage, 5417, 0); app.triggerJobs(); assertEquals(List.of(), tester.jobs().active()); assertEquals(Change.empty(), app.instance().change()); @@ -260,45 +260,45 @@ public class DeploymentTriggerTest { .build(); DeploymentContext app = tester.newDeploymentContext() .submit(appPackage); - Optional<ApplicationVersion> revision1 = app.lastSubmission(); + Optional<RevisionId> revision1 = app.lastSubmission(); app.submit(appPackage); - Optional<ApplicationVersion> revision2 = app.lastSubmission(); + Optional<RevisionId> revision2 = app.lastSubmission(); app.submit(appPackage); - Optional<ApplicationVersion> revision3 = app.lastSubmission(); + Optional<RevisionId> revision3 = app.lastSubmission(); app.submit(appPackage); - Optional<ApplicationVersion> revision4 = app.lastSubmission(); + Optional<RevisionId> revision4 = app.lastSubmission(); app.submit(appPackage); - Optional<ApplicationVersion> revision5 = app.lastSubmission(); + Optional<RevisionId> revision5 = app.lastSubmission(); // 5 revisions submitted; the first is rolling out, and the others are queued. tester.outstandingChangeDeployer().run(); - assertEquals(revision1, app.instance().change().application()); - assertEquals(revision2, app.deploymentStatus().outstandingChange(InstanceName.defaultName()).application()); + assertEquals(revision1, app.instance().change().revision()); + assertEquals(revision2, app.deploymentStatus().outstandingChange(InstanceName.defaultName()).revision()); // The second revision is set as the target by user interaction. tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(revision2.get())); tester.outstandingChangeDeployer().run(); - assertEquals(revision2, app.instance().change().application()); - assertEquals(revision3, app.deploymentStatus().outstandingChange(InstanceName.defaultName()).application()); + assertEquals(revision2, app.instance().change().revision()); + assertEquals(revision3, app.deploymentStatus().outstandingChange(InstanceName.defaultName()).revision()); // The second revision deploys completely, and the third starts rolling out. app.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3); tester.outstandingChangeDeployer().run(); tester.outstandingChangeDeployer().run(); - assertEquals(revision3, app.instance().change().application()); - assertEquals(revision4, app.deploymentStatus().outstandingChange(InstanceName.defaultName()).application()); + assertEquals(revision3, app.instance().change().revision()); + assertEquals(revision4, app.deploymentStatus().outstandingChange(InstanceName.defaultName()).revision()); // The third revision fails, and the fourth is chosen to replace it. app.triggerJobs().timeOutConvergence(systemTest); tester.outstandingChangeDeployer().run(); tester.outstandingChangeDeployer().run(); - assertEquals(revision4, app.instance().change().application()); - assertEquals(revision5, app.deploymentStatus().outstandingChange(InstanceName.defaultName()).application()); + assertEquals(revision4, app.instance().change().revision()); + assertEquals(revision5, app.deploymentStatus().outstandingChange(InstanceName.defaultName()).revision()); // Tests for outstanding change are relevant when current revision completes. app.runJob(systemTest).runJob(systemTest) @@ -306,7 +306,7 @@ public class DeploymentTriggerTest { .runJob(productionUsEast3); tester.outstandingChangeDeployer().run(); tester.outstandingChangeDeployer().run(); - assertEquals(revision5, app.instance().change().application()); + assertEquals(revision5, app.instance().change().revision()); assertEquals(Change.empty(), app.deploymentStatus().outstandingChange(InstanceName.defaultName())); app.runJob(productionUsEast3); } @@ -501,8 +501,8 @@ public class DeploymentTriggerTest { tester.clock().advance(Duration.ofHours(1)); app.submit(applicationPackage); app.runJob(productionUsWest1); - assertEquals(1, app.instanceJobs().get(productionUsWest1).lastSuccess().get().versions().targetApplication().buildNumber().getAsLong()); - assertEquals(2, app.deploymentStatus().outstandingChange(app.instance().name()).application().get().buildNumber().getAsLong()); + assertEquals(1, app.instanceJobs().get(productionUsWest1).lastSuccess().get().versions().targetRevision().number()); + assertEquals(2, app.deploymentStatus().outstandingChange(app.instance().name()).revision().get().number()); tester.triggerJobs(); // Platform upgrade keeps rolling, since it began before block window, and tests for the new revision have also started. @@ -564,7 +564,7 @@ public class DeploymentTriggerTest { tester.deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "mrTrigger", true, true, false); app.assertRunning(productionUsEast3); assertFalse(app.instance().jobPause(productionUsEast3).isPresent()); - assertEquals(app.deployment(productionUsEast3.zone(tester.controller().system())).version(), + assertEquals(app.deployment(productionUsEast3.zone()).version(), tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); } @@ -582,8 +582,8 @@ public class DeploymentTriggerTest { .runJob(stagingTest) .timeOutUpgrade(productionUsCentral1); - ApplicationVersion appVersion1 = app.lastSubmission().get(); - assertEquals(appVersion1, app.deployment(ZoneId.from("prod.us-central-1")).applicationVersion()); + RevisionId appVersion1 = app.lastSubmission().get(); + assertEquals(appVersion1, app.deployment(ZoneId.from("prod.us-central-1")).revision()); // Verify the application change is not removed when platform change is cancelled. tester.deploymentTrigger().cancelChange(app.instanceId(), PLATFORM); @@ -605,18 +605,25 @@ public class DeploymentTriggerTest { // Finally, the two production jobs complete, in order. app.runJob(productionUsCentral1).runJob(productionEuWest1); - assertEquals(appVersion1, app.deployment(ZoneId.from("prod.us-central-1")).applicationVersion()); + assertEquals(appVersion1, app.deployment(ZoneId.from("prod.us-central-1")).revision()); + } + + RevisionId latestDeployed(Instance instance) { + return instance.productionDeployments().values().stream() + .map(Deployment::revision) + .reduce((o, n) -> require(o.equals(n), n, "all versions should be equal, but got " + o + " and " + n)) + .orElseThrow(() -> new AssertionError("no versions deployed")); } @Test public void downgradingApplicationVersionWorks() { var app = tester.newDeploymentContext().submit().deploy(); - ApplicationVersion appVersion0 = app.lastSubmission().get(); - assertEquals(Optional.of(appVersion0), app.instance().latestDeployed()); + RevisionId appVersion0 = app.lastSubmission().get(); + assertEquals(appVersion0, latestDeployed(app.instance())); app.submit().deploy(); - ApplicationVersion appVersion1 = app.lastSubmission().get(); - assertEquals(Optional.of(appVersion1), app.instance().latestDeployed()); + RevisionId appVersion1 = app.lastSubmission().get(); + assertEquals(appVersion1, latestDeployed(app.instance())); // Downgrading application version. tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion0)); @@ -626,28 +633,27 @@ public class DeploymentTriggerTest { .runJob(productionUsEast3) .runJob(productionUsWest1); assertEquals(Change.empty(), app.instance().change()); - assertEquals(appVersion0, app.instance().deployments().get(productionUsEast3.zone(tester.controller().system())).applicationVersion()); - assertEquals(Optional.of(appVersion0), app.instance().latestDeployed()); + assertEquals(appVersion0, app.instance().deployments().get(productionUsEast3.zone()).revision()); + assertEquals(appVersion0, latestDeployed(app.instance())); } @Test public void settingANoOpChangeIsANoOp() { var app = tester.newDeploymentContext().submit(); - assertEquals(Optional.empty(), app.instance().latestDeployed()); app.deploy(); - ApplicationVersion appVersion0 = app.lastSubmission().get(); - assertEquals(Optional.of(appVersion0), app.instance().latestDeployed()); + RevisionId appVersion0 = app.lastSubmission().get(); + assertEquals(appVersion0, latestDeployed(app.instance())); app.submit().deploy(); - ApplicationVersion appVersion1 = app.lastSubmission().get(); - assertEquals(Optional.of(appVersion1), app.instance().latestDeployed()); + RevisionId appVersion1 = app.lastSubmission().get(); + assertEquals(appVersion1, latestDeployed(app.instance())); // Triggering a roll-out of an already deployed application is a no-op. assertEquals(Change.empty(), app.instance().change()); tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion1)); assertEquals(Change.empty(), app.instance().change()); - assertEquals(Optional.of(appVersion1), app.instance().latestDeployed()); + assertEquals(appVersion1, latestDeployed(app.instance())); } @Test @@ -680,7 +686,7 @@ public class DeploymentTriggerTest { tester.triggerJobs(); app1.jobAborted(systemTest).jobAborted(stagingTest); app1.runJob(systemTest).runJob(stagingTest).timeOutConvergence(productionUsCentral1); - assertEquals(version2, app1.deployment(productionUsCentral1.zone(main)).version()); + assertEquals(version2, app1.deployment(productionUsCentral1.zone()).version()); Instant triggered = app1.instanceJobs().get(productionUsCentral1).lastTriggered().get().start(); tester.clock().advance(Duration.ofHours(1)); @@ -696,9 +702,9 @@ public class DeploymentTriggerTest { assertEquals(triggered, app1.instanceJobs().get(productionUsCentral1).lastTriggered().get().start()); // Roll out a new application version, which gives a dual change -- this should trigger us-central-1, but only as long as it hasn't yet deployed there. - ApplicationVersion revision1 = app1.lastSubmission().get(); + RevisionId revision1 = app1.lastSubmission().get(); app1.submit(applicationPackage); - ApplicationVersion revision2 = app1.lastSubmission().get(); + RevisionId revision2 = app1.lastSubmission().get(); app1.runJob(systemTest) // Tests for new revision on version2 .runJob(stagingTest) .runJob(systemTest) // Tests for new revision on version1 @@ -706,14 +712,14 @@ public class DeploymentTriggerTest { assertEquals(Change.of(version1).with(revision2), app1.instance().change()); tester.triggerJobs(); app1.assertRunning(productionUsCentral1); - assertEquals(version2, app1.instance().deployments().get(productionUsCentral1.zone(main)).version()); - assertEquals(revision1, app1.deployment(productionUsCentral1.zone(main)).applicationVersion()); + assertEquals(version2, app1.instance().deployments().get(productionUsCentral1.zone()).version()); + assertEquals(revision1, app1.deployment(productionUsCentral1.zone()).revision()); assertTrue(triggered.isBefore(app1.instanceJobs().get(productionUsCentral1).lastTriggered().get().start())); // Change has a higher application version than what is deployed -- deployment should trigger. app1.timeOutUpgrade(productionUsCentral1); - assertEquals(version2, app1.deployment(productionUsCentral1.zone(main)).version()); - assertEquals(revision2, app1.deployment(productionUsCentral1.zone(main)).applicationVersion()); + assertEquals(version2, app1.deployment(productionUsCentral1.zone()).version()); + assertEquals(revision2, app1.deployment(productionUsCentral1.zone()).revision()); // Change is again strictly dominated, and us-central-1 is skipped, even though it is still failing. tester.clock().advance(Duration.ofHours(3)); // Enough time for retry @@ -746,8 +752,8 @@ public class DeploymentTriggerTest { app.runJob(systemTest).runJob(stagingTest); app.timeOutConvergence(productionEuWest1); tester.deploymentTrigger().cancelChange(app.instanceId(), PLATFORM); - assertEquals(v2, app.deployment(productionEuWest1.zone(main)).version()); - assertEquals(v1, app.deployment(productionUsEast3.zone(main)).version()); + assertEquals(v2, app.deployment(productionEuWest1.zone()).version()); + assertEquals(v1, app.deployment(productionUsEast3.zone()).version()); // New application version should run system and staging tests against both 6.1 and 6.2, in no particular order. app.submit(applicationPackage); @@ -772,8 +778,8 @@ public class DeploymentTriggerTest { app.failDeployment(productionEuWest1).failDeployment(productionUsEast3) .runJob(productionEuWest1).runJob(productionUsEast3); assertFalse(app.instance().change().hasTargets()); - assertEquals(2, app.instanceJobs().get(productionEuWest1).lastSuccess().get().versions().targetApplication().buildNumber().getAsLong()); - assertEquals(2, app.instanceJobs().get(productionUsEast3).lastSuccess().get().versions().targetApplication().buildNumber().getAsLong()); + assertEquals(2, app.instanceJobs().get(productionEuWest1).lastSuccess().get().versions().targetRevision().number()); + assertEquals(2, app.instanceJobs().get(productionUsEast3).lastSuccess().get().versions().targetRevision().number()); } @Test @@ -940,9 +946,9 @@ public class DeploymentTriggerTest { app3.abortJob(stagingTest); assertEquals(0, tester.jobs().active().size()); - assertTrue(app1.instance().change().application().isPresent()); - assertFalse(app2.instance().change().application().isPresent()); - assertFalse(app3.instance().change().application().isPresent()); + assertTrue(app1.instance().change().revision().isPresent()); + assertFalse(app2.instance().change().revision().isPresent()); + assertFalse(app3.instance().change().revision().isPresent()); tester.readyJobsTrigger().maintain(); app1.assertRunning(stagingTest); @@ -1007,12 +1013,12 @@ public class DeploymentTriggerTest { // Package is submitted, and change propagated to the two first instances. i1.submit(applicationPackage); - Optional<ApplicationVersion> v0 = i1.lastSubmission(); + Optional<RevisionId> v0 = i1.lastSubmission(); tester.outstandingChangeDeployer().run(); - assertEquals(v0, i1.instance().change().application()); - assertEquals(v0, i2.instance().change().application()); - assertEquals(Optional.empty(), i3.instance().change().application()); - assertEquals(Optional.empty(), i4.instance().change().application()); + assertEquals(v0, i1.instance().change().revision()); + assertEquals(v0, i2.instance().change().revision()); + assertEquals(Optional.empty(), i3.instance().change().revision()); + assertEquals(Optional.empty(), i4.instance().change().revision()); // Tests run in i4, as they're declared there, and i1 and i2 get to work i4.runJob(systemTest).runJob(stagingTest); @@ -1021,59 +1027,59 @@ public class DeploymentTriggerTest { // Since the post-deployment delay of i1 is incomplete, i3 doesn't yet get the change. tester.outstandingChangeDeployer().run(); - assertEquals(v0, i1.instance().latestDeployed()); - assertEquals(v0, i2.instance().latestDeployed()); - assertEquals(Optional.empty(), i1.instance().change().application()); - assertEquals(Optional.empty(), i2.instance().change().application()); - assertEquals(Optional.empty(), i3.instance().change().application()); - assertEquals(Optional.empty(), i4.instance().change().application()); + assertEquals(v0, Optional.of(latestDeployed(i1.instance()))); + assertEquals(v0, Optional.of(latestDeployed(i2.instance()))); + assertEquals(Optional.empty(), i1.instance().change().revision()); + assertEquals(Optional.empty(), i2.instance().change().revision()); + assertEquals(Optional.empty(), i3.instance().change().revision()); + assertEquals(Optional.empty(), i4.instance().change().revision()); // When the delay is done, i3 gets the change. tester.clock().advance(Duration.ofHours(6)); tester.outstandingChangeDeployer().run(); - assertEquals(Optional.empty(), i1.instance().change().application()); - assertEquals(Optional.empty(), i2.instance().change().application()); - assertEquals(v0, i3.instance().change().application()); - assertEquals(Optional.empty(), i4.instance().change().application()); + assertEquals(Optional.empty(), i1.instance().change().revision()); + assertEquals(Optional.empty(), i2.instance().change().revision()); + assertEquals(v0, i3.instance().change().revision()); + assertEquals(Optional.empty(), i4.instance().change().revision()); // v0 begins roll-out in i3, and v1 is submitted and rolls out in i1 and i2 some time later i3.runJob(productionUsEast3); // v0 tester.clock().advance(Duration.ofHours(12)); i1.submit(applicationPackage); - Optional<ApplicationVersion> v1 = i1.lastSubmission(); + Optional<RevisionId> v1 = i1.lastSubmission(); i4.runJob(systemTest).runJob(stagingTest); i1.runJob(productionUsEast3); // v1 i2.runJob(productionUsEast3); // v1 - assertEquals(v1, i1.instance().latestDeployed()); - assertEquals(v1, i2.instance().latestDeployed()); - assertEquals(Optional.empty(), i1.instance().change().application()); - assertEquals(Optional.empty(), i2.instance().change().application()); - assertEquals(v0, i3.instance().change().application()); - assertEquals(Optional.empty(), i4.instance().change().application()); + assertEquals(v1, Optional.of(latestDeployed(i1.instance()))); + assertEquals(v1, Optional.of(latestDeployed(i2.instance()))); + assertEquals(Optional.empty(), i1.instance().change().revision()); + assertEquals(Optional.empty(), i2.instance().change().revision()); + assertEquals(v0, i3.instance().change().revision()); + assertEquals(Optional.empty(), i4.instance().change().revision()); // After some time, v2 also starts rolling out to i1 and i2, but does not complete in i2 tester.clock().advance(Duration.ofHours(3)); i1.submit(applicationPackage); - Optional<ApplicationVersion> v2 = i1.lastSubmission(); + Optional<RevisionId> v2 = i1.lastSubmission(); i4.runJob(systemTest).runJob(stagingTest); i1.runJob(productionUsEast3); // v2 tester.clock().advance(Duration.ofHours(3)); // v1 is all done in i1 and i2, but does not yet roll out in i3; v2 is not completely rolled out there yet. tester.outstandingChangeDeployer().run(); - assertEquals(v0, i3.instance().change().application()); + assertEquals(v0, i3.instance().change().revision()); // i3 completes v0, which rolls out to i4; v1 is ready for i3, but v2 is not. i3.runJob(testUsEast3); - assertEquals(Optional.empty(), i3.instance().change().application()); + assertEquals(Optional.empty(), i3.instance().change().revision()); tester.outstandingChangeDeployer().run(); - assertEquals(v2, i1.instance().latestDeployed()); - assertEquals(v1, i2.instance().latestDeployed()); - assertEquals(v0, i3.instance().latestDeployed()); - assertEquals(Optional.empty(), i1.instance().change().application()); - assertEquals(v2, i2.instance().change().application()); - assertEquals(v1, i3.instance().change().application()); - assertEquals(v0, i4.instance().change().application()); + assertEquals(v2, Optional.of(latestDeployed(i1.instance()))); + assertEquals(v1, Optional.of(latestDeployed(i2.instance()))); + assertEquals(v0, Optional.of(latestDeployed(i3.instance()))); + assertEquals(Optional.empty(), i1.instance().change().revision()); + assertEquals(v2, i2.instance().change().revision()); + assertEquals(v1, i3.instance().change().revision()); + assertEquals(v0, i4.instance().change().revision()); } @Test @@ -1096,7 +1102,7 @@ public class DeploymentTriggerTest { DeploymentContext alpha = tester.newDeploymentContext("t", "a", "alpha"); DeploymentContext beta = tester.newDeploymentContext("t", "a", "beta"); alpha.submit(applicationPackage).deploy(); - Optional<ApplicationVersion> revision1 = alpha.lastSubmission(); + Optional<RevisionId> revision1 = alpha.lastSubmission(); Version version1 = new Version("7.1"); tester.controllerTester().upgradeSystem(version1); @@ -1119,7 +1125,7 @@ public class DeploymentTriggerTest { beta.assertNotRunning(testUsEast3); alpha.submit(applicationPackage); - Optional<ApplicationVersion> revision2 = alpha.lastSubmission(); + Optional<RevisionId> revision2 = alpha.lastSubmission(); assertEquals(Change.of(revision2.get()), alpha.instance().change()); assertEquals(Change.of(version1), beta.instance().change()); @@ -1371,7 +1377,7 @@ public class DeploymentTriggerTest { // Upgrade instance 1; upgrade rolls out first, with revision following. // The new platform won't roll out to the conservative instance until the normal one is upgraded. app1.submit(applicationPackage); - assertEquals(Change.of(version).with(app1.application().latestVersion().get()), app1.instance().change()); + assertEquals(Change.of(version).with(app1.application().revisions().last().get().id()), app1.instance().change()); // Upgrade platform. app2.runJob(systemTest); app1.runJob(stagingTest) @@ -1434,7 +1440,7 @@ public class DeploymentTriggerTest { tester.triggerJobs(); assertEquals(1, tester.jobs().active().size()); assertEquals(Change.empty(), app1.instance().change()); - assertEquals(Change.of(version).with(app1.application().latestVersion().get()), app2.instance().change()); + assertEquals(Change.of(version).with(app1.application().revisions().last().get().id()), app2.instance().change()); app2.runJob(productionEuWest1) .runJob(testEuWest1); @@ -1651,7 +1657,7 @@ public class DeploymentTriggerTest { " </prod>\n" + " </instance>\n" + " <instance id='gamma'>\n" + - " <upgrade revision-change='when-clear' revision-target='next' />\n" + + " <upgrade revision-change='when-clear' revision-target='next' min-risk='3' max-risk='6' />\n" + " <prod>\n" + " <region>us-east-3</region>\n" + " <test>us-east-3</test>\n" + @@ -1662,86 +1668,115 @@ public class DeploymentTriggerTest { var alpha = tester.newDeploymentContext("t", "a", "alpha"); var beta = tester.newDeploymentContext("t", "a", "beta"); var gamma = tester.newDeploymentContext("t", "a", "gamma"); - alpha.submit(appPackage).deploy(); + alpha.submit(appPackage, 0).deploy(); // revision2 is submitted, and rolls through alpha. var revision1 = alpha.lastSubmission(); - alpha.submit(appPackage); + alpha.submit(appPackage, 3); // Risk high enough that this may roll out alone to gamma. var revision2 = alpha.lastSubmission(); alpha.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3).runJob(testUsEast3); - assertEquals(Optional.empty(), alpha.instance().change().application()); + assertEquals(Optional.empty(), alpha.instance().change().revision()); // revision3 is submitted when revision2 is half-way. tester.outstandingChangeDeployer().run(); beta.runJob(productionUsEast3); - alpha.submit(appPackage); + alpha.submit(appPackage, 2); // Will only roll out to gamma together with the next revision. var revision3 = alpha.lastSubmission(); beta.runJob(testUsEast3); - assertEquals(Optional.empty(), beta.instance().change().application()); + assertEquals(Optional.empty(), beta.instance().change().revision()); - // revision3 is the target for alpha, beta is done, version1 is the target for gamma. + // revision3 is the target for alpha, beta is done, revision2 is the target for gamma. tester.outstandingChangeDeployer().run(); - assertEquals(revision3, alpha.instance().change().application()); - assertEquals(Optional.empty(), beta.instance().change().application()); - assertEquals(revision2, gamma.instance().change().application()); + assertEquals(revision3, alpha.instance().change().revision()); + assertEquals(Optional.empty(), beta.instance().change().revision()); + assertEquals(revision2, gamma.instance().change().revision()); // revision3 rolls to beta, then a couple of new revisions are submitted to alpha, and the latter is the new target. alpha.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3).runJob(testUsEast3); tester.outstandingChangeDeployer().run(); - assertEquals(Optional.empty(), alpha.instance().change().application()); - assertEquals(revision3, beta.instance().change().application()); + assertEquals(Optional.empty(), alpha.instance().change().revision()); + assertEquals(revision3, beta.instance().change().revision()); // revision5 supersedes revision4 - alpha.submit(appPackage); + alpha.submit(appPackage, 3); var revision4 = alpha.lastSubmission(); alpha.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3); - alpha.submit(appPackage); + alpha.submit(appPackage, 2); var revision5 = alpha.lastSubmission(); alpha.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3).runJob(testUsEast3); tester.outstandingChangeDeployer().run(); - assertEquals(Optional.empty(), alpha.instance().change().application()); - assertEquals(revision3, beta.instance().change().application()); + assertEquals(Optional.empty(), alpha.instance().change().revision()); + assertEquals(revision3, beta.instance().change().revision()); - // revision6 rolls through alpha, and becomes the next target for beta - alpha.submit(appPackage); + // revision6 rolls through alpha, and becomes the next target for beta, which also completes revision3. + alpha.submit(appPackage, 6); var revision6 = alpha.lastSubmission(); alpha.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3) .runJob(testUsEast3); beta.runJob(productionUsEast3).runJob(testUsEast3); tester.outstandingChangeDeployer().run(); - assertEquals(Optional.empty(), alpha.instance().change().application()); - assertEquals(revision6, beta.instance().change().application()); + assertEquals(Optional.empty(), alpha.instance().change().revision()); + assertEquals(revision6, beta.instance().change().revision()); - // revision6 rolls through beta, but revision3 is the next target for gamma with "exclusive" revision upgrades - alpha.jobAborted(stagingTest).runJob(stagingTest); - beta.runJob(productionUsEast3).runJob(testUsEast3); - - // revision 2 fails, but this does not bring on revision 3 + // revision 2 fails in gamma, but this does not bring on revision 3 gamma.failDeployment(productionUsEast3); tester.outstandingChangeDeployer().run(); - assertEquals(Optional.empty(), beta.instance().change().application()); - assertEquals(revision2, gamma.instance().change().application()); + assertEquals(revision2, gamma.instance().change().revision()); - // revision 2 completes + // revision 2 completes in gamma gamma.runJob(productionUsEast3) .runJob(testUsEast3); tester.outstandingChangeDeployer().run(); - assertEquals(Optional.empty(), alpha.instance().change().application()); - assertEquals(Optional.empty(), beta.instance().change().application()); - assertEquals(revision3, gamma.instance().change().application()); + assertEquals(Optional.empty(), alpha.instance().change().revision()); + assertEquals(Optional.empty(), gamma.instance().change().revision()); // no other revisions after 3 are ready, so gamma waits + + // revision6 rolls through beta, and revision3 is the next target for gamma with "when-clear" change-revision, now that 6 is blocking 4 and 5 + alpha.jobAborted(stagingTest).runJob(stagingTest); + beta.runJob(productionUsEast3).runJob(testUsEast3); + assertEquals(Optional.empty(), beta.instance().change().revision()); + + tester.outstandingChangeDeployer().run(); + assertEquals(Optional.empty(), alpha.instance().change().revision()); + assertEquals(Optional.empty(), beta.instance().change().revision()); + assertEquals(revision3, gamma.instance().change().revision()); // revision4 never became ready, but 5 did, so 4 is skipped, and 3 rolls out alone instead. // revision 6 is next, once 3 is done // revision 3 completes gamma.runJob(productionUsEast3) .runJob(testUsEast3); tester.outstandingChangeDeployer().run(); - assertEquals(revision6, gamma.instance().change().application()); + assertEquals(revision6, gamma.instance().change().revision()); + + // revision 7 becomes ready for gamma, but must wait for the idle time of 8 hours before being deployed + alpha.submit(appPackage, 1); + var revision7 = alpha.lastSubmission(); + alpha.deploy(); + tester.outstandingChangeDeployer(); + assertEquals(Change.empty(), gamma.instance().change()); + assertEquals(revision6.get(), gamma.deployment(ZoneId.from("prod.us-east-3")).revision()); + + tester.clock().advance(Duration.ofHours(8)); + tester.outstandingChangeDeployer().run(); + assertEquals(revision7, gamma.instance().change().revision()); + + // revision 8 is has too low risk to roll out on its own, but will start rolling immediately when revision 9 is submitted + gamma.deploy(); + alpha.submit(appPackage, 2); + var revision8 = alpha.lastSubmission(); + alpha.deploy(); + tester.outstandingChangeDeployer(); + assertEquals(Change.empty(), gamma.instance().change()); + assertEquals(revision7.get(), gamma.deployment(ZoneId.from("prod.us-east-3")).revision()); + + alpha.submit(appPackage, 5); + tester.outstandingChangeDeployer().run(); + assertEquals(revision8, gamma.instance().change().revision()); } @Test @@ -1900,55 +1935,51 @@ public class DeploymentTriggerTest { @Test public void mixedDirectAndPipelineJobsInProduction() { - ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("cd-us-east-1") - .region("cd-aws-us-east-1a") + ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("us-east-3") + .region("aws-us-east-1a") .build(); - var zones = List.of(ZoneId.from("test.cd-us-west-1"), - ZoneId.from("staging.cd-us-west-1"), - ZoneId.from("prod.cd-us-east-1"), - ZoneId.from("prod.cd-aws-us-east-1a")); - tester.controllerTester() - .setZones(zones, SystemName.cd) - .setRoutingMethod(zones, RoutingMethod.sharedLayer4); - tester.controllerTester().upgradeSystem(Version.fromString("6.1")); - tester.controllerTester().computeVersionStatus(); + ControllerTester wrapped = new ControllerTester(cd); + wrapped.upgradeSystem(Version.fromString("6.1")); + wrapped.computeVersionStatus(); + + DeploymentTester tester = new DeploymentTester(wrapped); var app = tester.newDeploymentContext(); - app.runJob(productionCdUsEast1, cdPackage); + app.runJob(productionUsEast3, cdPackage); app.submit(cdPackage); app.runJob(systemTest); // Staging test requires unknown initial version, and is broken. - tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false, true, true); - app.runJob(productionCdUsEast1) + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "user", false, true, true); + app.runJob(productionUsEast3) .abortJob(stagingTest) // Complete failing run. .runJob(stagingTest) // Run staging-test for production zone with no prior deployment. - .runJob(productionCdAwsUsEast1a); + .runJob(productionAwsUsEast1a); // Manually deploy to east again, then upgrade the system. - app.runJob(productionCdUsEast1, cdPackage); + app.runJob(productionUsEast3, cdPackage); var version = new Version("7.1"); tester.controllerTester().upgradeSystem(version); tester.upgrader().maintain(); // System and staging tests both require unknown versions, and are broken. - tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false, true, true); - app.runJob(productionCdUsEast1) + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "user", false, true, true); + app.runJob(productionUsEast3) .triggerJobs() .jobAborted(systemTest) .jobAborted(stagingTest) .runJob(systemTest) // Run test for aws zone again. .runJob(stagingTest) // Run test for aws zone again. - .runJob(productionCdAwsUsEast1a); + .runJob(productionAwsUsEast1a); // Deploy manually again, then submit new package. - app.runJob(productionCdUsEast1, cdPackage); + app.runJob(productionUsEast3, cdPackage); app.submit(cdPackage); app.triggerJobs().runJob(systemTest); // Staging test requires unknown initial version, and is broken. - tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false, true, true); - app.runJob(productionCdUsEast1) + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "user", false, true, true); + app.runJob(productionUsEast3) .jobAborted(stagingTest) .runJob(stagingTest) - .runJob(productionCdAwsUsEast1a); + .runJob(productionAwsUsEast1a); } @Test @@ -2001,12 +2032,12 @@ public class DeploymentTriggerTest { // Start upgrade, then receive new submission. Version version1 = new Version("7.8.9"); - ApplicationVersion build1 = app.lastSubmission().get(); + RevisionId build1 = app.lastSubmission().get(); tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); app.runJob(stagingTest); app.submit(); - ApplicationVersion build2 = app.lastSubmission().get(); + RevisionId build2 = app.lastSubmission().get(); assertNotEquals(build1, build2); // App now free to start system tests eagerly, for new submission. These should run assuming upgrade succeeds. @@ -2015,16 +2046,16 @@ public class DeploymentTriggerTest { assertEquals(version1, app.instanceJobs().get(stagingTest).lastCompleted().get().versions().targetPlatform()); assertEquals(build1, - app.instanceJobs().get(stagingTest).lastCompleted().get().versions().targetApplication()); + app.instanceJobs().get(stagingTest).lastCompleted().get().versions().targetRevision()); assertEquals(version1, app.instanceJobs().get(stagingTest).lastTriggered().get().versions().sourcePlatform().get()); assertEquals(build1, - app.instanceJobs().get(stagingTest).lastTriggered().get().versions().sourceApplication().get()); + app.instanceJobs().get(stagingTest).lastTriggered().get().versions().sourceRevision().get()); assertEquals(version1, app.instanceJobs().get(stagingTest).lastTriggered().get().versions().targetPlatform()); assertEquals(build2, - app.instanceJobs().get(stagingTest).lastTriggered().get().versions().targetApplication()); + app.instanceJobs().get(stagingTest).lastTriggered().get().versions().targetRevision()); // App completes upgrade, and outstanding change is triggered. This should let relevant, running jobs finish. app.runJob(systemTest) @@ -2113,7 +2144,7 @@ public class DeploymentTriggerTest { .build()); app.deploy(); assertEquals(version2, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); - assertEquals(version2, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetApplication().compileVersion().get()); + assertEquals(version2, app.application().revisions().get(tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 4e538acb8f2..031cdaa84ae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -12,20 +12,19 @@ import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Inspector; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo; +import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage; import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; @@ -34,6 +33,7 @@ import org.junit.Test; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -54,13 +54,16 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.app import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.time.temporal.ChronoUnit.SECONDS; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; @@ -94,7 +97,7 @@ public class InternalStepRunnerTest { tester.triggerJobs(); tester.runner().run(); DeploymentSpec spec = tester.configServer() - .application(app.testerId().id(), JobType.stagingTest.zone(system())).get() + .application(app.testerId().id(), DeploymentContext.stagingTest.zone()).get() .applicationPackage().deploymentSpec(); assertTrue(spec.instance(app.testerId().id().instance()).isPresent()); assertEquals("domain", spec.athenzDomain().get().value()); @@ -107,37 +110,26 @@ public class InternalStepRunnerTest { "Exception to retry", "test failure"); tester.configServer().throwOnNextPrepare(exception); - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage()); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); + tester.jobs().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), applicationPackage()); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().stepStatuses().get(Step.deployReal)); tester.configServer().throwOnNextPrepare(exception); tester.runner().run(); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().stepStatuses().get(Step.deployReal)); tester.clock().advance(Duration.ofHours(1).plusSeconds(1)); tester.configServer().throwOnNextPrepare(exception); tester.runner().run(); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); - assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().status()); + assertEquals(failed, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().stepStatuses().get(Step.deployReal)); + assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().status()); } @Test - // TODO jonmv: Change to only wait for restarts, and remove triggering of restarts from runner. public void restartsServicesAndWaitsForRestartAndReboot() { - RunId id = app.newRun(JobType.productionUsCentral1); - ZoneId zone = id.type().zone(system()); + RunId id = app.newRun(DeploymentContext.productionUsCentral1); + ZoneId zone = id.type().zone(); HostName host = tester.configServer().hostFor(instanceId, zone); - tester.configServer().setConfigChangeActions(new ConfigChangeActions(List.of(new RestartAction("cluster", - "container", - "search", - List.of(new ServiceInfo("queries", - "search", - "config", - host.value())), - List.of("Restart it!"))), - List.of(), - List.of())); tester.runner().run(); assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.deployReal)); @@ -156,28 +148,28 @@ public class InternalStepRunnerTest { @Test public void waitsForEndpointsAndTimesOut() { - app.newRun(JobType.systemTest); + app.newRun(DeploymentContext.systemTest); // Tester endpoint fails to show up for staging tests, and the real deployment for system tests. - var testZone = JobType.systemTest.zone(system()); - var stagingZone = JobType.stagingTest.zone(system()); + var testZone = DeploymentContext.systemTest.zone(); + var stagingZone = DeploymentContext.stagingTest.zone(); tester.newDeploymentContext(app.testerId().id()) .deferLoadBalancerProvisioningIn(testZone.environment()); tester.newDeploymentContext(app.instanceId()) .deferLoadBalancerProvisioningIn(stagingZone.environment()); tester.runner().run(); - tester.configServer().convergeServices(app.instanceId(), JobType.stagingTest.zone(system())); + tester.configServer().convergeServices(app.instanceId(), DeploymentContext.stagingTest.zone()); tester.runner().run(); - tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.instanceId(), JobType.stagingTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system())); + tester.configServer().convergeServices(app.instanceId(), DeploymentContext.systemTest.zone()); + tester.configServer().convergeServices(app.testerId().id(), DeploymentContext.systemTest.zone()); + tester.configServer().convergeServices(app.instanceId(), DeploymentContext.stagingTest.zone()); + tester.configServer().convergeServices(app.testerId().id(), DeploymentContext.stagingTest.zone()); tester.runner().run(); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).endpoint().plus(Duration.ofSeconds(1))); tester.runner().run(); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(failed, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); } @Test @@ -185,66 +177,66 @@ public class InternalStepRunnerTest { tester.controllerTester().upgradeSystem(new Version("7.1")); tester.controllerTester().computeVersionStatus(); tester.upgrader().maintain(); - app.newRun(JobType.systemTest); + app.newRun(DeploymentContext.systemTest); // Node is down too long in system test, and no nodes go down in staging. tester.runner().run(); - tester.configServer().setVersion(tester.controller().readSystemVersion(), app.testerId().id(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); - tester.configServer().setVersion(tester.controller().readSystemVersion(), app.testerId().id(), JobType.stagingTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system())); + tester.configServer().setVersion(tester.controller().readSystemVersion(), app.testerId().id(), DeploymentContext.systemTest.zone()); + tester.configServer().convergeServices(app.testerId().id(), DeploymentContext.systemTest.zone()); + tester.configServer().setVersion(tester.controller().readSystemVersion(), app.testerId().id(), DeploymentContext.stagingTest.zone()); + tester.configServer().convergeServices(app.testerId().id(), DeploymentContext.stagingTest.zone()); tester.runner().run(); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installTester)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), DeploymentContext.stagingTest).get().stepStatuses().get(Step.installTester)); - Node systemTestNode = tester.configServer().nodeRepository().list(JobType.systemTest.zone(system()), + Node systemTestNode = tester.configServer().nodeRepository().list(DeploymentContext.systemTest.zone(), NodeFilter.all().applications(app.instanceId())).iterator().next(); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().minus(Duration.ofSeconds(1))); - tester.configServer().nodeRepository().putNodes(JobType.systemTest.zone(system()), + tester.configServer().nodeRepository().putNodes(DeploymentContext.systemTest.zone(), Node.builder(systemTestNode) .serviceState(Node.ServiceState.allowedDown) .suspendedSince(tester.clock().instant()) .build()); tester.runner().run(); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.stagingTest).get().stepStatuses().get(Step.installInitialReal)); tester.clock().advance(Duration.ofSeconds(2)); tester.runner().run(); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(failed, tester.jobs().last(app.instanceId(), DeploymentContext.stagingTest).get().stepStatuses().get(Step.installInitialReal)); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).statelessNodesDown().minus(Duration.ofSeconds(3))); tester.runner().run(); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); tester.clock().advance(Duration.ofSeconds(2)); tester.runner().run(); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(failed, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); } @Test public void startingTestsFailsIfDeploymentExpires() { - app.newRun(JobType.systemTest); + app.newRun(DeploymentContext.systemTest); tester.runner().run(); - tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); + tester.configServer().convergeServices(app.instanceId(), DeploymentContext.systemTest.zone()); tester.runner().run(); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installTester)); - tester.applications().deactivate(app.instanceId(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); + tester.applications().deactivate(app.instanceId(), DeploymentContext.systemTest.zone()); + tester.configServer().convergeServices(app.testerId().id(), DeploymentContext.systemTest.zone()); tester.runner().run(); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.startTests)); - assertTrue(tester.jobs().last(app.instanceId(), JobType.systemTest).get().hasEnded()); - assertTrue(tester.jobs().last(app.instanceId(), JobType.systemTest).get().hasFailed()); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installTester)); + assertEquals(failed, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.startTests)); + assertTrue(tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().hasEnded()); + assertTrue(tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().hasFailed()); } @Test public void alternativeEndpointsAreDetected() { - var systemTestZone = JobType.systemTest.zone(system()); - var stagingZone = JobType.stagingTest.zone(system()); + var systemTestZone = DeploymentContext.systemTest.zone(); + var stagingZone = DeploymentContext.stagingTest.zone(); tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(systemTestZone), ZoneApiMock.from(stagingZone)); var applicationPackage = new ApplicationPackageBuilder() .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) @@ -256,15 +248,36 @@ public class InternalStepRunnerTest { .triggerJobs(); tester.runner().run(); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installTester)); app.flushDnsUpdates(); - tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); + tester.configServer().convergeServices(app.instanceId(), DeploymentContext.systemTest.zone()); + tester.configServer().convergeServices(app.testerId().id(), DeploymentContext.systemTest.zone()); tester.runner().run(); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installTester)); + } + + @Test + public void noTestsThenErrorIsError() { + RunId id = app.startSystemTestTests(); + Run run = tester.jobs().run(id).get(); + run = run.with(noTests, new LockedStep(() -> { }, Step.endTests)); + assertFalse(run.hasFailed()); + run = run.with(RunStatus.error, new LockedStep(() -> { }, Step.deactivateReal)); + assertTrue(run.hasFailed()); + assertEquals(RunStatus.error, run.status()); + } + + @Test + public void noTestsThenSuccessIsNoTests() { + RunId id = app.startSystemTestTests(); + tester.cloud().set(Status.NO_TESTS); + tester.runner().run(); + assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); + Run run = tester.jobs().run(id).get(); + assertEquals(noTests, run.status()); } @Test @@ -308,7 +321,7 @@ public class InternalStepRunnerTest { RunId id = app.startSystemTestTests(); tester.runner().run(); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); - var testZone = JobType.systemTest.zone(system()); + var testZone = DeploymentContext.systemTest.zone(); Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get(); assertEquals(app.instanceId().serializedForm(), configObject.field("application").asString()); assertEquals(testZone.value(), configObject.field("zone").asString()); @@ -361,7 +374,7 @@ public class InternalStepRunnerTest { assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.deployTester)); tester.clock().advance(JobRunner.jobTimeout); - var testZone = JobType.systemTest.zone(tester.controller().system()); + var testZone = DeploymentContext.systemTest.zone(); tester.runner().run(); app.flushDnsUpdates(); tester.configServer().convergeServices(app.instanceId(), testZone); @@ -388,28 +401,28 @@ public class InternalStepRunnerTest { @Test public void deployToDev() { - ZoneId zone = JobType.devUsEast1.zone(system()); - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage()); + ZoneId zone = DeploymentContext.devUsEast1.zone(); + tester.jobs().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), applicationPackage()); tester.runner().run(); - RunId id = tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().id(); + RunId id = tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().id(); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); Version version = new Version("7.8.9"); Future<?> concurrentDeployment = Executors.newSingleThreadExecutor().submit(() -> { - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.of(version), applicationPackage()); + tester.jobs().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.of(version), applicationPackage()); }); while ( ! concurrentDeployment.isDone()) tester.runner().run(); - assertEquals(id.number() + 1, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().id().number()); + assertEquals(id.number() + 1, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().id().number()); ApplicationPackage otherPackage = new ApplicationPackageBuilder().region("us-central-1").build(); - tester.jobs().deploy(app.instanceId(), JobType.perfUsEast3, Optional.empty(), otherPackage); + tester.jobs().deploy(app.instanceId(), DeploymentContext.perfUsEast3, Optional.empty(), otherPackage); tester.runner().run(); // Job run order determined by JobType enum order per application. tester.configServer().convergeServices(app.instanceId(), zone); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); assertEquals(applicationPackage().hash(), tester.configServer().application(app.instanceId(), zone).get().applicationPackage().hash()); - assertEquals(otherPackage.hash(), tester.configServer().application(app.instanceId(), JobType.perfUsEast3.zone(system())).get().applicationPackage().hash()); + assertEquals(otherPackage.hash(), tester.configServer().application(app.instanceId(), DeploymentContext.perfUsEast3.zone()).get().applicationPackage().hash()); tester.configServer().setVersion(version, app.instanceId(), zone); tester.runner().run(); @@ -419,7 +432,7 @@ public class InternalStepRunnerTest { @Test public void notificationIsSent() { - app.submit().failDeployment(JobType.systemTest); + app.submit().failDeployment(DeploymentContext.systemTest); MockMailer mailer = tester.controllerTester().serviceRegistry().mailer(); assertEquals(1, mailer.inbox("a@b").size()); assertEquals("Vespa application tenant.application: System test failing due to system error", @@ -429,12 +442,12 @@ public class InternalStepRunnerTest { mailer.inbox("b@a").get(0).subject()); // Re-run failing causes no additional email to be sent. - app.failDeployment(JobType.systemTest); + app.failDeployment(DeploymentContext.systemTest); assertEquals(1, mailer.inbox("a@b").size()); assertEquals(1, mailer.inbox("b@a").size()); // Failure with new package causes new email to be sent. - app.submit().failDeployment(JobType.systemTest); + app.submit().failDeployment(DeploymentContext.systemTest); assertEquals(2, mailer.inbox("a@b").size()); assertEquals(2, mailer.inbox("b@a").size()); } @@ -480,16 +493,17 @@ public class InternalStepRunnerTest { @Test public void realDeploymentRequiresForTesterCert() { - tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public); - var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), - ZoneApiMock.fromId("staging.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-us-east-1c")); - tester.controllerTester().zoneRegistry() - .setZones(zones) - .setRoutingMethod(zones, RoutingMethod.exclusive); + List<ZoneApiMock> zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), + ZoneApiMock.fromId("staging.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-us-east-1c")); + ControllerTester wrapped = new ControllerTester(SystemName.Public); + wrapped.zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.exclusive); + tester = new DeploymentTester(wrapped); tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values()); - ZoneId testZone = JobType.systemTest.zone(tester.controller().system()); - RunId id = app.newRun(JobType.systemTest); + app = tester.newDeploymentContext(); + RunId id = app.newRun(DeploymentContext.systemTest); tester.configServer().throwOnPrepare(instanceId -> { if (instanceId.instance().isTester()) throw new ConfigServerException(ConfigServerException.ErrorCode.PARENT_HOST_NOT_READY, "provisioning", "deploy tester"); @@ -501,7 +515,7 @@ public class InternalStepRunnerTest { List<X509Certificate> oldTrusted = new ArrayList<>(DeploymentContext.publicApplicationPackage().trustedCertificates()); X509Certificate oldCert = tester.jobs().run(id).get().testerCertificate().get(); oldTrusted.add(oldCert); - assertEquals(oldTrusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates()); + assertEquals(oldTrusted, tester.configServer().application(app.instanceId(), id.type().zone()).get().applicationPackage().trustedCertificates()); tester.configServer().throwOnNextPrepare(null); tester.runner().run(); @@ -511,25 +525,19 @@ public class InternalStepRunnerTest { List<X509Certificate> newTrusted = new ArrayList<>(DeploymentContext.publicApplicationPackage().trustedCertificates()); X509Certificate newCert = tester.jobs().run(id).get().testerCertificate().get(); newTrusted.add(newCert); - assertEquals(newTrusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates()); + assertEquals(newTrusted, tester.configServer().application(app.instanceId(), id.type().zone()).get().applicationPackage().trustedCertificates()); assertNotEquals(oldCert, newCert); } @Test public void certificateTimeoutAbortsJob() { - tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public); - var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), - ZoneApiMock.fromId("staging.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-us-east-1c")); - tester.controllerTester().zoneRegistry() - .setZones(zones) - .setRoutingMethod(zones, RoutingMethod.exclusive); - tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values()); + tester = new DeploymentTester(new ControllerTester(SystemName.Public)); + app = tester.newDeploymentContext(); RunId id = app.startSystemTestTests(); List<X509Certificate> trusted = new ArrayList<>(DeploymentContext.publicApplicationPackage().trustedCertificates()); trusted.add(tester.jobs().run(id).get().testerCertificate().get()); - assertEquals(trusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates()); + assertEquals(trusted, tester.configServer().application(app.instanceId(), id.type().zone()).get().applicationPackage().trustedCertificates()); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).testerCertificate().plus(Duration.ofSeconds(1))); tester.runner().run(); @@ -546,62 +554,4 @@ public class InternalStepRunnerTest { "3554970337.947820\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + "3554970337.947845\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"; - @Test - public void generates_correct_tester_flavor() { - DeploymentSpec spec = DeploymentSpec.fromXml("<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" + - " <instance id='first'>\n" + - " <test tester-flavor=\"d-6-16-100\" />\n" + - " <prod>\n" + - " <region active=\"true\">us-west-1</region>\n" + - " <test>us-west-1</test>\n" + - " </prod>\n" + - " </instance>\n" + - " <instance id='second'>\n" + - " <test />\n" + - " <staging />\n" + - " <prod tester-flavor=\"d-6-16-100\">\n" + - " <parallel>\n" + - " <region active=\"true\">us-east-3</region>\n" + - " <region active=\"true\">us-central-1</region>\n" + - " </parallel>\n" + - " <region active=\"true\">us-west-1</region>\n" + - " <test>us-west-1</test>\n" + - " </prod>\n" + - " </instance>\n" + - "</deployment>\n"); - - NodeResources firstResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("first")); - assertEquals(InternalStepRunner.DEFAULT_TESTER_RESOURCES, firstResources); - - NodeResources secondResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("second")); - assertEquals(6, secondResources.vcpu(), 1e-9); - assertEquals(16, secondResources.memoryGb(), 1e-9); - assertEquals(100, secondResources.diskGb(), 1e-9); - } - - @Test - public void generates_correct_services_xml() { - generates_correct_services_xml("test_runner_services.xml-cd"); - } - - private void generates_correct_services_xml(String filenameExpectedOutput) { - ControllerConfig.Steprunner.Testerapp config = new ControllerConfig.Steprunner.Testerapp.Builder().build(); - assertFile(filenameExpectedOutput, - new String(InternalStepRunner.servicesXml( - true, - false, - new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local), - config))); - } - - private void assertFile(String resourceName, String actualContent) { - try { - Path path = Paths.get("src/test/resources/").resolve(resourceName); - String expectedContent = new String(Files.readAllBytes(path)); - assertEquals(expectedContent, actualContent); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java index 61eca05cc67..59ee8cc6eae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import org.junit.Test; @@ -28,9 +27,9 @@ public class TestConfigSerializerTest { @Test public void testConfig() throws IOException { - ZoneId zone = JobType.systemTest.zone(SystemName.PublicCd); + ZoneId zone = DeploymentContext.systemTest.zone(); byte[] json = new TestConfigSerializer(SystemName.PublicCd).configJson(instanceId, - JobType.systemTest, + DeploymentContext.systemTest, true, Map.of(zone, List.of(Endpoint.of(ApplicationId.defaultId()) .target(EndpointId.of("ai"), ClusterSpec.Id.from("qrs"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java index 44bcbcfb4fa..b5fd3d750ef 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java @@ -28,11 +28,11 @@ public class NameServiceQueueTest { var r1 = new Record(Record.Type.CNAME, RecordName.from("cname.vespa.oath.cloud"), RecordData.from("example.com")); var r2 = new Record(Record.Type.TXT, RecordName.from("txt.example.com"), RecordData.from("text")); var r3 = List.of(new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), - new LatencyAliasTarget(HostName.from("alias1"), + new LatencyAliasTarget(HostName.of("alias1"), "dns-zone-01", ZoneId.from("prod", "us-north-1")).pack()), new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), - new LatencyAliasTarget(HostName.from("alias2"), + new LatencyAliasTarget(HostName.of("alias2"), "dns-zone-02", ZoneId.from("prod", "us-north-2")).pack())); var req1 = new CreateRecord(r1); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java index a1e7dd4efe3..8ed38761c95 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java @@ -5,9 +5,10 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -29,7 +30,7 @@ public class ApplicationStoreMock implements ApplicationStore { private static final byte[] tombstone = new byte[0]; - private final Map<ApplicationId, Map<ApplicationVersion, byte[]>> store = new ConcurrentHashMap<>(); + private final Map<ApplicationId, Map<RevisionId, byte[]>> store = new ConcurrentHashMap<>(); private final Map<DeploymentId, byte[]> devStore = new ConcurrentHashMap<>(); private final Map<ApplicationId, Map<Long, byte[]>> diffs = new ConcurrentHashMap<>(); private final Map<DeploymentId, Map<Long, byte[]>> devDiffs = new ConcurrentHashMap<>(); @@ -45,15 +46,14 @@ public class ApplicationStoreMock implements ApplicationStore { } @Override - public byte[] get(DeploymentId deploymentId, ApplicationVersion applicationVersion) { - if (applicationVersion.isDeployedDirectly()) + public byte[] get(DeploymentId deploymentId, RevisionId revisionId) { + if ( ! revisionId.isProduction()) return requireNonNull(devStore.get(deploymentId)); TenantAndApplicationId tenantAndApplicationId = TenantAndApplicationId.from(deploymentId.applicationId()); - byte[] bytes = store.get(appId(tenantAndApplicationId.tenant(), tenantAndApplicationId.application())).get(applicationVersion); + byte[] bytes = store.get(appId(tenantAndApplicationId.tenant(), tenantAndApplicationId.application())).get(revisionId); if (bytes == null) - throw new IllegalArgumentException("No application package found for " + tenantAndApplicationId + - " with version " + applicationVersion.id()); + throw new NotExistsException("No " + revisionId + " found for " + tenantAndApplicationId); return bytes; } @@ -71,50 +71,35 @@ public class ApplicationStoreMock implements ApplicationStore { @Override public Optional<byte[]> find(TenantName tenant, ApplicationName application, long buildNumber) { return store.getOrDefault(appId(tenant, application), Map.of()).entrySet().stream() - .filter(kv -> kv.getKey().buildNumber().orElse(Long.MIN_VALUE) == buildNumber) + .filter(kv -> kv.getKey().number() == buildNumber) .map(Map.Entry::getValue) .findFirst(); } @Override - public void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage, byte[] diff) { - store.computeIfAbsent(appId(tenant, application), __ -> new ConcurrentHashMap<>()).put(applicationVersion, applicationPackage); - applicationVersion.buildNumber().ifPresent(buildNumber -> - diffs.computeIfAbsent(appId(tenant, application), __ -> new ConcurrentHashMap<>()).put(buildNumber, diff)); + public void put(TenantName tenant, ApplicationName application, RevisionId revision, byte[] bytes, byte[] tests, byte[] diff) { + store.computeIfAbsent(appId(tenant, application), __ -> new ConcurrentHashMap<>()).put(revision, bytes); + store.computeIfAbsent(testerId(tenant, application), key -> new ConcurrentHashMap<>()) .put(revision, tests); + diffs.computeIfAbsent(appId(tenant, application), __ -> new ConcurrentHashMap<>()).put(revision.number(), diff); } @Override - public boolean prune(TenantName tenant, ApplicationName application, ApplicationVersion oldestToRetain) { - return store.containsKey(appId(tenant, application)) - && store.get(appId(tenant, application)).keySet().removeIf(version -> version.compareTo(oldestToRetain) < 0); + public void prune(TenantName tenant, ApplicationName application, RevisionId oldestToRetain) { + store.getOrDefault(appId(tenant, application), Map.of()).keySet().removeIf(version -> version.compareTo(oldestToRetain) < 0); + store.getOrDefault(testerId(tenant, application), Map.of()).keySet().removeIf(version -> version.compareTo(oldestToRetain) < 0); } @Override public void removeAll(TenantName tenant, ApplicationName application) { store.remove(appId(tenant, application)); + store.remove(testerId(tenant, application)); } @Override - public byte[] getTester(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion) { - return requireNonNull(store.get(testerId(tenant, application)).get(applicationVersion)); - } - - @Override - public void putTester(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] testerPackage) { - store.computeIfAbsent(testerId(tenant, application), key -> new ConcurrentHashMap<>()) - .put(applicationVersion, testerPackage); - } - - @Override - public boolean pruneTesters(TenantName tenant, ApplicationName application, ApplicationVersion oldestToRetain) { - return store.containsKey(testerId(tenant, application)) - && store.get(testerId(tenant, application)).keySet().removeIf(version -> version.compareTo(oldestToRetain) < 0); + public byte[] getTester(TenantName tenant, ApplicationName application, RevisionId revision) { + return requireNonNull(store.get(testerId(tenant, application)).get(revision)); } - @Override - public void removeAllTesters(TenantName tenant, ApplicationName application) { - store.remove(testerId(tenant, application)); - } @Override public Optional<byte[]> getDevDiff(DeploymentId deploymentId, long buildNumber) { @@ -128,10 +113,9 @@ public class ApplicationStoreMock implements ApplicationStore { } @Override - public void putDev(DeploymentId deploymentId, ApplicationVersion applicationVersion, byte[] applicationPackage, byte[] diff) { + public void putDev(DeploymentId deploymentId, RevisionId revision, byte[] applicationPackage, byte[] diff) { devStore.put(deploymentId, applicationPackage); - applicationVersion.buildNumber().ifPresent(buildNumber -> - devDiffs.computeIfAbsent(deploymentId, __ -> new ConcurrentHashMap<>()).put(buildNumber, diff)); + devDiffs.computeIfAbsent(deploymentId, __ -> new ConcurrentHashMap<>()).put(revision.number(), diff); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index e98ff71884d..1ed84659f58 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; +import ai.vespa.http.DomainName; +import ai.vespa.http.HttpURL.Path; import ai.vespa.http.HttpURL.Query; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; @@ -14,8 +16,6 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; -import ai.vespa.http.DomainName; -import ai.vespa.http.HttpURL.Path; import com.yahoo.text.Text; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; @@ -45,9 +45,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartF import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; -import com.yahoo.vespa.serviceview.bindings.ClusterView; -import com.yahoo.vespa.serviceview.bindings.ServiceView; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -74,6 +71,7 @@ import java.util.stream.Collectors; import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; import static com.yahoo.config.provision.NodeResources.StorageType.remote; +import static java.nio.charset.StandardCharsets.UTF_8; /** * @author mortent @@ -100,7 +98,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private Version lastPrepareVersion = null; private Consumer<ApplicationId> prepareException = null; - private ConfigChangeActions configChangeActions = null; private Supplier<String> log = () -> "INFO - All good"; @Inject @@ -108,11 +105,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.notController()); } - /** Sets the ConfigChangeActions that will be returned on next deployment. */ - public void setConfigChangeActions(ConfigChangeActions configChangeActions) { - this.configChangeActions = configChangeActions; - } - /** Assigns a reserved tenant node to the given deployment, with initial versions. */ public void provision(ZoneId zone, ApplicationId application, ClusterSpec.Id clusterId) { var current = new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1, slow, remote)); @@ -159,7 +151,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } public HostName hostFor(ApplicationId application, ZoneId zone) { - return HostName.from("host-" + application.serializedForm() + "-" + zone.value()); + return HostName.of("host-" + application.toFullString() + "-" + zone.value()); } public void bootstrap(List<ZoneId> zones, SystemApplication... applications) { @@ -176,8 +168,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer for (SystemApplication application : applications) { for (int i = 1; i <= 3; i++) { Node node = Node.builder() - .hostname(HostName.from("node-" + i + "-" + application.id().application() - .value() + "-" + zone.value())) + .hostname(HostName.of("node-" + i + "-" + application.id().application() + .value() + "-" + zone.value())) .state(Node.State.active) .type(application.nodeType()) .owner(application.id()) @@ -402,7 +394,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer putLoadBalancers(id.zoneId(), List.of(new LoadBalancer(UUID.randomUUID().toString(), id.applicationId(), cluster, - Optional.of(HostName.from("lb-0--" + id.applicationId().serializedForm() + "--" + id.zoneId().toString())), + Optional.of(HostName.of("lb-0--" + id.applicationId().toFullString() + "--" + id.zoneId().toString())), LoadBalancer.State.active, Optional.of("dns-zone-1")))); } @@ -430,10 +422,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer PrepareResponse prepareResponse = new PrepareResponse(); prepareResponse.message = "foo"; - prepareResponse.configChangeActions = configChangeActions != null - ? configChangeActions - : new ConfigChangeActions(List.of(), List.of(), List.of()); - setConfigChangeActions(null); + prepareResponse.configChangeActions = new ConfigChangeActions(List.of(), List.of(), List.of()); prepareResponse.tenant = new TenantId("tenant"); prepareResponse.log = warnings.getOrDefault(id, Collections.emptyList()); return prepareResponse; @@ -486,32 +475,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer removeLoadBalancers(deployment.applicationId(), deployment.zoneId()); } - // Returns a canned example response - @Override - public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, - String environment, String region) { - String cfgHostname = Text.format("https://cfg.%s.%s.test.vip:4443", environment, region); - String cfgServiceUrlPrefix = Text.format("%s/serviceview/v1/tenant/%s/application/%s/environment/%s/region/%s/instance/%s/service", - cfgHostname, tenantName, applicationName, - environment, region, instanceName); - ApplicationView applicationView = new ApplicationView(); - ClusterView cluster = new ClusterView(); - cluster.name = "cluster1"; - cluster.type = "content"; - cluster.url = cfgServiceUrlPrefix + "/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1"; - ServiceView service = new ServiceView(); - service.configId = "cluster1/storage/0"; - service.host = "host1"; - service.serviceName = "storagenode"; - service.serviceType = "storagenode"; - service.url = cfgServiceUrlPrefix + "/storagenode-awe3slno6mmq2fye191y324jl/state/v1/"; - cluster.services = new ArrayList<>(); - cluster.services.add(service); - applicationView.clusters = new ArrayList<>(); - applicationView.clusters.add(cluster); - return applicationView; - } - @Override public List<ClusterMetrics> getDeploymentMetrics(DeploymentId deployment) { return Collections.unmodifiableList(clusterMetrics.getOrDefault(deployment, List.of())); @@ -522,21 +485,14 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return this.protonMetrics; } - // Returns a canned example response @Override - public Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, Path restPath) { - Map<String,List<?>> root = new HashMap<>(); - List<Map<?,?>> resources = new ArrayList<>(); - Map<String,String> resource = new HashMap<>(); - resource.put("url", "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/filedistributorservice-dud1f4w037qdxdrn0ovxfdtgw/state/v1/config"); - resources.add(resource); - root.put("resources", resources); - return root; + public ProxyResponse getServiceNodePage(DeploymentId deployment, String serviceName, DomainName node, Path subPath, Query query) { + return new ProxyResponse((subPath + " and " + query).getBytes(UTF_8), "text/html", 200); } @Override - public ProxyResponse getServiceNodePage(DeploymentId deployment, String serviceName, DomainName node, Path subPath, Query query) { - return new ProxyResponse("<h1>OK</h1>".getBytes(StandardCharsets.UTF_8), "text/html", 200); + public ProxyResponse getServiceNodes(DeploymentId deployment) { + return new ProxyResponse("{\"json\":\"thank you very much\"}".getBytes(UTF_8), "application.json", 200); } @Override @@ -566,12 +522,12 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public InputStream getLogs(DeploymentId deployment, Map<String, String> queryParameters) { - return new ByteArrayInputStream(log.get().getBytes(StandardCharsets.UTF_8)); + return new ByteArrayInputStream(log.get().getBytes(UTF_8)); } @Override public ProxyResponse getApplicationPackageContent(DeploymentId deployment, Path path, URI requestUri) { - return new ProxyResponse(("{\"path\":\"/" + String.join("/", path.segments()) + "\"}").getBytes(StandardCharsets.UTF_8), "application/json", 200); + return new ProxyResponse(("{\"path\":\"/" + String.join("/", path.segments()) + "\"}").getBytes(UTF_8), "application/json", 200); } public void setLogStream(Supplier<String> log) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java index ef2b6f6f7e2..586056ce9dc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java @@ -3,9 +3,9 @@ package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.StringResponse; import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; -import com.yahoo.restapi.StringResponse; import java.io.InputStream; import java.util.Optional; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 7b67db39350..3f1ca3f9706 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -60,7 +60,7 @@ public class NodeRepositoryMock implements NodeRepository { @Override public void deleteNode(ZoneId zone, String hostname) { require(zone, hostname); - nodeRepository.get(zone).remove(HostName.from(hostname)); + nodeRepository.get(zone).remove(HostName.of(hostname)); } @Override @@ -246,8 +246,8 @@ public class NodeRepositoryMock implements NodeRepository { /** Add a fixed set of nodes to given zone */ public void addFixedNodes(ZoneId zone) { var nodeA = Node.builder() - .hostname(HostName.from("hostA")) - .parentHostname(HostName.from("parentHostA")) + .hostname(HostName.of("hostA")) + .parentHostname(HostName.of("parentHostA")) .state(Node.State.active) .type(NodeType.tenant) .owner(ApplicationId.from("tenant1", "app1", "default")) @@ -262,8 +262,8 @@ public class NodeRepositoryMock implements NodeRepository { .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) .build(); var nodeB = Node.builder() - .hostname(HostName.from("hostB")) - .parentHostname(HostName.from("parentHostB")) + .hostname(HostName.of("hostB")) + .parentHostname(HostName.of("parentHostB")) .state(Node.State.active) .type(NodeType.tenant) .owner(ApplicationId.from("tenant2", "app2", "default")) @@ -319,7 +319,7 @@ public class NodeRepositoryMock implements NodeRepository { } private Node require(ZoneId zone, String hostname) { - return require(zone, HostName.from(hostname)); + return require(zone, HostName.of(hostname)); } private Node require(ZoneId zone, HostName hostname) { @@ -329,7 +329,7 @@ public class NodeRepositoryMock implements NodeRepository { } private void patchNodes(ZoneId zone, String hostname, UnaryOperator<Node> patcher) { - patchNodes(zone, Optional.of(HostName.from(hostname)), patcher); + patchNodes(zone, Optional.of(HostName.of(hostname)), patcher); } private void patchNodes(DeploymentId deployment, Optional<HostName> hostname, UnaryOperator<Node> patcher) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 74c06d7ca1a..0afb1e67a85 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -4,8 +4,11 @@ package com.yahoo.vespa.hosted.controller.integration; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; +import com.yahoo.component.Version; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; @@ -46,6 +49,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainerMock; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.MockChangeRequestClient; +import java.time.Instant; + /** * A mock implementation of a {@link ServiceRegistry} for testing purposes. * @@ -54,11 +59,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.vcmr.MockChangeRequestC public class ServiceRegistryMock extends AbstractComponent implements ServiceRegistry { private final ManualClock clock = new ManualClock(); + private final ControllerVersion controllerVersion; private final ZoneRegistryMock zoneRegistryMock; private final ConfigServerMock configServerMock; private final MemoryNameService memoryNameService = new MemoryNameService(); private final MockMailer mockMailer = new MockMailer(); - private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(); + private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(clock); private final EndpointCertificateValidatorMock endpointCertificateValidatorMock = new EndpointCertificateValidatorMock(); private final MockMeteringClient mockMeteringClient = new MockMeteringClient(); private final MockContactRetriever mockContactRetriever = new MockContactRetriever(); @@ -91,6 +97,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg this.zoneRegistryMock = new ZoneRegistryMock(system); this.configServerMock = new ConfigServerMock(zoneRegistryMock); this.mockTesterCloud = new MockTesterCloud(nameService()); + this.clock.setInstant(Instant.ofEpochSecond(1600000000)); + this.controllerVersion = new ControllerVersion(Version.fromString("6.1.0"), "badb01", clock.instant()); } @Inject @@ -113,6 +121,16 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override + public ControllerVersion controllerVersion() { + return controllerVersion; + } + + @Override + public HostName getHostname() { + return HostName.of("test-controller"); + } + + @Override public MockMailer mailer() { return mockMailer; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index a4c30cca29e..5b8e25cbfe8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; @@ -41,9 +42,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); private final Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods = new HashMap<>(); private final Set<ZoneApi> reprovisionToUpgradeOs = new HashSet<>(); + private final SystemName system; // Don't even think about making it non-final! ƪ(`▿▿▿▿´ƪ) + + private List<? extends ZoneApi> zones; - private SystemName system; private UpgradePolicy upgradePolicy = null; /** @@ -53,11 +56,12 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry public ZoneRegistryMock(SystemName system) { this.system = system; if (system.isPublic()) { - this.zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), - ZoneApiMock.fromId("staging.aws-us-east-1c"), + this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"), + ZoneApiMock.fromId("staging.us-east-3"), ZoneApiMock.fromId("prod.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-eu-west-1a")); - setRoutingMethod(this.zones, RoutingMethod.exclusive); + ZoneApiMock.fromId("prod.aws-eu-west-1a"), + ZoneApiMock.fromId("dev.aws-us-east-1c")); + setRoutingMethod(this.zones, RoutingMethod.exclusive); } else { this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"), ZoneApiMock.fromId("staging.us-east-3"), @@ -95,11 +99,6 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return setZones(List.of(zone)); } - public ZoneRegistryMock setSystemName(SystemName system) { - this.system = system; - return this; - } - public ZoneRegistryMock setUpgradePolicy(UpgradePolicy upgradePolicy) { this.upgradePolicy = upgradePolicy; return this; @@ -202,6 +201,16 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override + public URI dashboardUrl(TenantName tenantName, ApplicationName applicationName) { + return URI.create("https://dashboard.tld/" + tenantName + "/" + applicationName); + } + + @Override + public URI dashboardUrl(TenantName tenantName) { + return URI.create("https://dashboard.tld/" + tenantName); + } + + @Override public URI dashboardUrl(RunId id) { return URI.create("https://dashboard.tld/" + id.application() + "/" + id.type().jobName() + "/" + id.number()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java index 0a2f5d9a236..c1d9c03819d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java @@ -9,8 +9,8 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.Test; @@ -83,6 +83,7 @@ public class ArchiveUriUpdaterTest { } private void deploy(DeploymentContext application, ZoneId zone) { - application.runJob(JobType.from(SystemName.Public, zone).orElseThrow(), new ApplicationPackage(new byte[0]), Version.fromString("7.1")); + application.runJob(JobType.deploymentTo(zone), new ApplicationPackage(new byte[0]), Version.fromString("7.1")); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java index 0e76c0375f2..7607a4f602d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java @@ -14,7 +14,7 @@ import java.time.Duration; import java.time.ZonedDateTime; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** * @author olaa diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java index 8daedc05e96..6ccd307f0d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java @@ -122,7 +122,7 @@ public class CloudEventTrackerTest { private Node createNode(String hostname, NodeType nodeType) { return Node.builder() - .hostname(HostName.from(hostname)) + .hostname(HostName.of(hostname)) .type(nodeType) .build(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java index ed00e7f5473..f768ab5e61b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; * @author ogronnesby */ public class CloudTrialExpirerTest { + private final ControllerTester tester = new ControllerTester(SystemName.PublicCd); private final DeploymentTester deploymentTester = new DeploymentTester(tester); private final CloudTrialExpirer expirer = new CloudTrialExpirer(tester.controller(), Duration.ofMinutes(5)); @@ -67,11 +68,21 @@ public class CloudTrialExpirerTest { @Test public void keep_inactive_trial_tenants_with_deployments() { registerTenant("with-deployments", "trial", Duration.ofDays(30)); - registerDeployment("with-deployments", "my-app", "default", "aws-us-east-1c"); + registerDeployment("with-deployments", "my-app", "default"); expirer.maintain(); assertPlan("with-deployments", "trial"); } + @Test + public void delete_tenants_with_applications_with_no_deployments() { + registerTenant("with-apps", "trial", Duration.ofDays(30)); + tester.createApplication("with-apps", "app1", "instance1"); + expirer.maintain(); + assertPlan("with-apps", "none"); + expirer.maintain(); + assertTrue(tester.controller().tenants().get("with-apps").isEmpty()); + } + private void registerTenant(String tenantName, String plan, Duration timeSinceLastLogin) { var name = TenantName.from(tenantName); tester.createTenant(tenantName, Tenant.Type.cloud); @@ -79,17 +90,12 @@ public class CloudTrialExpirerTest { tester.controller().tenants().updateLastLogin(name, List.of(LastLoginInfo.UserLevel.user), tester.controller().clock().instant().minus(timeSinceLastLogin)); } - private void registerDeployment(String tenantName, String appName, String instanceName, String regionName) { - var zone = ZoneApiMock.newBuilder() - .withSystem(tester.zoneRegistry().system()) - .withId("prod." + regionName) - .build(); - tester.zoneRegistry().setZones(zone); + private void registerDeployment(String tenantName, String appName, String instanceName) { var app = tester.createApplication(tenantName, appName, instanceName); var ctx = deploymentTester.newDeploymentContext(tenantName, appName, instanceName); var pkg = new ApplicationPackageBuilder() .instances("default") - .region(regionName) + .region("aws-us-east-1c") .trustDefaultCertificate() .build(); ctx.submit(pkg).deploy(); @@ -98,4 +104,5 @@ public class CloudTrialExpirerTest { private void assertPlan(String tenant, String planId) { assertEquals(planId, tester.serviceRegistry().billingController().getPlan(TenantName.from(tenant)).value()); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java index 10193b48837..294272041b6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock; import org.junit.Test; import java.time.Duration; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index 5f443759048..10a8bb79c2e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -5,10 +5,10 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; @@ -40,7 +40,7 @@ public class DeploymentExpirerTest { .build(); // Deploy dev - devApp.runJob(JobType.devUsEast1, appPackage); + devApp.runJob(DeploymentContext.devUsEast1, appPackage); // Deploy prod prodApp.submit(appPackage).deploy(); @@ -55,8 +55,8 @@ public class DeploymentExpirerTest { // Deploy dev unsuccessfully a few days before expiry tester.clock().advance(Duration.ofDays(12)); tester.configServer().throwOnNextPrepare(new RuntimeException(getClass().getSimpleName())); - tester.jobs().deploy(devApp.instanceId(), JobType.devUsEast1, Optional.empty(), appPackage); - Run lastRun = tester.jobs().last(devApp.instanceId(), JobType.devUsEast1).get(); + tester.jobs().deploy(devApp.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), appPackage); + Run lastRun = tester.jobs().last(devApp.instanceId(), DeploymentContext.devUsEast1).get(); assertSame(RunStatus.error, lastRun.status()); Deployment deployment = tester.applications().requireInstance(devApp.instanceId()) .deployments().get(devZone); @@ -72,7 +72,7 @@ public class DeploymentExpirerTest { // Dev application expires when enough time has passed since most recent attempt // Redeployments done by DeploymentUpgrader do not affect this tester.clock().advance(Duration.ofDays(12).plus(Duration.ofSeconds(1))); - tester.jobs().start(devApp.instanceId(), JobType.devUsEast1, lastRun.versions(), true, Optional.of("upgrade")); + tester.jobs().start(devApp.instanceId(), DeploymentContext.devUsEast1, lastRun.versions(), true, Optional.of("upgrade")); expirer.maintain(); assertEquals(0, permanentDeployments(devApp.instance())); assertEquals(1, permanentDeployments(prodApp.instance())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index 7c4203d253c..637a8832533 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -7,9 +7,9 @@ import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -21,9 +21,9 @@ import java.util.HashMap; import java.util.Map; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.canary; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.maxFailureAge; import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.maxInactivity; import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.upgradeGracePeriod; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index dc50b32b338..112519bb717 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -8,10 +8,10 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.Test; @@ -37,7 +37,7 @@ public class DeploymentMetricsMaintainerTest { @Test public void updates_metrics() { var application = tester.newDeploymentContext(); - application.runJob(JobType.devUsEast1, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); + application.runJob(DeploymentContext.devUsEast1, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); DeploymentMetricsMaintainer maintainer = maintainer(tester.controller()); Supplier<Application> app = application::application; @@ -51,7 +51,7 @@ public class DeploymentMetricsMaintainerTest { assertFalse("Never received any writes", deployment.get().activity().lastWritten().isPresent()); // Metrics are gathered and saved to application - application.runJob(JobType.devUsEast1, new ApplicationPackage(new byte[0]), Version.fromString("7.5.5")); + application.runJob(DeploymentContext.devUsEast1, new ApplicationPackage(new byte[0]), Version.fromString("7.5.5")); var metrics0 = Map.of(ClusterMetrics.QUERIES_PER_SECOND, 1D, ClusterMetrics.FEED_PER_SECOND, 2D, ClusterMetrics.DOCUMENT_COUNT, 3D, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java index cc428ebc94f..653ad2bb08a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java @@ -13,12 +13,11 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devUsEast1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentUpgrader.mostLikelyWeeHour; import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; /** diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 3b811d74f19..6bcb4284a14 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -2,8 +2,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock; @@ -17,9 +15,9 @@ import java.time.Duration; import java.util.List; import java.util.Optional; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; @@ -116,7 +114,7 @@ public class EndpointCertificateMaintainerTest { assertEquals(updatedMetadata.version(), originalMetadata.version()+1); // after another 4 days, we should force trigger deployment if it hasn't already happened - tester.clock().advance(Duration.ofDays(4)); + tester.clock().advance(Duration.ofDays(4).plusSeconds(1)); deploymentContext.assertNotRunning(productionUsWest1); assertEquals(1.0, maintainer.maintain(), 0.0000001); deploymentContext.assertRunning(productionUsWest1); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java index a06e15de3c7..890f0b41098 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java @@ -84,10 +84,10 @@ public class HostInfoUpdaterTest { // Updates node registered under a different hostname ZoneId zone = tester.zoneRegistry().zones().controllerUpgraded().all().ids().get(0); String hostnameSuffix = ".prod." + zone.value(); - Node configNode = Node.builder().hostname(HostName.from("cfg3" + hostnameSuffix)) + Node configNode = Node.builder().hostname(HostName.of("cfg3" + hostnameSuffix)) .type(NodeType.config) .build(); - Node configHost = Node.builder().hostname(HostName.from("cfghost3" + hostnameSuffix)) + Node configHost = Node.builder().hostname(HostName.of("cfghost3" + hostnameSuffix)) .type(NodeType.confighost) .build(); tester.serviceRegistry().configServer().nodeRepository().putNodes(zone, List.of(configNode, configHost)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 71a6fcb1d84..5fd1e8347ef 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -3,13 +3,12 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.JobMetrics; @@ -19,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Step.Status; import com.yahoo.vespa.hosted.controller.deployment.StepRunner; +import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import org.junit.Test; @@ -42,8 +42,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import java.util.stream.Collectors; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; @@ -77,10 +77,7 @@ public class JobRunnerTest { private static final ApplicationPackage applicationPackage = new ApplicationPackage(new byte[0]); private static final Versions versions = new Versions(Version.fromString("1.2.3"), - ApplicationVersion.from(new SourceRevision("repo", - "branch", - "bada55"), - 321), + RevisionId.forProduction(321), Optional.empty(), Optional.empty()); @@ -88,15 +85,16 @@ public class JobRunnerTest { public void multiThreadedExecutionFinishes() { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); - StepRunner stepRunner = (step, id) -> id.type() == stagingTest && step.get() == startTests? Optional.of(error) : Optional.of(running); + StepRunner stepRunner = (step, id) -> id.type().equals(stagingTest) && step.get() == startTests? Optional.of(error) : Optional.of(running); Phaser phaser = new Phaser(1); JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), phasedExecutor(phaser), stepRunner); TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + byte[] testPackageBytes = new byte[0]; + jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); - start(jobs, id, systemTest); + start(jobs, id, systemTest); try { start(jobs, id, systemTest); fail("Job is already running, so this should not be allowed!"); @@ -125,7 +123,8 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + byte[] testPackageBytes = new byte[0]; + jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); Supplier<Run> run = () -> jobs.last(id, systemTest).get(); start(jobs, id, systemTest); @@ -232,7 +231,8 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + byte[] testPackageBytes = new byte[0]; + jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); RunId runId = new RunId(id, systemTest, 1); start(jobs, id, systemTest); @@ -269,7 +269,8 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId instanceId = appId.defaultInstance(); JobId jobId = new JobId(instanceId, systemTest); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + byte[] testPackageBytes = new byte[0]; + jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); assertFalse(jobs.lastSuccess(jobId).isPresent()); for (int i = 0; i < jobs.historyLength(); i++) { @@ -364,7 +365,8 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + byte[] testPackageBytes = new byte[0]; + jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); start(jobs, id, systemTest); tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1))); @@ -381,7 +383,8 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + byte[] testPackageBytes = new byte[0]; + jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); for (Step step : JobProfile.of(systemTest).steps()) outcomes.put(step, running); @@ -409,6 +412,7 @@ public class JobRunnerTest { assertEquals(1, metric.getMetric(context::equals, JobMetrics.nodeAllocationFailure).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.endpointCertificateTimeout).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.testFailure).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.noTests).get().intValue()); } private void start(JobController jobs, ApplicationId id, JobType type) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 455e802e87b..64e3a95605a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -10,8 +10,11 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClientMock; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; @@ -37,9 +40,9 @@ import java.util.function.UnaryOperator; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -50,6 +53,7 @@ import static org.junit.Assert.assertTrue; public class MetricsReporterTest { private final MetricsMock metrics = new MetricsMock(); + private final ZmsClientMock zmsClient = new ZmsClientMock(new AthenzDbMock(), AthenzIdentities.from("mock.identity")); @Test public void audit_log_metric() { @@ -259,7 +263,7 @@ public class MetricsReporterTest { var context = tester.newDeploymentContext() .submit(applicationPackage) .deploy(); - assertEquals(1000, context.lastSubmission().get().buildTime().get().toEpochMilli()); + assertEquals(1000, context.application().revisions().get(context.lastSubmission().get()).buildTime().get().toEpochMilli()); MetricsReporter reporter = createReporter(tester.controller()); reporter.maintain(); @@ -483,7 +487,6 @@ public class MetricsReporterTest { @Test public void tenant_counter() { var tester = new ControllerTester(SystemName.Public); - tester.zoneRegistry().setSystemName(SystemName.Public); tester.createTenant("foo", Tenant.Type.cloud); tester.createTenant("bar", Tenant.Type.cloud); tester.createTenant("fix", Tenant.Type.cloud); @@ -554,6 +557,19 @@ public class MetricsReporterTest { assertEquals("Upgrade is overdue measure relative to window 3", Duration.ofHours(34).plusMinutes(30), metric.get()); } + @Test + public void zms_quota_metrics() { + var tester = new ControllerTester(); + var reporter = createReporter(tester.controller()); + reporter.maintain(); + + assertEquals(0.1, metrics.getMetric(d -> "subdomains".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get()); + assertEquals(0.2, metrics.getMetric(d -> "roles".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get()); + assertEquals(0.3, metrics.getMetric(d -> "policies".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get()); + assertEquals(0.4, metrics.getMetric(d -> "services".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get()); + assertEquals(0.5, metrics.getMetric(d -> "groups".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get()); + } + private void assertNodeCount(String metric, int n, Version version) { long nodeCount = metrics.getMetric((dimensions) -> version.toFullString().equals(dimensions.get("currentVersion")), metric) .stream() @@ -657,7 +673,7 @@ public class MetricsReporterTest { } private MetricsReporter createReporter(Controller controller) { - return new MetricsReporter(controller, metrics); + return new MetricsReporter(controller, metrics, zmsClient); } private static String appDimension(ApplicationId id) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 052a8bffab1..e989486c595 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -3,17 +3,13 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.deployment.Run; import org.junit.Test; -import java.time.Duration; -import java.util.List; import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -41,12 +37,12 @@ public class OutstandingChangeDeployerTest { assertFalse(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); app.submit(applicationPackage); - Optional<ApplicationVersion> revision = app.lastSubmission(); + Optional<RevisionId> revision = app.lastSubmission(); assertFalse(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); assertEquals(Change.of(version).with(revision.get()), app.instance().change()); app.submit(applicationPackage); - Optional<ApplicationVersion> outstanding = app.lastSubmission(); + Optional<RevisionId> outstanding = app.lastSubmission(); assertTrue(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); assertEquals(Change.of(version).with(revision.get()), app.instance().change()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggererTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggererTest.java index 76cc9f5773a..53e7dd7ca58 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggererTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggererTest.java @@ -22,7 +22,6 @@ import static java.time.DayOfWeek.MONDAY; import static java.time.DayOfWeek.THURSDAY; import static java.time.DayOfWeek.TUESDAY; import static java.time.DayOfWeek.WEDNESDAY; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index 6109890bae3..f9441f76a38 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -28,7 +28,9 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author olaa @@ -135,8 +137,8 @@ public class ResourceMeterMaintainerTest { Node.State.parked, Node.State.active) .map(state -> Node.builder() - .hostname(HostName.from("host" + state)) - .parentHostname(HostName.from("parenthost" + state)) + .hostname(HostName.of("host" + state)) + .parentHostname(HostName.of("parenthost" + state)) .state(state) .type(NodeType.tenant) .owner(ApplicationId.from("tenant1", "app1", "default")) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java index f58911a6114..1a6976034c5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -36,8 +36,8 @@ public class ResourceTagMaintainerTest { resourceTagMaintainer.maintain(); assertEquals(2, mockResourceTagger.getValues().size()); Map<HostName, ApplicationId> applicationForHost = mockResourceTagger.getValues().get(ZoneId.from("prod.region-2")); - assertEquals(ApplicationId.from("t1", "a1", "i1"), applicationForHost.get(HostName.from("parentHostA.prod.region-2"))); - assertEquals(SHARED_HOST_APPLICATION, applicationForHost.get(HostName.from("parentHostB.prod.region-2"))); + assertEquals(ApplicationId.from("t1", "a1", "i1"), applicationForHost.get(HostName.of("parentHostA.prod.region-2"))); + assertEquals(SHARED_HOST_APPLICATION, applicationForHost.get(HostName.of("parentHostB.prod.region-2"))); } private void setUpZones() { @@ -51,19 +51,19 @@ public class ResourceTagMaintainerTest { public void setNodes(ZoneId zone) { var hostA = Node.builder() - .hostname(HostName.from("parentHostA." + zone.value())) + .hostname(HostName.of("parentHostA." + zone.value())) .type(NodeType.host) .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) .build(); var nodeA = Node.builder() - .hostname(HostName.from("hostA." + zone.value())) + .hostname(HostName.of("hostA." + zone.value())) .type(NodeType.tenant) - .parentHostname(HostName.from("parentHostA." + zone.value())) + .parentHostname(HostName.of("parentHostA." + zone.value())) .owner(ApplicationId.from("tenant1", "app1", "default")) .build(); var hostB = Node.builder() - .hostname(HostName.from("parentHostB." + zone.value())) + .hostname(HostName.of("parentHostB." + zone.value())) .type(NodeType.host) .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) .build(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java index ccb2b6ebb74..bebecf8b52b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.RetriggerEntry; import org.junit.Test; @@ -34,11 +34,11 @@ public class RetriggerMaintainerTest { .build(); // Deploy app - devApp.runJob(JobType.devUsEast1, appPackage); + devApp.runJob(DeploymentContext.devUsEast1, appPackage); devApp.completeRollout(); // Trigger a run (to simulate a running job) - tester.deploymentTrigger().reTrigger(applicationId, JobType.devUsEast1, null); + tester.deploymentTrigger().reTrigger(applicationId, DeploymentContext.devUsEast1, null); // Add a job to the queue tester.deploymentTrigger().reTriggerOrAddToQueue(devApp.deploymentIdIn(ZoneId.from("dev", "us-east-1")), null); @@ -48,7 +48,7 @@ public class RetriggerMaintainerTest { assertEquals(1, retriggerEntries.size()); // Adding to queue triggers abort - devApp.jobAborted(JobType.devUsEast1); + devApp.jobAborted(DeploymentContext.devUsEast1); assertEquals(0, tester.jobs().active(applicationId).size()); // The maintainer runs and will actually trigger dev us-east, but keeps the entry in queue to verify it was actually run @@ -58,7 +58,7 @@ public class RetriggerMaintainerTest { assertEquals(1, tester.jobs().active(applicationId).size()); // Run outstanding jobs - devApp.runJob(JobType.devUsEast1); + devApp.runJob(DeploymentContext.devUsEast1); assertEquals(0, tester.jobs().active(applicationId).size()); // Run maintainer again, should find that the job has already run successfully and will remove the entry. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java index 0dbe31d921b..8b2bfe8ee95 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java @@ -35,7 +35,7 @@ public class SystemRoutingPolicyMaintainerTest { tester.configServer().putLoadBalancers(zone, List.of(new LoadBalancer("lb1", SystemApplication.configServer.id(), ClusterSpec.Id.from("config"), - Optional.of(HostName.from("lb1.example.com")), + Optional.of(HostName.of("lb1.example.com")), LoadBalancer.State.active, Optional.of("dns-zone-1")))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java index 81daf0cbcfe..7026d975010 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java @@ -5,9 +5,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.Test; @@ -35,11 +35,11 @@ public class TenantRoleMaintainerTest { .build(); // Deploy dev apps - devAppTenant1.runJob(JobType.devUsEast1, appPackage); - devAppTenant2.runJob(JobType.devUsEast1, appPackage); + devAppTenant1.runJob(DeploymentContext.devUsEast1, appPackage); + devAppTenant2.runJob(DeploymentContext.devUsEast1, appPackage); // Deploy perf apps - perfAppTenant1.runJob(JobType.perfUsEast3, appPackage); + perfAppTenant1.runJob(DeploymentContext.perfUsEast3, appPackage); // Deploy prod prodAppTenant2.submit(appPackage).deploy(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java index 08af46d8d33..c59155cb162 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java @@ -6,8 +6,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import org.junit.Test; @@ -33,7 +33,7 @@ public class TrafficShareUpdaterTest { ZoneId prod1 = ZoneId.from("prod", "ap-northeast-1"); ZoneId prod2 = ZoneId.from("prod", "us-east-3"); ZoneId prod3 = ZoneId.from("prod", "us-west-1"); - application.runJob(JobType.productionApNortheast1, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); + application.runJob(DeploymentContext.productionApNortheast1, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); // Single zone setQpsMetric(50.0, application.application().id().defaultInstance(), prod1, tester); @@ -42,7 +42,7 @@ public class TrafficShareUpdaterTest { assertTrafficFraction(1.0, 1.0, application.instanceId(), prod1, tester); // Two zones - application.runJob(JobType.productionUsEast3, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); + application.runJob(DeploymentContext.productionUsEast3, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); // - one cold setQpsMetric(50.0, application.application().id().defaultInstance(), prod1, tester); setQpsMetric(0.0, application.application().id().defaultInstance(), prod2, tester); @@ -59,7 +59,7 @@ public class TrafficShareUpdaterTest { assertTrafficFraction(0.47, 1.0, application.instanceId(), prod2, tester); // Three zones - application.runJob(JobType.productionUsWest1, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); + application.runJob(DeploymentContext.productionUsWest1, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); // - one cold setQpsMetric(53.0, application.application().id().defaultInstance(), prod1, tester); setQpsMetric(47.0, application.application().id().defaultInstance(), prod2, tester); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 78341682f75..185c1e8c891 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -5,10 +5,11 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -27,12 +28,12 @@ import java.util.OptionalInt; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devUsEast1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsCentral1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PIN; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; @@ -792,12 +793,12 @@ public class UpgraderTest { // New application change app.submit(applicationPackage("default")); - String applicationVersion = app.lastSubmission().get().id(); + RevisionId revision = app.lastSubmission().get(); // Application change recorded together with ongoing upgrade assertTrue("Change contains both upgrade and application change", app.instance().change().platform().get().equals(version) && - app.instance().change().application().get().id().equals(applicationVersion)); + app.instance().change().revision().get().equals(revision)); // Deployment completes app.runJob(systemTest).runJob(stagingTest) @@ -807,7 +808,7 @@ public class UpgraderTest { for (Deployment deployment : app.instance().deployments().values()) { assertEquals(version, deployment.version()); - assertEquals(applicationVersion, deployment.applicationVersion().id()); + assertEquals(revision, deployment.revision()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainerTest.java index e35c2058eb4..c1a9559ac46 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainerTest.java @@ -7,7 +7,7 @@ import org.junit.Test; import java.time.Duration; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** * @author olaa diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java index ad480bc1712..e7a6c3ea3c3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java @@ -37,9 +37,9 @@ public class VcmrMaintainerTest { private NodeRepositoryMock nodeRepo; private final ZoneId zoneId = ZoneId.from("prod.us-east-3"); private final ZoneId zone2 = ZoneId.from("prod.us-west-1"); - private final HostName host1 = HostName.from("host1"); - private final HostName host2 = HostName.from("host2"); - private final HostName host3 = HostName.from("host3"); + private final HostName host1 = HostName.of("host1"); + private final HostName host2 = HostName.of("host2"); + private final HostName host3 = HostName.of("host3"); private final String changeRequestId = "id123"; @Before @@ -227,6 +227,23 @@ public class VcmrMaintainerTest { assertEquals(State.PENDING_RETIREMENT, tenantAction2.getState()); } + @Test + public void out_of_sync_when_manual_reactivation() { + var nonRetiringNode = createNode(host1, NodeType.host, Node.State.active, false); + nodeRepo.putNodes(zoneId, nonRetiringNode); + + tester.curator().writeChangeRequest(inProgressChangeRequest()); + maintainer.maintain(); + + var writtenChangeRequest = tester.curator().readChangeRequest(changeRequestId).get(); + var actionPlan = writtenChangeRequest.getHostActionPlan(); + + var action = findHostAction(actionPlan, nonRetiringNode); + + assertEquals(State.OUT_OF_SYNC, action.getState()); + assertEquals(Status.OUT_OF_SYNC, writtenChangeRequest.getStatus()); + } + private VespaChangeRequest canceledChangeRequest() { return newChangeRequest(ChangeRequestSource.Status.CANCELED, State.RETIRED, State.RETIRING, ZonedDateTime.now()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index d0dbb23ad1b..4fbe21f11fb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -4,16 +4,18 @@ package com.yahoo.vespa.hosted.controller.notification; import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; +import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.notify.Notifier; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; @@ -63,12 +65,12 @@ public class NotificationsDbTest { notification(1201, Type.deployment, Level.error, NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), "instance msg"), notification(1301, Type.deployment, Level.warning, NotificationSource.from(new DeploymentId(ApplicationId.from(tenant.value(), "app2", "instance2"), ZoneId.from("prod", "us-north-2"))), "deployment msg"), notification(1401, Type.feedBlock, Level.error, NotificationSource.from(new DeploymentId(ApplicationId.from(tenant.value(), "app1", "instance1"), ZoneId.from("dev", "us-south-1")), ClusterSpec.Id.from("cluster1")), "cluster msg"), - notification(1501, Type.deployment, Level.warning, NotificationSource.from(new RunId(ApplicationId.from(tenant.value(), "app1", "instance1"), JobType.devUsEast1, 4)), "run id msg")); + notification(1501, Type.deployment, Level.warning, NotificationSource.from(new RunId(ApplicationId.from(tenant.value(), "app1", "instance1"), DeploymentContext.devUsEast1, 4)), "run id msg")); private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(12345)); - private final MockCuratorDb curatorDb = new MockCuratorDb(); + private final MockCuratorDb curatorDb = new MockCuratorDb(SystemName.Public); private final MockMailer mailer = new MockMailer(); - private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, mailer)); + private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, new ZoneRegistryMock(SystemName.cd), mailer)); @Test public void list_test() { @@ -76,8 +78,8 @@ public class NotificationsDbTest { assertEquals(notificationIndices(0, 1, 2, 3), notificationsDb.listNotifications(NotificationSource.from(tenant), true)); assertEquals(notificationIndices(2, 3), notificationsDb.listNotifications(NotificationSource.from(TenantAndApplicationId.from(tenant.value(), "app2")), false)); assertEquals(notificationIndices(4, 5), notificationsDb.listNotifications(NotificationSource.from(ApplicationId.from(tenant.value(), "app1", "instance1")), false)); - assertEquals(notificationIndices(5), notificationsDb.listNotifications(NotificationSource.from(new RunId(ApplicationId.from(tenant.value(), "app1", "instance1"), JobType.devUsEast1, 5)), false)); - assertEquals(List.of(), notificationsDb.listNotifications(NotificationSource.from(new RunId(ApplicationId.from(tenant.value(), "app1", "instance1"), JobType.productionUsEast3, 4)), false)); + assertEquals(notificationIndices(5), notificationsDb.listNotifications(NotificationSource.from(new RunId(ApplicationId.from(tenant.value(), "app1", "instance1"), DeploymentContext.devUsEast1, 5)), false)); + assertEquals(List.of(), notificationsDb.listNotifications(NotificationSource.from(new RunId(ApplicationId.from(tenant.value(), "app1", "instance1"), DeploymentContext.productionUsEast3, 4)), false)); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index e509a199c82..e60325a140a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -5,13 +5,15 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyUtils; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; @@ -22,6 +24,8 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; +import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; @@ -42,8 +46,6 @@ import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import static org.junit.Assert.assertEquals; @@ -83,26 +85,31 @@ public class ApplicationSerializerTest { OptionalLong projectId = OptionalLong.of(123L); + ApplicationId id1 = ApplicationId.from("t1", "a1", "i1"); + ApplicationId id3 = ApplicationId.from("t1", "a1", "i3"); List<Deployment> deployments = new ArrayList<>(); - ApplicationVersion applicationVersion1 = new ApplicationVersion(Optional.of(new SourceRevision("git@github:org/repo.git", "branch1", "commit1")), - OptionalLong.of(31), + ApplicationVersion applicationVersion1 = new ApplicationVersion(RevisionId.forProduction(31), + Optional.of(new SourceRevision("git@github:org/repo.git", "branch1", "commit1")), Optional.of("william@shakespeare"), Optional.of(Version.fromString("1.2.3")), Optional.of(Instant.ofEpochMilli(666)), Optional.empty(), Optional.of("best commit"), + Optional.of("hash1"), true, - Optional.of("hash1")); + false, + Optional.of("~(˘▾˘)~"), + 3); assertEquals("https://github/org/repo/tree/commit1", applicationVersion1.sourceUrl().get()); - ApplicationVersion applicationVersion2 = ApplicationVersion - .from(new SourceRevision("repo1", "branch1", "commit1"), 32, "a@b", - Version.fromString("6.3.1"), Instant.ofEpochMilli(496)); - SortedSet<ApplicationVersion> versions = new TreeSet<>(Set.of(applicationVersion2)); + ApplicationVersion applicationVersion2 = ApplicationVersion.from(RevisionId.forDevelopment(31, new JobId(id1, DeploymentContext.productionUsEast3)), + new SourceRevision("repo1", "branch1", "commit1"), "a@b", + Version.fromString("6.3.1"), + Instant.ofEpochMilli(496)); Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); - deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3), + deployments.add(new Deployment(zone1, applicationVersion1.id(), Version.fromString("1.2.3"), Instant.ofEpochMilli(3), DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty())); - deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), + deployments.add(new Deployment(zone2, applicationVersion2.id(), Version.fromString("1.2.3"), Instant.ofEpochMilli(5), new DeploymentMetrics(2, 3, 4, 5, 6, Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS)), Map.of(DeploymentMetrics.Warning.all, 3)), @@ -117,22 +124,20 @@ public class ApplicationSerializerTest { ZoneId.from("prod", "us-east-3"), RotationState.out), Instant.ofEpochMilli(42)))); - ApplicationId id1 = ApplicationId.from("t1", "a1", "i1"); - ApplicationId id3 = ApplicationId.from("t1", "a1", "i3"); + RevisionHistory revisions = RevisionHistory.ofRevisions(List.of(applicationVersion1), + Map.of(new JobId(id1, DeploymentContext.productionUsEast3), List.of(applicationVersion2))); List<Instance> instances = List.of(new Instance(id1, deployments, - Map.of(JobType.systemTest, Instant.ofEpochMilli(333)), + Map.of(DeploymentContext.systemTest, Instant.ofEpochMilli(333)), List.of(AssignedRotation.fromStrings("foo", "default", "my-rotation", Set.of("us-west-1"))), rotationStatus, - Change.of(new Version("6.1")), - Optional.of(applicationVersion2)), + Change.of(new Version("6.1"))), new Instance(id3, List.of(), Map.of(), List.of(), RotationStatus.EMPTY, - Change.of(Version.fromString("6.7")).withPin(), - Optional.empty())); + Change.of(Version.fromString("6.7")).withPin())); Application original = new Application(TenantAndApplicationId.from(id1), Instant.now().truncatedTo(ChronoUnit.MILLIS), @@ -145,23 +150,29 @@ public class ApplicationSerializerTest { new ApplicationMetrics(0.5, 0.9), Set.of(publicKey, otherPublicKey), projectId, - Optional.of(applicationVersion1), - versions, - instances); + revisions, + instances + ); Application serialized = APPLICATION_SERIALIZER.fromSlime(SlimeUtils.toJsonBytes(APPLICATION_SERIALIZER.toSlime(original))); assertEquals(original.id(), serialized.id()); assertEquals(original.createdAt(), serialized.createdAt()); - assertEquals(original.latestVersion(), serialized.latestVersion()); - assertEquals(original.latestVersion().get().authorEmail(), serialized.latestVersion().get().authorEmail()); - assertEquals(original.latestVersion().get().buildTime(), serialized.latestVersion().get().buildTime()); - assertEquals(original.latestVersion().get().sourceUrl(), serialized.latestVersion().get().sourceUrl()); - assertEquals(original.latestVersion().get().commit(), serialized.latestVersion().get().commit()); - assertEquals(original.latestVersion().get().bundleHash(), serialized.latestVersion().get().bundleHash()); - assertEquals(original.versions(), serialized.versions()); - assertEquals(original.versions(), serialized.versions()); - + assertEquals(applicationVersion1, serialized.revisions().last().get()); + assertEquals(applicationVersion1, serialized.revisions().get(serialized.instances().get(id1.instance()).deployments().get(zone1).revision())); + assertEquals(original.revisions().last(), serialized.revisions().last()); + assertEquals(original.revisions().last().get().authorEmail(), serialized.revisions().last().get().authorEmail()); + assertEquals(original.revisions().last().get().buildTime(), serialized.revisions().last().get().buildTime()); + assertEquals(original.revisions().last().get().sourceUrl(), serialized.revisions().last().get().sourceUrl()); + assertEquals(original.revisions().last().get().commit(), serialized.revisions().last().get().commit()); + assertEquals(original.revisions().last().get().bundleHash(), serialized.revisions().last().get().bundleHash()); + assertEquals(original.revisions().last().get().hasPackage(), serialized.revisions().last().get().hasPackage()); + assertEquals(original.revisions().last().get().shouldSkip(), serialized.revisions().last().get().shouldSkip()); + assertEquals(original.revisions().last().get().description(), serialized.revisions().last().get().description()); + assertEquals(original.revisions().last().get().risk(), serialized.revisions().last().get().risk()); + assertEquals(original.revisions().withPackage(), serialized.revisions().withPackage()); + assertEquals(original.revisions().production(), serialized.revisions().production()); + assertEquals(original.revisions().development(), serialized.revisions().development()); assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm()); assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm()); @@ -177,10 +188,10 @@ public class ApplicationSerializerTest { assertEquals(original.require(id1.instance()).deployments().get(zone1), serialized.require(id1.instance()).deployments().get(zone1)); assertEquals(original.require(id1.instance()).deployments().get(zone2), serialized.require(id1.instance()).deployments().get(zone2)); - assertEquals(original.require(id1.instance()).jobPause(JobType.systemTest), - serialized.require(id1.instance()).jobPause(JobType.systemTest)); - assertEquals(original.require(id1.instance()).jobPause(JobType.stagingTest), - serialized.require(id1.instance()).jobPause(JobType.stagingTest)); + assertEquals(original.require(id1.instance()).jobPause(DeploymentContext.systemTest), + serialized.require(id1.instance()).jobPause(DeploymentContext.systemTest)); + assertEquals(original.require(id1.instance()).jobPause(DeploymentContext.stagingTest), + serialized.require(id1.instance()).jobPause(DeploymentContext.stagingTest)); assertEquals(original.ownershipIssueId(), serialized.ownershipIssueId()); assertEquals(original.owner(), serialized.owner()); @@ -193,9 +204,6 @@ public class ApplicationSerializerTest { assertEquals(original.require(id1.instance()).change(), serialized.require(id1.instance()).change()); assertEquals(original.require(id3.instance()).change(), serialized.require(id3.instance()).change()); - assertEquals(original.require(id1.instance()).latestDeployed(), serialized.require(id1.instance()).latestDeployed()); - assertEquals(original.require(id3.instance()).latestDeployed(), serialized.require(id3.instance()).latestDeployed()); - // Test metrics assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE); assertEquals(original.metrics().writeServiceQuality(), serialized.metrics().writeServiceQuality(), Double.MIN_VALUE); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java index a9baeb9589d..0f7f97d333a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java @@ -2,11 +2,12 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Step; import org.junit.Test; @@ -28,11 +29,11 @@ public class BufferedLogStoreTest { public void chunkingAndFlush() { int chunkSize = 1 << 10; int maxChunks = 1 << 5; - CuratorDb buffer = new MockCuratorDb(); + CuratorDb buffer = new MockCuratorDb(SystemName.main); RunDataStore store = new MockRunDataStore(); BufferedLogStore logs = new BufferedLogStore(chunkSize, chunkSize * maxChunks, buffer, store); RunId id = new RunId(ApplicationId.from("tenant", "application", "instance"), - JobType.productionUsWest1, + DeploymentContext.productionUsWest1, 123); byte[] manyBytes = new byte[chunkSize / 2 + 1]; // One fits, and two (over-)fills. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ChangeRequestSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ChangeRequestSerializerTest.java index 29a722be76b..a52a6b81eb9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ChangeRequestSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ChangeRequestSerializerTest.java @@ -12,7 +12,7 @@ import java.time.Instant; import java.time.ZonedDateTime; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** * @author olaa diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializerTest.java index 1bcaad9ab8e..20cc82f0143 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializerTest.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import org.junit.Test; import java.time.Instant; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java index 383f5038416..c4aa2b5a18b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java @@ -7,7 +7,7 @@ import org.junit.Test; import java.util.List; import java.util.Optional; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; public class EndpointCertificateMetadataSerializerTest { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java index 4ef09bbfce0..5ecc22ffc5e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java @@ -32,11 +32,11 @@ public class NameServiceQueueSerializerTest { new CreateRecord(record1), new CreateRecords(List.of(record2)), new CreateRecords(List.of(new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), - new LatencyAliasTarget(HostName.from("alias1"), + new LatencyAliasTarget(HostName.of("alias1"), "dns-zone-01", ZoneId.from("prod", "us-north-1")).pack()), new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), - new LatencyAliasTarget(HostName.from("alias2"), + new LatencyAliasTarget(HostName.of("alias2"), "dns-zone-02", ZoneId.from("prod", "us-north-2")).pack())) ), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java index cbb595d2a3b..370b1cbe02c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java @@ -5,9 +5,9 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import org.junit.Test; @@ -25,6 +25,7 @@ public class NotificationsSerializerTest { @Test public void serialization_test() throws IOException { + NotificationsSerializer serializer = new NotificationsSerializer(); TenantName tenantName = TenantName.from("tenant1"); List<Notification> notifications = List.of( new Notification(Instant.ofEpochSecond(1234), @@ -35,10 +36,10 @@ public class NotificationsSerializerTest { new Notification(Instant.ofEpochSecond(2345), Notification.Type.deployment, Notification.Level.error, - NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app1", "instance1"), JobType.systemTest, 12)), + NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app1", "instance1"), DeploymentContext.systemTest, 12)), List.of("Failed to deploy: Node allocation failure"))); - Slime serialized = NotificationsSerializer.toSlime(notifications); + Slime serialized = serializer.toSlime(notifications); assertEquals("{\"notifications\":[" + "{" + "\"at\":1234000," + @@ -53,11 +54,12 @@ public class NotificationsSerializerTest { "\"messages\":[\"Failed to deploy: Node allocation failure\"]," + "\"application\":\"app1\"," + "\"instance\":\"instance1\"," + - "\"jobId\":\"system-test\"," + + "\"jobId\":\"test.us-east-1\"," + "\"runNumber\":12" + "}]}", new String(SlimeUtils.toJsonBytes(serialized))); - List<Notification> deserialized = NotificationsSerializer.fromSlime(tenantName, serialized); + List<Notification> deserialized = serializer.fromSlime(tenantName, serialized); assertEquals(notifications, deserialized); } + }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java index 482303a0e49..d50c071d98e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java @@ -30,12 +30,12 @@ public class OsVersionStatusSerializerTest { Map<OsVersion, List<NodeVersion>> versions = new LinkedHashMap<>(); versions.put(new OsVersion(version1, CloudName.defaultName()), List.of( - new NodeVersion(HostName.from("node1"), ZoneId.from("prod", "us-west"), version1, version2, Optional.of(Instant.ofEpochMilli(11))), - new NodeVersion(HostName.from("node2"), ZoneId.from("prod", "us-east"), version1, version2, Optional.of(Instant.ofEpochMilli(22))) + new NodeVersion(HostName.of("node1"), ZoneId.from("prod", "us-west"), version1, version2, Optional.of(Instant.ofEpochMilli(11))), + new NodeVersion(HostName.of("node2"), ZoneId.from("prod", "us-east"), version1, version2, Optional.of(Instant.ofEpochMilli(22))) )); versions.put(new OsVersion(version2, CloudName.defaultName()), List.of( - new NodeVersion(HostName.from("node3"), ZoneId.from("prod", "us-west"), version2, version2, Optional.of(Instant.ofEpochMilli(33))), - new NodeVersion(HostName.from("node4"), ZoneId.from("prod", "us-east"), version2, version2, Optional.of(Instant.ofEpochMilli(44))) + new NodeVersion(HostName.of("node3"), ZoneId.from("prod", "us-west"), version2, version2, Optional.of(Instant.ofEpochMilli(33))), + new NodeVersion(HostName.of("node4"), ZoneId.from("prod", "us-east"), version2, version2, Optional.of(Instant.ofEpochMilli(44))) )); OsVersionStatusSerializer serializer = new OsVersionStatusSerializer(new OsVersionSerializer(), new NodeVersionSerializer()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index 422188420bd..2e8ba84d4ff 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -38,13 +38,13 @@ public class RoutingPolicySerializerTest { ClusterSpec.Id.from("my-cluster2"), ZoneId.from("prod", "us-north-2")); var policies = List.of(new RoutingPolicy(id1, - HostName.from("long-and-ugly-name"), + HostName.of("long-and-ugly-name"), Optional.of("zone1"), instanceEndpoints, applicationEndpoints, new RoutingPolicy.Status(true, RoutingStatus.DEFAULT)), new RoutingPolicy(id2, - HostName.from("long-and-ugly-name-2"), + HostName.of("long-and-ugly-name-2"), Optional.empty(), instanceEndpoints, Set.of(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 237d54db20c..fd0ea50e50b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -6,11 +6,10 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.JobProfile; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; @@ -23,9 +22,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; -import java.util.Collections; +import java.util.List; import java.util.Optional; +import static com.yahoo.config.provision.SystemName.main; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; @@ -45,6 +45,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static com.yahoo.vespa.hosted.controller.deployment.Step.startStagingSetup; import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -55,7 +56,7 @@ public class RunSerializerTest { private static final RunSerializer serializer = new RunSerializer(); private static final Path runFile = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json"); private static final RunId id = new RunId(ApplicationId.from("tenant", "application", "default"), - JobType.productionUsEast3, + DeploymentContext.productionUsEast3, 112358); private static final Instant start = Instant.parse("2007-12-03T10:15:30.00Z"); @@ -82,30 +83,12 @@ public class RunSerializerTest { assertEquals(running, run.status()); assertEquals(3, run.lastTestLogEntry()); assertEquals(new Version(1, 2, 3), run.versions().targetPlatform()); - ApplicationVersion applicationVersion = ApplicationVersion.from(Optional.of(new SourceRevision("git@github.com:user/repo.git", - "master", - "f00bad")), - 123, - Optional.of("a@b"), - Optional.of(Version.fromString("6.3.1")), - Optional.of(Instant.ofEpochMilli(100)), - Optional.empty(), - Optional.empty(), - true, - Optional.empty()); - assertEquals(applicationVersion, run.versions().targetApplication()); - assertEquals(applicationVersion.authorEmail(), run.versions().targetApplication().authorEmail()); - assertEquals(applicationVersion.buildTime(), run.versions().targetApplication().buildTime()); - assertEquals(applicationVersion.compileVersion(), run.versions().targetApplication().compileVersion()); - assertEquals("f00bad", run.versions().targetApplication().commit().get()); - assertEquals("https://github.com/user/repo/tree/f00bad", run.versions().targetApplication().sourceUrl().get()); + RevisionId revision1 = RevisionId.forDevelopment(123, id.job()); + RevisionId revision2 = RevisionId.forProduction(122); + assertEquals(revision1, run.versions().targetRevision()); assertEquals("because", run.reason().get()); assertEquals(new Version(1, 2, 2), run.versions().sourcePlatform().get()); - assertEquals(ApplicationVersion.from(new SourceRevision("git@github.com:user/repo.git", - "master", - "badb17"), - 122), - run.versions().sourceApplication().get()); + assertEquals(revision2, run.versions().sourceRevision().get()); assertEquals(Optional.of(new ConvergenceSummary(1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233)), run.convergenceSummary()); assertEquals(X509CertificateUtils.fromPem("-----BEGIN CERTIFICATE-----\n" + @@ -143,7 +126,7 @@ public class RunSerializerTest { assertEquals(aborted, run.status()); assertTrue(run.hasEnded()); - Run phoenix = serializer.runsFromSlime(serializer.toSlime(Collections.singleton(run))).get(id); + Run phoenix = serializer.runsFromSlime(serializer.toSlime(List.of(run))).get(id); assertEquals(run.id(), phoenix.id()); assertEquals(run.start(), phoenix.start()); assertEquals(run.end(), phoenix.end()); @@ -156,6 +139,9 @@ public class RunSerializerTest { assertEquals(run.isDryRun(), phoenix.isDryRun()); assertEquals(run.reason(), phoenix.reason()); + assertEquals(new String(SlimeUtils.toJsonBytes(serializer.toSlime(run).get(), false), UTF_8), + new String(SlimeUtils.toJsonBytes(serializer.toSlime(phoenix).get(), false), UTF_8)); + Run initial = Run.initial(id, run.versions(), run.isRedeployment(), run.start(), JobProfile.production, Optional.empty()); assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializerTest.java index 127f6b0dcbf..8b2c291f751 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializerTest.java @@ -22,7 +22,7 @@ import java.security.cert.X509Certificate; import java.time.Instant; import java.time.temporal.ChronoUnit; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; public class SupportAccessSerializerTest { @@ -37,41 +37,41 @@ public class SupportAccessSerializerTest { .withAllowedUntil(hour(36), "andreer", hour(30)); @Language("JSON") - private final String expectedWithCertificates = "{\n" + - " \"history\": [\n" + - " {\n" + - " \"state\": \"allowed\",\n" + - " \"at\": \"1970-01-02T06:00:00Z\",\n" + - " \"until\": \"1970-01-02T12:00:00Z\",\n" + - " \"by\": \"andreer\"\n" + - " },\n" + - " {\n" + - " \"state\": \"disallowed\",\n" + - " \"at\": \"1970-01-01T22:00:00Z\",\n" + - " \"by\": \"andreer\"\n" + - " },\n" + - " {\n" + - " \"state\": \"allowed\",\n" + - " \"at\": \"1970-01-01T02:00:00Z\",\n" + - " \"until\": \"1970-01-02T00:00:00Z\",\n" + - " \"by\": \"andreer\"\n" + - " }\n" + - " ],\n" + - " \"grants\": [\n" + - " {\n" + - " \"requestor\": \"mortent\",\n" + - " \"certificate\": \"" + toPem(cert_7_to_19) + "\",\n" + - " \"notBefore\": \"1970-01-01T07:00:00Z\",\n" + - " \"notAfter\": \"1970-01-01T19:00:00Z\"\n" + - " },\n" + - " {\n" + - " \"requestor\": \"mortent\",\n" + - " \"certificate\": \"" + toPem(cert_3_to_4) + "\",\n" + - " \"notBefore\": \"1970-01-01T03:00:00Z\",\n" + - " \"notAfter\": \"1970-01-01T04:00:00Z\"\n" + - " }\n" + - " ]\n" + - "}\n"; + private final String expectedWithCertificates = "{\n" + + " \"history\": [\n" + + " {\n" + + " \"state\": \"allowed\",\n" + + " \"at\": \"1970-01-02T06:00:00Z\",\n" + + " \"until\": \"1970-01-02T12:00:00Z\",\n" + + " \"by\": \"andreer\"\n" + + " },\n" + + " {\n" + + " \"state\": \"disallowed\",\n" + + " \"at\": \"1970-01-01T22:00:00Z\",\n" + + " \"by\": \"andreer\"\n" + + " },\n" + + " {\n" + + " \"state\": \"allowed\",\n" + + " \"at\": \"1970-01-01T02:00:00Z\",\n" + + " \"until\": \"1970-01-02T00:00:00Z\",\n" + + " \"by\": \"andreer\"\n" + + " }\n" + + " ],\n" + + " \"grants\": [\n" + + " {\n" + + " \"requestor\": \"mortent\",\n" + + " \"certificate\": \"" + toPem(cert_7_to_19) + "\",\n" + + " \"notBefore\": \"1970-01-01T07:00:00Z\",\n" + + " \"notAfter\": \"1970-01-01T19:00:00Z\"\n" + + " },\n" + + " {\n" + + " \"requestor\": \"mortent\",\n" + + " \"certificate\": \"" + toPem(cert_3_to_4) + "\",\n" + + " \"notBefore\": \"1970-01-01T03:00:00Z\",\n" + + " \"notAfter\": \"1970-01-01T04:00:00Z\"\n" + + " }\n" + + " ]\n" + + "}\n"; public String toPem(X509Certificate cert) { return X509CertificateUtils.toPem(cert).replace("\n", "\\n"); @@ -81,14 +81,12 @@ public class SupportAccessSerializerTest { public void serialize_default() { var slime = SupportAccessSerializer.serializeCurrentState(SupportAccess.DISALLOWED_NO_HISTORY, Instant.EPOCH); assertSerialized(slime, "{\n" + - " \"state\": {\n" + - " \"supportAccess\": \"NOT_ALLOWED\"\n" + - " },\n" + - " \"history\": [\n" + - " ],\n" + - " \"grants\": [\n" + - " ]\n" + - "}\n"); + " \"state\": {\n" + + " \"supportAccess\": \"NOT_ALLOWED\"\n" + + " },\n" + + " \"history\": [ ],\n" + + " \"grants\": [ ]\n" + + "}\n"); } @Test @@ -101,46 +99,45 @@ public class SupportAccessSerializerTest { public void serialize_with_status() { var slime = SupportAccessSerializer.serializeCurrentState(supportAccessExample, hour(12)); assertSerialized(slime, - "{\n" + - " \"state\": {\n" + - " \"supportAccess\": \"ALLOWED\",\n" + - " \"until\": \"1970-01-02T12:00:00Z\",\n" + - " \"by\": \"andreer\"\n" + - " },\n" + - " \"history\": [\n" + - " {\n" + - " \"state\": \"allowed\",\n" + - " \"at\": \"1970-01-02T06:00:00Z\",\n" + - " \"until\": \"1970-01-02T12:00:00Z\",\n" + - " \"by\": \"andreer\"\n" + - " },\n" + - " {\n" + - " \"state\": \"disallowed\",\n" + - " \"at\": \"1970-01-01T22:00:00Z\",\n" + - " \"by\": \"andreer\"\n" + - " },\n" + - " {\n" + - " \"state\": \"allowed\",\n" + - " \"at\": \"1970-01-01T02:00:00Z\",\n" + - " \"until\": \"1970-01-02T00:00:00Z\",\n" + - " \"by\": \"andreer\"\n" + - " },\n" + - " {\n" + - " \"state\": \"grant\",\n" + - " \"at\": \"1970-01-01T03:00:00Z\",\n" + - " \"until\": \"1970-01-01T04:00:00Z\",\n" + - " \"by\": \"mortent\"\n" + - " }\n" + - " ],\n" + - " \"grants\": [\n" + - " {\n" + - " \"requestor\": \"mortent\",\n" + - " \"notBefore\": \"1970-01-01T07:00:00Z\",\n" + - " \"notAfter\": \"1970-01-01T19:00:00Z\"\n" + - " }" + - "\n" + - " ]\n" + - "}\n"); + "{\n" + + " \"state\": {\n" + + " \"supportAccess\": \"ALLOWED\",\n" + + " \"until\": \"1970-01-02T12:00:00Z\",\n" + + " \"by\": \"andreer\"\n" + + " },\n" + + " \"history\": [\n" + + " {\n" + + " \"state\": \"allowed\",\n" + + " \"at\": \"1970-01-02T06:00:00Z\",\n" + + " \"until\": \"1970-01-02T12:00:00Z\",\n" + + " \"by\": \"andreer\"\n" + + " },\n" + + " {\n" + + " \"state\": \"disallowed\",\n" + + " \"at\": \"1970-01-01T22:00:00Z\",\n" + + " \"by\": \"andreer\"\n" + + " },\n" + + " {\n" + + " \"state\": \"allowed\",\n" + + " \"at\": \"1970-01-01T02:00:00Z\",\n" + + " \"until\": \"1970-01-02T00:00:00Z\",\n" + + " \"by\": \"andreer\"\n" + + " },\n" + + " {\n" + + " \"state\": \"grant\",\n" + + " \"at\": \"1970-01-01T03:00:00Z\",\n" + + " \"until\": \"1970-01-01T04:00:00Z\",\n" + + " \"by\": \"mortent\"\n" + + " }\n" + + " ],\n" + + " \"grants\": [\n" + + " {\n" + + " \"requestor\": \"mortent\",\n" + + " \"notBefore\": \"1970-01-01T07:00:00Z\",\n" + + " \"notAfter\": \"1970-01-01T19:00:00Z\"\n" + + " }\n" + + " ]\n" + + "}\n"); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index e202d4a687d..73cfc6ad2f3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -55,7 +55,7 @@ public class VersionStatusSerializerTest { private static List<NodeVersion> nodeVersions(Version version, Version wantedVersion, String... hostnames) { var nodeVersions = new ArrayList<NodeVersion>(); for (var hostname : hostnames) { - nodeVersions.add(new NodeVersion(HostName.from(hostname), ZoneId.from("prod", "us-north-1"), version, wantedVersion, Optional.empty())); + nodeVersions.add(new NodeVersion(HostName.of(hostname), ZoneId.from("prod", "us-north-1"), version, wantedVersion, Optional.empty())); } return nodeVersions; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json index 29f748d5408..ec36f52c23a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json @@ -294,7 +294,7 @@ "deploymentJobs": { "jobStatus": [ { - "jobType": "staging-test", + "jobType": "staging.zone", "pausedUntil": 321 } ] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 85881fbfdbc..1216bcefab6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -1,7 +1,7 @@ [ { "id": "tenant:application:default", - "type": "production-us-east-3", + "type": "prod.us-east-3", "number": 112358, "start": 1196676930000, "sleepUntil": 1196676930100, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java index dd43f419624..c4fbf1aa3a5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; +import ai.vespa.http.HttpURL.Path; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.github.tomakehurst.wiremock.stubbing.Scenario; import com.yahoo.config.provision.SystemName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import ai.vespa.http.HttpURL.Path; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.yolean.concurrent.Sleeper; import org.apache.http.protocol.HttpContext; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java index 5827ef676d7..e6de60c859b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import com.yahoo.jdisc.http.HttpRequest; import ai.vespa.http.HttpURL.Path; +import com.yahoo.jdisc.http.HttpRequest; import org.junit.Test; import java.net.URI; @@ -19,9 +19,9 @@ public class ProxyRequestTest { @Test public void testBadUri() { - assertThrows("Request path '/path' does not end with proxy path '/zone/v2/'", - IllegalArgumentException.class, - () -> testRequest("http://domain.tld/path", "/zone/v2/")); + assertEquals("Request path '/path' does not end with proxy path '/zone/v2/'", + assertThrows(IllegalArgumentException.class, + () -> testRequest("http://domain.tld/path", "/zone/v2/")).getMessage()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java index 599827f2d03..32fe8ddecff 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import com.yahoo.jdisc.http.HttpRequest; import ai.vespa.http.HttpURL.Path; +import com.yahoo.jdisc.http.HttpRequest; import org.junit.Test; import java.io.ByteArrayOutputStream; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index ca27e9d23cd..07f00e8c989 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -9,7 +9,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.container.http.filter.FilterChainRepository; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; import com.yahoo.jdisc.http.filter.SecurityRequestFilterChain; -import com.yahoo.test.json.JsonTestHelper; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -17,20 +17,19 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; -import org.junit.ComparisonFailure; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.charset.CharacterCodingException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Optional; import java.util.function.Supplier; -import java.util.regex.Pattern; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * Provides testing of JSON container responses @@ -39,6 +38,8 @@ import static org.junit.Assert.assertEquals; */ public class ContainerTester { + private static final boolean writeResponses = false; + private final JDisc container; private final String responseFilePath; @@ -97,34 +98,31 @@ public class ContainerTester { private void assertResponse(Request request, File responseFile, int expectedStatusCode, boolean removeWhitespace, boolean compareJson) { String expectedResponse = readTestFile(responseFile.toString()); expectedResponse = include(expectedResponse); - if (removeWhitespace) expectedResponse = expectedResponse.replaceAll("(\"[^\"]*\")|\\s*", "$1"); // Remove whitespace FilterResult filterResult = invokeSecurityFilters(request); request = filterResult.request; Response response = filterResult.response != null ? filterResult.response : container.handleRequest(request); String responseString; try { responseString = response.getBodyAsString(); - } catch (CharacterCodingException e) { + } + catch (CharacterCodingException e) { throw new UncheckedIOException(e); } - if (expectedResponse.contains("(ignore)")) { - // Convert expected response to a literal pattern and replace any ignored field with a pattern that matches - // until the first stop character - String stopCharacters = "[^,:\\\\[\\\\]{}]"; - String expectedResponsePattern = Pattern.quote(expectedResponse) - .replaceAll("\"?\\(ignore\\)\"?", "\\\\E" + - stopCharacters + "*\\\\Q"); - if (!Pattern.matches(expectedResponsePattern, responseString)) { - throw new ComparisonFailure(responseFile + " (with ignored fields)", - expectedResponsePattern, responseString); + try { + if (responseFile.toString().endsWith(".json")) { + byte[] expected = SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlimeOrThrow(expectedResponse).get(), false); + byte[] actual = SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlimeOrThrow(responseString).get(), false); + if (writeResponses) writeTestFile(responseFile.toString(), actual); + else assertEquals(new String(expected, UTF_8), new String(actual, UTF_8)); } - } else { - if (compareJson) { - JsonTestHelper.assertJsonEquals(expectedResponse, responseString); - } else { - assertEquals(responseFile.toString(), expectedResponse, responseString); + else { // Not JSON? Let's do a verbatim comparison, then ... + if (writeResponses) writeTestFile(responseFile.toString(), responseString.getBytes(UTF_8)); + else assertEquals(expectedResponse, responseString); } } + catch (IOException e) { + fail("failed writing JSON: " + e); + } assertEquals("Status code", expectedStatusCode, response.getStatus()); } @@ -142,7 +140,7 @@ public class ContainerTester { public void assertResponse(Supplier<Request> request, String expectedResponse, int expectedStatusCode) { assertResponse(request, - (response) -> assertEquals(expectedResponse, new String(response.getBody(), StandardCharsets.UTF_8)), + (response) -> assertEquals(expectedResponse, new String(response.getBody(), UTF_8)), expectedStatusCode); } @@ -193,6 +191,14 @@ public class ContainerTester { return prefix + includedContent + postFix; } + private void writeTestFile(String name, byte[] content) { + try { + Files.write(Paths.get(responseFilePath, name), content); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + private String readTestFile(String name) { try { return new String(Files.readAllBytes(Paths.get(responseFilePath, name))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java index 2fd8026319b..324c9706df9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi; import ai.vespa.hosted.api.MultiPartStreamer; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.api.integration.user.User; +import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; @@ -12,8 +12,6 @@ import com.yahoo.yolean.Exceptions; import java.nio.charset.StandardCharsets; import java.security.Principal; -import java.util.HashMap; -import java.util.Map; import java.util.Set; import java.util.function.Supplier; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 6e81c4280c2..33b7500ceac 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -302,10 +302,11 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { @Test public void archive_uri_test() { - new DeploymentTester(new ControllerTester(tester)) - .newDeploymentContext(ApplicationId.from(tenantName, applicationName, InstanceName.defaultName())) - .submit() - .deploy(); + ControllerTester wrapped = new ControllerTester(tester); + wrapped.upgradeSystem(Version.fromString("7.1")); + new DeploymentTester(wrapped).newDeploymentContext(ApplicationId.from(tenantName, applicationName, InstanceName.defaultName())) + .submit() + .deploy(); tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), (response) -> assertFalse(response.getBodyAsString().contains("archiveAccessRole")), @@ -365,7 +366,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { request("/application/v4/tenant/scoober/application/unique/submit", POST) .data(data) .roles(Set.of(Role.developer(tenantName))), - "{\"message\":\"Application package version: 1.0.1-commit1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + "{\"message\":\"application build 1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isPresent()); } @@ -401,7 +402,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { .build(); new ControllerTester(tester).upgradeSystem(new Version("6.1")); tester.controller().jobController().deploy(ApplicationId.from("scoober", "albums", "default"), - JobType.productionAwsUsEast1c, + JobType.prod("aws-us-east-1c"), Optional.empty(), applicationPackage); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 67b201bdc9d..6bfbb044944 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -42,21 +41,17 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationActio import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; -import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; @@ -83,6 +78,7 @@ import java.io.File; import java.math.BigInteger; import java.net.URI; import java.security.cert.X509Certificate; +import java.time.Duration; import java.time.Instant; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; @@ -262,7 +258,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(entity) .userIdentity(HOSTED_VESPA_OPERATOR), "{\"message\":\"Deployment started in run 1 of production-us-east-3 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); - app1.runJob(JobType.productionUsEast3); + app1.runJob(DeploymentContext.productionUsEast3); tester.controller().applications().deactivate(app1.instanceId(), ZoneId.from("prod", "us-east-3")); // POST (deploy) an application to start a manual deployment to dev @@ -270,13 +266,13 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(entity) .userIdentity(USER_ID), "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); - app1.runJob(JobType.devUsEast1); + app1.runJob(DeploymentContext.devUsEast1); // POST (deploy) a job to restart a manual deployment to dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1", POST) .userIdentity(USER_ID), "{\"message\":\"Triggered dev-us-east-1 for tenant1.application1.instance1\"}"); - app1.runJob(JobType.devUsEast1); + app1.runJob(DeploymentContext.devUsEast1); // GET dev application package tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1/package", GET) @@ -333,9 +329,9 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(applicationPackageInstance1, 123)), - "{\"message\":\"Application package version: 1.0.1-commit1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + "{\"message\":\"application build 1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); - app1.runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsCentral1); + app1.runJob(DeploymentContext.systemTest).runJob(DeploymentContext.stagingTest).runJob(DeploymentContext.productionUsCentral1); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .withoutAthenzIdentity() @@ -362,7 +358,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(applicationPackage, 1000)), - "{\"message\":\"Application package version: 1.0.1-commit1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + "{\"message\":\"application build 1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); deploymentTester.triggerJobs(); @@ -371,7 +367,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("{ \"skipTests\": true, \"skipRevision\": true, \"skipUpgrade\": true }") .userIdentity(USER_ID), "{\"message\":\"Triggered production-us-west-1 for tenant2.application2.instance1, without revision and platform upgrade\"}"); - app2.runJob(JobType.productionUsWest1); + app2.runJob(DeploymentContext.productionUsWest1); // POST a re-triggering to force a production job to start with previous parameters tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/instance1/job/production-us-west-1", POST) @@ -522,18 +518,18 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST a roll-out of the latest application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/application", POST) .userIdentity(USER_ID), - "{\"message\":\"Triggered application change to 1.0.1-commit1 for tenant1.application1.instance1\"}"); + "{\"message\":\"Triggered revision change to build 1 for tenant1.application1.instance1\"}"); // POST a roll-out of a given revision tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/application", POST) .data("{ \"build\": 1 }") .userIdentity(USER_ID), - "{\"message\":\"Triggered application change to 1.0.1-commit1 for tenant1.application1.instance1\"}"); + "{\"message\":\"Triggered revision change to build 1 for tenant1.application1.instance1\"}"); // DELETE (cancel) ongoing change tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", DELETE) .userIdentity(HOSTED_VESPA_OPERATOR), - "{\"message\":\"Changed deployment from 'application change to 1.0.1-commit1' to 'no change' for tenant1.application1.instance1\"}"); + "{\"message\":\"Changed deployment from 'revision change to build 1' to 'no change' for tenant1.application1.instance1\"}"); // DELETE (cancel) again is a no-op tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", DELETE) @@ -634,13 +630,13 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"enabled\":true,\"clusters\":[{\"name\":\"cluster\",\"pending\":[{\"type\":\"type\",\"requiredGeneration\":100}],\"ready\":[{\"type\":\"type\",\"readyAtMillis\":345,\"startedAtMillis\":456,\"endedAtMillis\":567,\"state\":\"failed\",\"message\":\"(#`д´)ノ\",\"progress\":0.1,\"speed\":1.0}]}]}"); // POST to request a service dump - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/node/host-tenant1:application1:instance1-prod.us-central-1/service-dump", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/node/host-tenant1.application1.instance1-prod.us-central-1/service-dump", POST) .userIdentity(HOSTED_VESPA_OPERATOR) .data("{\"configId\":\"default/container.1\",\"artifacts\":[\"jvm-dump\"],\"dumpOptions\":{\"duration\":30}}"), "{\"message\":\"Request created\"}"); // GET to get status of service dump - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/node/host-tenant1:application1:instance1-prod.us-central-1/service-dump", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/node/host-tenant1.application1.instance1-prod.us-central-1/service-dump", GET) .userIdentity(HOSTED_VESPA_OPERATOR), "{\"createdMillis\":" + tester.controller().clock().millis() + ",\"configId\":\"default/container.1\"" + ",\"artifacts\":[\"jvm-dump\"],\"dumpOptions\":{\"duration\":30}}"); @@ -698,14 +694,16 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), new File("suspended.json")); - // GET services - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service", GET) + + // GET service/state/v1 + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/service/storagenode/host.com/state/v1/?foo=bar", GET) .userIdentity(USER_ID), - new File("services.json")); - // GET service - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", GET) + new File("service")); + + // GET orchestrator + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/orchestrator", GET) .userIdentity(USER_ID), - new File("service.json")); + "{\"json\":\"thank you very much\"}"); // DELETE application with active deployments fails tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) @@ -731,11 +729,11 @@ public class ApplicationApiTest extends ControllerContainerTest { // Setup for test config tests tester.controller().jobController().deploy(ApplicationId.from("tenant1", "application1", "default"), - JobType.productionUsCentral1, + DeploymentContext.productionUsCentral1, Optional.empty(), applicationPackageDefault); tester.controller().jobController().deploy(ApplicationId.from("tenant1", "application1", "my-user"), - JobType.devUsEast1, + DeploymentContext.devUsEast1, Optional.empty(), applicationPackageDefault); @@ -777,7 +775,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(packageWithService, 123)), - "{\"message\":\"Application package version: 1.0.2-commit1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + "{\"message\":\"application build 2, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/diff/2", GET).userIdentity(HOSTED_VESPA_OPERATOR), (response) -> assertTrue(response.getBodyAsString(), @@ -822,7 +820,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .screwdriverIdentity(SCREWDRIVER_ID) .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(streamer::data))) .data(streamer), - "{\"message\":\"Application package version: 1.0.3-commit1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + "{\"message\":\"application build 3, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); // Sixth attempt has a multi-instance deployment spec, and is accepted. ApplicationPackage multiInstanceSpec = new ApplicationPackageBuilder() @@ -835,9 +833,14 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(multiInstanceSpec, 123)), - "{\"message\":\"Application package version: 1.0.4-commit1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + "{\"message\":\"application build 4, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + // DELETE submitted build, to mark it as non-deployable + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit/2", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"Marked build '2' as non-deployable\"}"); + // GET deployment job overview, after triggering system and staging test jobs. assertEquals(2, tester.controller().applications().deploymentTrigger().triggerReadyJobs()); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job", GET) @@ -1053,6 +1056,7 @@ public class ApplicationApiTest extends ControllerContainerTest { public void testDeployWithApplicationPackage() { // Setup addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); + deploymentTester.controllerTester().upgradeController(new Version("6.2")); // POST (deploy) a system application with an application package MultiPartStreamer noAppEntity = createApplicationDeployData(Optional.empty()); @@ -1088,7 +1092,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Create tenant and deploy var app = deploymentTester.newDeploymentContext("tenant1", "application1", "instance1"); app.submit(applicationPackage).deploy(); - tester.controller().jobController().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage); + tester.controller().jobController().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), applicationPackage); assertEquals(Set.of(ZoneId.from("prod.us-west-1"), ZoneId.from("prod.us-east-3"), ZoneId.from("prod.eu-west-1"), ZoneId.from("dev.us-east-1")), app.instance().deployments().keySet()); @@ -1194,7 +1198,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET non-existent application package tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET).userIdentity(HOSTED_VESPA_OPERATOR), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"No application package has been submitted for 'tenant1.application1'\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"no application package has been submitted for tenant1.application1\"}", 404); // GET non-existent application package of specific build @@ -1202,11 +1206,11 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(applicationPackageInstance1, 1000)), - "{\"message\":\"Application package version: 1.0.1-commit1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + "{\"message\":\"application build 1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET) .properties(Map.of("build", "42")) .userIdentity(HOSTED_VESPA_OPERATOR), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"No application package found for 'tenant1.application1' with build number 42\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"No build 42 found for tenant1.application1\"}", 404); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deployment", DELETE).userIdentity(USER_ID).oAuthCredentials(OKTA_CREDENTIALS), "{\"message\":\"All deployments removed\"}"); @@ -1215,7 +1219,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET) .properties(Map.of("build", "foobar")) .userIdentity(HOSTED_VESPA_OPERATOR), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid build number: For input string: \\\"foobar\\\"\"}", + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"invalid value for request parameter 'build': For input string: \\\"foobar\\\"\"}", 400); // POST (deploy) an application to legacy deploy path @@ -1417,7 +1421,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit/", POST) .data(createApplicationSubmissionData(applicationPackage, 123)) .screwdriverIdentity(screwdriverId), - "{\"message\":\"Application package version: 1.0.1-commit1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + "{\"message\":\"application build 1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); } @Test @@ -1606,11 +1610,13 @@ public class ApplicationApiTest extends ControllerContainerTest { ); // Should be 1 available grant + tester.serviceRegistry().clock().advance(Duration.ofSeconds(1)); + now = tester.serviceRegistry().clock().instant(); List<SupportAccessGrant> activeGrants = tester.controller().supportAccess().activeGrantsFor(new DeploymentId(ApplicationId.fromSerializedForm("tenant1:application1:instance1"), zone)); assertEquals(1, activeGrants.size()); // Adding grant should trigger job - app.assertRunning(JobType.productionUsWest1); + app.assertRunning(DeploymentContext.productionUsWest1); // DELETE removes access String disallowedResponse = grantResponse @@ -1622,7 +1628,7 @@ public class ApplicationApiTest extends ControllerContainerTest { ); // Revoking access should trigger job - app.assertRunning(JobType.productionUsWest1); + app.assertRunning(DeploymentContext.productionUsWest1); // Should be no available grant activeGrants = tester.controller().supportAccess().activeGrantsFor(new DeploymentId(ApplicationId.fromSerializedForm("tenant1:application1:instance1"), zone)); @@ -1630,32 +1636,6 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void testServiceView() { - createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); - String serviceApi="/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service"; - // Not allowed to request apis not listed in feature flag allowed-service-view-apis. e.g /document/v1 - tester.assertResponse(request(serviceApi + "/storagenode-awe3slno6mmq2fye191y324jl/document/v1/", GET) - .userIdentity(USER_ID) - .oAuthCredentials(OKTA_CREDENTIALS), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Nothing at path '/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/document/v1/'\"}", - 404); - - // Test path traversal - tester.assertResponse(request(serviceApi + "/storagenode-awe3slno6mmq2fye191y324jl/state/v1/../../document/v1/", GET) - .userIdentity(USER_ID) - .oAuthCredentials(OKTA_CREDENTIALS), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Nothing at path '/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/document/v1/'\"}", - 404); - - // Test urlencoded path traversal - tester.assertResponse(request(serviceApi + "/storagenode-awe3slno6mmq2fye191y324jl/state%2Fv1%2F..%2F..%2Fdocument%2Fv1%2F", GET) - .userIdentity(USER_ID) - .oAuthCredentials(OKTA_CREDENTIALS), - accessDenied, - 403); - } - - @Test public void create_application_on_deploy() { // Setup createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); @@ -1715,7 +1695,8 @@ public class ApplicationApiTest extends ControllerContainerTest { static MultiPartStreamer createApplicationSubmissionData(ApplicationPackage applicationPackage, long projectId) { return new MultiPartStreamer().addJson(EnvironmentResource.SUBMIT_OPTIONS, "{\"repository\":\"repository1\",\"branch\":\"master\",\"commit\":\"commit1\"," - + "\"projectId\":" + projectId + ",\"authorEmail\":\"a@b\"}") + + "\"projectId\":" + projectId + ",\"authorEmail\":\"a@b\"," + + "\"description\":\"my best commit yet\",\"risk\":9001}") .addBytes(EnvironmentResource.APPLICATION_ZIP, applicationPackage.zippedContent()) .addBytes(EnvironmentResource.APPLICATION_TEST_ZIP, "content".getBytes()); } @@ -1855,7 +1836,7 @@ public class ApplicationApiTest extends ControllerContainerTest { Notification.Level.warning, "Something something deprecated..."); tester.controller().notificationsDb().setNotification( - NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app2", "instance1"), JobType.systemTest, 12)), + NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app2", "instance1"), DeploymentContext.systemTest, 12)), Notification.Type.deployment, Notification.Level.error, "Failed to deploy: Node allocation failure"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 4935ab22586..d2698afdc48 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -4,18 +4,20 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.component.Version; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.test.json.JsonTestHelper; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.notification.Notification.Type; +import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import org.junit.Test; import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -25,18 +27,19 @@ import java.time.Instant; import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devAwsUsEast2a; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsCentral1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devAwsUsEast2a; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devUsEast1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsCentral1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsCentral1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; /** @@ -65,14 +68,14 @@ public class JobControllerApiHandlerHelperTest { // Revision 1 gets deployed everywhere. app.submit(applicationPackage).deploy(); - ApplicationVersion revision1 = app.lastSubmission().get(); + RevisionId revision1 = app.lastSubmission().get(); assertEquals(1000, tester.application().projectId().getAsLong()); // System test includes test report assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), systemTest).get().id(), "0"), "system-test-log.json"); tester.clock().advance(Duration.ofMillis(1000)); // Revision 2 gets deployed everywhere except in us-east-3. - ApplicationVersion revision2 = app.submit(applicationPackage).lastSubmission().get(); + RevisionId revision2 = app.submit(applicationPackage).lastSubmission().get(); app.runJob(systemTest); app.runJob(stagingTest); app.runJob(productionUsCentral1); @@ -89,9 +92,9 @@ public class JobControllerApiHandlerHelperTest { tester.clock().advance(Duration.ofHours(4).plusSeconds(1)); tester.runner().run(); assertEquals(installationFailed, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); - assertEquals(revision2, app.deployment(productionUsCentral1.zone(tester.controller().system())).applicationVersion()); - assertEquals(revision1, app.deployment(productionUsEast3.zone(tester.controller().system())).applicationVersion()); - assertEquals(revision2, app.deployment(productionUsWest1.zone(tester.controller().system())).applicationVersion()); + assertEquals(revision2, app.deployment(productionUsCentral1.zone()).revision()); + assertEquals(revision1, app.deployment(productionUsEast3.zone()).revision()); + assertEquals(revision2, app.deployment(productionUsWest1.zone()).revision()); tester.clock().advance(Duration.ofMillis(1000)); @@ -104,7 +107,7 @@ public class JobControllerApiHandlerHelperTest { assertEquals(running, tester.jobs().last(app.instanceId(), stagingTest).get().status()); // Staging deployment expires and the job fails, and is immediately retried. - tester.controller().applications().deactivate(app.instanceId(), stagingTest.zone(tester.controller().system())); + tester.controller().applications().deactivate(app.instanceId(), stagingTest.zone()); tester.runner().run(); assertEquals(installationFailed, tester.jobs().last(app.instanceId(), stagingTest).get().status()); @@ -113,7 +116,7 @@ public class JobControllerApiHandlerHelperTest { tester.triggerJobs(); tester.runner().run(); assertEquals(running, tester.jobs().last(app.instanceId(), stagingTest).get().status()); - tester.controller().applications().deactivate(app.instanceId(), stagingTest.zone(tester.controller().system())); + tester.controller().applications().deactivate(app.instanceId(), stagingTest.zone()); tester.runner().run(); assertEquals(installationFailed, tester.jobs().last(app.instanceId(), stagingTest).get().status()); @@ -131,14 +134,14 @@ public class JobControllerApiHandlerHelperTest { // Only us-east-3 is verified, on revision1. // staging-test has 5 runs: one success without sources on revision1, one success from revision1 to revision2, // one success from revision2 to revision3 and two failures from revision1 to revision3. - assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(app.instanceId(), stagingTest), Optional.empty(), URI.create("https://some.url:43/root")), "staging-runs.json"); + assertResponse(JobControllerApiHandlerHelper.runResponse(app.application(), tester.jobs().runs(app.instanceId(), stagingTest), Optional.empty(), URI.create("https://some.url:43/root")), "staging-runs.json"); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), stagingTest).get().id(), "0"), "staging-test-log.json"); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), productionUsEast3).get().id(), "0"), "us-east-3-log-without-first.json"); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root/")), "overview.json"); var userApp = tester.newDeploymentContext(app.instanceId().tenant().value(), app.instanceId().application().value(), "user"); userApp.runJob(devAwsUsEast2a, applicationPackage); - assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), Optional.empty(), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json"); + assertResponse(JobControllerApiHandlerHelper.runResponse(app.application(), tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), Optional.empty(), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json"); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), userApp.instanceId(), URI.create("https://some.url:43/root/")), "overview-user-instance.json"); assertResponse(JobControllerApiHandlerHelper.overviewResponse(tester.controller(), app.application().id(), URI.create("https://some.url:43/root/")), "deployment-overview-2.json"); } @@ -149,8 +152,8 @@ public class JobControllerApiHandlerHelperTest { var app = tester.newDeploymentContext(); tester.clock().setInstant(Instant.EPOCH); - ZoneId zone = JobType.devUsEast1.zone(tester.controller().system()); - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage()); + ZoneId zone = DeploymentContext.devUsEast1.zone(); + tester.jobs().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), applicationPackage()); tester.configServer().setLogStream(() -> "1554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), devUsEast1).get().id(), null), "dev-us-east-1-log-first-part.json"); @@ -159,7 +162,7 @@ public class JobControllerApiHandlerHelperTest { tester.runner().run(); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), devUsEast1).get().id(), "8"), "dev-us-east-1-log-second-part.json"); - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage()); + tester.jobs().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), applicationPackage()); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root")), "dev-overview.json"); } @@ -189,17 +192,16 @@ public class JobControllerApiHandlerHelperTest { "jobs-direct-deployment.json"); } - private void compare(HttpResponse response, String expected) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - response.render(baos); - JsonTestHelper.assertJsonEquals(baos.toString(), expected); - } - private void assertResponse(HttpResponse response, String fileName) { try { Path path = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/").resolve(fileName); - String expected = Files.readString(path); - compare(response, expected); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.render(baos); + byte[] actualJson = SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlimeOrThrow(baos.toByteArray()).get(), false); + // Files.write(path, actualJson); + byte[] expected = Files.readAllBytes(path); + assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlimeOrThrow(expected).get(), false), UTF_8), + new String(actualJson, UTF_8)); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java deleted file mode 100644 index c69cd51e20d..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.io.IOUtils; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.serviceview.bindings.ApplicationView; -import com.yahoo.vespa.serviceview.bindings.ClusterView; -import com.yahoo.vespa.serviceview.bindings.ServiceView; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.util.Collections; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class ServiceApiResponseTest { - - private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/"; - - @Test - public void testServiceViewResponse() throws URISyntaxException, IOException { - ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - ApplicationId.from("tenant1", "application1", "default"), - Collections.singletonList(new URI("config-server1")), - new URI("http://server1:4080/request/path?foo=bar")); - ApplicationView applicationView = new ApplicationView(); - ClusterView clusterView = new ClusterView(); - clusterView.type = "container"; - clusterView.name = "cluster1"; - clusterView.url = "cluster-url"; - ServiceView serviceView = new ServiceView(); - serviceView.url = null; - serviceView.serviceType = "container"; - serviceView.serviceName = "service1"; - serviceView.configId = "configId1"; - serviceView.host = "host1"; - serviceView.legacyStatusPages = "legacyPages"; - clusterView.services = Collections.singletonList(serviceView); - applicationView.clusters = Collections.singletonList(clusterView); - response.setResponse(applicationView); - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - response.render(stream); - Slime responseSlime = SlimeUtils.jsonToSlime(stream.toByteArray()); - Slime expectedSlime = SlimeUtils.jsonToSlime(IOUtils.readFile(new File(responseFiles + "service-api-response.json")).getBytes(StandardCharsets.UTF_8)); - - assertEquals("service-api-response.json", - new String(SlimeUtils.toJsonBytes(expectedSlime), StandardCharsets.UTF_8), - new String(SlimeUtils.toJsonBytes(responseSlime), StandardCharsets.UTF_8)); - } - - @Test - public void testServiceViewResponseWithURLs() throws URISyntaxException, IOException { - ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - ApplicationId.from("tenant2", "application2", "default"), - Collections.singletonList(new URI("http://cfg1.test/")), - new URI("http://cfg1.test/serviceview/v1/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/service/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1")); - ApplicationView applicationView = new ApplicationView(); - ClusterView clusterView = new ClusterView(); - clusterView.type = "container"; - clusterView.name = "cluster1"; - clusterView.url = "http://cfg1.test/serviceview/v1/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/service/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1/health"; - ServiceView serviceView = new ServiceView(); - serviceView.url = null; - serviceView.serviceType = "container"; - serviceView.serviceName = "service1"; - serviceView.configId = "configId1"; - serviceView.host = "host1"; - serviceView.legacyStatusPages = "legacyPages"; - clusterView.services = Collections.singletonList(serviceView); - applicationView.clusters = Collections.singletonList(clusterView); - response.setResponse(applicationView); - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - response.render(stream); - Slime responseSlime = SlimeUtils.jsonToSlime(stream.toByteArray()); - Slime expectedSlime = SlimeUtils.jsonToSlime(IOUtils.readFile(new File(responseFiles + "service-api-response-with-urls.json")).getBytes(StandardCharsets.UTF_8)); - - assertEquals("service-api-response.json", - new String(SlimeUtils.toJsonBytes(expectedSlime), StandardCharsets.UTF_8), - new String(SlimeUtils.toJsonBytes(responseSlime), StandardCharsets.UTF_8)); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json index fc40a9ce692..d90e03a1439 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json @@ -1,7 +1,7 @@ { "clusters": { "default": { - "type":"container", + "type": "container", "min": { "nodes": 2, "groups": 1, @@ -13,7 +13,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": "(ignore)" + "cost": 0.09 }, "max": { "nodes": 2, @@ -26,7 +26,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": "(ignore)" + "cost": 0.35 }, "current": { "nodes": 2, @@ -39,7 +39,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": "(ignore)" + "cost": 0.18 }, "target": { "nodes": 2, @@ -52,7 +52,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": "(ignore)" + "cost": 0.24 }, "utilization": { "cpu": 0.1, @@ -78,7 +78,7 @@ "diskSpeed": "fast", "storageType": "any" }, - "cost": "(ignore)" + "cost": 0.0 }, "to": { "nodes": 2, @@ -91,7 +91,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": "(ignore)" + "cost": 0.18 }, "at": 1234, "completion": 2234 @@ -101,7 +101,7 @@ "autoscalingStatus": "Cluster is ideally scaled", "scalingDuration": 360000, "maxQueryGrowthRate": 0.7, - "currentQueryFractionOfMax":0.3 + "currentQueryFractionOfMax": 0.3 } } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json index 2479f127f92..74b9abb1e3b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json @@ -1,12 +1,12 @@ [ { "tenant": "tenant1", - "application":"application1", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "application": "application1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", "instances": [ { - "instance":"instance1", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" + "instance": "instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json index 422c8c122fa..88a92bf7810 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json @@ -1,7 +1,7 @@ { "nodes": [ { - "hostname": "host-tenant1:application1:instance1-prod.us-central-1", + "hostname": "host-tenant1.application1.instance1-prod.us-central-1", "state": "active", "orchestration": "unorchestrated", "version": "6.1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json index 2ee72f150e5..d124d80fe03 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json @@ -2,11 +2,11 @@ "tenant": "tenant1", "application": "application1", "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/job/", - "instances": [], - "pemDeployKeys": [], + "instances": [ ], + "pemDeployKeys": [ ], "metrics": { "queryServiceQuality": 0.0, "writeServiceQuality": 0.0 }, - "activity": {} + "activity": { } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json index c8f5b7bf50a..019fdcb553a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json @@ -13,7 +13,7 @@ "instances": [ { "instance": "default", - "deployments": [] + "deployments": [ ] }, { "instance": "instance1", @@ -25,9 +25,9 @@ "commit": "commit1" } }, - "changeBlockers": [], + "changeBlockers": [ ], "rotationId": "rotation-id-2", - "deployments": [] + "deployments": [ ] } ], "pemDeployKeys": [ @@ -37,5 +37,5 @@ "queryServiceQuality": 0.0, "writeServiceQuality": 0.0 }, - "activity": {} + "activity": { } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json index 7aae1815dac..0b673a5852e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json @@ -12,7 +12,7 @@ "instances": [ { "instance": "default", - "deployments": [] + "deployments": [ ] }, { "instance": "instance1", @@ -24,15 +24,15 @@ "commit": "commit1" } }, - "changeBlockers": [], + "changeBlockers": [ ], "rotationId": "rotation-id-2", - "deployments": [] + "deployments": [ ] } ], - "pemDeployKeys": [], + "pemDeployKeys": [ ], "metrics": { "queryServiceQuality": 0.0, "writeServiceQuality": 0.0 }, - "activity": {} + "activity": { } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json index 06b48064b94..f2736102968 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json @@ -1,9 +1,9 @@ { - "revisionId":"(ignore)", - "applicationZipSize":"(ignore)", - "prepareMessages":[], - "configChangeActions":{ - "restart":[], - "refeed":[] + "revisionId": "da39a3ee5e6b4b0d3255bfef95601890afd80709", + "applicationZipSize": 0, + "prepareMessages": [ ], + "configChangeActions": { + "restart": [ ], + "refeed": [ ] } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json index 74f41524d3e..5268b7b56cb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json @@ -17,9 +17,10 @@ "clusters": "http://localhost:8080/application/v4/tenant/scoober/application/albums/instance/default/environment/prod/region/aws-us-east-1c/clusters", "nodes": "http://localhost:8080/zone/v2/prod/aws-us-east-1c/nodes/v2/node/?recursive=true&application=scoober.albums.default", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=aws-us-east-1c&application=scoober.albums", - "version": "(ignore)", + "version": "7.1.0", "revision": "1.0.1-commit1", - "deployTimeEpochMs": "(ignore)", + "build": 1, + "deployTimeEpochMs": 1600000000000, "screwdriverId": "1000", "applicationVersion": { "build": 1, @@ -28,8 +29,8 @@ "commit": "commit1" }, "status": "complete", - "quota": "(ignore)", - "activity": {}, + "quota": 1.062, + "activity": { }, "metrics": { "queriesPerSecond": 0.0, "writesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted-2.json index 8ea3f318d1d..74d5bf454aa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted-2.json @@ -1,4 +1,4 @@ { "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1.myuser. This may take about 15 minutes the first time.", "run": 1 -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index a1ce3aa240e..481997102d2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -5,7 +5,7 @@ "steps": [ { "type": "instance", - "dependencies": [], + "dependencies": [ ], "declared": true, "instance": "default", "readyAt": 0, @@ -24,7 +24,12 @@ "upgrade": true, "available": [ { - "platform": "7.1.0" + "platform": "7.1.0", + "upgrade": true + }, + { + "platform": "6.1.0", + "upgrade": false } ], "blockers": [ @@ -95,6 +100,22 @@ "sourceUrl": "repository1/tree/commit1", "commit": "commit1" } + }, + { + "application": { + "build": 2, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } + }, + { + "application": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } } ], "blockers": [ @@ -130,7 +151,7 @@ }, { "type": "test", - "dependencies": [], + "dependencies": [ ], "declared": false, "instance": "default", "readyAt": 0, @@ -138,7 +159,7 @@ "url": "https://some.url:43/instance/default/job/system-test", "environment": "test", "region": "test.us-east-1", - "toRun": [], + "toRun": [ ], "runs": [ { "id": 3, @@ -318,7 +339,7 @@ }, { "type": "test", - "dependencies": [], + "dependencies": [ ], "declared": true, "instance": "default", "readyAt": 15153000, @@ -769,7 +790,7 @@ "sourceUrl": "repository1/tree/commit1", "commit": "commit1" }, - "toRun": [], + "toRun": [ ], "runs": [ { "id": 3, @@ -1230,19 +1251,22 @@ "build": 3, "compileVersion": "6.1.0", "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" + "commit": "commit1", + "deployable": true }, { "build": 2, "compileVersion": "6.1.0", "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" + "commit": "commit1", + "deployable": true }, { "build": 1, "compileVersion": "6.1.0", "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" + "commit": "commit1", + "deployable": true } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 782819ef6c6..81d363aa3e8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -5,7 +5,7 @@ "steps": [ { "type": "instance", - "dependencies": [], + "dependencies": [ ], "declared": true, "instance": "instance1", "readyAt": 0, @@ -20,14 +20,15 @@ "latestVersions": { "platform": { "platform": "6.1.0", - "at": "(ignore)", + "at": 1600000000000, "upgrade": true, "available": [ { - "platform": "6.1.0" + "platform": "6.1.0", + "upgrade": true } ], - "blockers": [] + "blockers": [ ] }, "application": { "application": { @@ -46,15 +47,23 @@ "sourceUrl": "repository1/tree/commit1", "commit": "commit1" } + }, + { + "application": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } } ], - "blockers": [] + "blockers": [ ] } } }, { "type": "test", - "dependencies": [], + "dependencies": [ ], "declared": false, "instance": "instance1", "readyAt": 0, @@ -62,12 +71,12 @@ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", "environment": "test", "region": "test.us-east-1", - "toRun": [], + "toRun": [ ], "runs": [ { "id": 2, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2", - "start": "(ignore)", + "start": 1600000000000, "status": "running", "versions": { "targetPlatform": "6.1.0", @@ -124,8 +133,8 @@ { "id": 1, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1", - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -183,7 +192,7 @@ }, { "type": "test", - "dependencies": [], + "dependencies": [ ], "declared": false, "instance": "instance1", "readyAt": 0, @@ -191,12 +200,12 @@ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test", "environment": "staging", "region": "staging.us-east-3", - "toRun": [], + "toRun": [ ], "runs": [ { "id": 2, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/2", - "start": "(ignore)", + "start": 1600000000000, "status": "running", "versions": { "targetPlatform": "6.1.0", @@ -269,8 +278,8 @@ { "id": 1, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/1", - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -370,8 +379,8 @@ { "id": 1, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/run/1", - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -427,7 +436,7 @@ { "id": 1, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/run/1", - "start": "(ignore)", + "start": 1600000000000, "status": "running", "versions": { "targetPlatform": "6.1.0", @@ -483,7 +492,7 @@ { "id": 2, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/2", - "start": "(ignore)", + "start": 1600000000000, "status": "running", "versions": { "targetPlatform": "6.1.0", @@ -512,8 +521,8 @@ { "id": 1, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1", - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -547,18 +556,19 @@ ], "declared": true, "instance": "instance2", - "deploying": {}, + "deploying": { }, "latestVersions": { "platform": { "platform": "6.1.0", - "at": "(ignore)", + "at": 1600000000000, "upgrade": true, "available": [ { - "platform": "6.1.0" + "platform": "6.1.0", + "upgrade": true } ], - "blockers": [] + "blockers": [ ] }, "application": { "application": { @@ -580,14 +590,6 @@ }, { "application": { - "build": 2, - "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - } - }, - { - "application": { "build": 1, "compileVersion": "6.1.0", "sourceUrl": "repository1/tree/commit1", @@ -595,7 +597,7 @@ } } ], - "blockers": [] + "blockers": [ ] } } }, @@ -623,7 +625,7 @@ } } ], - "runs": [] + "runs": [ ] }, { "type": "deployment", @@ -649,7 +651,7 @@ } } ], - "runs": [] + "runs": [ ] }, { "type": "deployment", @@ -675,7 +677,7 @@ } } ], - "runs": [] + "runs": [ ] } ], "builds": [ @@ -683,26 +685,37 @@ "build": 4, "compileVersion": "6.1.0", "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" + "commit": "commit1", + "description": "my best commit yet", + "risk": 9001, + "deployable": true }, { "build": 3, "compileVersion": "6.1.0", "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" + "commit": "commit1", + "description": "my best commit yet", + "risk": 9001, + "deployable": false }, { "build": 2, "compileVersion": "6.1.0", "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" + "commit": "commit1", + "description": "my best commit yet", + "risk": 9001, + "deployable": false }, { "build": 1, "compileVersion": "6.1.0", "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" + "commit": "commit1", + "description": "my best commit yet", + "risk": 9001, + "deployable": true } ] } - diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json index 3f70ae1e303..7244cf1493a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -14,12 +14,13 @@ "legacy": false } ], - "clusters":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/clusters", + "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/clusters", "nodes": "http://localhost:8080/zone/v2/prod/us-west-1/nodes/v2/node/?recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-west-1&application=tenant1.application1.instance1", - "version": "(ignore)", + "version": "6.1.0", "revision": "1.0.1-commit1", - "deployTimeEpochMs": "(ignore)", + "build": 1, + "deployTimeEpochMs": 1600000000000, "screwdriverId": "1000", "applicationVersion": { "build": 1, @@ -28,8 +29,8 @@ "commit": "commit1" }, "status": "complete", - "quota": "(ignore)", - "activity": {}, + "quota": 1.062, + "activity": { }, "metrics": { "queriesPerSecond": 0.0, "writesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json index 860fe541682..7244cf1493a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json @@ -14,12 +14,13 @@ "legacy": false } ], - "clusters":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/clusters", + "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/clusters", "nodes": "http://localhost:8080/zone/v2/prod/us-west-1/nodes/v2/node/?recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-west-1&application=tenant1.application1.instance1", "version": "6.1.0", "revision": "1.0.1-commit1", - "deployTimeEpochMs": "(ignore)", + "build": 1, + "deployTimeEpochMs": 1600000000000, "screwdriverId": "1000", "applicationVersion": { "build": 1, @@ -28,8 +29,8 @@ "commit": "commit1" }, "status": "complete", - "quota": "(ignore)", - "activity": {}, + "quota": 1.062, + "activity": { }, "metrics": { "queriesPerSecond": 0.0, "writesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 315b1af25c7..1448b79385b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -33,9 +33,10 @@ "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/clusters", "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/?recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", - "version": "(ignore)", - "revision": "(ignore)", - "deployTimeEpochMs": "(ignore)", + "version": "6.1.0", + "revision": "1.0.1-commit1", + "build": 1, + "deployTimeEpochMs": 1600000000000, "screwdriverId": "123", "endpointStatus": [ { @@ -43,7 +44,7 @@ "rotationId": "rotation-id-1", "clusterId": "foo", "status": "UNKNOWN", - "lastUpdated": "(ignore)" + "lastUpdated": 0 } ], "applicationVersion": { @@ -53,7 +54,7 @@ "commit": "commit1" }, "status": "complete", - "quota": "(ignore)", + "quota": 1.062, "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json index e2d7423f161..ea0e6a4b442 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -4,20 +4,22 @@ "jobName": "dev-us-east-1", "runs": [ { + "id": 2, + "url": "https://some.url:43/root/run/2", + "start": 0, + "status": "running", "versions": { + "targetPlatform": "6.1.0", "targetApplication": { "build": 2, "compileVersion": "6.1.0" }, - "targetPlatform": "6.1.0", + "sourcePlatform": "6.1.0", "sourceApplication": { "build": 1, "compileVersion": "6.1.0" - }, - "sourcePlatform": "6.1.0" + } }, - "start": 0, - "id": 2, "steps": [ { "name": "deployReal", @@ -31,21 +33,21 @@ "name": "copyVespaLogs", "status": "unfinished" } - ], - "url": "https://some.url:43/root/run/2", - "status": "running" + ] }, { + "id": 1, + "url": "https://some.url:43/root/run/1", + "start": 0, + "end": 0, + "status": "success", "versions": { + "targetPlatform": "6.1.0", "targetApplication": { "build": 1, "compileVersion": "6.1.0" - }, - "targetPlatform": "6.1.0" + } }, - "start": 0, - "end": 0, - "id": 1, "steps": [ { "name": "deployReal", @@ -59,9 +61,7 @@ "name": "copyVespaLogs", "status": "succeeded" } - ], - "url": "https://some.url:43/root/run/1", - "status": "success" + ] } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json index 3ef993c6589..63869ecfba8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json @@ -6,7 +6,7 @@ { "at": 0, "type": "info", - "message": "Deploying platform version 6.1 and application version 1.0.1 ..." + "message": "Deploying platform version 6.1 and application dev build 1 for dev-us-east-1 of default ..." }, { "at": 0, @@ -28,7 +28,7 @@ { "at": 0, "type": "info", - "message": "host-tenant:application:default-dev.us-east-1: unorchestrated" + "message": "host-tenant.application.default-dev.us-east-1: unorchestrated" }, { "at": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json index 2df97a6c765..4b97410b21c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json @@ -1 +1,3 @@ -{"message":"Successfully set tenant1.application1.instance1 in prod.us-west-1 in service"} +{ + "message": "Successfully set tenant1.application1.instance1 in prod.us-west-1 in service" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json index 6a41b0000e4..e7f5c2407bd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json @@ -1 +1,3 @@ -{"message":"Successfully set tenant1.application1.instance1 in prod.us-west-1 out of service"} +{ + "message": "Successfully set tenant1.application1.instance1 in prod.us-west-1 out of service" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-reference-default.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-reference-default.json index 7117cc22507..cf964b0a1ae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-reference-default.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-reference-default.json @@ -1,6 +1,6 @@ { "tenant": "tenant1", - "application":"application1", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default" + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-reference.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-reference.json index 60243633614..e25f992ab25 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-reference.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-reference.json @@ -1,6 +1,6 @@ { "tenant": "tenant1", - "application":"application1", - "instance":"instance1", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" + "application": "application1", + "instance": "instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json index afac12a191b..3d722168d52 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json @@ -6,7 +6,7 @@ "sourceUrl": "repository1/tree/commit1", "commit": "commit1", "projectId": 1000, - "changeBlockers": [], + "changeBlockers": [ ], "instances": [ { "environment": "prod", @@ -15,10 +15,10 @@ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1" } ], - "pemDeployKeys": [], + "pemDeployKeys": [ ], "metrics": { "queryServiceQuality": 0.0, "writeServiceQuality": 0.0 }, - "activity": {} + "activity": { } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json index b98de97856d..b94d8fda83a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json @@ -57,7 +57,7 @@ "rotationId": "rotation-id-1", "clusterId": "foo", "status": "UNKNOWN", - "lastUpdated": "(ignore)" + "lastUpdated": 0 } ], "environment": "prod", @@ -74,7 +74,7 @@ "region": "us-west-1" } ], - "pemDeployKeys": [], + "pemDeployKeys": [ ], "metrics": { "queryServiceQuality": 0.5, "writeServiceQuality": 0.7 diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json index a697c667ab0..4bd328f605f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json @@ -41,8 +41,122 @@ ], "rotationId": "rotation-id-1", "instances": [ - @include(dev-us-east-1.json), - @include(prod-us-central-1.json), + { + "tenant": "tenant1", + "application": "application1", + "instance": "instance1", + "environment": "dev", + "region": "us-east-1", + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1.application1.tenant1.us-east-1.dev.vespa.oath.cloud/", + "scope": "zone", + "routingMethod": "sharedLayer4", + "legacy": false + } + ], + "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/dev/region/us-east-1/clusters", + "nodes": "http://localhost:8080/zone/v2/dev/us-east-1/nodes/v2/node/?recursive=true&application=tenant1.application1.instance1", + "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-east-1&application=tenant1.application1.instance1", + "version": "6.1.0", + "revision": "1.0.1", + "build": 1, + "deployTimeEpochMs": 1600000000000, + "screwdriverId": "123", + "status": "complete", + "quota": 1.062, + "activity": { + "lastQueried": 1527848130000, + "lastWritten": 1527848130000, + "lastQueriesPerSecond": 1.0, + "lastWritesPerSecond": 2.0 + }, + "metrics": { + "queriesPerSecond": 1.0, + "writesPerSecond": 2.0, + "documentCount": 3.0, + "queryLatencyMillis": 4.0, + "writeLatencyMillis": 5.0, + "lastUpdated": 123123 + } + }, + { + "bcpStatus": { + "rotationStatus": "UNKNOWN" + }, + "tenant": "tenant1", + "application": "application1", + "instance": "instance1", + "environment": "prod", + "region": "us-central-1", + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1.application1.tenant1.us-central-1.vespa.oath.cloud/", + "scope": "zone", + "routingMethod": "sharedLayer4", + "legacy": false + }, + { + "cluster": "foo", + "tls": true, + "url": "https://instance1.application1.tenant1.global.vespa.oath.cloud/", + "scope": "global", + "routingMethod": "sharedLayer4", + "legacy": false + }, + { + "cluster": "foo", + "tls": true, + "url": "https://a0.application1.tenant1.us-central-1-r.vespa.oath.cloud/", + "scope": "application", + "routingMethod": "sharedLayer4", + "legacy": false + } + ], + "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/clusters", + "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/?recursive=true&application=tenant1.application1.instance1", + "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", + "version": "6.1.0", + "revision": "1.0.1-commit1", + "build": 1, + "deployTimeEpochMs": 1600000000000, + "screwdriverId": "123", + "endpointStatus": [ + { + "endpointId": "default", + "rotationId": "rotation-id-1", + "clusterId": "foo", + "status": "UNKNOWN", + "lastUpdated": 0 + } + ], + "applicationVersion": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "status": "complete", + "quota": 1.062, + "activity": { + "lastQueried": 1527848130000, + "lastWritten": 1527848130000, + "lastQueriesPerSecond": 1.0, + "lastWritesPerSecond": 2.0 + }, + "metrics": { + "queriesPerSecond": 1.0, + "writesPerSecond": 2.0, + "documentCount": 3.0, + "queryLatencyMillis": 4.0, + "writeLatencyMillis": 5.0, + "lastUpdated": 123123 + } + }, { "environment": "prod", "region": "us-east-3" @@ -52,7 +166,7 @@ "region": "us-west-1" } ], - "pemDeployKeys": [], + "pemDeployKeys": [ ], "metrics": { "queryServiceQuality": 0.5, "writeServiceQuality": 0.7 diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json index 25f6ed21466..ed916b1406c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json @@ -1,3 +1,3 @@ { - "deployment": [] + "deployment": [ ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index e8c2660f999..12430b67539 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -6,8 +6,8 @@ { "id": 2, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/run/2", - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -15,7 +15,7 @@ "build": 1, "compileVersion": "6.1.0" }, - "sourcePlatform":"6.1.0", + "sourcePlatform": "6.1.0", "sourceApplication": { "build": 1, "compileVersion": "6.1.0" @@ -39,8 +39,8 @@ { "id": 1, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/run/1", - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success", "versions": { "targetPlatform": "6.1.0", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-applicationPackage.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-applicationPackage.json index c0833ae0f05..82725213aaa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-applicationPackage.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-applicationPackage.json @@ -1,14 +1,14 @@ { "notifications": [ { - "at": "(ignore)", + "at": 1600000000000, "level": "warning", "type": "applicationPackage", "tenant": "tenant1", "application": "app1" }, { - "at": "(ignore)", + "at": 1600000000000, "level": "warning", "type": "applicationPackage", "tenant": "tenant2", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json index 277831f2a9f..556440c40d5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json @@ -1,7 +1,7 @@ { "notifications": [ { - "at": "(ignore)", + "at": 1600000000000, "level": "error", "type": "deployment", "messages": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json index 33755843486..1a731dfe4a9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json @@ -1,7 +1,7 @@ { "notifications": [ { - "at": "(ignore)", + "at": 1600000000000, "level": "warning", "type": "applicationPackage", "messages": [ @@ -10,7 +10,7 @@ "application": "app1" }, { - "at": "(ignore)", + "at": 1600000000000, "level": "error", "type": "deployment", "messages": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index b0556e39630..6522d91800c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -4,16 +4,18 @@ "jobName": "dev-aws-us-east-2a", "runs": [ { + "id": 1, + "url": "https://some.url:43/root//run/1", + "start": 14503000, + "end": 14503000, + "status": "success", "versions": { + "targetPlatform": "7.1.0", "targetApplication": { "build": 1, "compileVersion": "6.1.0" - }, - "targetPlatform": "7.1.0" + } }, - "start": 14503000, - "end": 14503000, - "id": 1, "steps": [ { "name": "deployReal", @@ -27,9 +29,7 @@ "name": "copyVespaLogs", "status": "succeeded" } - ], - "url": "https://some.url:43/root//run/1", - "status": "success" + ] } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json index 25f6ed21466..ed916b1406c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json @@ -1,3 +1,3 @@ { - "deployment": [] + "deployment": [ ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json index 99ab2b34615..bfbeda9233d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json @@ -1,5 +1,222 @@ - [ - @include(tenant1-recursive.json), - @include(tenant2.json) + { + "tenant": "tenant1", + "type": "ATHENS", + "athensDomain": "domain1", + "property": "property1", + "applications": [ + { + "tenant": "tenant1", + "application": "application1", + "instance": "instance1", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1", + "projectId": 123, + "deploying": { + "revision": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } + }, + "changeBlockers": [ + { + "versions": true, + "revisions": false, + "timeZone": "UTC", + "days": [ + 1, + 2, + 3, + 4, + 5 + ], + "hours": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8 + ] + } + ], + "rotationId": "rotation-id-1", + "instances": [ + { + "tenant": "tenant1", + "application": "application1", + "instance": "instance1", + "environment": "dev", + "region": "us-east-1", + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1.application1.tenant1.us-east-1.dev.vespa.oath.cloud/", + "scope": "zone", + "routingMethod": "sharedLayer4", + "legacy": false + } + ], + "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/dev/region/us-east-1/clusters", + "nodes": "http://localhost:8080/zone/v2/dev/us-east-1/nodes/v2/node/?recursive=true&application=tenant1.application1.instance1", + "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-east-1&application=tenant1.application1.instance1", + "version": "6.1.0", + "revision": "1.0.1", + "build": 1, + "deployTimeEpochMs": 1600000000000, + "screwdriverId": "123", + "status": "complete", + "quota": 1.062, + "activity": { + "lastQueried": 1527848130000, + "lastWritten": 1527848130000, + "lastQueriesPerSecond": 1.0, + "lastWritesPerSecond": 2.0 + }, + "metrics": { + "queriesPerSecond": 1.0, + "writesPerSecond": 2.0, + "documentCount": 3.0, + "queryLatencyMillis": 4.0, + "writeLatencyMillis": 5.0, + "lastUpdated": 123123 + } + }, + { + "bcpStatus": { + "rotationStatus": "UNKNOWN" + }, + "tenant": "tenant1", + "application": "application1", + "instance": "instance1", + "environment": "prod", + "region": "us-central-1", + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1.application1.tenant1.us-central-1.vespa.oath.cloud/", + "scope": "zone", + "routingMethod": "sharedLayer4", + "legacy": false + }, + { + "cluster": "foo", + "tls": true, + "url": "https://instance1.application1.tenant1.global.vespa.oath.cloud/", + "scope": "global", + "routingMethod": "sharedLayer4", + "legacy": false + }, + { + "cluster": "foo", + "tls": true, + "url": "https://a0.application1.tenant1.us-central-1-r.vespa.oath.cloud/", + "scope": "application", + "routingMethod": "sharedLayer4", + "legacy": false + } + ], + "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/clusters", + "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/?recursive=true&application=tenant1.application1.instance1", + "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", + "version": "6.1.0", + "revision": "1.0.1-commit1", + "build": 1, + "deployTimeEpochMs": 1600000000000, + "screwdriverId": "123", + "endpointStatus": [ + { + "endpointId": "default", + "rotationId": "rotation-id-1", + "clusterId": "foo", + "status": "UNKNOWN", + "lastUpdated": 0 + } + ], + "applicationVersion": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "status": "complete", + "quota": 1.062, + "activity": { + "lastQueried": 1527848130000, + "lastWritten": 1527848130000, + "lastQueriesPerSecond": 1.0, + "lastWritesPerSecond": 2.0 + }, + "metrics": { + "queriesPerSecond": 1.0, + "writesPerSecond": 2.0, + "documentCount": 3.0, + "queryLatencyMillis": 4.0, + "writeLatencyMillis": 5.0, + "lastUpdated": 123123 + } + }, + { + "environment": "prod", + "region": "us-east-3" + }, + { + "environment": "prod", + "region": "us-west-1" + } + ], + "pemDeployKeys": [ ], + "metrics": { + "queryServiceQuality": 0.5, + "writeServiceQuality": 0.7 + }, + "activity": { + "lastQueried": 1527848130000, + "lastWritten": 1527848130000, + "lastQueriesPerSecond": 1.0, + "lastWritesPerSecond": 2.0 + }, + "ownershipIssueId": "321", + "owner": "owner-username", + "deploymentIssueId": "123" + } + ], + "metaData": { + "createdAtMillis": 1600000000000, + "lastDeploymentToDevMillis": 1600000000000, + "lastSubmissionToProdMillis": 1000 + } + }, + { + "tenant": "tenant2", + "type": "ATHENS", + "athensDomain": "domain2", + "property": "property2", + "propertyId": "1234", + "propertyUrl": "www.properties.tld/1234", + "contactsUrl": "www.contacts.tld/1234", + "issueCreationUrl": "www.issues.tld/1234", + "contacts": [ + [ + "alice" + ], + [ + "bob" + ] + ], + "applications": [ ], + "metaData": { + "createdAtMillis": 1600000000000, + "lastLoginByUserMillis": 1234, + "lastLoginByAdministratorMillis": 1234 + } + } ] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json index e8f7839e7cd..8c4851413a7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json @@ -1,5 +1,45 @@ [ - @include(tenant-with-application-with-metadata.json), - @include(tenant2.json) + { + "tenant": "tenant1", + "type": "ATHENS", + "athensDomain": "domain1", + "property": "property1", + "applications": [ + { + "tenant": "tenant1", + "application": "application1", + "instance": "instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" + } + ], + "metaData": { + "createdAtMillis": 1600000000000, + "lastDeploymentToDevMillis": 1600000000000, + "lastSubmissionToProdMillis": 1000 + } + }, + { + "tenant": "tenant2", + "type": "ATHENS", + "athensDomain": "domain2", + "property": "property2", + "propertyId": "1234", + "propertyUrl": "www.properties.tld/1234", + "contactsUrl": "www.contacts.tld/1234", + "issueCreationUrl": "www.issues.tld/1234", + "contacts": [ + [ + "alice" + ], + [ + "bob" + ] + ], + "applications": [ ], + "metaData": { + "createdAtMillis": 1600000000000, + "lastLoginByUserMillis": 1234, + "lastLoginByAdministratorMillis": 1234 + } + } ] - diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json index d63a7ba7d56..224b38f5f19 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json @@ -1,7 +1,7 @@ { - "resources":[ + "resources": [ { - "url":"http://localhost:8080/application/v4/tenant/" + "url": "http://localhost:8080/application/v4/tenant/" } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service new file mode 100644 index 00000000000..834a13b8d7a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service @@ -0,0 +1 @@ +path '/state/v1/' and query 'foo=bar%3F&forwarded-url=http%3A%2F%2Flocalhost%3A8080%2Fapplication%2Fv4%2Ftenant%2Ftenant1%2Fapplication%2Fapplication1%2Finstance%2Finstance1%2Fenvironment%2Fprod%2Fregion%2Fus-central-1%2Fservice%2Fstoragenode%2Fhost.com%2Fstate%2Fv1%2F'
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json deleted file mode 100644 index 0e610c4d4b2..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "clusters": [ - { - "name": "cluster1", - "type": "container", - "url": "http://cfg1.test/serviceview/v1/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/service/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1/health", - "services": [ - { - "url": null, - "serviceType": "container", - "serviceName": "service1", - "configId": "configId1", - "host": "host1" - } - ] - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json deleted file mode 100644 index 3380eb26911..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "clusters": [ - { - "name": "cluster1", - "type": "container", - "url": "cluster-url", - "services": [ - { - "url": null, - "serviceType": "container", - "serviceName": "service1", - "configId": "configId1", - "host": "host1" - } - ] - } - ] -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json deleted file mode 100644 index 8fb64d65ff8..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "resources": [ - { - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/filedistributorservice-dud1f4w037qdxdrn0ovxfdtgw/state/v1/config" - } - ] -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json deleted file mode 100644 index 1a434afafbb..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "clusters": [ - { - "name": "cluster1", - "type": "content", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1", - "services": [ - { - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", - "serviceType": "storagenode", - "serviceName": "storagenode", - "configId": "cluster1/storage/0", - "host": "host1" - } - ] - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index 6b1d48f4a08..ba65b962a73 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -18,7 +18,7 @@ { "at": 14503000, "type": "info", - "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" + "message": "host-tenant.application.default-t-staging.us-east-3: unorchestrated" }, { "at": 14503000, @@ -33,7 +33,7 @@ { "at": 14503000, "type": "info", - "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" + "message": "host-tenant.application.default-t-staging.us-east-3: unorchestrated" }, { "at": 14503000, @@ -48,7 +48,7 @@ { "at": 14503000, "type": "info", - "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" + "message": "host-tenant.application.default-t-staging.us-east-3: unorchestrated" }, { "at": 14503000, @@ -63,7 +63,7 @@ { "at": 14503000, "type": "info", - "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" + "message": "host-tenant.application.default-t-staging.us-east-3: unorchestrated" }, { "at": 14503000, @@ -80,7 +80,7 @@ { "at": 14503000, "type": "info", - "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..." + "message": "Deploying platform version 6.1 and application build 1 ..." }, { "at": 14503000, @@ -102,7 +102,7 @@ { "at": 14503000, "type": "info", - "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" + "message": "host-tenant.application.default-staging.us-east-3: unorchestrated" }, { "at": 14503000, @@ -127,7 +127,7 @@ { "at": 14503000, "type": "debug", - "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" + "message": "host-tenant.application.default-staging.us-east-3: unorchestrated" }, { "at": 14503000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/suspended.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/suspended.json index a360474da49..0b855edb2b2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/suspended.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/suspended.json @@ -1,3 +1,3 @@ { - "suspended":false -}
\ No newline at end of file + "suspended": false +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json index 377b8c6ed69..3b505bc11fd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json @@ -4,351 +4,351 @@ "log": { "deployTester": [ { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Deploying the tester container on platform 6.1 ..." }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Deployment successful." }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "foo" } ], "installTester": [ { - "at": "(ignore)", + "at": 1600000000000, "type": "info", - "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated" + "message": "host-tenant1.application1.instance1-t-test.us-east-1: unorchestrated" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "--- platform vespa/vespa:6.1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", - "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated" + "message": "host-tenant1.application1.instance1-t-test.us-east-1: unorchestrated" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "--- platform vespa/vespa:6.1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", - "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated" + "message": "host-tenant1.application1.instance1-t-test.us-east-1: unorchestrated" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "--- platform vespa/vespa:6.1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", - "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated" + "message": "host-tenant1.application1.instance1-t-test.us-east-1: unorchestrated" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "--- platform vespa/vespa:6.1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", - "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated" + "message": "host-tenant1.application1.instance1-t-test.us-east-1: unorchestrated" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "--- platform vespa/vespa:6.1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Tester container successfully installed!" } ], "deployReal": [ { - "at": "(ignore)", + "at": 1600000000000, "type": "info", - "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..." + "message": "Deploying platform version 6.1 and application build 1 ..." }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Deployment successful." }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "foo" } ], "installReal": [ { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "######## Details for all nodes ########" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "host-tenant1.application1.instance1-test.us-east-1: unorchestrated" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "--- platform vespa/vespa:6.1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Waiting for convergence of 1 services across 1 nodes" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "1 application services upgrading" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "host-tenant1.application1.instance1-test.us-east-1: unorchestrated" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Waiting for convergence of 1 services across 1 nodes" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "1 application services upgrading" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "host-tenant1.application1.instance1-test.us-east-1: unorchestrated" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Waiting for convergence of 1 services across 1 nodes" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "1 application services upgrading" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "host-tenant1.application1.instance1-test.us-east-1: unorchestrated" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Waiting for convergence of 1 services across 1 nodes" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "1 application services upgrading" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "host-tenant1.application1.instance1-test.us-east-1: unorchestrated" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Waiting for convergence of 1 services across 1 nodes" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "1 application services upgrading" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "host-tenant1.application1.instance1-test.us-east-1: unorchestrated" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Waiting for convergence of 1 services across 1 nodes" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "1 application services upgrading" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "host-tenant1.application1.instance1-test.us-east-1: unorchestrated" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Found endpoints:" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "- test.us-east-1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": " |-- https://instance1.application1.tenant1.us-east-1.test.vespa.oath.cloud/ (cluster 'default')" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Installation succeeded!" } ], "startTests": [ { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Attempting to find endpoints ..." }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Found endpoints:" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "- test.us-east-1" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": " |-- https://instance1.application1.tenant1.us-east-1.test.vespa.oath.cloud/ (cluster 'default')" }, { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Starting tests ..." } ], "endTests": [ { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Tests completed successfully." } ], "deactivateReal": [ { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Deactivating deployment of tenant1.application1.instance1 in test.us-east-1 ..." } ], "deactivateTester": [ { - "at": "(ignore)", + "at": 1600000000000, "type": "info", "message": "Deactivating tester of tenant1.application1.instance1 in test.us-east-1 ..." } @@ -358,43 +358,43 @@ "steps": { "deployTester": { "status": "succeeded", - "startMillis": "(ignore)" + "startMillis": 1600000000000 }, "installTester": { "status": "succeeded", - "startMillis": "(ignore)" + "startMillis": 1600000000000 }, "deployReal": { "status": "succeeded", - "startMillis": "(ignore)" + "startMillis": 1600000000000 }, "installReal": { "status": "succeeded", - "startMillis": "(ignore)" + "startMillis": 1600000000000 }, "startTests": { "status": "succeeded", - "startMillis": "(ignore)" + "startMillis": 1600000000000 }, "endTests": { "status": "succeeded", - "startMillis": "(ignore)" + "startMillis": 1600000000000 }, "copyVespaLogs": { "status": "succeeded", - "startMillis": "(ignore)" + "startMillis": 1600000000000 }, "deactivateReal": { "status": "succeeded", - "startMillis": "(ignore)" + "startMillis": 1600000000000 }, "deactivateTester": { "status": "succeeded", - "startMillis": "(ignore)" + "startMillis": 1600000000000 }, "report": { "status": "succeeded", - "startMillis": "(ignore)" + "startMillis": 1600000000000 } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json index c0988e8c301..1ac4658ce10 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json @@ -3,7 +3,7 @@ { "id": 2, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2", - "start": "(ignore)", + "start": 1600000000000, "status": "running", "versions": { "targetPlatform": "6.1.0", @@ -60,8 +60,8 @@ { "id": 1, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1", - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success", "versions": { "targetPlatform": "6.1.0", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json index f675825c3b6..e31058d349b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json @@ -18,7 +18,7 @@ { "at": 0, "type": "info", - "message": "host-tenant:application:default-t-test.us-east-1: unorchestrated" + "message": "host-tenant.application.default-t-test.us-east-1: unorchestrated" }, { "at": 0, @@ -33,7 +33,7 @@ { "at": 0, "type": "info", - "message": "host-tenant:application:default-t-test.us-east-1: unorchestrated" + "message": "host-tenant.application.default-t-test.us-east-1: unorchestrated" }, { "at": 0, @@ -48,7 +48,7 @@ { "at": 0, "type": "info", - "message": "host-tenant:application:default-t-test.us-east-1: unorchestrated" + "message": "host-tenant.application.default-t-test.us-east-1: unorchestrated" }, { "at": 0, @@ -63,7 +63,7 @@ { "at": 0, "type": "info", - "message": "host-tenant:application:default-t-test.us-east-1: unorchestrated" + "message": "host-tenant.application.default-t-test.us-east-1: unorchestrated" }, { "at": 0, @@ -78,7 +78,7 @@ { "at": 0, "type": "info", - "message": "host-tenant:application:default-t-test.us-east-1: unorchestrated" + "message": "host-tenant.application.default-t-test.us-east-1: unorchestrated" }, { "at": 0, @@ -95,7 +95,7 @@ { "at": 0, "type": "info", - "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..." + "message": "Deploying platform version 6.1 and application build 1 ..." }, { "at": 0, @@ -117,7 +117,7 @@ { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "host-tenant.application.default-test.us-east-1: unorchestrated" }, { "at": 0, @@ -142,7 +142,7 @@ { "at": 0, "type": "debug", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "host-tenant.application.default-test.us-east-1: unorchestrated" }, { "at": 0, @@ -167,7 +167,7 @@ { "at": 0, "type": "debug", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "host-tenant.application.default-test.us-east-1: unorchestrated" }, { "at": 0, @@ -192,7 +192,7 @@ { "at": 0, "type": "debug", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "host-tenant.application.default-test.us-east-1: unorchestrated" }, { "at": 0, @@ -217,7 +217,7 @@ { "at": 0, "type": "debug", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "host-tenant.application.default-test.us-east-1: unorchestrated" }, { "at": 0, @@ -242,7 +242,7 @@ { "at": 0, "type": "debug", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "host-tenant.application.default-test.us-east-1: unorchestrated" }, { "at": 0, @@ -267,7 +267,7 @@ { "at": 0, "type": "debug", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "host-tenant.application.default-test.us-east-1: unorchestrated" }, { "at": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json index b02b1ea2565..006e5158168 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json @@ -6,12 +6,12 @@ "applications": [ { "tenant": "tenant1", - "application":"application1", - "instance":"instance1", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" + "application": "application1", + "instance": "instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" } ], "metaData": { - "createdAtMillis": "(ignore)" + "createdAtMillis": 1600000000000 } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-empty-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-empty-application.json index 58c76b8227e..d08cd640890 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-empty-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-empty-application.json @@ -10,7 +10,7 @@ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1" } ], - "metaData":{ - "createdAtMillis": "(ignore)" + "metaData": { + "createdAtMillis": 1600000000000 } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json index bd77a68a1eb..72aa268163a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json @@ -4,8 +4,8 @@ "athensDomain": "domain2", "property": "property2", "propertyId": "1234", - "applications": [], + "applications": [ ], "metaData": { - "createdAtMillis": "(ignore)" + "createdAtMillis": 1600000000000 } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json index 33ed505ce35..7f94c1ea7b3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json @@ -4,7 +4,7 @@ "athensDomain": "domain1", "property": "property1", "applications": [ ], - "metaData":{ - "createdAtMillis": "(ignore)" + "metaData": { + "createdAtMillis": 1600000000000 } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-deleted.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-deleted.json index 1c4b76932ac..b3eabe5db96 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-deleted.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-deleted.json @@ -1,9 +1,9 @@ { "tenant": "tenant1", "type": "DELETED", - "applications": [], + "applications": [ ], "metaData": { - "createdAtMillis": "(ignore)", - "deletedAtMillis": "(ignore)" + "createdAtMillis": 1600000000000, + "deletedAtMillis": 1600000000000 } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json index 551b26c8513..054aaec0fcd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json @@ -4,11 +4,193 @@ "athensDomain": "domain1", "property": "property1", "applications": [ - @include(instance1-recursive.json) + { + "tenant": "tenant1", + "application": "application1", + "instance": "instance1", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1", + "projectId": 123, + "deploying": { + "revision": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } + }, + "changeBlockers": [ + { + "versions": true, + "revisions": false, + "timeZone": "UTC", + "days": [ + 1, + 2, + 3, + 4, + 5 + ], + "hours": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8 + ] + } + ], + "rotationId": "rotation-id-1", + "instances": [ + { + "tenant": "tenant1", + "application": "application1", + "instance": "instance1", + "environment": "dev", + "region": "us-east-1", + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1.application1.tenant1.us-east-1.dev.vespa.oath.cloud/", + "scope": "zone", + "routingMethod": "sharedLayer4", + "legacy": false + } + ], + "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/dev/region/us-east-1/clusters", + "nodes": "http://localhost:8080/zone/v2/dev/us-east-1/nodes/v2/node/?recursive=true&application=tenant1.application1.instance1", + "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-east-1&application=tenant1.application1.instance1", + "version": "6.1.0", + "revision": "1.0.1", + "build": 1, + "deployTimeEpochMs": 1600000000000, + "screwdriverId": "123", + "status": "complete", + "quota": 1.062, + "activity": { + "lastQueried": 1527848130000, + "lastWritten": 1527848130000, + "lastQueriesPerSecond": 1.0, + "lastWritesPerSecond": 2.0 + }, + "metrics": { + "queriesPerSecond": 1.0, + "writesPerSecond": 2.0, + "documentCount": 3.0, + "queryLatencyMillis": 4.0, + "writeLatencyMillis": 5.0, + "lastUpdated": 123123 + } + }, + { + "bcpStatus": { + "rotationStatus": "UNKNOWN" + }, + "tenant": "tenant1", + "application": "application1", + "instance": "instance1", + "environment": "prod", + "region": "us-central-1", + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1.application1.tenant1.us-central-1.vespa.oath.cloud/", + "scope": "zone", + "routingMethod": "sharedLayer4", + "legacy": false + }, + { + "cluster": "foo", + "tls": true, + "url": "https://instance1.application1.tenant1.global.vespa.oath.cloud/", + "scope": "global", + "routingMethod": "sharedLayer4", + "legacy": false + }, + { + "cluster": "foo", + "tls": true, + "url": "https://a0.application1.tenant1.us-central-1-r.vespa.oath.cloud/", + "scope": "application", + "routingMethod": "sharedLayer4", + "legacy": false + } + ], + "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/clusters", + "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/?recursive=true&application=tenant1.application1.instance1", + "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", + "version": "6.1.0", + "revision": "1.0.1-commit1", + "build": 1, + "deployTimeEpochMs": 1600000000000, + "screwdriverId": "123", + "endpointStatus": [ + { + "endpointId": "default", + "rotationId": "rotation-id-1", + "clusterId": "foo", + "status": "UNKNOWN", + "lastUpdated": 0 + } + ], + "applicationVersion": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "status": "complete", + "quota": 1.062, + "activity": { + "lastQueried": 1527848130000, + "lastWritten": 1527848130000, + "lastQueriesPerSecond": 1.0, + "lastWritesPerSecond": 2.0 + }, + "metrics": { + "queriesPerSecond": 1.0, + "writesPerSecond": 2.0, + "documentCount": 3.0, + "queryLatencyMillis": 4.0, + "writeLatencyMillis": 5.0, + "lastUpdated": 123123 + } + }, + { + "environment": "prod", + "region": "us-east-3" + }, + { + "environment": "prod", + "region": "us-west-1" + } + ], + "pemDeployKeys": [ ], + "metrics": { + "queryServiceQuality": 0.5, + "writeServiceQuality": 0.7 + }, + "activity": { + "lastQueried": 1527848130000, + "lastWritten": 1527848130000, + "lastQueriesPerSecond": 1.0, + "lastWritesPerSecond": 2.0 + }, + "ownershipIssueId": "321", + "owner": "owner-username", + "deploymentIssueId": "123" + } ], "metaData": { - "createdAtMillis": "(ignore)", - "lastDeploymentToDevMillis": "(ignore)", + "createdAtMillis": 1600000000000, + "lastDeploymentToDevMillis": 1600000000000, "lastSubmissionToProdMillis": 1000 } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1.json index a105a194974..a29e86b0aa2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1.json @@ -3,8 +3,8 @@ "type": "ATHENS", "athensDomain": "domain2", "property": "property1", - "applications": [], + "applications": [ ], "metaData": { - "createdAtMillis": "(ignore)" + "createdAtMillis": 1600000000000 } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json index 497d80c96a5..b04ac57d804 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json @@ -15,10 +15,10 @@ "bob" ] ], - "applications": [], + "applications": [ ], "metaData": { - "createdAtMillis": "(ignore)", + "createdAtMillis": 1600000000000, "lastLoginByUserMillis": 1234, "lastLoginByAdministratorMillis": 1234 } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/responses/athensDomain-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/responses/athensDomain-list.json index 913b8fab62f..9a456fd0433 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/responses/athensDomain-list.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/responses/athensDomain-list.json @@ -4,4 +4,4 @@ "domain2", "vespa.vespa.tenants.sandbox" ] -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/responses/property-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/responses/property-list.json index 596dea037bd..2931fc8b162 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/responses/property-list.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/responses/property-list.json @@ -1,6 +1,12 @@ { "properties": [ - {"propertyid": "1234", "property": "foo"}, - {"propertyid": "4321", "property": "bar"} + { + "propertyid": "1234", + "property": "foo" + }, + { + "propertyid": "4321", + "property": "bar" + } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/responses/root.json index 1947a27467f..737991f29c9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/responses/root.json @@ -1,10 +1,10 @@ { - "resources":[ + "resources": [ { - "url":"http://localhost:8080/athenz/v1/domains/" + "url": "http://localhost:8080/athenz/v1/domains/" }, { - "url":"http://localhost:8080/athenz/v1/properties/" + "url": "http://localhost:8080/athenz/v1/properties/" } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java index 74ed1a0bb10..0ef242aed16 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java @@ -26,8 +26,12 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; -import static com.yahoo.application.container.handler.Request.Method.*; -import static org.junit.Assert.*; +import static com.yahoo.application.container.handler.Request.Method.DELETE; +import static com.yahoo.application.container.handler.Request.Method.GET; +import static com.yahoo.application.container.handler.Request.Method.PATCH; +import static com.yahoo.application.container.handler.Request.Method.POST; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author olaa @@ -101,7 +105,7 @@ public class BillingApiHandlerTest extends ControllerContainerCloudTest { billingController.setPlan(tenant, PlanId.from("some-plan"), true); var request = request("/billing/v1/tenant/tenant1/billing?until=2020-05-28").roles(tenantRole); - tester.assertResponse(request, new File("tenant-billing-view")); + tester.assertResponse(request, new File("tenant-billing-view.json")); } @@ -117,7 +121,7 @@ public class BillingApiHandlerTest extends ControllerContainerCloudTest { tester.assertResponse(request, accessDenied, 403); request.roles(financeAdmin); - tester.assertResponse(request, new File("invoice-creation-response")); + tester.assertResponse(request, new File("invoice-creation-response.json")); bills = billingController.getBillsForTenant(tenant); assertEquals(1, bills.size()); @@ -152,7 +156,7 @@ public class BillingApiHandlerTest extends ControllerContainerCloudTest { request = request("/billing/v1/invoice/tenant/tenant1/line-item") .roles(financeAdmin); - tester.assertResponse(request, new File("line-item-list")); + tester.assertResponse(request, new File("line-item-list.json")); } @Test @@ -183,7 +187,7 @@ public class BillingApiHandlerTest extends ControllerContainerCloudTest { var request = request("/billing/v1/billing?until=2020-05-28").roles(financeAdmin); - tester.assertResponse(request, new File("billing-all-tenants")); + tester.assertResponse(request, new File("billing-all-tenants.json")); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants deleted file mode 100644 index 5c61dc6e86e..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants +++ /dev/null @@ -1,58 +0,0 @@ -{ - "until":"2020-05-28", - "tenants":[ - { - "tenant":"tenant1", - "plan":"some-plan", - "planName":"Plan with id: some-plan", - "collection": "AUTO", - "current":{ - "amount":"123.00", - "status":"accrued", - "from":"2020-05-23", - "items":[ - { - "id":"some-id", - "description":"description", - "amount":"123.00", - "plan":"some-plan", - "planName":"Plan with id: some-plan" - } - ] - }, - "additional": - { - "items":[ - { - "id":"line-item-id", - "description":"support", - "amount":"42.00", - "plan":"some-plan", - "planName":"Plan with id: some-plan" - } - ] - } - }, - { - "tenant":"tenant2", - "plan":"some-plan", - "planName":"Plan with id: some-plan", - "collection": "AUTO", - "current":{ - "amount":"123.00", - "status":"accrued", - "from":"2020-05-23", - "items":[ - { - "id":"some-id", - "description":"description", - "amount":"123.00", - "plan":"some-plan", - "planName":"Plan with id: some-plan" - } - ] - }, - "additional":{"items":[]} - } - ] -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json new file mode 100644 index 00000000000..fe89ef246bb --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json @@ -0,0 +1,59 @@ +{ + "until": "2020-05-28", + "tenants": [ + { + "tenant": "tenant1", + "plan": "some-plan", + "planName": "Plan with id: some-plan", + "collection": "AUTO", + "current": { + "amount": "123.00", + "status": "accrued", + "from": "2020-05-23", + "items": [ + { + "id": "some-id", + "description": "description", + "amount": "123.00", + "plan": "some-plan", + "planName": "Plan with id: some-plan" + } + ] + }, + "additional": { + "items": [ + { + "id": "line-item-id", + "description": "support", + "amount": "42.00", + "plan": "some-plan", + "planName": "Plan with id: some-plan" + } + ] + } + }, + { + "tenant": "tenant2", + "plan": "some-plan", + "planName": "Plan with id: some-plan", + "collection": "AUTO", + "current": { + "amount": "123.00", + "status": "accrued", + "from": "2020-05-23", + "items": [ + { + "id": "some-id", + "description": "description", + "amount": "123.00", + "plan": "some-plan", + "planName": "Plan with id: some-plan" + } + ] + }, + "additional": { + "items": [ ] + } + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/invoice-creation-response b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/invoice-creation-response deleted file mode 100644 index fe9e8486e9b..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/invoice-creation-response +++ /dev/null @@ -1 +0,0 @@ -{"message":"Created invoice with ID id-123","id":"id-123"}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/invoice-creation-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/invoice-creation-response.json new file mode 100644 index 00000000000..49fde010c58 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/invoice-creation-response.json @@ -0,0 +1,4 @@ +{ + "message": "Created invoice with ID id-123", + "id": "id-123" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list deleted file mode 100644 index 98c2046afd8..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list +++ /dev/null @@ -1,11 +0,0 @@ -{ - "lineItems":[ - { - "id":"line-item-id", - "description":"some description", - "amount":"123.45", - "plan":"some-plan", - "planName":"Plan with id: some-plan" - } - ] -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list.json new file mode 100644 index 00000000000..e8404b12dd8 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list.json @@ -0,0 +1,11 @@ +{ + "lineItems": [ + { + "id": "line-item-id", + "description": "some description", + "amount": "123.45", + "plan": "some-plan", + "planName": "Plan with id: some-plan" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view deleted file mode 100644 index e5588c45677..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view +++ /dev/null @@ -1,45 +0,0 @@ -{ - "until":"2020-05-28", - "plan":"some-plan", - "planName":"Plan with id: some-plan", - "current":{ - "amount":"123.00", - "status":"accrued", - "from":"2020-05-23", - "items":[ - { - "id":"some-id", - "description":"description", - "amount":"123.00", - "plan":"some-plan", - "planName":"Plan with id: some-plan" - } - ] - }, - "additional":{"items":[]}, - "bills":[ - { - "id":"id-1", - "from":"2020-05-23", - "to":"2020-05-28", - "amount":"123.00", - "status":"OPEN", - "statusHistory":[ - { - "at":"2020-05-23", - "status":"OPEN" - } - ], - "items":[ - { - "id":"some-id", - "description":"description", - "amount":"123.00", - "plan":"some-plan", - "planName":"Plan with id: some-plan" - } - ] - } - ], - "collection":"AUTO" -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json new file mode 100644 index 00000000000..953b946c329 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json @@ -0,0 +1,47 @@ +{ + "until": "2020-05-28", + "plan": "some-plan", + "planName": "Plan with id: some-plan", + "current": { + "amount": "123.00", + "status": "accrued", + "from": "2020-05-23", + "items": [ + { + "id": "some-id", + "description": "description", + "amount": "123.00", + "plan": "some-plan", + "planName": "Plan with id: some-plan" + } + ] + }, + "additional": { + "items": [ ] + }, + "bills": [ + { + "id": "id-1", + "from": "2020-05-23", + "to": "2020-05-28", + "amount": "123.00", + "status": "OPEN", + "statusHistory": [ + { + "at": "2020-05-23", + "status": "OPEN" + } + ], + "items": [ + { + "id": "some-id", + "description": "description", + "amount": "123.00", + "plan": "some-plan", + "planName": "Plan with id: some-plan" + } + ] + } + ], + "collection": "AUTO" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/initial.json index cf349e06cff..1fb8ad8be17 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/initial.json @@ -24,4 +24,4 @@ } ] } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/vcmrs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/vcmrs.json index 54d4ea8bcbd..4ae079ebfb4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/vcmrs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/vcmrs.json @@ -37,4 +37,4 @@ ] } ] -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index 8a6244e19a0..94ca4268000 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -186,7 +185,7 @@ public class ControllerApiTest extends ControllerContainerTest { tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/access/requests/"+hostedOperator.getName(), requestBody, Request.Method.POST), "{\"message\":\"Unable to approve membership request\"}", 400); - tester.controller().supportAccess().allow(deployment, Instant.now().plus(Duration.ofHours(1)), "tenantx"); + tester.controller().supportAccess().allow(deployment, tester.controller().clock().instant().plus(Duration.ofHours(1)), "tenantx"); tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/access/requests/"+hostedOperator.getName(), requestBody, Request.Method.POST), "{\"members\":[\"user.alice\"]}"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 0dad88e645b..79e11fa1140 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -85,7 +85,7 @@ "name": "ResourceTagMaintainer" }, { - "name":"RetriggerMaintainer" + "name": "RetriggerMaintainer" }, { "name": "SystemRoutingPolicyMaintainer" @@ -94,7 +94,7 @@ "name": "SystemUpgrader" }, { - "name":"TenantRoleMaintainer" + "name": "TenantRoleMaintainer" }, { "name": "TrafficShareUpdater" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json index 1008ada6def..cf7738efe7f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json @@ -6,7 +6,7 @@ "cpu": 12.0, "memory": 48.0, "disk": 1200.0, - "architecture":"arm64" + "architecture": "arm64" }, { "applicationId": "tenant.app.instance", @@ -15,6 +15,6 @@ "cpu": 24.0, "memory": 96.0, "disk": 2400.0, - "architecture":"x86_64" + "architecture": "x86_64" } -]
\ No newline at end of file +] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/stats.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/stats.json index 1c9230798dd..673767c13a0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/stats.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/stats.json @@ -61,4 +61,4 @@ ] } ] -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index cf6453235d3..8db6bdf9a4a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -2,9 +2,9 @@ package com.yahoo.vespa.hosted.controller.restapi.deployment; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -36,20 +36,24 @@ public class BadgeApiTest extends ControllerContainerTest { .build(); application.submit(applicationPackage).deploy(); application.submit(applicationPackage) - .runJob(JobType.systemTest) - .runJob(JobType.stagingTest) - .runJob(JobType.productionUsWest1) - .runJob(JobType.productionAwsUsEast1a) - .runJob(JobType.testUsWest1) - .runJob(JobType.productionApSoutheast1) - .failDeployment(JobType.testApSoutheast1); + .runJob(DeploymentContext.systemTest) + .runJob(DeploymentContext.stagingTest) + .runJob(DeploymentContext.productionUsWest1) + .runJob(DeploymentContext.productionAwsUsEast1a) + .runJob(DeploymentContext.testUsWest1) + .runJob(DeploymentContext.productionApSoutheast1) + .failDeployment(DeploymentContext.testApSoutheast1); application.submit(applicationPackage) - .runJob(JobType.systemTest) - .runJob(JobType.stagingTest); + .failTests(DeploymentContext.systemTest, true) + .runJob(DeploymentContext.stagingTest); for (int i = 0; i < 31; i++) - application.failDeployment(JobType.productionUsWest1); + if ((i & 1) == 0) + application.failDeployment(DeploymentContext.productionUsWest1); + else + application.triggerJobs().abortJob(DeploymentContext.productionUsWest1); application.triggerJobs(); - tester.controller().applications().deploymentTrigger().reTrigger(application.instanceId(), JobType.testEuWest1, "reason"); + tester.controller().applications().deploymentTrigger().reTrigger(application.instanceId(), DeploymentContext.systemTest, "reason"); + tester.controller().applications().deploymentTrigger().reTrigger(application.instanceId(), DeploymentContext.testEuWest1, "reason"); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default"), Files.readString(Paths.get(responseFiles + "overview.svg")), 200); @@ -60,7 +64,7 @@ public class BadgeApiTest extends ControllerContainerTest { // New change not reflected before cache entry expires. tester.serviceRegistry().clock().advance(Duration.ofSeconds(59)); - application.runJob(JobType.productionUsWest1); + application.runJob(DeploymentContext.productionUsWest1); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=32"), Files.readString(Paths.get(responseFiles + "history.svg")), 200); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index ca0db13b3d1..d837b9e8264 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -6,9 +6,9 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -48,7 +48,7 @@ public class DeploymentApiTest extends ControllerContainerTest { // Deploy application without any declared jobs on the oldest version. var oldAppWithoutDeployment = deploymentTester.newDeploymentContext("tenant4", "application4", "default"); - oldAppWithoutDeployment.submit().failDeployment(JobType.systemTest); + oldAppWithoutDeployment.submit().failDeployment(DeploymentContext.systemTest); oldAppWithoutDeployment.submit(emptyPackage); // System upgrades to 5.0 for the other applications. @@ -61,8 +61,8 @@ public class DeploymentApiTest extends ControllerContainerTest { var otherProductionApp = deploymentTester.newDeploymentContext("tenant2", "application2", "i2"); var appWithoutDeployments = deploymentTester.newDeploymentContext("tenant3", "application3", "default"); failingApp.submit(applicationPackage).deploy(); - productionApp.submit(multiInstancePackage).runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1); - otherProductionApp.runJob(JobType.productionUsWest1); + productionApp.submit(multiInstancePackage).runJob(DeploymentContext.systemTest).runJob(DeploymentContext.stagingTest).runJob(DeploymentContext.productionUsWest1); + otherProductionApp.runJob(DeploymentContext.productionUsWest1); // Deploy once so that job information is stored, then remove the deployment by submitting an empty deployment spec. appWithoutDeployments.submit(applicationPackage).deploy(); @@ -75,13 +75,13 @@ public class DeploymentApiTest extends ControllerContainerTest { // Applications upgrade, 1/2 succeed deploymentTester.upgrader().maintain(); deploymentTester.triggerJobs(); - productionApp.runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1); - failingApp.failDeployment(JobType.systemTest).failDeployment(JobType.stagingTest); + productionApp.runJob(DeploymentContext.systemTest).runJob(DeploymentContext.stagingTest).runJob(DeploymentContext.productionUsWest1); + failingApp.failDeployment(DeploymentContext.systemTest).failDeployment(DeploymentContext.stagingTest); deploymentTester.upgrader().maintain(); deploymentTester.triggerJobs(); // Application fails application change - productionApp.submit(multiInstancePackage).failDeployment(JobType.systemTest); + productionApp.submit(multiInstancePackage).failDeployment(DeploymentContext.systemTest); tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller()))); tester.assertResponse(operatorRequest("http://localhost:8080/deployment/v1/"), new File("root.json")); @@ -97,8 +97,8 @@ public class DeploymentApiTest extends ControllerContainerTest { version.isControllerVersion(), version.isSystemVersion(), version.isReleased(), - List.of(new NodeVersion(HostName.from("config1.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Optional.empty()), - new NodeVersion(HostName.from("config2.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Optional.empty())), + List.of(new NodeVersion(HostName.of("config1.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Optional.empty()), + new NodeVersion(HostName.of("config2.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Optional.empty())), version.confidence() ); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history.svg index 0e30796bae2..c0566ade33a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history.svg +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history.svg @@ -33,9 +33,15 @@ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> + <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bd890b' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> <stop offset='0' stop-color='#ab83ff' /> - <stop offset='1' stop-color='#00f244' /> + <stop offset='1' stop-color='#00f844' /> <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> @@ -44,100 +50,100 @@ </clipPath> <g clip-path='url(#rounded)'> <rect x='653.26809109179' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='646.1879570885093' rx='3' width='13.080134003280707' height='20' fill='#00f244'/> + <rect x='646.1879570885093' rx='3' width='13.080134003280707' height='20' fill='#00f844'/> <rect x='646.1879570885093' rx='3' width='13.080134003280707' height='20' fill='url(#shade)'/> <rect x='646.4043039211865' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='639.1078230852286' rx='3' width='13.296480835957981' height='20' fill='#00f244'/> + <rect x='639.1078230852286' rx='3' width='13.296480835957981' height='20' fill='#00f844'/> <rect x='639.1078230852286' rx='3' width='13.296480835957981' height='20' fill='url(#shade)'/> <rect x='639.3307808029524' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='631.8113422492706' rx='3' width='13.519438553681752' height='20' fill='#bf103c'/> <rect x='631.8113422492706' rx='3' width='13.519438553681752' height='20' fill='url(#shade)'/> <rect x='632.0411128600877' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='624.2919036955889' rx='3' width='13.749209164498811' height='20' fill='#bf103c'/> + <rect x='624.2919036955889' rx='3' width='13.749209164498811' height='20' fill='#bd890b'/> <rect x='624.2919036955889' rx='3' width='13.749209164498811' height='20' fill='url(#shade)'/> <rect x='624.5286953802824' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='616.5426945310901' rx='3' width='13.986000849192376' height='20' fill='#bf103c'/> <rect x='616.5426945310901' rx='3' width='13.986000849192376' height='20' fill='url(#shade)'/> <rect x='616.7867218317995' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='608.5566936818977' rx='3' width='14.230028149901685' height='20' fill='#bf103c'/> + <rect x='608.5566936818977' rx='3' width='14.230028149901685' height='20' fill='#bd890b'/> <rect x='608.5566936818977' rx='3' width='14.230028149901685' height='20' fill='url(#shade)'/> <rect x='608.8081776965013' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='600.326665531996' rx='3' width='14.48151216450522' height='20' fill='#bf103c'/> <rect x='600.326665531996' rx='3' width='14.48151216450522' height='20' fill='url(#shade)'/> <rect x='600.5858341144344' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='591.8451533674908' rx='3' width='14.740680746943664' height='20' fill='#bf103c'/> + <rect x='591.8451533674908' rx='3' width='14.740680746943664' height='20' fill='#bd890b'/> <rect x='591.8451533674908' rx='3' width='14.740680746943664' height='20' fill='url(#shade)'/> <rect x='592.1122413342111' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='583.1044726205471' rx='3' width='15.007768713664104' height='20' fill='#bf103c'/> <rect x='583.1044726205471' rx='3' width='15.007768713664104' height='20' fill='url(#shade)'/> <rect x='583.3797219632555' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='574.096703906883' rx='3' width='15.283018056372542' height='20' fill='#bf103c'/> + <rect x='574.096703906883' rx='3' width='15.283018056372542' height='20' fill='#bd890b'/> <rect x='574.096703906883' rx='3' width='15.283018056372542' height='20' fill='url(#shade)'/> <rect x='574.380364011798' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='564.8136858505105' rx='3' width='15.566678161287442' height='20' fill='#bf103c'/> <rect x='564.8136858505105' rx='3' width='15.566678161287442' height='20' fill='url(#shade)'/> <rect x='565.1060137243161' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='555.2470076892231' rx='3' width='15.859006035092985' height='20' fill='#bf103c'/> + <rect x='555.2470076892231' rx='3' width='15.859006035092985' height='20' fill='#bd890b'/> <rect x='555.2470076892231' rx='3' width='15.859006035092985' height='20' fill='url(#shade)'/> <rect x='555.5482681919269' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='545.3880016541301' rx='3' width='16.16026653779677' height='20' fill='#bf103c'/> <rect x='545.3880016541301' rx='3' width='16.16026653779677' height='20' fill='url(#shade)'/> <rect x='545.6984677390362' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='535.2277351163333' rx='3' width='16.470732622702883' height='20' fill='#bf103c'/> + <rect x='535.2277351163333' rx='3' width='16.470732622702883' height='20' fill='#bd890b'/> <rect x='535.2277351163333' rx='3' width='16.470732622702883' height='20' fill='url(#shade)'/> <rect x='535.5476880773482' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='524.7570024936304' rx='3' width='16.790685583717845' height='20' fill='#bf103c'/> <rect x='524.7570024936304' rx='3' width='16.790685583717845' height='20' fill='url(#shade)'/> <rect x='525.0867322201259' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='513.9663169099125' rx='3' width='17.12041531021341' height='20' fill='#bf103c'/> + <rect x='513.9663169099125' rx='3' width='17.12041531021341' height='20' fill='#bd890b'/> <rect x='513.9663169099125' rx='3' width='17.12041531021341' height='20' fill='url(#shade)'/> <rect x='514.3061221493763' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='502.84590159969906' rx='3' width='17.460220549677203' height='20' fill='#bf103c'/> <rect x='502.84590159969906' rx='3' width='17.460220549677203' height='20' fill='url(#shade)'/> <rect x='503.196090228411' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='491.38568105002184' rx='3' width='17.810409178389147' height='20' fill='#bf103c'/> + <rect x='491.38568105002184' rx='3' width='17.810409178389147' height='20' fill='#bd890b'/> <rect x='491.38568105002184' rx='3' width='17.810409178389147' height='20' fill='url(#shade)'/> <rect x='491.7465703520016' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='479.5752718716327' rx='3' width='18.171298480368904' height='20' fill='#bf103c'/> <rect x='479.5752718716327' rx='3' width='18.171298480368904' height='20' fill='url(#shade)'/> <rect x='479.9471888261109' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='467.40397339126383' rx='3' width='18.543215434847077' height='20' fill='#bf103c'/> + <rect x='467.40397339126383' rx='3' width='18.543215434847077' height='20' fill='#bd890b'/> <rect x='467.40397339126383' rx='3' width='18.543215434847077' height='20' fill='url(#shade)'/> <rect x='467.7872549689374' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='454.86075795641676' rx='3' width='18.92649701252067' height='20' fill='#bf103c'/> <rect x='454.86075795641676' rx='3' width='18.92649701252067' height='20' fill='url(#shade)'/> <rect x='455.25575142475725' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='441.9342609438961' rx='3' width='19.32149048086113' height='20' fill='#bf103c'/> + <rect x='441.9342609438961' rx='3' width='19.32149048086113' height='20' fill='#bd890b'/> <rect x='441.9342609438961' rx='3' width='19.32149048086113' height='20' fill='url(#shade)'/> <rect x='442.3413241817867' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='428.61277046303496' rx='3' width='19.728553718751726' height='20' fill='#bf103c'/> <rect x='428.61277046303496' rx='3' width='19.728553718751726' height='20' fill='url(#shade)'/> <rect x='429.03227228502243' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='414.8842167442832' rx='3' width='20.148055540739207' height='20' fill='#bf103c'/> + <rect x='414.8842167442832' rx='3' width='20.148055540739207' height='20' fill='#bd890b'/> <rect x='414.8842167442832' rx='3' width='20.148055540739207' height='20' fill='url(#shade)'/> <rect x='415.31653723473755' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='400.736161203544' rx='3' width='20.58037603119359' height='20' fill='#bf103c'/> <rect x='400.736161203544' rx='3' width='20.58037603119359' height='20' fill='url(#shade)'/> <rect x='401.1816920610293' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='386.1557851723504' rx='3' width='21.025906888678875' height='20' fill='#bf103c'/> + <rect x='386.1557851723504' rx='3' width='21.025906888678875' height='20' fill='#bd890b'/> <rect x='386.1557851723504' rx='3' width='21.025906888678875' height='20' fill='url(#shade)'/> <rect x='386.61493006451815' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='371.12987828367153' rx='3' width='21.485051780846597' height='20' fill='#bf103c'/> <rect x='371.12987828367153' rx='3' width='21.485051780846597' height='20' fill='url(#shade)'/> <rect x='371.60305321299876' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='355.6448265028249' rx='3' width='21.958226710173836' height='20' fill='#bf103c'/> + <rect x='355.6448265028249' rx='3' width='21.958226710173836' height='20' fill='#bd890b'/> <rect x='355.6448265028249' rx='3' width='21.958226710173836' height='20' fill='url(#shade)'/> <rect x='356.13246018352817' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='339.68659979265107' rx='3' width='22.445860390877083' height='20' fill='#bf103c'/> <rect x='339.68659979265107' rx='3' width='22.445860390877083' height='20' fill='url(#shade)'/> <rect x='340.18913403911733' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='323.24073940177396' rx='3' width='22.948394637343355' height='20' fill='#bf103c'/> + <rect x='323.24073940177396' rx='3' width='22.948394637343355' height='20' fill='#bd890b'/> <rect x='323.24073940177396' rx='3' width='22.948394637343355' height='20' fill='url(#shade)'/> <rect x='323.7586295288612' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='306.2923447644306' rx='3' width='23.466284764430597' height='20' fill='#bf103c'/> <rect x='306.2923447644306' rx='3' width='23.466284764430597' height='20' fill='url(#shade)'/> <rect x='306.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='288.82606' rx='3' width='24.0' height='20' fill='#bf103c'/> + <rect x='288.82606' rx='3' width='24.0' height='20' fill='#bd890b'/> <rect x='288.82606' rx='3' width='24.0' height='20' fill='url(#shade)'/> <rect x='288.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#run-on-failure)'/> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history2.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history2.svg index 73d65b08b69..e527fa8d80f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history2.svg +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/history2.svg @@ -33,9 +33,15 @@ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> + <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bd890b' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> <stop offset='0' stop-color='#ab83ff' /> - <stop offset='1' stop-color='#00f244' /> + <stop offset='1' stop-color='#00f844' /> <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> @@ -44,103 +50,103 @@ </clipPath> <g clip-path='url(#rounded)'> <rect x='653.26809109179' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='646.1879570885093' rx='3' width='13.080134003280707' height='20' fill='#00f244'/> + <rect x='646.1879570885093' rx='3' width='13.080134003280707' height='20' fill='#00f844'/> <rect x='646.1879570885093' rx='3' width='13.080134003280707' height='20' fill='url(#shade)'/> <rect x='646.4043039211865' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='639.1078230852286' rx='3' width='13.296480835957981' height='20' fill='#bf103c'/> <rect x='639.1078230852286' rx='3' width='13.296480835957981' height='20' fill='url(#shade)'/> <rect x='639.3307808029524' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='631.8113422492706' rx='3' width='13.519438553681752' height='20' fill='#bf103c'/> + <rect x='631.8113422492706' rx='3' width='13.519438553681752' height='20' fill='#bd890b'/> <rect x='631.8113422492706' rx='3' width='13.519438553681752' height='20' fill='url(#shade)'/> <rect x='632.0411128600877' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='624.2919036955889' rx='3' width='13.749209164498811' height='20' fill='#bf103c'/> <rect x='624.2919036955889' rx='3' width='13.749209164498811' height='20' fill='url(#shade)'/> <rect x='624.5286953802824' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='616.5426945310901' rx='3' width='13.986000849192376' height='20' fill='#bf103c'/> + <rect x='616.5426945310901' rx='3' width='13.986000849192376' height='20' fill='#bd890b'/> <rect x='616.5426945310901' rx='3' width='13.986000849192376' height='20' fill='url(#shade)'/> <rect x='616.7867218317995' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='608.5566936818977' rx='3' width='14.230028149901685' height='20' fill='#bf103c'/> <rect x='608.5566936818977' rx='3' width='14.230028149901685' height='20' fill='url(#shade)'/> <rect x='608.8081776965013' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='600.326665531996' rx='3' width='14.48151216450522' height='20' fill='#bf103c'/> + <rect x='600.326665531996' rx='3' width='14.48151216450522' height='20' fill='#bd890b'/> <rect x='600.326665531996' rx='3' width='14.48151216450522' height='20' fill='url(#shade)'/> <rect x='600.5858341144344' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='591.8451533674908' rx='3' width='14.740680746943664' height='20' fill='#bf103c'/> <rect x='591.8451533674908' rx='3' width='14.740680746943664' height='20' fill='url(#shade)'/> <rect x='592.1122413342111' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='583.1044726205471' rx='3' width='15.007768713664104' height='20' fill='#bf103c'/> + <rect x='583.1044726205471' rx='3' width='15.007768713664104' height='20' fill='#bd890b'/> <rect x='583.1044726205471' rx='3' width='15.007768713664104' height='20' fill='url(#shade)'/> <rect x='583.3797219632555' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='574.096703906883' rx='3' width='15.283018056372542' height='20' fill='#bf103c'/> <rect x='574.096703906883' rx='3' width='15.283018056372542' height='20' fill='url(#shade)'/> <rect x='574.380364011798' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='564.8136858505105' rx='3' width='15.566678161287442' height='20' fill='#bf103c'/> + <rect x='564.8136858505105' rx='3' width='15.566678161287442' height='20' fill='#bd890b'/> <rect x='564.8136858505105' rx='3' width='15.566678161287442' height='20' fill='url(#shade)'/> <rect x='565.1060137243161' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='555.2470076892231' rx='3' width='15.859006035092985' height='20' fill='#bf103c'/> <rect x='555.2470076892231' rx='3' width='15.859006035092985' height='20' fill='url(#shade)'/> <rect x='555.5482681919269' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='545.3880016541301' rx='3' width='16.16026653779677' height='20' fill='#bf103c'/> + <rect x='545.3880016541301' rx='3' width='16.16026653779677' height='20' fill='#bd890b'/> <rect x='545.3880016541301' rx='3' width='16.16026653779677' height='20' fill='url(#shade)'/> <rect x='545.6984677390362' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='535.2277351163333' rx='3' width='16.470732622702883' height='20' fill='#bf103c'/> <rect x='535.2277351163333' rx='3' width='16.470732622702883' height='20' fill='url(#shade)'/> <rect x='535.5476880773482' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='524.7570024936304' rx='3' width='16.790685583717845' height='20' fill='#bf103c'/> + <rect x='524.7570024936304' rx='3' width='16.790685583717845' height='20' fill='#bd890b'/> <rect x='524.7570024936304' rx='3' width='16.790685583717845' height='20' fill='url(#shade)'/> <rect x='525.0867322201259' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='513.9663169099125' rx='3' width='17.12041531021341' height='20' fill='#bf103c'/> <rect x='513.9663169099125' rx='3' width='17.12041531021341' height='20' fill='url(#shade)'/> <rect x='514.3061221493763' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='502.84590159969906' rx='3' width='17.460220549677203' height='20' fill='#bf103c'/> + <rect x='502.84590159969906' rx='3' width='17.460220549677203' height='20' fill='#bd890b'/> <rect x='502.84590159969906' rx='3' width='17.460220549677203' height='20' fill='url(#shade)'/> <rect x='503.196090228411' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='491.38568105002184' rx='3' width='17.810409178389147' height='20' fill='#bf103c'/> <rect x='491.38568105002184' rx='3' width='17.810409178389147' height='20' fill='url(#shade)'/> <rect x='491.7465703520016' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='479.5752718716327' rx='3' width='18.171298480368904' height='20' fill='#bf103c'/> + <rect x='479.5752718716327' rx='3' width='18.171298480368904' height='20' fill='#bd890b'/> <rect x='479.5752718716327' rx='3' width='18.171298480368904' height='20' fill='url(#shade)'/> <rect x='479.9471888261109' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='467.40397339126383' rx='3' width='18.543215434847077' height='20' fill='#bf103c'/> <rect x='467.40397339126383' rx='3' width='18.543215434847077' height='20' fill='url(#shade)'/> <rect x='467.7872549689374' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='454.86075795641676' rx='3' width='18.92649701252067' height='20' fill='#bf103c'/> + <rect x='454.86075795641676' rx='3' width='18.92649701252067' height='20' fill='#bd890b'/> <rect x='454.86075795641676' rx='3' width='18.92649701252067' height='20' fill='url(#shade)'/> <rect x='455.25575142475725' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='441.9342609438961' rx='3' width='19.32149048086113' height='20' fill='#bf103c'/> <rect x='441.9342609438961' rx='3' width='19.32149048086113' height='20' fill='url(#shade)'/> <rect x='442.3413241817867' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='428.61277046303496' rx='3' width='19.728553718751726' height='20' fill='#bf103c'/> + <rect x='428.61277046303496' rx='3' width='19.728553718751726' height='20' fill='#bd890b'/> <rect x='428.61277046303496' rx='3' width='19.728553718751726' height='20' fill='url(#shade)'/> <rect x='429.03227228502243' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='414.8842167442832' rx='3' width='20.148055540739207' height='20' fill='#bf103c'/> <rect x='414.8842167442832' rx='3' width='20.148055540739207' height='20' fill='url(#shade)'/> <rect x='415.31653723473755' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='400.736161203544' rx='3' width='20.58037603119359' height='20' fill='#bf103c'/> + <rect x='400.736161203544' rx='3' width='20.58037603119359' height='20' fill='#bd890b'/> <rect x='400.736161203544' rx='3' width='20.58037603119359' height='20' fill='url(#shade)'/> <rect x='401.1816920610293' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='386.1557851723504' rx='3' width='21.025906888678875' height='20' fill='#bf103c'/> <rect x='386.1557851723504' rx='3' width='21.025906888678875' height='20' fill='url(#shade)'/> <rect x='386.61493006451815' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='371.12987828367153' rx='3' width='21.485051780846597' height='20' fill='#bf103c'/> + <rect x='371.12987828367153' rx='3' width='21.485051780846597' height='20' fill='#bd890b'/> <rect x='371.12987828367153' rx='3' width='21.485051780846597' height='20' fill='url(#shade)'/> <rect x='371.60305321299876' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='355.6448265028249' rx='3' width='21.958226710173836' height='20' fill='#bf103c'/> <rect x='355.6448265028249' rx='3' width='21.958226710173836' height='20' fill='url(#shade)'/> <rect x='356.13246018352817' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='339.68659979265107' rx='3' width='22.445860390877083' height='20' fill='#bf103c'/> + <rect x='339.68659979265107' rx='3' width='22.445860390877083' height='20' fill='#bd890b'/> <rect x='339.68659979265107' rx='3' width='22.445860390877083' height='20' fill='url(#shade)'/> <rect x='340.18913403911733' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='323.24073940177396' rx='3' width='22.948394637343355' height='20' fill='#bf103c'/> <rect x='323.24073940177396' rx='3' width='22.948394637343355' height='20' fill='url(#shade)'/> <rect x='323.7586295288612' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='306.2923447644306' rx='3' width='23.466284764430597' height='20' fill='#bf103c'/> + <rect x='306.2923447644306' rx='3' width='23.466284764430597' height='20' fill='#bd890b'/> <rect x='306.2923447644306' rx='3' width='23.466284764430597' height='20' fill='url(#shade)'/> <rect x='306.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='288.82606' rx='3' width='24.0' height='20' fill='#bf103c'/> <rect x='288.82606' rx='3' width='24.0' height='20' fill='url(#shade)'/> <rect x='288.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='#00f244'/> + <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='#00f844'/> <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#shade)'/> <rect width='169.18729000000002' height='20' fill='#404040'/> <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg index dde2b740e37..a0005ed6d76 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg @@ -33,9 +33,15 @@ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> + <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bd890b' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> <stop offset='0' stop-color='#ab83ff' /> - <stop offset='1' stop-color='#00f244' /> + <stop offset='1' stop-color='#00f844' /> <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> @@ -45,21 +51,21 @@ <g clip-path='url(#rounded)'> <rect x='757.7809900000001' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='725.59036' rx='3' width='38.19063' height='20' fill='url(#run-on-success)'/> - <polygon points='635.8470950000001 0 635.8470950000001 20 734.59036 20 742.59036 0' fill='#00f244'/> + <polygon points='635.8470950000001 0 635.8470950000001 20 734.59036 20 742.59036 0' fill='#00f844'/> <rect x='635.8470950000001' rx='3' width='131.74345499999998' height='20' fill='url(#shade)'/> <rect x='635.8470950000001' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='603.656465' rx='3' width='38.19063' height='20' fill='#bf103c'/> - <polygon points='486.981225 0 486.981225 20 612.656465 20 620.656465 0' fill='#00f244'/> + <polygon points='486.981225 0 486.981225 20 612.656465 20 620.656465 0' fill='#00f844'/> <rect x='486.981225' rx='3' width='158.67543' height='20' fill='url(#shade)'/> <rect x='486.981225' rx='3' width='8' height='20' fill='url(#shadow)'/> <rect x='348.865175' rx='3' width='144.11604999999997' height='20' fill='url(#run-on-success)'/> <rect x='358.865175' rx='3' width='134.11604999999997' height='20' fill='url(#shade)'/> <rect x='358.865175' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='326.674545' rx='3' width='38.19063' height='20' fill='#00f244'/> + <rect x='326.674545' rx='3' width='38.19063' height='20' fill='#00f844'/> <polygon points='237.71563000000003 0 237.71563000000003 20 335.674545 20 343.674545 0' fill='url(#run-on-failure)'/> <rect x='237.71563000000003' rx='3' width='130.959105' height='20' fill='url(#shade)'/> <rect x='237.71563000000003' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='153.18729000000002' rx='3' width='90.52834000000001' height='20' fill='#00f244'/> + <rect x='153.18729000000002' rx='3' width='90.52834000000001' height='20' fill='url(#run-on-warning)'/> <rect x='163.18729000000002' rx='3' width='80.52834000000001' height='20' fill='url(#shade)'/> <rect width='169.18729000000002' height='20' fill='#404040'/> <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index 211aa57d8ed..13a6391d5da 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -7,8 +7,8 @@ "date": 0, "controllerVersion": false, "systemVersion": false, - "configServers": [], - "failingApplications": [], + "configServers": [ ], + "failingApplications": [ ], "productionApplications": [ { "tenant": "tenant1", @@ -29,7 +29,7 @@ "productionSuccesses": 1 } ], - "deployingApplications": [], + "deployingApplications": [ ], "applications": [ { "tenant": "tenant1", @@ -40,11 +40,11 @@ "jobs": [ { "name": "system-test", - "coolingDownUntil": "(ignore)" + "coolingDownUntil": 1600000000000 }, { "name": "staging-test", - "coolingDownUntil": "(ignore)" + "coolingDownUntil": 1600000000000 }, { "name": "production-us-west-1" @@ -54,8 +54,8 @@ "production-us-west-1": { "success": { "number": 1, - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success" } } @@ -64,8 +64,8 @@ "production-us-west-1": { "success": { "number": 1, - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success" } } @@ -79,12 +79,6 @@ "upgradePolicy": "default", "jobs": [ { - "name": "system-test" - }, - { - "name": "staging-test" - }, - { "name": "production-us-west-1" } ], @@ -92,8 +86,8 @@ "production-us-west-1": { "success": { "number": 1, - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success" } } @@ -102,8 +96,8 @@ "production-us-west-1": { "success": { "number": 1, - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success" } } @@ -193,12 +187,12 @@ "jobs": [ { "name": "system-test", - "coolingDownUntil": "(ignore)", + "coolingDownUntil": 1600000000000, "pending": "application" }, { "name": "staging-test", - "coolingDownUntil": "(ignore)", + "coolingDownUntil": 1600000000000, "pending": "platform" }, { @@ -210,26 +204,26 @@ "system-test": { "failing": { "number": 2, - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "error" }, "running": { "number": 3, - "start": "(ignore)", + "start": 1600000000000, "status": "running" } }, "staging-test": { "failing": { "number": 2, - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "error" }, "running": { "number": 3, - "start": "(ignore)", + "start": 1600000000000, "status": "running" } } @@ -238,26 +232,26 @@ "system-test": { "failing": { "number": 2, - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "error" }, "running": { "number": 3, - "start": "(ignore)", + "start": 1600000000000, "status": "running" } }, "staging-test": { "failing": { "number": 2, - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "error" }, "running": { "number": 3, - "start": "(ignore)", + "start": 1600000000000, "status": "running" } } @@ -272,7 +266,7 @@ "jobs": [ { "name": "system-test", - "coolingDownUntil": "(ignore)", + "coolingDownUntil": 1600000000000, "pending": "application" }, { @@ -288,35 +282,35 @@ "system-test": { "failing": { "number": 3, - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "error" } }, "staging-test": { "running": { "number": 3, - "start": "(ignore)", + "start": 1600000000000, "status": "running" } }, "production-us-west-1": { "success": { "number": 2, - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success" } } }, "upgradeRuns": { - "system-test": {}, - "staging-test": {}, + "system-test": { }, + "staging-test": { }, "production-us-west-1": { "success": { "number": 2, - "start": "(ignore)", - "end": "(ignore)", + "start": 1600000000000, + "end": 1600000000000, "status": "success" } } @@ -330,12 +324,6 @@ "upgradePolicy": "default", "jobs": [ { - "name": "system-test" - }, - { - "name": "staging-test" - }, - { "name": "production-us-west-1", "pending": "platform" } @@ -344,7 +332,7 @@ "production-us-west-1": { "running": { "number": 2, - "start": "(ignore)", + "start": 1600000000000, "status": "running" } } @@ -353,7 +341,7 @@ "production-us-west-1": { "running": { "number": 2, - "start": "(ignore)", + "start": 1600000000000, "status": "running" } } @@ -365,25 +353,21 @@ "jobs": [ "system-test", "staging-test", - "production-us-east-3", - "test-us-east-3", - "production-us-west-1", - "test-us-west-1", - "production-us-central-1", - "test-us-central-1", "production-ap-northeast-1", "test-ap-northeast-1", "production-ap-northeast-2", "test-ap-northeast-2", "production-ap-southeast-1", "test-ap-southeast-1", - "production-eu-west-1", - "test-eu-west-1", "production-aws-us-east-1a", "test-aws-us-east-1a", - "production-aws-us-west-2a", - "test-aws-us-west-2a", - "production-aws-us-east-1b", - "test-aws-us-east-1b" + "production-eu-west-1", + "test-eu-west-1", + "production-us-central-1", + "test-us-central-1", + "production-us-east-3", + "test-us-east-3", + "production-us-west-1", + "test-us-west-1" ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg index 3bcbed97499..0bdaa3f30ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg @@ -33,9 +33,15 @@ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> + <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bd890b' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> <stop offset='0' stop-color='#ab83ff' /> - <stop offset='1' stop-color='#00f244' /> + <stop offset='1' stop-color='#00f844' /> <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> @@ -44,7 +50,7 @@ </clipPath> <g clip-path='url(#rounded)'> <rect x='288.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='#00f244'/> + <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='#00f844'/> <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#shade)'/> <rect width='169.18729000000002' height='20' fill='#404040'/> <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg index 27e967f8e46..1a38228e75d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg @@ -33,9 +33,15 @@ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> + <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bd890b' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> <stop offset='0' stop-color='#ab83ff' /> - <stop offset='1' stop-color='#00f244' /> + <stop offset='1' stop-color='#00f844' /> <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java index eab3a37a9c3..be3a50c38b0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -66,8 +66,7 @@ public class ControllerAuthorizationFilterTest { @Test public void unprivilegedInPublic() { - ControllerTester tester = new ControllerTester(); - tester.zoneRegistry().setSystemName(SystemName.Public); + ControllerTester tester = new ControllerTester(SystemName.Public); SecurityContext securityContext = new SecurityContext(() -> "user", Set.of(Role.everyone())); ControllerAuthorizationFilter filter = createFilter(tester); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java index 407d9f8765c..c7071cfb68c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java @@ -3,9 +3,6 @@ package com.yahoo.vespa.hosted.controller.restapi.filter; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.TenantName; -import com.yahoo.container.jdisc.RequestHandlerTestDriver; -import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; @@ -19,7 +16,6 @@ import java.util.Set; import static com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo.UserLevel.administrator; import static com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo.UserLevel.developer; import static com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo.UserLevel.user; - import static org.junit.Assert.assertEquals; public class LastLoginUpdateFilterTest { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java index 37150b7e6f0..62ec8f4222c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.role.Role; import org.junit.Test; import java.io.ByteArrayOutputStream; @@ -14,6 +13,9 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.Set; +import static com.yahoo.slime.SlimeUtils.jsonToSlimeOrThrow; +import static com.yahoo.slime.SlimeUtils.toJsonBytes; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; /** @@ -46,10 +48,9 @@ public class TsdbQueryRewriterTest { byte[] data = Files.readAllBytes(Paths.get("src/test/resources/horizon", initialFilename)); data = TsdbQueryRewriter.rewrite(data, tenants, operator, SystemName.Public); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new JsonFormat(false).encode(baos, SlimeUtils.jsonToSlime(data)); - String expectedJson = Files.readString(Paths.get("src/test/resources/horizon", expectedFilename)); + String actualJson = new String(toJsonBytes(jsonToSlimeOrThrow(data).get(), false), UTF_8); + String expectedJson = new String(toJsonBytes(jsonToSlimeOrThrow(Files.readAllBytes(Paths.get("src/test/resources/horizon", expectedFilename))).get(), false), UTF_8); - assertEquals(expectedJson, baos.toString()); + assertEquals(expectedJson, actualJson); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index cf4deb7b4bf..5c210616cb1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -47,13 +47,17 @@ public class OsApiTest extends ControllerContainerTest { private ContainerTester tester; private List<OsUpgrader> osUpgraders; + @Override + protected SystemName system() { + return SystemName.cd; + } + @Before public void before() { tester = new ContainerTester(container, responses); tester.serviceRegistry().clock().setInstant(Instant.ofEpochMilli(1234)); addUserToHostedOperatorRole(operator); - zoneRegistryMock().setSystemName(SystemName.cd) - .setZones(zone1, zone2, zone3) + zoneRegistryMock().setZones(zone1, zone2, zone3) .reprovisionToUpgradeOsIn(zone3) .setOsUpgradePolicy(cloud1, UpgradePolicy.builder().upgrade(zone1).upgrade(zone2).build()) .setOsUpgradePolicy(cloud2, UpgradePolicy.builder().upgrade(zone3).build()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json index 4e026a2c881..7a8eef983f1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json @@ -164,7 +164,7 @@ "upgradeBudget": "PT24H", "scheduledAt": 1234, "cloud": "cloud2", - "nodes": [] + "nodes": [ ] } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java index de7f2515979..ec3328fbf61 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java @@ -6,7 +6,7 @@ import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.hosted.controller.api.integration.user.User; +import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.yolean.Exceptions; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json index 59519c33d06..5301e8398eb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json @@ -7,7 +7,7 @@ "region": "us-west-1", "status": "in", "agent": "operator", - "changedAt": "(ignore)" + "changedAt": 1600000000000 } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json index e95d9bcdc42..5383eb7f806 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json @@ -7,7 +7,7 @@ "region": "us-west-1", "status": "in", "agent": "system", - "changedAt": "(ignore)" + "changedAt": 0 } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json index 49b85775e63..889aa199279 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json @@ -7,7 +7,7 @@ "region": "us-west-1", "status": "out", "agent": "operator", - "changedAt": "(ignore)" + "changedAt": 1600000000000 } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json index abf0a46ae3e..0aa6d79ee63 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json @@ -4,5 +4,5 @@ "region": "us-west-1", "status": "in", "agent": "operator", - "changedAt": "(ignore)" + "changedAt": 1600000000000 } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json index 8328e1ffab1..cb3fb75f5cb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json @@ -4,5 +4,5 @@ "region": "us-west-1", "status": "in", "agent": "system", - "changedAt": "(ignore)" + "changedAt": 0 } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json index d86ca2d56e6..1602ca76a9d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json @@ -4,5 +4,5 @@ "region": "us-west-1", "status": "out", "agent": "operator", - "changedAt": "(ignore)" + "changedAt": 1600000000000 } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json index 06fb2b92c53..77c5d544f6d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json @@ -7,7 +7,7 @@ "region": "us-east-3", "status": "in", "agent": "unknown", - "changedAt": "(ignore)" + "changedAt": 1497618757000 }, { "routingMethod": "sharedLayer4", @@ -16,7 +16,7 @@ "region": "us-west-1", "status": "in", "agent": "unknown", - "changedAt": "(ignore)" + "changedAt": 1497618757000 } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json index 5de12d9b1ec..6ff308679f6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json @@ -7,7 +7,7 @@ "region": "us-east-3", "status": "in", "agent": "unknown", - "changedAt": "(ignore)" + "changedAt": 1497618757000 }, { "routingMethod": "sharedLayer4", @@ -16,7 +16,7 @@ "region": "us-west-1", "status": "in", "agent": "unknown", - "changedAt": "(ignore)" + "changedAt": 1497618757000 }, { "routingMethod": "sharedLayer4", @@ -25,7 +25,7 @@ "region": "us-east-3", "status": "in", "agent": "unknown", - "changedAt": "(ignore)" + "changedAt": 1497618757000 }, { "routingMethod": "sharedLayer4", @@ -34,7 +34,7 @@ "region": "us-west-1", "status": "in", "agent": "unknown", - "changedAt": "(ignore)" + "changedAt": 1497618757000 } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json index 4eb51c1e907..e7c4f5842f5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json @@ -7,7 +7,7 @@ "region": "us-west-1", "status": "in", "agent": "operator", - "changedAt": "(ignore)" + "changedAt": 1600000000000 } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json index 615ce4b4a6e..4a774c7b850 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json @@ -7,7 +7,7 @@ "region": "us-west-1", "status": "in", "agent": "unknown", - "changedAt": "(ignore)" + "changedAt": 1497618757000 } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json index 816bc810048..16ac4eb907d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json @@ -7,7 +7,7 @@ "region": "us-west-1", "status": "out", "agent": "operator", - "changedAt": "(ignore)" + "changedAt": 1600000000000 } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json index 8460cc5ec8a..a6b873b5be5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json @@ -4,5 +4,5 @@ "region": "us-west-1", "status": "in", "agent": "operator", - "changedAt": "(ignore)" + "changedAt": 0 } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json index 8460cc5ec8a..a6b873b5be5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json @@ -4,5 +4,5 @@ "region": "us-west-1", "status": "in", "agent": "operator", - "changedAt": "(ignore)" + "changedAt": 0 } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json index 88fddcbd955..3fd71a731eb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json @@ -4,5 +4,5 @@ "region": "us-west-1", "status": "out", "agent": "operator", - "changedAt": "(ignore)" + "changedAt": 0 } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResultTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResultTest.java index b4a58567284..c157cb3516f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResultTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResultTest.java @@ -11,7 +11,9 @@ import org.junit.Test; import java.util.List; -import static com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.*; +import static com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.FlagDataChange; +import static com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.OperationError; +import static com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.merge; import static org.assertj.core.api.Assertions.assertThat; /** diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java index cdf3674633c..1cd4b5e5541 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.user.User; +import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import org.junit.Test; @@ -58,7 +58,7 @@ public class UserApiOnPremTest extends ControllerContainerTest { .forEach(d -> d.admin(AthenzIdentities.from("domain1.bob"))); tester.assertResponse(createUserRequest(user, operator), - new File("user-without-applications.json")); + new File("on-prem-user-without-applications.json")); tester.assertResponse(createUserRequest(user, tenantAdmin), new File("user-with-applications-athenz.json")); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index a93e9f55e30..d0c95fe593e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; -import com.yahoo.vespa.hosted.controller.api.integration.user.User; +import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-roles.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-roles.json index ca437dba761..8497358fe40 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-roles.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-roles.json @@ -1,19 +1,19 @@ { "tenant": "my-tenant", "application": "my-app", - "roleNames": [], + "roleNames": [ ], "users": [ { "name": "administrator@tenant", "email": "administrator@tenant", "verified": false, - "roles": {} + "roles": { } }, { "name": "developer@tenant", "email": "developer@tenant", "verified": false, - "roles": {} + "roles": { } } ] -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json index b7d48f283f3..dffb0c90df1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json @@ -6,4 +6,3 @@ } ] } - diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/on-prem-user-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/on-prem-user-without-applications.json new file mode 100644 index 00000000000..405ed9627c3 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/on-prem-user-without-applications.json @@ -0,0 +1,27 @@ +{ + "isPublic": false, + "isCd": false, + "hasTrialCapacity": true, + "user": { + "name": "Joe Developer", + "email": "dev@domail", + "nickname": "dev", + "verified": false + }, + "tenants": { }, + "operator": [ + "hostedOperator", + "hostedSupporter", + "hostedAccountant" + ], + "flags": [ + { + "id": "enable-public-signup-flow", + "rules": [ + { + "value": false + } + ] + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json index 56104d626dd..2a33f35545c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json @@ -1,8 +1,8 @@ { "name": "", "email": "", - "website":"", + "website": "", "contactName": "administrator", "contactEmail": "administrator@tenant", - "contacts": [] -}
\ No newline at end of file + "contacts": [ ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-roles.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-roles.json index bc921e4bdf4..5aca5de95fc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-roles.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-roles.json @@ -45,4 +45,4 @@ } } ] -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json index 54585767d51..3237e99783d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json @@ -10,12 +10,13 @@ { "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n", "user": "developer@tenant" - }], - "secretStores": [], + } + ], + "secretStores": [ ], "integrations": { "aws": { "tenantRole": "my-tenant-tenant-role", - "accounts": [] + "accounts": [ ] } }, "quota": { @@ -30,7 +31,7 @@ "url": "http://localhost:8080/application/v4/tenant/my-tenant/application/my-app" } ], - "metaData":{ - "createdAtMillis": "(ignore)" + "metaData": { + "createdAtMillis": 1600000000000 } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json index 1cd2fb41263..0cc8ba2cd9e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json @@ -40,6 +40,6 @@ } ], "metaData": { - "createdAtMillis": "(ignore)" + "createdAtMillis": 1600000000000 } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json index 14b900caf50..3153c6e218a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json @@ -2,12 +2,12 @@ "tenant": "my-tenant", "type": "CLOUD", "creator": "administrator@tenant", - "pemDeveloperKeys": [], - "secretStores": [], + "pemDeveloperKeys": [ ], + "secretStores": [ ], "integrations": { "aws": { "tenantRole": "my-tenant-tenant-role", - "accounts": [] + "accounts": [ ] } }, "quota": { @@ -15,8 +15,8 @@ "budgetUsed": 0.0, "clusterSize": 5 }, - "applications": [], - "metaData":{ - "createdAtMillis": "(ignore)" + "applications": [ ], + "metaData": { + "createdAtMillis": 1600000000000 } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json index 0211f595ce7..14f78e14c5b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json @@ -1,7 +1,7 @@ { "isPublic": false, "isCd": false, - "hasTrialCapacity": (ignore), + "hasTrialCapacity": true, "user": { "name": "Joe Developer", "email": "dev@domail", @@ -10,7 +10,7 @@ }, "tenants": { "sandbox": { - "supported": (ignore), + "supported": false, "roles": [ "administrator", "developer", @@ -18,7 +18,7 @@ ] }, "tenant1": { - "supported": (ignore), + "supported": false, "roles": [ "administrator", "developer", @@ -26,7 +26,7 @@ ] }, "tenant2": { - "supported": (ignore), + "supported": false, "roles": [ "administrator", "developer", @@ -34,5 +34,14 @@ ] } }, - "flags": [{"id":"enable-public-signup-flow","rules":[{"value":false}]}] + "flags": [ + { + "id": "enable-public-signup-flow", + "rules": [ + { + "value": false + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json index 76904bf9bb4..39bdc7cd275 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json @@ -6,7 +6,7 @@ "name": "Joe Developer", "email": "dev@domail", "nickname": "dev", - "verified":false + "verified": false }, "tenants": { "sandbox": { @@ -29,5 +29,14 @@ ] } }, - "flags": [{"id":"enable-public-signup-flow","rules":[{"value":false}]}] + "flags": [ + { + "id": "enable-public-signup-flow", + "rules": [ + { + "value": false + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-supported-tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-supported-tenant.json index a40354a9e71..5ae67f19382 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-supported-tenant.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-supported-tenant.json @@ -32,4 +32,4 @@ ] } ] -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json index 9f9578e6ed8..3ad692d9768 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json @@ -1,18 +1,27 @@ { - "isPublic": (ignore), - "isCd": (ignore), - "hasTrialCapacity": (ignore), + "isPublic": true, + "isCd": false, + "hasTrialCapacity": true, "user": { "name": "Joe Developer", "email": "dev@domail", "nickname": "dev", - "verified":false + "verified": false }, - "tenants": {}, + "tenants": { }, "operator": [ "hostedOperator", "hostedSupporter", "hostedAccountant" ], - "flags": [{"id":"enable-public-signup-flow","rules":[{"value":false}]}] + "flags": [ + { + "id": "enable-public-signup-flow", + "rules": [ + { + "value": false + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-trial-capacity-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-trial-capacity-cloud.json index 2b98a75068a..ef1305647e1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-trial-capacity-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-trial-capacity-cloud.json @@ -6,8 +6,17 @@ "name": "Joe Developer", "email": "dev@domail", "nickname": "dev", - "verified":false + "verified": false }, - "tenants": {}, - "flags": [{"id":"enable-public-signup-flow","rules":[{"value":false}]}] -}
\ No newline at end of file + "tenants": { }, + "flags": [ + { + "id": "enable-public-signup-flow", + "rules": [ + { + "value": false + } + ] + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 1919de33e8b..303230b91ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; @@ -322,7 +321,7 @@ public class RoutingPoliciesTest { @Test public void zone_routing_policies_without_dns_update() { - var tester = new RoutingPoliciesTester(new DeploymentTester(), SystemName.main, false); + var tester = new RoutingPoliciesTester(new DeploymentTester(), false); var context = tester.newDeploymentContext("tenant1", "app1", "default"); tester.provisionLoadBalancers(1, context.instanceId(), true, zone1, zone2); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); @@ -410,10 +409,7 @@ public class RoutingPoliciesTest { var context = tester.newDeploymentContext("tenant1", "app1", "default"); context.submit(applicationPackage).deploy(); var zone = ZoneId.from("dev", "us-east-1"); - var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); - tester.controllerTester().serviceRegistry().zoneRegistry() - .setZones(zoneApi) - .exclusiveRoutingIn(zoneApi); + tester.controllerTester().setRoutingMethod(List.of(zone), RoutingMethod.exclusive); var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud"); assertEquals(prodRecords, tester.recordNames()); @@ -453,7 +449,7 @@ public class RoutingPoliciesTest { tester.controllerTester().configServer().removeLoadBalancers(context.instanceId(), zone1); // Load balancer for the same application is provisioned again, but with a different hostname - var newHostname = HostName.from("new-hostname"); + var newHostname = HostName.of("new-hostname"); var loadBalancer = new LoadBalancer("LB-0-Z-" + zone1.value(), context.instanceId(), ClusterSpec.Id.from("c0"), @@ -624,7 +620,7 @@ public class RoutingPoliciesTest { // Application starts deployment context = context.submit(applicationPackage); - for (var testJob : List.of(JobType.systemTest, JobType.stagingTest)) { + for (var testJob : List.of(DeploymentContext.systemTest, DeploymentContext.stagingTest)) { context = context.runJob(testJob); // Since runJob implicitly tears down the deployment and immediately deletes DNS records associated with the // deployment, we consume only one DNS update at a time here @@ -708,7 +704,7 @@ public class RoutingPoliciesTest { List<Record> records = tester.controllerTester().nameService().findRecords(Record.Type.CNAME, name); assertEquals(1, records.size()); - assertEquals(RecordData.from("lb-0--hosted-vespa:zone-config-servers:default--prod.us-west-1."), + assertEquals(RecordData.from("lb-0--hosted-vespa.zone-config-servers.default--prod.us-west-1."), records.get(0).data()); } @@ -859,10 +855,10 @@ public class RoutingPoliciesTest { for (int i = 0; i < count; i++) { HostName lbHostname; if (shared) { - lbHostname = HostName.from("shared-lb--" + zone.value()); + lbHostname = HostName.of("shared-lb--" + zone.value()); } else { - lbHostname = HostName.from("lb-" + i + "--" + application.serializedForm() + - "--" + zone.value()); + lbHostname = HostName.of("lb-" + i + "--" + application.toFullString() + + "--" + zone.value()); } loadBalancers.add( new LoadBalancer("LB-" + i + "-Z-" + zone.value(), @@ -879,8 +875,8 @@ public class RoutingPoliciesTest { var sharedRegion = RegionName.from("aws-us-east-1c"); return List.of(ZoneId.from(Environment.prod, sharedRegion), ZoneId.from(Environment.prod, RegionName.from("aws-eu-west-1a")), - ZoneId.from(Environment.staging, sharedRegion), - ZoneId.from(Environment.test, sharedRegion)); + ZoneId.from(Environment.staging, RegionName.from("us-east-3")), + ZoneId.from(Environment.test, RegionName.from("us-east-1"))); } private static class RoutingPoliciesTester { @@ -892,23 +888,21 @@ public class RoutingPoliciesTest { } public RoutingPoliciesTester(SystemName system) { - this(new DeploymentTester(system.isPublic() - ? new ControllerTester(new RotationsConfig.Builder().build(), system) - : new ControllerTester()), - system, + this(new DeploymentTester(system.isPublic() ? new ControllerTester(new RotationsConfig.Builder().build(), system) + : new ControllerTester(system)), true); } - public RoutingPoliciesTester(DeploymentTester tester, SystemName system, boolean exclusiveRouting) { + public RoutingPoliciesTester(DeploymentTester tester, boolean exclusiveRouting) { this.tester = tester; List<ZoneId> zones; - if (system.isPublic()) { + if (tester.controller().system().isPublic()) { zones = publicZones(); } else { zones = new ArrayList<>(tester.controllerTester().zoneRegistry().zones().all().ids()); // Default zones zones.add(zone4); // Missing from default ZoneRegistryMock zones } - tester.controllerTester().setZones(zones, system); + tester.controllerTester().setZones(zones); if (exclusiveRouting) { tester.controllerTester().setRoutingMethod(zones, RoutingMethod.exclusive); } @@ -985,7 +979,7 @@ public class RoutingPoliciesTest { deploymentsByDnsName.forEach((dnsName, deployments) -> { Set<String> weightedTargets = deployments.stream() .map(d -> "weighted/lb-" + loadBalancerId + "--" + - d.applicationId().serializedForm() + "--" + d.zoneId().value() + + d.applicationId().toFullString() + "--" + d.zoneId().value() + "/dns-zone-1/" + d.zoneId().value() + "/" + deploymentWeights.get(d)) .collect(Collectors.toSet()); assertEquals(dnsName + " has expected targets", weightedTargets, aliasDataOf(dnsName)); @@ -1009,7 +1003,7 @@ public class RoutingPoliciesTest { zonesByRegionEndpoint.forEach((regionEndpoint, zonesInRegion) -> { Set<String> weightedTargets = zonesInRegion.stream() .map(z -> "weighted/lb-" + loadBalancerId + "--" + - instance.serializedForm() + "--" + z.value() + + instance.toFullString() + "--" + z.value() + "/dns-zone-1/" + z.value() + "/" + zoneWeights.get(z)) .collect(Collectors.toSet()); assertEquals("Region endpoint " + regionEndpoint + " points to load balancer", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java index d7847da2404..277c8e1ef85 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; @@ -115,7 +114,7 @@ public class RotationRepositoryTest { // We're now out of rotations and next deployment fails var application3 = tester.newDeploymentContext("tenant3", "app3", "default"); application3.submit(applicationPackage) - .runJobExpectingFailure(JobType.systemTest, Optional.of("out of rotations")); + .runJobExpectingFailure(DeploymentContext.systemTest, Optional.of("out of rotations")); } @Test @@ -124,7 +123,7 @@ public class RotationRepositoryTest { .globalServiceId("foo") .region("us-east-3") .build(); - application.submit(applicationPackage).runJobExpectingFailure(JobType.systemTest, Optional.of("less than 2 prod zones are defined")); + application.submit(applicationPackage).runJobExpectingFailure(DeploymentContext.systemTest, Optional.of("less than 2 prod zones are defined")); } @Test @@ -149,10 +148,10 @@ public class RotationRepositoryTest { ZoneApiMock.fromId("staging.cd-us-west-1"), ZoneApiMock.fromId("prod.cd-us-east-1"), ZoneApiMock.fromId("prod.cd-us-west-1")); + DeploymentTester tester = new DeploymentTester(new ControllerTester(rotationsConfig, SystemName.cd)); tester.controllerTester().zoneRegistry() .setZones(zones) - .setRoutingMethod(zones, RoutingMethod.sharedLayer4) - .setSystemName(SystemName.cd); + .setRoutingMethod(zones, RoutingMethod.sharedLayer4); tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.notController()); var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); application2.submit(applicationPackage).deploy(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index ec8d5c35049..7a137d4e410 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -2,18 +2,19 @@ package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; -import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.Run; @@ -29,10 +30,10 @@ import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static java.util.stream.Collectors.toSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -57,10 +58,11 @@ public class VersionStatusTest { @Test public void testSystemVersionIsControllerVersionIfConfigServersAreNewer() { ControllerTester tester = new ControllerTester(); - Version largerThanCurrent = new Version(Vtag.currentVersion.getMajor() + 1); + Version controllerVersion = tester.controller().readVersionStatus().controllerVersion().get().versionNumber(); + Version largerThanCurrent = new Version(controllerVersion.getMajor() + 1); tester.upgradeSystemApplications(largerThanCurrent); VersionStatus versionStatus = VersionStatus.compute(tester.controller()); - assertEquals(Vtag.currentVersion, versionStatus.systemVersion().get().versionNumber()); + assertEquals(controllerVersion, versionStatus.systemVersion().get().versionNumber()); } @Test @@ -82,12 +84,13 @@ public class VersionStatusTest { @Test public void testControllerVersionIsVersionOfOldestController() { - HostName controller1 = HostName.from("controller-1"); - HostName controller2 = HostName.from("controller-2"); - HostName controller3 = HostName.from("controller-3"); + HostName controller1 = HostName.of("controller-1"); + HostName controller2 = HostName.of("controller-2"); + HostName controller3 = HostName.of("controller-3"); MockCuratorDb db = new MockCuratorDb(Stream.of(controller1, controller2, controller3) .map(hostName -> hostName.value() + ":2222") - .collect(Collectors.joining(","))); + .collect(Collectors.joining(",")), + SystemName.main); ControllerTester tester = new ControllerTester(db); writeControllerVersion(controller1, Version.fromString("6.2"), db); @@ -491,12 +494,13 @@ public class VersionStatusTest { @Test public void testCommitDetailsPreservation() { - HostName controller1 = HostName.from("controller-1"); - HostName controller2 = HostName.from("controller-2"); - HostName controller3 = HostName.from("controller-3"); + HostName controller1 = HostName.of("controller-1"); + HostName controller2 = HostName.of("controller-2"); + HostName controller3 = HostName.of("controller-3"); MockCuratorDb db = new MockCuratorDb(Stream.of(controller1, controller2, controller3) .map(hostName -> hostName.value() + ":2222") - .collect(Collectors.joining(","))); + .collect(Collectors.joining(",")), + SystemName.main); DeploymentTester tester = new DeploymentTester(new ControllerTester(db)); // Commit details are set for initial version diff --git a/controller-server/src/test/resources/testConfig.json b/controller-server/src/test/resources/testConfig.json index 7b91b4930a1..5c3d5942001 100644 --- a/controller-server/src/test/resources/testConfig.json +++ b/controller-server/src/test/resources/testConfig.json @@ -1,20 +1,20 @@ { "application": "tenant:application:default", - "zone": "test.aws-us-east-1c", + "zone": "test.us-east-1", "system": "publiccd", "isCI": true, "endpoints": { - "test.aws-us-east-1c": [ + "test.us-east-1": [ "https://ai.default.default.global.vespa.oath.cloud/" ] }, "zoneEndpoints": { - "test.aws-us-east-1c": { + "test.us-east-1": { "ai": "https://ai.default.default.global.vespa.oath.cloud/" } }, "clusters": { - "test.aws-us-east-1c": [ + "test.us-east-1": [ "facts" ] } |