diff options
author | Øyvind Grønnesby <oyving@yahooinc.com> | 2022-05-30 12:58:57 +0200 |
---|---|---|
committer | Øyvind Grønnesby <oyving@yahooinc.com> | 2022-05-30 12:58:57 +0200 |
commit | 1cce4cc2640fe7bce984b808b1571555f5df0c30 (patch) | |
tree | 65c8a1830e8e94c368fcd418dd0f7cabd94439a6 /controller-server | |
parent | 9fa7903909043d7b855f7e3ba315050ba5a12597 (diff) | |
parent | 970cd02608c75df65bcfee3b061f906a39566825 (diff) |
Merge remote-tracking branch 'origin/master' into ogronnesby/contact-info-resources
Conflicts:
controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
Diffstat (limited to 'controller-server')
39 files changed, 485 insertions, 234 deletions
diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 5cf53929a98..ff7367e03b2 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -102,12 +102,6 @@ </dependency> <dependency> - <groupId>javax.ws.rs</groupId> - <artifactId>javax.ws.rs-api</artifactId> - <scope>provided</scope> - </dependency> - - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>flags</artifactId> <version>${project.version}</version> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index 7a0e60aacb4..4f58e87035b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -8,11 +8,11 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.security.KeyUtils; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; @@ -128,26 +128,26 @@ public abstract class LockedTenant { private final BiMap<PublicKey, Principal> developerKeys; private final TenantInfo info; private final List<TenantSecretStore> tenantSecretStores; - private final Optional<String> archiveAccessRole; + private final ArchiveAccess archiveAccess; private Cloud(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info, - List<TenantSecretStore> tenantSecretStores, Optional<String> archiveAccessRole) { + List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess) { super(name, createdAt, lastLoginInfo); this.developerKeys = ImmutableBiMap.copyOf(developerKeys); this.creator = creator; this.info = info; this.tenantSecretStores = tenantSecretStores; - this.archiveAccessRole = archiveAccessRole; + this.archiveAccess = archiveAccess; } private Cloud(CloudTenant tenant) { - this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccessRole()); + this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess()); } @Override public CloudTenant get() { - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccessRole); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess); } public Cloud withDeveloperKey(PublicKey key, Principal principal) { @@ -155,38 +155,38 @@ public abstract class LockedTenant { if (keys.containsKey(key)) throw new IllegalArgumentException("Key " + KeyUtils.toPem(key) + " is already owned by " + keys.get(key)); keys.put(key, principal); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccessRole); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess); } public Cloud withoutDeveloperKey(PublicKey key) { BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys); keys.remove(key); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccessRole); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess); } public Cloud withInfo(TenantInfo newInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccessRole); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccess); } @Override public LockedTenant with(LastLoginInfo lastLoginInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccessRole); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess); } public Cloud withSecretStore(TenantSecretStore tenantSecretStore) { ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); secretStores.add(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccessRole); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess); } public Cloud withoutSecretStore(TenantSecretStore tenantSecretStore) { ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); secretStores.remove(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccessRole); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess); } - public Cloud withArchiveAccessRole(Optional<String> role) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, role); + public Cloud withArchiveAccess(ArchiveAccess archiveAccess) { + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java index 21914b87818..0a0adcfc252 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -46,7 +46,7 @@ public class CuratorArchiveBucketDb { return getBucketNameFromCache(zoneId, tenant) .or(() -> findAndUpdateArchiveUriCache(zoneId, tenant, buckets(zoneId))) .or(() -> createIfMissing ? Optional.of(assignToBucket(zoneId, tenant)) : Optional.empty()) - .map(bucketName -> URI.create(Text.format("s3://%s/%s/", bucketName, tenant.value()))); + .map(bucketName -> archiveService.bucketURI(zoneId, bucketName, tenant)); } private String assignToBucket(ZoneId zoneId, TenantName tenant) { @@ -57,7 +57,7 @@ public class CuratorArchiveBucketDb { .orElseGet(() -> { // If not, find an existing bucket with space Optional<ArchiveBucket> unfilledBucket = zoneBuckets.stream() - .filter(bucket -> bucket.tenants().size() < tenantsPerBucket().orElse(Integer.MAX_VALUE)) + .filter(bucket -> archiveService.canAddTenantToBucket(zoneId, bucket)) .findAny(); // And place the tenant in that bucket. @@ -94,23 +94,6 @@ public class CuratorArchiveBucketDb { return bucketName; } - private OptionalInt tenantsPerBucket() { - if (system.isPublic()) { - /* - * Due to policy limits, we can't put data for more than this many tenants in a bucket. - * Policy size limit is 20kb, about 550 bytes for non-tenant related policies. Each tenant - * needs about 500 + len(role_arn) bytes, we limit role_arn to 100 characters, so we can - * fit about (20k - 550) / 600 ~ 32 tenants per bucket. - */ - return OptionalInt.of(30); - } else { - /* - * The S3 policies in main/cd have a fixed size. - */ - return OptionalInt.empty(); - } - } - private Optional<String> getBucketNameFromCache(ZoneId zoneId, TenantName tenantName) { return Optional.ofNullable(archiveUriCache.get(zoneId)).map(map -> map.get(tenantName)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index e4869cc9bf4..3bc2aec95ff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.restapi.RestApiException; import com.yahoo.text.Text; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -33,7 +34,6 @@ import com.yahoo.vespa.hosted.controller.security.TenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import javax.ws.rs.ForbiddenException; import java.time.Instant; import java.util.Arrays; import java.util.List; @@ -197,7 +197,7 @@ public class AthenzFacade implements AccessControl { } catch (ZmsClientException e) { if (e.getErrorCode() == com.yahoo.jdisc.Response.Status.FORBIDDEN) - throw new ForbiddenException("Not authorized to create application", e); + throw new RestApiException.Forbidden("Not authorized to create application", e); else throw e; } @@ -289,7 +289,7 @@ public class AthenzFacade implements AccessControl { private void verifyIsDomainAdmin(AthenzIdentity identity, AthenzDomain domain) { log("getMembership(domain=%s, role=%s, principal=%s)", domain, "admin", identity); if ( ! zmsClient.getMembership(new AthenzRole(domain, "admin"), identity)) - throw new ForbiddenException( + throw new RestApiException.Forbidden( Text.format("The user '%s' is not admin in Athenz domain '%s'", identity.getFullName(), domain.getName())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index be07a2b0cb1..f7452323955 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -7,9 +7,7 @@ import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; -import com.yahoo.text.Text; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -148,9 +146,11 @@ public class DeploymentTrigger { return; } - applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> + applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { + if (application.get().deploymentSpec().instance(id.instance()).isPresent()) applications().store(application.with(id.instance(), - instance -> withRemainingChange(instance, instance.change(), jobs.deploymentStatus(application.get()))))); + instance -> withRemainingChange(instance, instance.change(), jobs.deploymentStatus(application.get())))); + }); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index fb7a3fc4ba5..632eb556444 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -662,7 +662,10 @@ public class InternalStepRunner implements StepRunner { "or a Java test bundle under 'components/' with at least one test with the annotation " + "for this suite. See docs.vespa.ai/en/testing.html for details."); controller.jobController().updateTestReport(id); - return Optional.of(noTests); + + DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); + boolean requireTests = spec.steps().stream().anyMatch(step -> step.concerns(id.type().environment())); + return Optional.of(requireTests ? testFailure : noTests); } case SUCCESS: logger.log("Tests completed successfully."); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 30f16acf77d..9331c9942b6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -69,8 +69,12 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup; import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static java.time.temporal.ChronoUnit.SECONDS; +import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; import static java.util.function.Predicate.not; import static java.util.logging.Level.INFO; @@ -179,11 +183,27 @@ public class JobController { if (deployment.isEmpty() || deployment.get().at().isBefore(run.start())) return run; - Instant from = run.lastVespaLogTimestamp().isAfter(deployment.get().at()) ? run.lastVespaLogTimestamp() : deployment.get().at(); + Instant deployedAt = run.stepInfo(installInitialReal).or(() -> run.stepInfo(installReal)).flatMap(StepInfo::startTime).orElseThrow(); + Instant from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.minusSeconds(10); List<LogEntry> log = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() .getLogs(new DeploymentId(id.application(), zone), Map.of("from", Long.toString(from.toEpochMilli()))), from); + + if (run.hasStep(installTester) && run.versions().targetPlatform().isAfter(new Version("7.590"))) { // todo jonmv: remove + deployedAt = run.stepInfo(installTester).flatMap(StepInfo::startTime).orElseThrow(); + from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.minusSeconds(10); + List<LogEntry> testerLog = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() + .getLogs(new DeploymentId(id.tester().id(), zone), + Map.of("from", Long.toString(from.toEpochMilli()))), + from); + + Instant justNow = controller.clock().instant().minusSeconds(2); + log = Stream.concat(log.stream(), testerLog.stream()) + .filter(entry -> entry.at().isBefore(justNow)) + .sorted(comparing(LogEntry::at)) + .collect(toUnmodifiableList()); + } if (log.isEmpty()) return run; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 03cc6c6ba8d..7ffaaabb1a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -45,8 +45,9 @@ public class Run { // For deserialisation only -- do not use! public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, boolean isRedeployment, Instant start, Optional<Instant> end, - Optional<Instant> sleepUntil, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince, - Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate, boolean dryRun, Optional<String> reason) { + Optional<Instant> sleepUntil, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, + Optional<Instant> noNodesDownSince, Optional<ConvergenceSummary> convergenceSummary, + Optional<X509Certificate> testerCertificate, boolean dryRun, Optional<String> reason) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); this.versions = versions; @@ -67,8 +68,9 @@ public class Run { public static Run initial(RunId id, Versions versions, boolean isRedeployment, Instant now, JobProfile profile, Optional<String> triggeredBy) { EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class); profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step))); - return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), Optional.empty(), running, - -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty(), profile == JobProfile.developmentDryRun, triggeredBy); + return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), + Optional.empty(), running, -1, Instant.EPOCH, Optional.empty(), Optional.empty(), + Optional.empty(), profile == JobProfile.developmentDryRun, triggeredBy); } /** Returns a new Run with the status of the given completed step set accordingly. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 379cc9c4f0a..ec4d138c44f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -65,7 +65,7 @@ public enum Step { deactivateReal(true, deployInitialReal, deployReal, endTests, copyVespaLogs), /** Deactivate the tester. */ - deactivateTester(true, deployTester, endTests), + deactivateTester(true, deployTester, endTests, copyVespaLogs), /** Report completion to the deployment orchestration machinery. */ report(true, installReal, deactivateReal, deactivateTester); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index bd69ea41b05..788360996ff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -45,47 +46,34 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { @Override protected double maintain() { - // Count buckets - so we can alert if we get close to the account limit of 1000 - zoneRegistry.zonesIncludingSystem().all().ids().forEach(zoneId -> - metric.set(bucketCountMetricName, archiveBucketDb.buckets(zoneId).size(), - metric.createContext(Map.of("zone", zoneId.value())))); + // Count buckets - so we can alert if we get close to the AWS account limit of 1000 + zoneRegistry.zonesIncludingSystem().all().zones().forEach(z -> + metric.set(bucketCountMetricName, archiveBucketDb.buckets(z.getVirtualId()).size(), + metric.createContext(Map.of( + "zone", z.getVirtualId().value(), + "cloud", z.getCloudName().value())))); zoneRegistry.zonesIncludingSystem().controllerUpgraded().zones().forEach(z -> { - ZoneId zoneId = z.getVirtualId(); - try { - var tenantArchiveAccessRoles = cloudTenantArchiveExternalAccessRoles(); - archiveBucketDb.buckets(zoneId).forEach(archiveBucket -> - archiveService.updateBucketPolicy(zoneId, archiveBucket, - Maps.filterEntries(tenantArchiveAccessRoles, - entry -> archiveBucket.tenants().contains(entry.getKey()))) - ); - Map<String, List<ArchiveBucket>> bucketsPerKey = archiveBucketDb.buckets(zoneId).stream() - .collect(groupingBy(ArchiveBucket::keyArn)); - bucketsPerKey.forEach((keyArn, buckets) -> { - Set<String> authorizedIamRolesForKey = buckets.stream() - .flatMap(b -> b.tenants().stream()) - .filter(tenantArchiveAccessRoles::containsKey) - .map(tenantArchiveAccessRoles::get) - .collect(Collectors.toSet()); - archiveService.updateKeyPolicy(zoneId, keyArn, authorizedIamRolesForKey); - }); - } catch (Exception e) { - throw new RuntimeException("Failed to maintain archive access in " + zoneId.value(), e); - } - } - ); + ZoneId zoneId = z.getVirtualId(); + try { + var tenantArchiveAccessRoles = cloudTenantArchiveExternalAccessRoles(); + var buckets = archiveBucketDb.buckets(zoneId); + archiveService.updatePolicies(zoneId, buckets, tenantArchiveAccessRoles); + } catch (Exception e) { + throw new RuntimeException("Failed to maintain archive access in " + zoneId.value(), e); + } + }); return 1.0; } - private Map<TenantName, String> cloudTenantArchiveExternalAccessRoles() { + private Map<TenantName, ArchiveAccess> cloudTenantArchiveExternalAccessRoles() { List<Tenant> tenants = controller().tenants().asList(); return tenants.stream() .filter(t -> t instanceof CloudTenant) .map(t -> (CloudTenant) t) - .filter(t -> t.archiveAccessRole().isPresent()) .collect(Collectors.toUnmodifiableMap( - Tenant::name, cloudTenant -> cloudTenant.archiveAccessRole().orElseThrow())); + Tenant::name, cloudTenant -> cloudTenant.archiveAccess())); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index 36ab2e6f384..42821ea8fe2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -40,15 +40,13 @@ public class ArchiveUriUpdater extends ControllerMaintainer { protected double maintain() { Map<ZoneId, Set<TenantName>> tenantsByZone = new HashMap<>(); - tenantsByZone.put(controller().zoneRegistry().systemZone().getVirtualId(), - new HashSet<>(INFRASTRUCTURE_TENANTS)); + controller().zoneRegistry().zonesIncludingSystem().reachable().zones().forEach( + z -> tenantsByZone.put(z.getVirtualId(), new HashSet<>(INFRASTRUCTURE_TENANTS))); for (var application : applications.asList()) { for (var instance : application.instances().values()) { for (var deployment : instance.deployments().values()) { - tenantsByZone - .computeIfAbsent(deployment.zone(), zone -> new HashSet<>(INFRASTRUCTURE_TENANTS)) - .add(instance.id().tenant()); + tenantsByZone.get(deployment.zone()).add(instance.id().tenant()); } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 369699eb3a3..b4fac8074aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -83,7 +83,6 @@ public class JobRunner extends ControllerMaintainer { jobs.abort(run.id(), "job timeout of " + jobTimeout + " reached"); advance(jobs.run(run.id()).get()); }); - else if (run.readySteps().isEmpty()) executors.execute(() -> finish(run.id())); else if (run.hasFailed() || run.sleepUntil().map(sleepUntil -> ! sleepUntil.isAfter(controller().clock().instant())).orElse(true)) @@ -93,7 +92,8 @@ public class JobRunner extends ControllerMaintainer { private void finish(RunId id) { try { jobs.finish(id); - controller().applications().deploymentTrigger().notifyOfCompletion(id.application()); + if ( ! id.type().environment().isManuallyDeployed()) + controller().applications().deploymentTrigger().notifyOfCompletion(id.application()); } catch (TimeoutException e) { // One of the steps are still being run — that's ok, we'll try to finish the run again later. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java index 269864ad641..a4c7c50085c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java @@ -13,7 +13,6 @@ import java.util.stream.Collectors; /** * (de)serializes tenant/bucket mappings for a zone - * <p> * * @author andreer */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index dd28978d948..fcc6d99aec2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.SystemName; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -214,7 +213,7 @@ class RunSerializer { run.sleepUntil().ifPresent(end -> runObject.setLong(sleepingUntilField, end.toEpochMilli())); runObject.setString(statusField, valueOf(run.status())); runObject.setLong(lastTestRecordField, run.lastTestLogEntry()); - runObject.setLong(lastVespaLogTimestampField, Instant.EPOCH.until(run.lastVespaLogTimestamp(), ChronoUnit.MICROS)); + if (run.lastVespaLogTimestamp().isAfter(Instant.EPOCH)) runObject.setLong(lastVespaLogTimestampField, Instant.EPOCH.until(run.lastVespaLogTimestamp(), ChronoUnit.MICROS)); run.noNodesDownSince().ifPresent(noNodesDownSince -> runObject.setLong(noNodesDownSinceField, noNodesDownSince.toEpochMilli())); run.convergenceSummary().ifPresent(convergenceSummary -> toSlime(convergenceSummary, runObject.setArray(convergenceSummaryField))); run.testerCertificate().ifPresent(certificate -> runObject.setString(testerCertificateField, X509CertificateUtils.toPem(certificate))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 00e38abcba7..e7cf0c34511 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInf import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; @@ -77,6 +78,10 @@ public class TenantSerializer { private static final String lastLoginInfoField = "lastLoginInfo"; private static final String secretStoresField = "secretStores"; private static final String archiveAccessRoleField = "archiveAccessRole"; + private static final String archiveAccessField = "archiveAccess"; + private static final String awsArchiveAccessRoleField = "awsArchiveAccessRole"; + private static final String gcpArchiveAccessMemberField = "gcpArchiveAccessMember"; + private static final String awsIdField = "awsId"; private static final String roleField = "role"; @@ -117,7 +122,13 @@ public class TenantSerializer { toSlime(legacyBillingInfo, root.setObject(billingInfoField)); toSlime(tenant.info(), root); toSlime(tenant.tenantSecretStores(), root); - tenant.archiveAccessRole().ifPresent(role -> root.setString(archiveAccessRoleField, role)); + toSlime(tenant.archiveAccess(), root); + } + + private void toSlime(ArchiveAccess archiveAccess, Cursor root) { + Cursor object = root.setObject(archiveAccessField); + archiveAccess.awsRole().ifPresent(role -> object.setString(awsArchiveAccessRoleField, role)); + archiveAccess.gcpMember().ifPresent(member -> object.setString(gcpArchiveAccessMemberField, member)); } private void toSlime(DeletedTenant tenant, Cursor root) { @@ -175,8 +186,8 @@ public class TenantSerializer { BiMap<PublicKey, Principal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField)); TenantInfo info = tenantInfoFromSlime(tenantObject.field(tenantInfoField)); List<TenantSecretStore> tenantSecretStores = secretStoresFromSlime(tenantObject.field(secretStoresField)); - Optional<String> archiveAccessRole = SlimeUtils.optionalString(tenantObject.field(archiveAccessRoleField)); - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccessRole); + ArchiveAccess archiveAccess = archiveAccessFromSlime(tenantObject); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess); } private DeletedTenant deletedTenantFrom(Inspector tenantObject) { @@ -195,6 +206,22 @@ public class TenantSerializer { return keys.build(); } + ArchiveAccess archiveAccessFromSlime(Inspector tenantObject) { + // TODO(enygaard, 2022-05-24): Remove when all tenants have been rewritten to use ArchiveAccess object + Optional<String> archiveAccessRole = SlimeUtils.optionalString(tenantObject.field(archiveAccessRoleField)); + if (archiveAccessRole.isPresent()) { + return new ArchiveAccess().withAWSRole(archiveAccessRole.get()); + } + Inspector object = tenantObject.field(archiveAccessField); + if (!object.valid()) { + return new ArchiveAccess(); + } + Optional<String> awsArchiveAccessRole = SlimeUtils.optionalString(object.field(awsArchiveAccessRoleField)); + Optional<String> gcpArchiveAccessMember = SlimeUtils.optionalString(object.field(gcpArchiveAccessMemberField)); + return new ArchiveAccess() + .withAWSRole(awsArchiveAccessRole) + .withGCPMember(gcpArchiveAccessMember); + } TenantInfo tenantInfoFromSlime(Inspector infoObject) { if (!infoObject.valid()) return TenantInfo.empty(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java index 886dc27b404..c57133d8efd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java @@ -29,7 +29,9 @@ public class ProxyResponse extends HttpResponse { super(statusResponse); this.contentType = contentType; - String configServerPrefix = HttpURL.from(configServer).withPath(Path.empty()).asURI().toString(); + // Configserver always serves from 4443, therefore all responses will have port 4443 in them, + // but the request URI (loadbalancer/VIP) is not always 4443 + String configServerPrefix = HttpURL.from(configServer).withPort(4443).withPath(Path.empty()).asURI().toString(); String controllerRequestPrefix = controllerRequest.getControllerPrefixUri().toString(); bodyResponseRewritten = bodyResponse.replace(configServerPrefix, controllerRequestPrefix); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d781ea96b47..7af0f962b33 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -35,6 +35,7 @@ import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; +import com.yahoo.restapi.RestApiException; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.security.KeyUtils; import com.yahoo.slime.Cursor; @@ -108,6 +109,7 @@ import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; @@ -122,9 +124,6 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; -import javax.ws.rs.ForbiddenException; -import javax.ws.rs.InternalServerErrorException; -import javax.ws.rs.NotAuthorizedException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -209,10 +208,10 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); } } - catch (ForbiddenException e) { + catch (RestApiException.Forbidden e) { return ErrorResponse.forbidden(Exceptions.toMessageString(e)); } - catch (NotAuthorizedException e) { + catch (RestApiException.Unauthorized e) { return ErrorResponse.unauthorized(Exceptions.toMessageString(e)); } catch (NotExistsException e) { @@ -305,7 +304,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/info/profile")) return putTenantInfo(path.get("tenant"), request, this::putTenantInfoProfile); if (path.matches("/application/v4/tenant/{tenant}/info/billing")) return putTenantInfo(path.get("tenant"), request, this::putTenantInfoBilling); if (path.matches("/application/v4/tenant/{tenant}/info/contacts")) return putTenantInfo(path.get("tenant"), request, this::putTenantInfoContacts); - if (path.matches("/application/v4/tenant/{tenant}/archive-access")) return allowArchiveAccess(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/archive-access")) return allowAwsArchiveAccess(path.get("tenant"), request); // TODO(enygaard, 2022-05-25) Remove when no longer used by console + if (path.matches("/application/v4/tenant/{tenant}/archive-access/aws")) return allowAwsArchiveAccess(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/archive-access/gcp")) return allowGcpArchiveAccess(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/secret-store/{name}")) return addSecretStore(path.get("tenant"), path.get("name"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); @@ -353,7 +354,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/access/managed/operator")) return removeManagedAccess(path.get("tenant")); if (path.matches("/application/v4/tenant/{tenant}/key")) return removeDeveloperKey(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/archive-access")) return removeArchiveAccess(path.get("tenant")); + if (path.matches("/application/v4/tenant/{tenant}/archive-access")) return removeAwsArchiveAccess(path.get("tenant")); // TODO(enygaard, 2022-05-25) Remove when no longer used by console + if (path.matches("/application/v4/tenant/{tenant}/archive-access/aws")) return removeAwsArchiveAccess(path.get("tenant")); + if (path.matches("/application/v4/tenant/{tenant}/archive-access/gcp")) return removeGcpArchiveAccess(path.get("tenant")); if (path.matches("/application/v4/tenant/{tenant}/secret-store/{name}")) return deleteSecretStore(path.get("tenant"), path.get("name"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deployment")) return removeAllProdDeployments(path.get("tenant"), path.get("application")); @@ -1174,7 +1177,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse allowArchiveAccess(String tenantName, HttpRequest request) { + private HttpResponse allowAwsArchiveAccess(String tenantName, HttpRequest request) { if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); @@ -1182,27 +1185,62 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { var role = mandatory("role", data).asString(); if (role.isBlank()) { - return ErrorResponse.badRequest("Archive access role can't be whitespace only"); + return ErrorResponse.badRequest("AWS archive access role can't be whitespace only"); } controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withArchiveAccessRole(Optional.of(role)); + var access = lockedTenant.get().archiveAccess(); + lockedTenant = lockedTenant.withArchiveAccess(access.withAWSRole(role)); controller.tenants().store(lockedTenant); }); - return new MessageResponse("Archive access role set to '" + role + "' for tenant " + tenantName + "."); + return new MessageResponse("AWS archive access role set to '" + role + "' for tenant " + tenantName + "."); } - private HttpResponse removeArchiveAccess(String tenantName) { + private HttpResponse removeAwsArchiveAccess(String tenantName) { if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withArchiveAccessRole(Optional.empty()); + var access = lockedTenant.get().archiveAccess(); + lockedTenant = lockedTenant.withArchiveAccess(access.removeAWSRole()); controller.tenants().store(lockedTenant); }); - return new MessageResponse("Archive access role removed for tenant " + tenantName + "."); + return new MessageResponse("AWS archive access role removed for tenant " + tenantName + "."); + } + + private HttpResponse allowGcpArchiveAccess(String tenantName, HttpRequest request) { + if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) + throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); + + var data = toSlime(request.getData()).get(); + var member = mandatory("member", data).asString(); + + if (member.isBlank()) { + return ErrorResponse.badRequest("GCP archive access role can't be whitespace only"); + } + + controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, lockedTenant -> { + var access = lockedTenant.get().archiveAccess(); + lockedTenant = lockedTenant.withArchiveAccess(access.withGCPMember(member)); + controller.tenants().store(lockedTenant); + }); + + return new MessageResponse("GCP archive access member set to '" + member + "' for tenant " + tenantName + "."); + } + + private HttpResponse removeGcpArchiveAccess(String tenantName) { + if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) + throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); + + controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, lockedTenant -> { + var access = lockedTenant.get().archiveAccess(); + lockedTenant = lockedTenant.withArchiveAccess(access.removeGCPMember()); + controller.tenants().store(lockedTenant); + }); + + return new MessageResponse("GCP archive access member removed for tenant " + tenantName + "."); } private HttpResponse patchApplication(String tenantName, String applicationName, HttpRequest request) { @@ -2500,7 +2538,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { log.warning(String.format("Failed to get quota for tenant %s: %s", tenant.name(), Exceptions.toMessageString(e))); } - cloudTenant.archiveAccessRole().ifPresent(role -> object.setString("archiveAccessRole", role)); + // TODO(enygaard, 2022-05-25) Remove when console is using new archive access structure + cloudTenant.archiveAccess().awsRole().ifPresent(role -> object.setString("archiveAccessRole", role)); + toSlime(cloudTenant.archiveAccess(), object.setObject("archiveAccess")); break; } @@ -2531,6 +2571,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { tenantMetaDataToSlime(tenant, applications, object.setObject("metaData")); } + private void toSlime(ArchiveAccess archiveAccess, Cursor object) { + archiveAccess.awsRole().ifPresent(role -> object.setString("awsRole", role)); + archiveAccess.gcpMember().ifPresent(member -> object.setString("gcpMember", member)); + } + private void toSlime(Quota quota, QuotaUsage usage, Cursor object) { quota.budget().ifPresentOrElse( budget -> object.setDouble("budget", budget.doubleValue()), @@ -2676,7 +2721,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private static Principal requireUserPrincipal(HttpRequest request) { Principal principal = request.getJDiscRequest().getUserPrincipal(); - if (principal == null) throw new InternalServerErrorException("Expected a user principal"); + if (principal == null) throw new RestApiException.InternalServerError("Expected a user principal"); return principal; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java index c72d8ceb089..a3b77e22f1d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java @@ -9,6 +9,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; +import com.yahoo.restapi.RestApiException; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -24,7 +25,6 @@ import com.yahoo.vespa.hosted.controller.maintenance.ChangeManagementAssessor; import com.yahoo.vespa.hosted.controller.persistence.ChangeRequestSerializer; import com.yahoo.yolean.Exceptions; -import javax.ws.rs.BadRequestException; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -96,13 +96,13 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { try { return SlimeUtils.jsonToSlime(request.getData().readAllBytes()).get(); } catch (IOException e) { - throw new BadRequestException("Failed to parse request body"); + throw new RestApiException.BadRequest("Failed to parse request body"); } } private static Inspector getInspectorFieldOrThrow(Inspector inspector, String field) { if (!inspector.field(field).valid()) - throw new BadRequestException("Field " + field + " cannot be null"); + throw new RestApiException.BadRequest("Field " + field + " cannot be null"); return inspector.field(field); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 346e61c907c..3ca6335dcb7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -12,6 +12,7 @@ import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; +import com.yahoo.restapi.RestApiException; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.SlimeUtils; @@ -27,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import com.yahoo.yolean.Exceptions; -import javax.ws.rs.InternalServerErrorException; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; @@ -203,7 +203,7 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { private static Principal requireUserPrincipal(HttpRequest request) { Principal principal = request.getJDiscRequest().getUserPrincipal(); - if (principal == null) throw new InternalServerErrorException("Expected a user principal"); + if (principal == null) throw new RestApiException.InternalServerError("Expected a user principal"); return principal; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index 87691d2927a..81e9b1972b9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.security; import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.TenantName; +import com.yahoo.restapi.RestApiException; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; @@ -22,7 +23,6 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import javax.ws.rs.ForbiddenException; import java.time.Instant; import java.util.List; import java.util.stream.Collectors; @@ -76,7 +76,7 @@ public class CloudAccessControl implements AccessControl { var trialTenants = billingController.tenantsWithPlan(tenantNames, trialPlanId).size(); if (maxTrialTenants.value() >= 0 && maxTrialTenants.value() <= trialTenants) { - throw new ForbiddenException("Too many tenants with trial plans, please contact the Vespa support team"); + throw new RestApiException.Forbidden("Too many tenants with trial plans, please contact the Vespa support team"); } } @@ -84,11 +84,11 @@ public class CloudAccessControl implements AccessControl { if (allowedByPrivilegedRole(auth0Credentials)) return; if (!allowedByFeatureFlag(auth0Credentials)) { - throw new ForbiddenException("You are not currently permitted to create tenants. Please contact the Vespa team to request access."); + throw new RestApiException.Forbidden("You are not currently permitted to create tenants. Please contact the Vespa team to request access."); } if(administeredTenants(auth0Credentials) >= 3) { - throw new ForbiddenException("You are already administering 3 tenants. If you need more, please contact the Vespa team."); + throw new RestApiException.Forbidden("You are already administering 3 tenants. If you need more, please contact the Vespa team."); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java index 1a052b6a578..4c3d76b1b17 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java @@ -32,7 +32,7 @@ public class CuratorArchiveBucketDbTest { assertEquals(Optional.of(URI.create("s3://existingBucket/default/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.defaultName(), true)); // Assigns to existing bucket while there is space - IntStream.range(0, 29).forEach(i -> + IntStream.range(0, 4).forEach(i -> assertEquals( Optional.of(URI.create("s3://existingBucket/tenant" + i + "/")), bucketDb .archiveUriFor(ZoneId.defaultId(), TenantName.from("tenant" + i), true))); @@ -47,7 +47,7 @@ public class CuratorArchiveBucketDbTest { assertEquals(Optional.empty(), bucketDb.archiveUriFor(ZoneId.from("prod.us-east-3"), TenantName.from("newTenant"), false)); // Lists all buckets by zone - Set<TenantName> existingBucketTenants = Streams.concat(Stream.of(TenantName.defaultName()), IntStream.range(0, 29).mapToObj(i -> TenantName.from("tenant" + i))).collect(Collectors.toUnmodifiableSet()); + Set<TenantName> existingBucketTenants = Streams.concat(Stream.of(TenantName.defaultName()), IntStream.range(0, 4).mapToObj(i -> TenantName.from("tenant" + i))).collect(Collectors.toUnmodifiableSet()); assertEquals( Set.of( new ArchiveBucket("existingBucket", "keyArn").withTenants(existingBucketTenants), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index ae4b6259da1..27951d95689 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -445,7 +445,6 @@ public class DeploymentContext { tester.cloud().set(noTests ? Status.NO_TESTS : Status.FAILURE); runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasEnded()); - assertEquals(noTests, jobs.run(id).get().hasSucceeded()); assertTrue(configServer().nodeRepository().list(job.type().zone(), NodeFilter.all().applications(TesterId.of(instanceId).id())).isEmpty()); return this; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index beda9bd551d..d602ee8cde3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -2267,4 +2267,16 @@ public class DeploymentTriggerTest { assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); } + @Test + public void testNoTests() { + DeploymentContext app = tester.newDeploymentContext(); + app.submit(new ApplicationPackageBuilder().systemTest().region("us-east-3").build()); + + // Declared tests must have run actual tests to succeed. + app.failTests(systemTest, true); + assertFalse(tester.jobs().last(app.instanceId(), systemTest).get().hasSucceeded()); + app.failTests(stagingTest, true); + assertTrue(tester.jobs().last(app.instanceId(), stagingTest).get().hasSucceeded()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 031cdaa84ae..ae08d1b3a22 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; @@ -24,23 +23,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud. import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage; -import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -60,7 +50,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.time.temporal.ChronoUnit.SECONDS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -455,6 +444,7 @@ public class InternalStepRunnerTest { @Test public void vespaLogIsCopied() { // Tests fail. We should get logs. This fails too, on the first attempt. + tester.controllerTester().computeVersionStatus(); RunId id = app.startSystemTestTests(); tester.cloud().set(TesterCloud.Status.ERROR); tester.configServer().setLogStream(() -> { throw new ConfigServerException(ConfigServerException.ErrorCode.NOT_FOUND, "404", "context"); }); @@ -464,31 +454,43 @@ public class InternalStepRunnerTest { assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.copyVespaLogs)); assertTestLogEntries(id, Step.copyVespaLogs, new LogEntry(lastId + 2, tester.clock().instant(), info, - "Found no logs, but will retry"), - new LogEntry(lastId + 4, tester.clock().instant(), info, "Found no logs, but will retry")); // Config servers now provide the log, and we get it. - tester.configServer().setLogStream(() -> vespaLog); + tester.configServer().setLogStream(() -> vespaLog(tester.clock().instant())); tester.runner().run(); assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); assertTestLogEntries(id, Step.copyVespaLogs, new LogEntry(lastId + 2, tester.clock().instant(), info, "Found no logs, but will retry"), - new LogEntry(lastId + 4, tester.clock().instant(), info, - "Found no logs, but will retry"), - new LogEntry(lastId + 5, Instant.EPOCH.plus(3554970337935104L, ChronoUnit.MICROS), info, + new LogEntry(lastId + 3, tester.clock().instant().minusSeconds(4), info, + "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), + new LogEntry(lastId + 4, tester.clock().instant().minusSeconds(4), info, + "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), + /* + new LogEntry(lastId + 5, tester.clock().instant().minusSeconds(4), info, + "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), + new LogEntry(lastId + 6, tester.clock().instant().minusSeconds(4), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), - new LogEntry(lastId + 6, Instant.EPOCH.plus(3554970337947777L, ChronoUnit.MICROS), info, + */ + new LogEntry(lastId + 5, tester.clock().instant().minusSeconds(3), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), - new LogEntry(lastId + 7, Instant.EPOCH.plus(3554970337947820L, ChronoUnit.MICROS), info, + new LogEntry(lastId + 6, tester.clock().instant().minusSeconds(3), warning, + "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstderr\n" + + "java.lang.NullPointerException\n\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\n\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)")); + /* + new LogEntry(lastId + 9, tester.clock().instant().minusSeconds(3), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), - new LogEntry(lastId + 8, Instant.EPOCH.plus(3554970337947845L, ChronoUnit.MICROS), warning, + new LogEntry(lastId + 10, tester.clock().instant().minusSeconds(3), warning, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstderr\n" + "java.lang.NullPointerException\n\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\n\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)")); + */ } @Test @@ -548,10 +550,12 @@ public class InternalStepRunnerTest { assertEquals(List.of(entries), tester.jobs().details(id).get().get(step)); } - private static final String vespaLog = "-1554970337.084804\t17480180-v6-3.ostk.bm2.prod.ne1.yahoo.com\t5549/832\tcontainer\tContainer.com.yahoo.container.jdisc.ConfiguredApplication\tinfo\tSwitching to the latest deployed set of configurations and components. Application switch number: 2\n" + - "3554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + - "3554970337.947777\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + - "3554970337.947820\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + - "3554970337.947845\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"; + private static String vespaLog(Instant now) { + return "-1\t17480180-v6-3.ostk.bm2.prod.ne1.yahoo.com\t5549/832\tcontainer\tContainer.com.yahoo.container.jdisc.ConfiguredApplication\tinfo\tSwitching to the latest deployed set of configurations and components. Application switch number: 2\n" + + (now.getEpochSecond() - 4) + "." + now.getNano() / 1000 + "\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + + (now.getEpochSecond() - 4) + "." + now.getNano() / 1000 + "\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + + (now.getEpochSecond() - 3) + "." + now.getNano() / 1000 + "\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + + (now.getEpochSecond() - 3) + "." + now.getNano() / 1000 + "\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"; + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java index 5571f957e83..3535417c586 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import org.junit.Test; @@ -33,24 +34,24 @@ public class ArchiveAccessMaintainerTest { String tenant1role = "arn:aws:iam::123456789012:role/my-role"; String tenant2role = "arn:aws:iam::210987654321:role/my-role"; var tenant1 = createTenantWithAccessRole(tester, "tenant1", tenant1role); - createTenantWithAccessRole(tester, "tenant2", tenant2role); + var tenant2 = createTenantWithAccessRole(tester, "tenant2", tenant2role); ZoneId testZone = ZoneId.from("prod.aws-us-east-1c"); tester.controller().archiveBucketDb().archiveUriFor(testZone, tenant1, true); var testBucket = new ArchiveBucket("bucketName", "keyArn").withTenant(tenant1); MockArchiveService archiveService = (MockArchiveService) tester.controller().serviceRegistry().archiveService(); - assertNull(archiveService.authorizedIamRolesForBucket.get(testBucket)); - assertNull(archiveService.authorizedIamRolesForKey.get(testBucket.keyArn())); + + assertEquals(0, archiveService.authorizeAccessByTenantName.size()); MockMetric metric = new MockMetric(); new ArchiveAccessMaintainer(tester.controller(), metric, Duration.ofMinutes(10)).maintain(); - assertEquals(Map.of(tenant1, tenant1role), archiveService.authorizedIamRolesForBucket.get(testBucket)); - assertEquals(Set.of(tenant1role), archiveService.authorizedIamRolesForKey.get(testBucket.keyArn())); + assertEquals(new ArchiveAccess().withAWSRole(tenant1role), archiveService.authorizeAccessByTenantName.get(tenant1)); + assertEquals(new ArchiveAccess().withAWSRole(tenant2role), archiveService.authorizeAccessByTenantName.get(tenant2)); var expected = Map.of("archive.bucketCount", tester.controller().zoneRegistry().zonesIncludingSystem().all().ids().stream() .collect(Collectors.toMap( - zone -> Map.of("zone", zone.value()), + zone -> Map.of("zone", zone.value(), "cloud", "default"), zone -> zone.equals(testZone) ? 1d : 0d))); assertEquals(expected, metric.metrics()); @@ -59,7 +60,7 @@ public class ArchiveAccessMaintainerTest { private TenantName createTenantWithAccessRole(ControllerTester tester, String tenantName, String role) { var tenant = tester.createTenant(tenantName, Tenant.Type.cloud); tester.controller().tenants().lockOrThrow(tenant, LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withArchiveAccessRole(Optional.of(role)); + lockedTenant = lockedTenant.withArchiveAccess(new ArchiveAccess().withAWSRole(role)); tester.controller().tenants().store(lockedTenant); }); return tenant; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java index c1d9c03819d..d208657c1c4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java @@ -40,27 +40,24 @@ public class ArchiveUriUpdaterTest { var application = tester.newDeploymentContext(tenant1.value(), "app1", "instance1"); ZoneId zone = ZoneId.from("prod", "aws-us-east-1c"); - // Initially we should not set any archive URIs as the archive service does not return any + // Initially we should only is the bucket for hosted-vespa tenant updater.maintain(); - assertArchiveUris(Map.of(), zone); - // but the controller zone is always present - assertArchiveUris(Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/hosted-vespa/"), - ZoneId.from("prod", "controller")); + assertArchiveUris(Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/hosted-vespa/"), zone); + assertArchiveUris(Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/hosted-vespa/"), ZoneId.from("prod", "controller")); // Archive service now has URI for tenant1, but tenant1 is not deployed in zone setBucketNameInService(Map.of(tenant1, "uri-1"), zone); - setBucketNameInService(Map.of(tenantInfra, "uri-3"), zone); updater.maintain(); - assertArchiveUris(Map.of(), zone); + assertArchiveUris(Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/hosted-vespa/"), zone); deploy(application, zone); updater.maintain(); - assertArchiveUris(Map.of(tenant1, "s3://uri-1/tenant1/", tenantInfra, "s3://uri-3/hosted-vespa/"), zone); + assertArchiveUris(Map.of(tenant1, "s3://uri-1/tenant1/", tenantInfra, "s3://bucketName/hosted-vespa/"), zone); // URI for tenant1 should be updated and removed for tenant2 setArchiveUriInNodeRepo(Map.of(tenant1, "wrong-uri", tenant2, "uri-2"), zone); updater.maintain(); - assertArchiveUris(Map.of(tenant1, "s3://uri-1/tenant1/", tenantInfra, "s3://uri-3/hosted-vespa/"), zone); + assertArchiveUris(Map.of(tenant1, "s3://uri-1/tenant1/", tenantInfra, "s3://bucketName/hosted-vespa/"), zone); } private void assertArchiveUris(Map<TenantName, String> expectedUris, ZoneId zone) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 5fd1e8347ef..b9ced334e5a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -93,8 +93,7 @@ public class JobRunnerTest { ApplicationId id = appId.defaultInstance(); byte[] testPackageBytes = new byte[0]; jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); - - start(jobs, id, systemTest); + start(jobs, id, systemTest); try { start(jobs, id, systemTest); fail("Job is already running, so this should not be allowed!"); @@ -106,12 +105,16 @@ public class JobRunnerTest { assertFalse(jobs.last(id, systemTest).get().hasEnded()); assertTrue(jobs.last(id, stagingTest).get().stepStatuses().values().stream().allMatch(unfinished::equals)); assertFalse(jobs.last(id, stagingTest).get().hasEnded()); - runner.maintain(); + runner.maintain(); phaser.arriveAndAwaitAdvance(); assertTrue(jobs.last(id, systemTest).get().stepStatuses().values().stream().allMatch(succeeded::equals)); - assertTrue(jobs.last(id, stagingTest).get().hasEnded()); assertTrue(jobs.last(id, stagingTest).get().hasFailed()); + + runner.maintain(); + phaser.arriveAndAwaitAdvance(); + assertTrue(jobs.last(id, systemTest).get().hasEnded()); + assertTrue(jobs.last(id, stagingTest).get().hasEnded()); } @Test @@ -165,8 +168,8 @@ public class JobRunnerTest { outcomes.put(endTests, testFailure); runner.maintain(); assertTrue(run.get().hasFailed()); - assertEquals(List.of(copyVespaLogs, deactivateTester), run.get().readySteps()); - assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal, startTests, endTests, copyVespaLogs, deactivateTester); + assertEquals(List.of(copyVespaLogs), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal, startTests, endTests, copyVespaLogs); outcomes.put(copyVespaLogs, running); runner.maintain(); @@ -442,8 +445,8 @@ public class JobRunnerTest { @Override public void execute(Runnable command) { phaser.register(); delegate.execute(() -> { - command.run(); - phaser.arriveAndDeregister(); + try { command.run(); } + finally { phaser.arriveAndDeregister(); } }); } }; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index 75dbebe96ff..5666f8bafd8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; @@ -60,7 +61,7 @@ public class NotificationsDbTest { List.of(TenantContacts.Audience.NOTIFICATIONS), email)))), List.of(), - Optional.empty()); + new ArchiveAccess()); private static final List<Notification> notifications = List.of( notification(1001, Type.deployment, Level.error, NotificationSource.from(tenant), "tenant msg"), notification(1101, Type.applicationPackage, Level.warning, NotificationSource.from(TenantAndApplicationId.from(tenant.value(), "app1")), "app msg"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index fd0ea50e50b..6a01a70eb98 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -25,7 +25,6 @@ import java.time.Instant; import java.util.List; import java.util.Optional; -import static com.yahoo.config.provision.SystemName.main; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; @@ -132,6 +131,7 @@ public class RunSerializerTest { assertEquals(run.end(), phoenix.end()); assertEquals(run.status(), phoenix.status()); assertEquals(run.lastTestLogEntry(), phoenix.lastTestLogEntry()); + assertEquals(run.lastVespaLogTimestamp(), phoenix.lastVespaLogTimestamp()); assertEquals(run.noNodesDownSince(), phoenix.noNodesDownSince()); assertEquals(run.testerCertificate(), phoenix.testerCertificate()); assertEquals(run.versions(), phoenix.versions()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index e0d14f19f21..a9e633a78d6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -6,12 +6,14 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.security.KeyUtils; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; @@ -101,7 +103,7 @@ public class TenantSerializerTest { otherPublicKey, new SimplePrincipal("jane")), TenantInfo.empty(), List.of(), - Optional.empty() + new ArchiveAccess() ); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.name(), serialized.name()); @@ -123,13 +125,61 @@ public class TenantSerializerTest { new TenantSecretStore("ss1", "123", "role1"), new TenantSecretStore("ss2", "124", "role2") ), - Optional.of("arn:aws:iam::123456789012:role/my-role") + new ArchiveAccess().withAWSRole("arn:aws:iam::123456789012:role/my-role") ); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.info(), serialized.info()); assertEquals(tenant.tenantSecretStores(), serialized.tenantSecretStores()); } + @Test + public void cloud_tenant_with_old_archive_access_serialization() { + var json = "{\n" + + " \"name\": \"elderly-lady\",\n" + + " \"type\": \"cloud\",\n" + + " \"createdAt\": 1234,\n" + + " \"lastLoginInfo\": {\n" + + " \"user\": 123,\n" + + " \"developer\": 456\n" + + " },\n" + + " \"creator\": \"foobar-user\",\n" + + " \"pemDeveloperKeys\": [\n" + + " {\n" + + " \"key\": \"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\\n-----END PUBLIC KEY-----\\n\",\n" + + " \"user\": \"joe\"\n" + + " },\n" + + " {\n" + + " \"key\": \"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\\n-----END PUBLIC KEY-----\\n\",\n" + + " \"user\": \"jane\"\n" + + " }\n" + + " ],\n" + + " \"billingInfo\": {\n" + + " \"customerId\": \"customer\",\n" + + " \"productCode\": \"Vespa\"\n" + + " },\n" + + " \"archiveAccessRole\": \"arn:aws:iam::123456789012:role/my-role\"\n" + + "}"; + var tenant = (CloudTenant) serializer.tenantFrom(SlimeUtils.jsonToSlime(json)); + assertEquals("arn:aws:iam::123456789012:role/my-role", tenant.archiveAccess().awsRole().get()); + assertFalse(tenant.archiveAccess().gcpMember().isPresent()); + } + + @Test + public void cloud_tenant_with_archive_access() { + CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"), + Instant.ofEpochMilli(1234L), + lastLoginInfo(123L, 456L, null), + Optional.of(new SimplePrincipal("foobar-user")), + ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"), + otherPublicKey, new SimplePrincipal("jane")), + TenantInfo.empty(), + List.of(), + new ArchiveAccess().withAWSRole("arn:aws:iam::123456789012:role/my-role").withGCPMember("user:foo@example.com") + ); + CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + assertEquals(serialized.archiveAccess().awsRole().get(), "arn:aws:iam::123456789012:role/my-role"); + assertEquals(serialized.archiveAccess().gcpMember().get(), "user:foo@example.com"); + } @Test public void cloud_tenant_with_tenant_info_partial() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java index 32fe8ddecff..845d007c154 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java @@ -24,7 +24,7 @@ public class ProxyResponseTest { Map.of(), null, List.of(URI.create("http://example.com")), Path.parse("configserver")); ProxyResponse proxyResponse = new ProxyResponse( request, - "response link is http://configserver:1234/bla/bla/", + "response link is http://configserver:4443/bla/bla/", 200, URI.create("http://configserver:1234"), "application/json"); @@ -42,7 +42,7 @@ public class ProxyResponseTest { Map.of(), null, List.of(URI.create("http://example.com")), Path.parse("configserver")); ProxyResponse proxyResponse = new ProxyResponse( request, - "response link is http://configserver:1234/bla/bla/", + "response link is http://configserver:4443/bla/bla/", 200, URI.create("http://configserver:1234"), "application/json"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 065492e47ec..26dd6335ab8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.restapi.RestApiException; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; @@ -29,7 +30,6 @@ import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import org.junit.Before; import org.junit.Test; -import javax.ws.rs.ForbiddenException; import java.io.File; import java.util.Collections; import java.util.Optional; @@ -267,7 +267,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { try { tester.controller().tenants().create(tenantSpec("tenant2"), credentials("administrator")); fail("Should not be allowed to create tenant that exceed trial limit"); - } catch (ForbiddenException e) { + } catch (RestApiException.Forbidden e) { assertEquals("Too many tenants with trial plans, please contact the Vespa support team", e.getMessage()); } } @@ -361,9 +361,40 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { .data("{\"role\":\"dummy\"}").roles(Role.administrator(tenantName)), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid archive access role 'dummy': Must match expected pattern: 'arn:aws:iam::\\\\d{12}:.+'\"}", 400); + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access/aws", PUT) + .data("{\"role\":\"arn:aws:iam::123456789012:role/my-role\"}").roles(Role.administrator(tenantName)), + "{\"message\":\"AWS archive access role set to 'arn:aws:iam::123456789012:role/my-role' for tenant scoober.\"}", 200); + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertTrue(response.getBodyAsString().contains("\"archiveAccessRole\":\"arn:aws:iam::123456789012:role/my-role\"")), + 200); + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access/aws", DELETE).roles(Role.administrator(tenantName)), + "{\"message\":\"AWS archive access role removed for tenant scoober.\"}", 200); + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertFalse(response.getBodyAsString().contains("\"archiveAccessRole\":\"arn:aws:iam::123456789012:role/my-role\"")), + 200); + + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access/gcp", PUT) + .data("{\"member\":\"user:test@example.com\"}").roles(Role.administrator(tenantName)), + "{\"message\":\"GCP archive access member set to 'user:test@example.com' for tenant scoober.\"}", 200); + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertTrue(response.getBodyAsString().contains("\"gcpMember\":\"user:test@example.com\"")), + 200); + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access/gcp", DELETE).roles(Role.administrator(tenantName)), + "{\"message\":\"GCP archive access member removed for tenant scoober.\"}", 200); + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertFalse(response.getBodyAsString().contains("\"gcpMember\":\"user:test@example.com\"")), + 200); + + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access", PUT) + .data("{\"role\":\"arn:aws:iam::123456789012:role/my-role\"}").roles(Role.administrator(tenantName)), + "{\"message\":\"AWS archive access role set to 'arn:aws:iam::123456789012:role/my-role' for tenant scoober.\"}", 200); + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertTrue(response.getBodyAsString().contains("\"archiveAccessRole\":\"arn:aws:iam::123456789012:role/my-role\"")), + 200); + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access", PUT) .data("{\"role\":\"arn:aws:iam::123456789012:role/my-role\"}").roles(Role.administrator(tenantName)), - "{\"message\":\"Archive access role set to 'arn:aws:iam::123456789012:role/my-role' for tenant scoober.\"}", 200); + "{\"message\":\"AWS archive access role set to 'arn:aws:iam::123456789012:role/my-role' for tenant scoober.\"}", 200); tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), (response) -> assertTrue(response.getBodyAsString().contains("\"archiveAccessRole\":\"arn:aws:iam::123456789012:role/my-role\"")), 200); @@ -373,7 +404,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { new File("deployment-cloud.json")); tester.assertResponse(request("/application/v4/tenant/scoober/archive-access", DELETE).roles(Role.administrator(tenantName)), - "{\"message\":\"Archive access role removed for tenant scoober.\"}", 200); + "{\"message\":\"AWS archive access role removed for tenant scoober.\"}", 200); tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), (response) -> assertFalse(response.getBodyAsString().contains("archiveAccessRole")), 200); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index 8db6bdf9a4a..279bd289c00 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -26,8 +26,7 @@ public class BadgeApiTest extends ControllerContainerTest { public void testBadgeApi() throws IOException { ContainerTester tester = new ContainerTester(container, responseFiles); var application = new DeploymentTester(new ControllerTester(tester)).newDeploymentContext("tenant", "application", "default"); - ApplicationPackage applicationPackage = new ApplicationPackageBuilder().systemTest() - .parallel("us-west-1", "aws-us-east-1a") + ApplicationPackage applicationPackage = new ApplicationPackageBuilder().parallel("us-west-1", "aws-us-east-1a") .test("us-west-1") .region("ap-southeast-1") .test("ap-southeast-1") @@ -59,6 +58,8 @@ public class BadgeApiTest extends ControllerContainerTest { Files.readString(Paths.get(responseFiles + "overview.svg")), 200); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=0"), Files.readString(Paths.get(responseFiles + "single-running.svg")), 200); + tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/system-test"), + Files.readString(Paths.get(responseFiles + "running-test.svg")), 200); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=32"), Files.readString(Paths.get(responseFiles + "history.svg")), 200); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg index a0005ed6d76..46e4acaace6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg @@ -1,4 +1,4 @@ -<svg xmlns='http://www.w3.org/2000/svg' width='763.7809900000001' height='20' role='img' aria-label='Deployment Status'> +<svg xmlns='http://www.w3.org/2000/svg' width='689.25265' height='20' role='img' aria-label='Deployment Status'> <title>Deployment Status</title> <linearGradient id='light' x2='0' y2='100%'> <stop offset='0' stop-color='#fff' stop-opacity='.5'/> @@ -46,32 +46,29 @@ <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> <clipPath id='rounded'> - <rect width='763.7809900000001' height='20' rx='3' fill='#fff'/> + <rect width='689.25265' height='20' rx='3' fill='#fff'/> </clipPath> <g clip-path='url(#rounded)'> - <rect x='757.7809900000001' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='725.59036' rx='3' width='38.19063' height='20' fill='url(#run-on-success)'/> - <polygon points='635.8470950000001 0 635.8470950000001 20 734.59036 20 742.59036 0' fill='#00f844'/> - <rect x='635.8470950000001' rx='3' width='131.74345499999998' height='20' fill='url(#shade)'/> - <rect x='635.8470950000001' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='603.656465' rx='3' width='38.19063' height='20' fill='#bf103c'/> - <polygon points='486.981225 0 486.981225 20 612.656465 20 620.656465 0' fill='#00f844'/> - <rect x='486.981225' rx='3' width='158.67543' height='20' fill='url(#shade)'/> - <rect x='486.981225' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='348.865175' rx='3' width='144.11604999999997' height='20' fill='url(#run-on-success)'/> - <rect x='358.865175' rx='3' width='134.11604999999997' height='20' fill='url(#shade)'/> - <rect x='358.865175' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='326.674545' rx='3' width='38.19063' height='20' fill='#00f844'/> - <polygon points='237.71563000000003 0 237.71563000000003 20 335.674545 20 343.674545 0' fill='url(#run-on-failure)'/> - <rect x='237.71563000000003' rx='3' width='130.959105' height='20' fill='url(#shade)'/> - <rect x='237.71563000000003' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='153.18729000000002' rx='3' width='90.52834000000001' height='20' fill='url(#run-on-warning)'/> - <rect x='163.18729000000002' rx='3' width='80.52834000000001' height='20' fill='url(#shade)'/> + <rect x='683.25265' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='651.06202' rx='3' width='38.19063' height='20' fill='url(#run-on-success)'/> + <polygon points='561.318755 0 561.318755 20 660.06202 20 668.06202 0' fill='#00f844'/> + <rect x='561.318755' rx='3' width='131.74345499999998' height='20' fill='url(#shade)'/> + <rect x='561.318755' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='529.128125' rx='3' width='38.19063' height='20' fill='#bf103c'/> + <polygon points='412.452885 0 412.452885 20 538.128125 20 546.128125 0' fill='#00f844'/> + <rect x='412.452885' rx='3' width='158.67543' height='20' fill='url(#shade)'/> + <rect x='412.452885' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='274.336835' rx='3' width='144.11604999999997' height='20' fill='url(#run-on-success)'/> + <rect x='284.336835' rx='3' width='134.11604999999997' height='20' fill='url(#shade)'/> + <rect x='284.336835' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='252.146205' rx='3' width='38.19063' height='20' fill='#00f844'/> + <polygon points='163.18729000000002 0 163.18729000000002 20 261.146205 20 269.146205 0' fill='url(#run-on-failure)'/> + <rect x='163.18729000000002' rx='3' width='130.959105' height='20' fill='url(#shade)'/> <rect width='169.18729000000002' height='20' fill='#404040'/> <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> <rect width='2' height='20' fill='url(#left-light)'/> - <rect x='761.7809900000001' width='2' height='20' fill='url(#right-shadow)'/> - <rect width='763.7809900000001' height='20' fill='url(#light)'/> + <rect x='687.25265' width='2' height='20' fill='url(#right-shadow)'/> + <rect width='689.25265' height='20' fill='url(#light)'/> </g> <g fill='#fff' text-anchor='middle' font-family='Verdana,Geneva,DejaVu Sans,sans-serif' text-rendering='geometricPrecision' font-size='11'> <svg x='6.5' y='3.0' width='16.0' height='16.0' viewBox='0 0 150 150'> @@ -96,29 +93,27 @@ </svg> <text font-size='11' x='96.09364500000001' y='15' fill='#000' fill-opacity='.4' textLength='135.18729000000002'>tenant.application.default</text> <text font-size='11' x='95.59364500000001' y='14' fill='#fff' textLength='135.18729000000002'>tenant.application.default</text> - <text font-size='11' x='206.95146000000003' y='15' fill='#000' fill-opacity='.4' textLength='62.52834000000001'>system-test</text> - <text font-size='11' x='206.45146000000003' y='14' fill='#fff' textLength='62.52834000000001'>system-test</text> - <text font-size='11' x='276.60659250000003' y='15' fill='#000' fill-opacity='.4' textLength='52.781925'>us-west-1</text> - <text font-size='11' x='276.10659250000003' y='14' fill='#fff' textLength='52.781925'>us-west-1</text> - <text font-size='9' x='323.08605000000006' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='322.58605000000006' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='351.26986000000005' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text> - <text font-size='9' x='350.76986000000005' y='14' fill='#fff' textLength='16.190630000000002'>test</text> - <text font-size='11' x='412.334705' y='15' fill='#000' fill-opacity='.4' textLength='81.93905999999998'>aws-us-east-1a</text> - <text font-size='11' x='411.834705' y='14' fill='#fff' textLength='81.93905999999998'>aws-us-east-1a</text> - <text font-size='9' x='473.39273000000003' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='472.89273000000003' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> - <text font-size='11' x='539.73035' y='15' fill='#000' fill-opacity='.4' textLength='80.49825'>ap-southeast-1</text> - <text font-size='11' x='539.23035' y='14' fill='#fff' textLength='80.49825'>ap-southeast-1</text> - <text font-size='9' x='600.06797' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='599.56797' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='628.25178' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text> - <text font-size='9' x='627.75178' y='14' fill='#fff' textLength='16.190630000000002'>test</text> - <text font-size='11' x='675.1302325' y='15' fill='#000' fill-opacity='.4' textLength='53.566275'>eu-west-1</text> - <text font-size='11' x='674.6302325' y='14' fill='#fff' textLength='53.566275'>eu-west-1</text> - <text font-size='9' x='722.0018650000001' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='721.5018650000001' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='750.1856750000001' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text> - <text font-size='9' x='749.6856750000001' y='14' fill='#fff' textLength='16.190630000000002'>test</text> + <text font-size='11' x='202.07825250000002' y='15' fill='#000' fill-opacity='.4' textLength='52.781925'>us-west-1</text> + <text font-size='11' x='201.57825250000002' y='14' fill='#fff' textLength='52.781925'>us-west-1</text> + <text font-size='9' x='248.55771000000001' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='248.05771000000001' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='276.74152000000004' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text> + <text font-size='9' x='276.24152000000004' y='14' fill='#fff' textLength='16.190630000000002'>test</text> + <text font-size='11' x='337.806365' y='15' fill='#000' fill-opacity='.4' textLength='81.93905999999998'>aws-us-east-1a</text> + <text font-size='11' x='337.306365' y='14' fill='#fff' textLength='81.93905999999998'>aws-us-east-1a</text> + <text font-size='9' x='398.86439' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='398.36439' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> + <text font-size='11' x='465.20201' y='15' fill='#000' fill-opacity='.4' textLength='80.49825'>ap-southeast-1</text> + <text font-size='11' x='464.70201' y='14' fill='#fff' textLength='80.49825'>ap-southeast-1</text> + <text font-size='9' x='525.53963' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='525.03963' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='553.72344' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text> + <text font-size='9' x='553.22344' y='14' fill='#fff' textLength='16.190630000000002'>test</text> + <text font-size='11' x='600.6018925' y='15' fill='#000' fill-opacity='.4' textLength='53.566275'>eu-west-1</text> + <text font-size='11' x='600.1018925' y='14' fill='#fff' textLength='53.566275'>eu-west-1</text> + <text font-size='9' x='647.473525' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='646.973525' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='675.657335' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text> + <text font-size='9' x='675.157335' y='14' fill='#fff' textLength='16.190630000000002'>test</text> </g> </svg> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/running-test.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/running-test.svg new file mode 100644 index 00000000000..9463c01e8ad --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/running-test.svg @@ -0,0 +1,93 @@ +<svg xmlns='http://www.w3.org/2000/svg' width='300.38627894289516' height='20' role='img' aria-label='Deployment Status'> + <title>Deployment Status</title> + <linearGradient id='light' x2='0' y2='100%'> + <stop offset='0' stop-color='#fff' stop-opacity='.5'/> + <stop offset='.1' stop-color='#fff' stop-opacity='.15'/> + <stop offset='.9' stop-color='#000' stop-opacity='.15'/> + <stop offset='1' stop-color='#000' stop-opacity='.5'/> + </linearGradient> + <linearGradient id='left-light' x2='100%' y2='0'> + <stop offset='0' stop-color='#fff' stop-opacity='.3'/> + <stop offset='.5' stop-color='#fff' stop-opacity='.1'/> + <stop offset='1' stop-color='#fff' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='right-shadow' x2='100%' y2='0'> + <stop offset='0' stop-color='#000' stop-opacity='.0'/> + <stop offset='.5' stop-color='#000' stop-opacity='.1'/> + <stop offset='1' stop-color='#000' stop-opacity='.3'/> + </linearGradient> + <linearGradient id='shadow' x2='100%' y2='0'> + <stop offset='0' stop-color='#222' stop-opacity='.3'/> + <stop offset='.625' stop-color='#555' stop-opacity='.3'/> + <stop offset='.9' stop-color='#555' stop-opacity='.05'/> + <stop offset='1' stop-color='#555' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='shade' x2='100%' y2='0'> + <stop offset='0' stop-color='#000' stop-opacity='.20'/> + <stop offset='0.05' stop-color='#000' stop-opacity='.10'/> + <stop offset='1' stop-color='#000' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='run-on-failure' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bf103c' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bd890b' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#00f844' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <clipPath id='rounded'> + <rect width='300.38627894289516' height='20' rx='3' fill='#fff'/> + </clipPath> + <g clip-path='url(#rounded)'> + <rect x='297.7936599677133' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='267.7546449838567' rx='3' width='36.03901498385664' height='20' fill='#00f844'/> + <rect x='267.7546449838567' rx='3' width='36.03901498385664' height='20' fill='url(#shade)'/> + <rect x='271.5979829411765' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='237.71563000000003' rx='3' width='39.88235294117647' height='20' fill='#00f844'/> + <rect x='237.71563000000003' rx='3' width='39.88235294117647' height='20' fill='url(#shade)'/> + <rect x='237.71563000000003' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='163.18729000000002' rx='3' width='80.52834000000001' height='20' fill='url(#run-on-warning)'/> + <rect x='163.18729000000002' rx='3' width='80.52834000000001' height='20' fill='url(#shade)'/> + <rect width='169.18729000000002' height='20' fill='#404040'/> + <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> + <rect width='2' height='20' fill='url(#left-light)'/> + <rect x='298.38627894289516' width='2' height='20' fill='url(#right-shadow)'/> + <rect width='300.38627894289516' height='20' fill='url(#light)'/> + </g> + <g fill='#fff' text-anchor='middle' font-family='Verdana,Geneva,DejaVu Sans,sans-serif' text-rendering='geometricPrecision' font-size='11'> + <svg x='6.5' y='3.0' width='16.0' height='16.0' viewBox='0 0 150 150'> + <polygon fill='#402a14' fill-opacity='0.5' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/> + <polygon fill='#402a14' fill-opacity='0.5' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/> + <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/> + <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/> + </svg> + <svg x='6.0' y='2.0' width='16.0' height='16.0' viewBox='0 0 150 150'> + <linearGradient id='yellow-shaded' x1='91.17' y1='44.83' x2='136.24' y2='73.4' gradientUnits='userSpaceOnUse'> + <stop offset='0.01' stop-color='#c6783e'/> + <stop offset='0.54' stop-color='#ff9750'/> + </linearGradient> + <linearGradient id='blue-shaded' x1='60.71' y1='104.56' x2='-15.54' y2='63' gradientUnits='userSpaceOnUse'> + <stop offset='0' stop-color='#005a8e'/> + <stop offset='0.54' stop-color='#1a7db6'/> + </linearGradient> + <polygon fill='#ff9d4b' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/> + <polygon fill='url(#yellow-shaded)' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/> + <polygon fill='#1a7db6' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/> + <polygon fill='url(#blue-shaded)' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/> + </svg> + <text font-size='11' x='96.09364500000001' y='15' fill='#000' fill-opacity='.4' textLength='135.18729000000002'>tenant.application.default</text> + <text font-size='11' x='95.59364500000001' y='14' fill='#fff' textLength='135.18729000000002'>tenant.application.default</text> + <text font-size='11' x='206.95146000000003' y='15' fill='#000' fill-opacity='.4' textLength='62.52834000000001'>system-test</text> + <text font-size='11' x='206.45146000000003' y='14' fill='#fff' textLength='62.52834000000001'>system-test</text> + </g> +</svg> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index 15c7dbf73ab..9024d7c8e7e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.restapi.ApplicationRequestToDiscFilterRequestWrapper; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; @@ -76,7 +77,7 @@ public class SignatureFilterTest { ImmutableBiMap.of(), TenantInfo.empty(), List.of(), - Optional.empty())); + new ArchiveAccess())); tester.curator().writeApplication(new Application(appId, tester.clock().instant())); } @@ -122,7 +123,7 @@ public class SignatureFilterTest { ImmutableBiMap.of(publicKey, () -> "user"), TenantInfo.empty(), List.of(), - Optional.empty())); + new ArchiveAccess())); verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), new SecurityContext(new SimplePrincipal("user"), Set.of(Role.reader(id.tenant()), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json index 3237e99783d..f980f9231f3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json @@ -24,6 +24,7 @@ "budgetUsed": 0.0, "clusterSize": 5 }, + "archiveAccess": { }, "applications": [ { "tenant": "my-tenant", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json index 0cc8ba2cd9e..1152033791b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json @@ -32,6 +32,7 @@ "budgetUsed": 0.0, "clusterSize": 5 }, + "archiveAccess": { }, "applications": [ { "tenant": "my-tenant", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json index 3153c6e218a..631346181a1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json @@ -15,6 +15,7 @@ "budgetUsed": 0.0, "clusterSize": 5 }, + "archiveAccess": { }, "applications": [ ], "metaData": { "createdAtMillis": 1600000000000 |