diff options
author | Øyvind Grønnesby <oyving@verizonmedia.com> | 2021-10-08 12:50:02 +0200 |
---|---|---|
committer | Øyvind Grønnesby <oyving@verizonmedia.com> | 2021-10-08 12:50:02 +0200 |
commit | 77ef010b968fca024196928eff7fba6ea092bc6e (patch) | |
tree | 78cece60ce6a2fc273afac191062145bd6f2363d /controller-server | |
parent | 66f9d8574f87089d18df88358b381b4890ce7fe5 (diff) | |
parent | daaf9321988ba1ba0fd81ee3853fdd51dc692f26 (diff) |
Merge remote-tracking branch 'origin/master' into ogronnesby/billing-service
Diffstat (limited to 'controller-server')
305 files changed, 3780 insertions, 2488 deletions
diff --git a/controller-server/pom.xml b/controller-server/pom.xml index a0d05588a0d..8615b8672c1 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -1,5 +1,5 @@ <?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 @@ -158,13 +158,6 @@ </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>http-utils</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - </dependency> - - <dependency> <!-- required by java-jwt --> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index f8624b40737..bbf76fcb480 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index e9b4b25672a..861d101ae3f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.aws.TenantRoles; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; @@ -36,6 +35,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; @@ -44,8 +44,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.application.ActivateResult; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackageValidator; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageValidator; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.DeploymentQuotaCalculator; @@ -80,7 +80,6 @@ import java.time.Instant; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -277,8 +276,9 @@ public class ApplicationController { /** Reads the oldest installed platform for the given application and zone from the node repo of that zone. */ private Optional<Version> oldestInstalledPlatform(JobId job) { return configServer.nodeRepository().list(job.type().zone(controller.system()), - job.application(), - EnumSet.of(active, reserved)) + NodeFilter.all() + .applications(job.application()) + .states(active, reserved)) .stream() .map(Node::currentVersion) .filter(version -> ! version.isEmpty()) @@ -331,11 +331,6 @@ public class ApplicationController { }); } - /** Fetches the requested application package from the artifact store(s). */ - public ApplicationPackage getApplicationPackage(ApplicationId id, ApplicationVersion version) { - return new ApplicationPackage(applicationStore.get(id.tenant(), id.application(), version)); - } - /** Returns given application with a new instance */ public LockedApplication withNewInstance(LockedApplication application, ApplicationId instance) { if (instance.instance().isTester()) @@ -362,7 +357,6 @@ public class ApplicationController { try (Lock deploymentLock = lockForDeployment(job.application(), zone)) { Set<ContainerEndpoint> containerEndpoints; Optional<EndpointCertificateMetadata> endpointCertificateMetadata; - Optional<TenantRoles> tenantRoles = Optional.empty(); Run run = controller.jobController().last(job) .orElseThrow(() -> new IllegalStateException("No known run of '" + job + "'")); @@ -372,7 +366,7 @@ public class ApplicationController { Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform()); ApplicationVersion revision = run.versions().sourceApplication().filter(__ -> deploySourceVersions).orElse(run.versions().targetApplication()); - ApplicationPackage applicationPackage = getApplicationPackage(job.application(), zone, revision); + ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(new DeploymentId(job.application(), zone), revision)); try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); @@ -390,7 +384,8 @@ public class ApplicationController { } // Release application lock while doing the deployment, which is a lengthy task. // Carry out deployment without holding the application lock. - ActivateResult result = deploy(job.application(), applicationPackage, zone, platform, containerEndpoints, endpointCertificateMetadata, tenantRoles); + ActivateResult result = deploy(job.application(), applicationPackage, zone, platform, containerEndpoints, + endpointCertificateMetadata, run.isDryRun()); // Record the quota usage for this application var quotaUsage = deploymentQuotaUsage(zone, job.application()); @@ -472,7 +467,7 @@ public class ApplicationController { ApplicationPackage applicationPackage = new ApplicationPackage( artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); - return deploy(application.id(), applicationPackage, zone, version, Set.of(), /* No application cert */ Optional.empty(), Optional.empty()); + return deploy(application.id(), applicationPackage, zone, version, Set.of(), /* No application cert */ Optional.empty(), false); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -480,13 +475,13 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, Version platform) { - return deploy(tester.id(), applicationPackage, zone, platform, Set.of(), /* No application cert for tester*/ Optional.empty(), Optional.empty()); + return deploy(tester.id(), applicationPackage, zone, platform, Set.of(), /* No application cert for tester*/ Optional.empty(), false); } private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, - Optional<TenantRoles> tenantRoles) { + boolean dryRun) { try { Optional<DockerImage> dockerImageRepo = Optional.ofNullable( dockerImageRepoFlag @@ -520,7 +515,8 @@ public class ApplicationController { ConfigServer.PreparedApplication preparedApplication = configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform, endpoints, endpointCertificateMetadata, dockerImageRepo, domain, - tenantRoles, deploymentQuota, tenantSecretStores, operatorCertificates)); + deploymentQuota, tenantSecretStores, operatorCertificates, + dryRun)); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); @@ -829,11 +825,6 @@ public class ApplicationController { return DeploymentQuotaCalculator.calculateQuotaUsage(application); } - private ApplicationPackage getApplicationPackage(ApplicationId application, ZoneId zone, ApplicationVersion revision) { - return new ApplicationPackage(revision.isUnknown() ? applicationStore.getDev(application, zone) - : applicationStore.get(application.tenant(), application.application(), revision)); - } - /* * Get the AthenzUser from this principal or Optional.empty if this does not represent a user. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 65e046b448d..4dd380324fa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -35,6 +35,7 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import com.yahoo.vespa.serviceview.bindings.ApplicationView; +import com.yahoo.yolean.concurrent.Sleeper; import java.time.Clock; import java.time.Duration; @@ -73,6 +74,7 @@ public class Controller extends AbstractComponent { private final TenantController tenantController; private final JobController jobController; private final Clock clock; + private final Sleeper sleeper; private final ZoneRegistry zoneRegistry; private final ServiceRegistry serviceRegistry; private final AuditLogger auditLogger; @@ -97,18 +99,20 @@ public class Controller extends AbstractComponent { MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, ControllerConfig controllerConfig) { this(curator, rotationsConfig, accessControl, com.yahoo.net.HostName::getLocalhost, flagSource, - mavenRepository, serviceRegistry, metric, secretStore, controllerConfig); + mavenRepository, serviceRegistry, metric, secretStore, controllerConfig, Sleeper.DEFAULT); } public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, Supplier<String> hostnameSupplier, FlagSource flagSource, MavenRepository mavenRepository, - ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, ControllerConfig controllerConfig) { + ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, + ControllerConfig controllerConfig, Sleeper sleeper) { this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null"); this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); this.serviceRegistry = Objects.requireNonNull(serviceRegistry, "ServiceRegistry cannot be null"); this.zoneRegistry = Objects.requireNonNull(serviceRegistry.zoneRegistry(), "ZoneRegistry cannot be null"); this.clock = Objects.requireNonNull(serviceRegistry.clock(), "Clock cannot be null"); + this.sleeper = Objects.requireNonNull(sleeper); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null"); this.metric = Objects.requireNonNull(metric, "Metric cannot be null"); @@ -158,6 +162,8 @@ public class Controller extends AbstractComponent { public Clock clock() { return clock; } + public Sleeper sleeper() { return sleeper; } + public ZoneRegistry zoneRegistry() { return zoneRegistry; } public NameServiceForwarder nameServiceForwarder() { return nameServiceForwarder; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index 72d3cf7723b..ea2bcfcac4b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 92633fa969c..13ef1339e44 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.yahoo.config.application.api.DeploymentSpec; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index f25f1c64372..3f4ce7cc49b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.google.common.collect.BiMap; @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; @@ -47,9 +48,10 @@ public abstract class LockedTenant { static LockedTenant of(Tenant tenant, Lock lock) { switch (tenant.type()) { - case athenz: return new Athenz((AthenzTenant) tenant); - case cloud: return new Cloud((CloudTenant) tenant); - default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.getClass().getName() + "'."); + case athenz: return new Athenz((AthenzTenant) tenant); + case cloud: return new Cloud((CloudTenant) tenant); + case deleted: return new Deleted((DeletedTenant) tenant); + default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.getClass().getName() + "'."); } } @@ -58,6 +60,10 @@ public abstract class LockedTenant { public abstract LockedTenant with(LastLoginInfo lastLoginInfo); + public Deleted deleted(Instant deletedAt) { + return new Deleted(new DeletedTenant(name, createdAt, deletedAt)); + } + @Override public String toString() { return "tenant '" + name + "'"; @@ -183,4 +189,26 @@ public abstract class LockedTenant { } } + + /** A locked DeletedTenant. */ + public static class Deleted extends LockedTenant { + + private final Instant deletedAt; + + private Deleted(DeletedTenant tenant) { + super(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo()); + this.deletedAt = tenant.deletedAt(); + } + + @Override + public DeletedTenant get() { + return new DeletedTenant(name, createdAt, deletedAt); + } + + @Override + public LockedTenant with(LastLoginInfo lastLoginInfo) { + return this; + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java index 02fa49d04d0..f2cb4346b6a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.yahoo.text.Text; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 047738784b7..4506362418b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -1,13 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Text; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.concurrent.Once; @@ -16,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; +import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -26,6 +24,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -63,11 +62,17 @@ public class TenantController { }); } - /** Returns a list of all known tenants sorted by name */ + /** Returns a list of all known, non-deleted tenants sorted by name */ public List<Tenant> asList() { + return asList(false); + } + + /** Returns a list of all known tenants sorted by name */ + public List<Tenant> asList(boolean includeDeleted) { return curator.readTenants().stream() - .sorted(Comparator.comparing(Tenant::name)) - .collect(Collectors.toList()); + .filter(tenant -> tenant.type() != Tenant.Type.deleted || includeDeleted) + .sorted(Comparator.comparing(Tenant::name)) + .collect(Collectors.toList()); } /** Locks a tenant for modification and applies the given action. */ @@ -110,21 +115,22 @@ public class TenantController { /** Create a tenant, provided the given credentials are valid. */ public void create(TenantSpec tenantSpec, Credentials credentials) { try (Lock lock = lock(tenantSpec.tenant())) { - requireNonExistent(tenantSpec.tenant()); TenantId.validate(tenantSpec.tenant().value()); + requireNonExistent(tenantSpec.tenant()); curator.writeTenant(accessControl.createTenant(tenantSpec, controller.clock().instant(), credentials, asList())); - try { - controller.serviceRegistry().roleService().createTenantRole(tenantSpec.tenant()); - } catch (Exception e) { - throw new RuntimeException("Unable to create tenant role for tenant: " + tenantSpec.tenant()); - } + // We should create tenant roles here but it takes too long - assuming the TenantRoleMaintainer will do it Soon™ } } /** Find tenant by name */ public Optional<Tenant> get(TenantName name) { - return curator.readTenant(name); + return get(name, false); + } + + public Optional<Tenant> get(TenantName name, boolean includeDeleted) { + return curator.readTenant(name) + .filter(tenant -> tenant.type() != Tenant.Type.deleted || includeDeleted); } /** Find tenant by name */ @@ -157,22 +163,28 @@ public class TenantController { } /** Deletes the given tenant. */ - public void delete(TenantName tenant, Credentials credentials) { + public void delete(TenantName tenant, Supplier<Credentials> credentials, boolean forget) { try (Lock lock = lock(tenant)) { - require(tenant); - if ( ! controller.applications().asList(tenant).isEmpty()) - throw new IllegalArgumentException("Could not delete tenant '" + tenant.value() - + "': This tenant has active applications"); - - curator.removeTenant(tenant); - accessControl.deleteTenant(tenant, credentials); - controller.notificationsDb().removeNotifications(NotificationSource.from(tenant)); + Tenant oldTenant = get(tenant, true) + .orElseThrow(() -> new NotExistsException("Could not delete tenant '" + tenant + "': Tenant not found")); + + if (oldTenant.type() != Tenant.Type.deleted) { + if (!controller.applications().asList(tenant).isEmpty()) + throw new IllegalArgumentException("Could not delete tenant '" + tenant.value() + + "': This tenant has active applications"); + + accessControl.deleteTenant(tenant, credentials.get()); + controller.notificationsDb().removeNotifications(NotificationSource.from(tenant)); + } + + if (forget) curator.removeTenant(tenant); + else curator.writeTenant(new DeletedTenant(tenant, oldTenant.createdAt(), controller.clock().instant())); } } private void requireNonExistent(TenantName name) { if (SystemApplication.TENANT.equals(name) - || get(name).isPresent() + || get(name, true).isPresent() // Underscores are allowed in existing tenant names, but tenants with - and _ cannot co-exist. E.g. // my-tenant cannot be created if my_tenant exists. || get(name.value().replace('-', '_')).isPresent()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ActivateResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ActivateResult.java index 20400be0d1f..989a0de0f83 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ActivateResult.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ActivateResult.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java index 8917c489c65..09acb12d660 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 7ca6cb1e20b..a6e53fead37 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import com.yahoo.collections.AbstractFilteringList; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java index 33eafecf60a..244fb952b3f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; @@ -36,7 +36,7 @@ public final class Change { private Change(Optional<Version> platform, Optional<ApplicationVersion> application, boolean pinned) { this.platform = requireNonNull(platform, "platform cannot be null"); this.application = requireNonNull(application, "application cannot be null"); - if (application.isPresent() && application.get().isUnknown()) { + if (application.isPresent() && (application.get().isUnknown() || application.get().isDeployedDirectly())) { throw new IllegalArgumentException("Application version to deploy must be a known version"); } this.pinned = pinned; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 43ce466e8e6..6081c3323c8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java index 71f0d64c43a..fc2ac94ffed 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java index 094cb9a19b0..1e0946d07be 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java index c17530fd9e2..481f57e13f1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.application.api.DeploymentSpec; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index 4954628a46a..0e8a59134ed 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import com.google.common.collect.ImmutableMap; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/QuotaUsage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/QuotaUsage.java index 1e070d5a66b..2e0a2d48b78 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/QuotaUsage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/QuotaUsage.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import java.util.Objects; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java index 4dbce299b5d..569ea0bfb1f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * Core application model * diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index c29bc3f3f5e..9767ef59252 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; import com.google.common.hash.Hashing; import com.yahoo.component.Version; @@ -251,10 +251,11 @@ public class ApplicationPackage { private Map<Path, Optional<byte[]>> read(Collection<String> names) { var entries = new ZipStreamReader(new ByteArrayInputStream(zip), name -> names.contains(withoutLegacyDir(name)), - maxSize) + maxSize, + true) .entries().stream() .collect(toMap(entry -> Paths.get(withoutLegacyDir(entry.zipEntry().getName())).normalize(), - entry -> Optional.of(entry.content()))); + ZipStreamReader.ZipEntryWithContent::content)); names.stream().map(Paths::get).forEach(path -> entries.putIfAbsent(path.normalize(), Optional.empty())); return entries; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java new file mode 100644 index 00000000000..97810b9de80 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java @@ -0,0 +1,112 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.vespa.hosted.controller.application.pkg.ZipStreamReader.ZipEntryWithContent; + +/** + * @author freva + */ +public class ApplicationPackageDiff { + + public static byte[] diffAgainstEmpty(ApplicationPackage right) { + byte[] emptyZip = new byte[]{80, 75, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + return diff(new ApplicationPackage(emptyZip), right); + } + + public static byte[] diff(ApplicationPackage left, ApplicationPackage right) { + return diff(left, right, 10 << 20, 1 << 20, 10 << 20); + } + + static byte[] diff(ApplicationPackage left, ApplicationPackage right, int maxFileSizeToDiff, int maxDiffSizePerFile, int maxTotalDiffSize) { + if (Arrays.equals(left.zippedContent(), right.zippedContent())) return "No diff\n".getBytes(StandardCharsets.UTF_8); + + Map<String, ZipEntryWithContent> leftContents = readContents(left, maxFileSizeToDiff); + Map<String, ZipEntryWithContent> rightContents = readContents(right, maxFileSizeToDiff); + + StringBuilder sb = new StringBuilder(); + List<String> files = Stream.of(leftContents, rightContents) + .flatMap(contents -> contents.keySet().stream()) + .sorted() + .distinct() + .collect(Collectors.toList()); + for (String file : files) { + if (sb.length() > maxTotalDiffSize) + sb.append("--- ").append(file).append('\n').append("Diff skipped: Total diff size >").append(maxTotalDiffSize).append("B)\n\n"); + else + diff(Optional.ofNullable(leftContents.get(file)), Optional.ofNullable(rightContents.get(file)), maxDiffSizePerFile) + .ifPresent(diff -> sb.append("--- ").append(file).append('\n').append(diff).append('\n')); + } + + return (sb.length() == 0 ? "No diff\n" : sb.toString()).getBytes(StandardCharsets.UTF_8); + } + + private static Optional<String> diff(Optional<ZipEntryWithContent> left, Optional<ZipEntryWithContent> right, int maxDiffSizePerFile) { + Optional<byte[]> leftContent = left.flatMap(ZipEntryWithContent::content); + Optional<byte[]> rightContent = right.flatMap(ZipEntryWithContent::content); + if (leftContent.isPresent() && rightContent.isPresent() && Arrays.equals(leftContent.get(), rightContent.get())) + return Optional.empty(); + + if (Stream.of(left, right).flatMap(Optional::stream).anyMatch(entry -> entry.content().isEmpty())) + return Optional.of(String.format("Diff skipped: File too large (%s -> %s)\n", + left.map(e -> e.size() + "B").orElse("new file"), right.map(e -> e.size() + "B").orElse("file deleted"))); + + if (Stream.of(leftContent, rightContent).flatMap(Optional::stream).anyMatch(c -> isBinary(c))) + return Optional.of(String.format("Diff skipped: File is binary (%s -> %s)\n", + left.map(e -> e.size() + "B").orElse("new file"), right.map(e -> e.size() + "B").orElse("file deleted"))); + + return LinesComparator.diff( + leftContent.map(c -> lines(c)).orElseGet(List::of), + rightContent.map(c -> lines(c)).orElseGet(List::of)) + .map(diff -> diff.length() > maxDiffSizePerFile ? "Diff skipped: Diff too large (" + diff.length() + "B)\n" : diff); + } + + private static Map<String, ZipEntryWithContent> readContents(ApplicationPackage app, int maxFileSizeToDiff) { + return new ZipStreamReader(new ByteArrayInputStream(app.zippedContent()), entry -> true, maxFileSizeToDiff, false).entries().stream() + .collect(Collectors.toMap(entry -> entry.zipEntry().getName(), e -> e)); + } + + private static List<String> lines(byte[] data) { + List<String> lines = new ArrayList<>(Math.min(16, data.length / 100)); + try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(data), StandardCharsets.UTF_8))) { + String line; + while ((line = bufferedReader.readLine()) != null) { + lines.add(line); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return lines; + } + + private static boolean isBinary(byte[] data) { + if (data.length == 0) return false; + + int lengthToCheck = Math.min(data.length, 10000); + int ascii = 0; + + for (int i = 0; i < lengthToCheck; i++) { + byte b = data[i]; + if (b < 0x9) return true; + + // TAB, newline/line feed, carriage return + if (b == 0x9 || b == 0xA || b == 0xD) ascii++; + else if (b >= 0x20 && b <= 0x7E) ascii++; + } + + return (double) ascii / lengthToCheck < 0.95; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java index bb2d8b3c553..3dc95251eb8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java @@ -1,5 +1,5 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +package com.yahoo.vespa.hosted.controller.application.pkg; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; @@ -14,6 +14,7 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import java.time.Instant; @@ -80,7 +81,7 @@ public class ApplicationPackageValidator { for (var endpoint : instance.endpoints()) { var clouds = new HashSet<CloudName>(); for (var region : endpoint.regions()) { - for (ZoneApi zone : controller.zoneRegistry().zones().all().in(region).zones()) { + for (ZoneApi zone : controller.zoneRegistry().zones().all().in(Environment.prod).in(region).zones()) { clouds.add(zone.getCloudName()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/LinesComparator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/LinesComparator.java new file mode 100644 index 00000000000..8b4791c6b1b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/LinesComparator.java @@ -0,0 +1,246 @@ +/* + * Line based variant of Apache commons-text StringComparator + * https://github.com/apache/commons-text/blob/3b1a0a5a47ee9fa2b36f99ca28e2e1d367a10a11/src/main/java/org/apache/commons/text/diff/StringsComparator.java + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.yahoo.vespa.hosted.controller.application.pkg; + +import com.yahoo.collections.Pair; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * <p> + * It is guaranteed that the comparisons will always be done as + * {@code o1.equals(o2)} where {@code o1} belongs to the first + * sequence and {@code o2} belongs to the second sequence. This can + * be important if subclassing is used for some elements in the first + * sequence and the {@code equals} method is specialized. + * </p> + * <p> + * Comparison can be seen from two points of view: either as giving the smallest + * modification allowing to transform the first sequence into the second one, or + * as giving the longest sequence which is a subsequence of both initial + * sequences. The {@code equals} method is used to compare objects, so any + * object can be put into sequences. Modifications include deleting, inserting + * or keeping one object, starting from the beginning of the first sequence. + * </p> + * <p> + * This class implements the comparison algorithm, which is the very efficient + * algorithm from Eugene W. Myers + * <a href="http://www.cis.upenn.edu/~bcpierce/courses/dd/papers/diff.ps"> + * An O(ND) Difference Algorithm and Its Variations</a>. + */ +public class LinesComparator { + + private final List<String> left; + private final List<String> right; + private final int[] vDown; + private final int[] vUp; + + private LinesComparator(List<String> left, List<String> right) { + this.left = left; + this.right = right; + + int size = left.size() + right.size() + 2; + vDown = new int[size]; + vUp = new int[size]; + } + + private void buildScript(int start1, int end1, int start2, int end2, List<Pair<LineOperation, String>> result) { + Snake middle = getMiddleSnake(start1, end1, start2, end2); + + if (middle == null + || middle.start == end1 && middle.diag == end1 - end2 + || middle.end == start1 && middle.diag == start1 - start2) { + + int i = start1; + int j = start2; + while (i < end1 || j < end2) { + if (i < end1 && j < end2 && left.get(i).equals(right.get(j))) { + result.add(new Pair<>(LineOperation.keep, left.get(i))); + ++i; + ++j; + } else { + if (end1 - start1 > end2 - start2) { + result.add(new Pair<>(LineOperation.delete, left.get(i))); + ++i; + } else { + result.add(new Pair<>(LineOperation.insert, right.get(j))); + ++j; + } + } + } + + } else { + buildScript(start1, middle.start, start2, middle.start - middle.diag, result); + for (int i = middle.start; i < middle.end; ++i) { + result.add(new Pair<>(LineOperation.keep, left.get(i))); + } + buildScript(middle.end, end1, middle.end - middle.diag, end2, result); + } + } + + private Snake buildSnake(final int start, final int diag, final int end1, final int end2) { + int end = start; + while (end - diag < end2 && end < end1 && left.get(end).equals(right.get(end - diag))) { + ++end; + } + return new Snake(start, end, diag); + } + + private Snake getMiddleSnake(final int start1, final int end1, final int start2, final int end2) { + final int m = end1 - start1; + final int n = end2 - start2; + if (m == 0 || n == 0) { + return null; + } + + final int delta = m - n; + final int sum = n + m; + final int offset = (sum % 2 == 0 ? sum : sum + 1) / 2; + vDown[1 + offset] = start1; + vUp[1 + offset] = end1 + 1; + + for (int d = 0; d <= offset; ++d) { + // Down + for (int k = -d; k <= d; k += 2) { + // First step + + final int i = k + offset; + if (k == -d || k != d && vDown[i - 1] < vDown[i + 1]) { + vDown[i] = vDown[i + 1]; + } else { + vDown[i] = vDown[i - 1] + 1; + } + + int x = vDown[i]; + int y = x - start1 + start2 - k; + + while (x < end1 && y < end2 && left.get(x).equals(right.get(y))) { + vDown[i] = ++x; + ++y; + } + // Second step + if (delta % 2 != 0 && delta - d <= k && k <= delta + d) { + if (vUp[i - delta] <= vDown[i]) { // NOPMD + return buildSnake(vUp[i - delta], k + start1 - start2, end1, end2); + } + } + } + + // Up + for (int k = delta - d; k <= delta + d; k += 2) { + // First step + final int i = k + offset - delta; + if (k == delta - d || k != delta + d && vUp[i + 1] <= vUp[i - 1]) { + vUp[i] = vUp[i + 1] - 1; + } else { + vUp[i] = vUp[i - 1]; + } + + int x = vUp[i] - 1; + int y = x - start1 + start2 - k; + while (x >= start1 && y >= start2 && left.get(x).equals(right.get(y))) { + vUp[i] = x--; + y--; + } + // Second step + if (delta % 2 == 0 && -d <= k && k <= d) { + if (vUp[i] <= vDown[i + delta]) { // NOPMD + return buildSnake(vUp[i], k + start1 - start2, end1, end2); + } + } + } + } + + // this should not happen + throw new RuntimeException("Internal Error"); + } + + private static class Snake { + private final int start; + private final int end; + private final int diag; + + private Snake(int start, int end, int diag) { + this.start = start; + this.end = end; + this.diag = diag; + } + } + + private enum LineOperation { + keep(" "), delete("- "), insert("+ "); + private final String prefix; + LineOperation(String prefix) { + this.prefix = prefix; + } + } + + /** @return line-based diff in unified format. Empty contents are identical. */ + public static Optional<String> diff(List<String> left, List<String> right) { + List<Pair<LineOperation, String>> changes = new ArrayList<>(Math.max(left.size(), right.size())); + new LinesComparator(left, right).buildScript(0, left.size(), 0, right.size(), changes); + + // After we have a list of keep, delete, insert for each line from left and right input, generate a unified + // diff by printing all delete and insert operations with contextLines of keep lines before and after. + // Make sure the change windows are non-overlapping by continuously growing the window + int contextLines = 3; + List<int[]> changeWindows = new ArrayList<>(); + int[] last = null; + for (int i = 0, leftIndex = 0, rightIndex = 0; i < changes.size(); i++) { + if (changes.get(i).getFirst() == LineOperation.keep) { + leftIndex++; + rightIndex++; + continue; + } + + // We found a new change and it is too far away from the previous change to be combined into the same window + if (last == null || i - last[1] > contextLines) { + last = new int[]{Math.max(i - contextLines, 0), Math.min(i + contextLines + 1, changes.size()), Math.max(leftIndex - contextLines, 0), Math.max(rightIndex - contextLines, 0)}; + changeWindows.add(last); + } else // otherwise, extend the previous change window + last[1] = Math.min(i + contextLines + 1, changes.size()); + + if (changes.get(i).getFirst() == LineOperation.delete) leftIndex++; + else rightIndex++; + } + if (changeWindows.isEmpty()) return Optional.empty(); + + StringBuilder sb = new StringBuilder(); + for (int[] changeWindow: changeWindows) { + int start = changeWindow[0], end = changeWindow[1], leftIndex = changeWindow[2], rightIndex = changeWindow[3]; + Map<LineOperation, Long> counts = IntStream.range(start, end) + .mapToObj(i -> changes.get(i).getFirst()) + .collect(Collectors.groupingBy(i -> i, Collectors.counting())); + sb.append("@@ -").append(leftIndex + 1).append(',').append(end - start - counts.getOrDefault(LineOperation.insert, 0L)) + .append(" +").append(rightIndex + 1).append(',').append(end - start - counts.getOrDefault(LineOperation.delete, 0L)).append(" @@\n"); + for (int i = start; i < end; i++) + sb.append(changes.get(i).getFirst().prefix).append(changes.get(i).getSecond()).append('\n'); + } + return Optional.of(sb.toString()); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java index 4f01df21430..7ddd0af7a7a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java @@ -1,5 +1,5 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +package com.yahoo.vespa.hosted.controller.application.pkg; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -10,6 +10,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.function.Predicate; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -23,16 +24,15 @@ public class ZipStreamReader { private final List<ZipEntryWithContent> entries = new ArrayList<>(); private final int maxEntrySizeInBytes; - public ZipStreamReader(InputStream input, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes) { + public ZipStreamReader(InputStream input, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes, boolean throwIfEntryExceedsMaxSize) { this.maxEntrySizeInBytes = maxEntrySizeInBytes; try (ZipInputStream zipInput = new ZipInputStream(input)) { ZipEntry zipEntry; while (null != (zipEntry = zipInput.getNextEntry())) { if (!entryNameMatcher.test(requireName(zipEntry.getName()))) continue; - entries.add(new ZipEntryWithContent(zipEntry, readContent(zipInput))); + entries.add(readContent(zipEntry, zipInput, throwIfEntryExceedsMaxSize)); } - } catch (IOException e) { throw new UncheckedIOException("IO error reading zip content", e); } @@ -59,7 +59,7 @@ public class ZipStreamReader { } } - private byte[] readContent(ZipInputStream zipInput) { + private ZipEntryWithContent readContent(ZipEntry zipEntry, ZipInputStream zipInput, boolean throwIfEntryExceedsMaxSize) { try (ByteArrayOutputStream bis = new ByteArrayOutputStream()) { byte[] buffer = new byte[2048]; int read; @@ -67,12 +67,15 @@ public class ZipStreamReader { while ( -1 != (read = zipInput.read(buffer))) { size += read; if (size > maxEntrySizeInBytes) { - throw new IllegalArgumentException("Entry in zip content exceeded size limit of " + - maxEntrySizeInBytes + " bytes"); - } - bis.write(buffer, 0, read); + if (throwIfEntryExceedsMaxSize) throw new IllegalArgumentException( + "Entry in zip content exceeded size limit of " + maxEntrySizeInBytes + " bytes"); + } else bis.write(buffer, 0, read); } - return bis.toByteArray(); + + boolean hasContent = size <= maxEntrySizeInBytes; + return new ZipEntryWithContent(zipEntry, + Optional.of(bis).filter(__ -> hasContent).map(ByteArrayOutputStream::toByteArray), + size); } catch (IOException e) { throw new UncheckedIOException("Failed reading from zipped content", e); } @@ -96,16 +99,19 @@ public class ZipStreamReader { public static class ZipEntryWithContent { private final ZipEntry zipEntry; - private final byte[] content; + private final Optional<byte[]> content; + private final long size; - public ZipEntryWithContent(ZipEntry zipEntry, byte[] content) { + public ZipEntryWithContent(ZipEntry zipEntry, Optional<byte[]> content, long size) { this.zipEntry = zipEntry; this.content = content; + this.size = size; } public ZipEntry zipEntry() { return zipEntry; } - public byte[] content() { return content; } - + public byte[] contentOrThrow() { return content.orElseThrow(); } + public Optional<byte[]> content() { return content; } + public long size() { return size; } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java index 0f5c2123325..a7555307a59 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -1,9 +1,13 @@ // Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.archive; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; @@ -40,22 +44,32 @@ public class CuratorArchiveBucketDb { private final ArchiveService archiveService; private final CuratorDb curatorDb; - private final boolean enabled; + private final BooleanFlag enableFlag; + private final SystemName system; public CuratorArchiveBucketDb(Controller controller) { this.archiveService = controller.serviceRegistry().archiveService(); this.curatorDb = controller.curator(); - this.enabled = controller.zoneRegistry().system().isPublic(); + this.enableFlag = Flags.ENABLE_ONPREM_TENANT_S3_ARCHIVE.bindTo(controller.flagSource()); + this.system = controller.zoneRegistry().system(); } public Optional<URI> archiveUriFor(ZoneId zoneId, TenantName tenant) { - if (enabled) { + if (enabled(zoneId, tenant)) { return Optional.of(URI.create(Text.format("s3://%s/%s/", findOrAssignBucket(zoneId, tenant), tenant.value()))); } else { return Optional.empty(); } } + private boolean enabled(ZoneId zone, TenantName tenant) { + return system.isPublic() || + enableFlag + .with(FetchVector.Dimension.ZONE_ID, zone.value()) + .with(FetchVector.Dimension.TENANT_ID, tenant.value()) + .value(); + } + private String findOrAssignBucket(ZoneId zoneId, TenantName tenant) { return getBucketNameFromCache(zoneId, tenant) .or(() -> findAndUpdateArchiveUriCache(zoneId, tenant, buckets(zoneId))) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/HostedAthenzIdentities.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/HostedAthenzIdentities.java index 08dbdff13db..ef2f24bbf6d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/HostedAthenzIdentities.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/HostedAthenzIdentities.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz; import com.yahoo.vespa.athenz.api.AthenzDomain; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/config/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/config/package-info.java index f3f2acf1e01..02df7e5c2cf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/config/package-info.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/config/package-info.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * Required for using {@link com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig} outside controller-server module. * diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java index 8604e5e48fe..6b93229dffe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.impl; import com.google.inject.Inject; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 8cc02b5bf69..d116ef3333c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.impl; import com.google.common.cache.CacheBuilder; @@ -164,9 +164,20 @@ public class AthenzFacade implements AccessControl { @Override public void deleteTenant(TenantName tenant, Credentials credentials) { AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; - - log("deleteTenancy(tenantDomain=%s, service=%s)", athenzCredentials.domain(), service); - zmsClient.deleteTenancy(athenzCredentials.domain(), service, athenzCredentials.identityToken(), athenzCredentials.accessToken()); + AthenzDomain tenantDomain = athenzCredentials.domain(); + log("deleteTenancy(tenantDomain=%s, service=%s)", tenantDomain, service); + try { + zmsClient.deleteTenancy(tenantDomain, service, athenzCredentials.identityToken(), athenzCredentials.accessToken()); + } catch (ZmsClientException e) { + if (e.getErrorCode() == 404) { + log.log(Level.WARNING, + "Failed to cleanup tenant " + tenant.value() + " with domain '" + tenantDomain.getName() + + "' in Athenz due to non-existing tenant domain", + e); + } else { + throw e; + } + } } @Override @@ -196,8 +207,19 @@ public class AthenzFacade implements AccessControl { AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; log("deleteProviderResourceGroup(tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s)", athenzCredentials.domain(), service.getDomain().getName(), service.getName(), id.application()); - zmsClient.deleteProviderResourceGroup(athenzCredentials.domain(), service, id.application().value(), - athenzCredentials.identityToken(), athenzCredentials.accessToken()); + try { + zmsClient.deleteProviderResourceGroup(athenzCredentials.domain(), service, id.application().value(), + athenzCredentials.identityToken(), athenzCredentials.accessToken()); + } catch (ZmsClientException e) { + if (e.getErrorCode() == 404) { + log.log(Level.WARNING, + "Failed to cleanup application '" + id.serialized() + + "' in Athenz due to non-existing tenant domain or resource group", + e); + } else { + throw e; + } + } } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index 1447e1809a4..684648ed70a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.certificate; import com.yahoo.config.application.api.DeploymentInstanceSpec; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Once.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Once.java index 81ddd8d2d70..dbcd8bf1459 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Once.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Once.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.concurrent; import java.time.Duration; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java index 61135613e98..7ab895654f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.application.api.DeploymentInstanceSpec; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 4e6df1921b6..a34217d2226 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.application.api.DeploymentInstanceSpec; @@ -105,16 +105,6 @@ public class DeploymentTrigger { instance = instance.withChange(instance.change().with(outstanding.application().get())); return instance.withChange(remainingChange(instance, status)); }); - - // Abort irrelevant, running jobs to get new application out faster. - Map<JobId, List<Versions>> newJobsToRun = jobs.deploymentStatus(application.get()).jobsToRun(); - for (Run run : jobs.active(application.get().id().instance(instanceName))) { - if ( ! run.id().type().environment().isManuallyDeployed() - && newJobsToRun.getOrDefault(run.id().job(), List.of()).stream() - .noneMatch(versions -> versions.targetsMatch(run.versions()) - && versions.sourcesMatchIfPresent(run.versions()))) - jobs.abort(run.id()); - } } } applications().store(application); @@ -190,8 +180,11 @@ public class DeploymentTrigger { public List<JobId> forceTrigger(ApplicationId applicationId, JobType jobType, String user, boolean requireTests) { Application application = applications().requireApplication(TenantAndApplicationId.from(applicationId)); Instance instance = application.require(applicationId.instance()); - DeploymentStatus status = jobs.deploymentStatus(application); JobId job = new JobId(instance.id(), jobType); + if (job.type().environment().isManuallyDeployed()) + return forceTriggerManualJob(job); + + DeploymentStatus status = jobs.deploymentStatus(application); Versions versions = Versions.from(instance.change(), application, status.deploymentFor(job), controller.readSystemVersion()); Map<JobId, List<Versions>> jobs = status.testJobs(Map.of(job, versions)); if (jobs.isEmpty() || ! requireTests) @@ -202,6 +195,16 @@ public class DeploymentTrigger { return List.copyOf(jobs.keySet()); } + private List<JobId> forceTriggerManualJob(JobId job) { + Run last = jobs.last(job).orElseThrow(() -> new IllegalArgumentException(job + " has never been run")); + Versions target = new Versions(controller.readSystemVersion(), + last.versions().targetApplication(), + Optional.of(last.versions().targetPlatform()), + Optional.of(last.versions().targetApplication())); + jobs.start(job.application(), job.type(), target, true); + return List.of(job); + } + /** Retrigger job. If the job is already running, it will be canceled, and retrigger enqueued. */ public Optional<JobId> reTriggerOrAddToQueue(DeploymentId deployment) { JobType jobType = JobType.from(controller.system(), deployment.zoneId()) @@ -289,10 +292,6 @@ public class DeploymentTrigger { return controller.applications(); } - private Optional<Deployment> deploymentFor(Instance instance, JobType jobType) { - return Optional.ofNullable(instance.deployments().get(jobType.zone(controller.system()))); - } - // ---------- Ready job computation ---------- /** Returns the set of all jobs which have changes to propagate from the upstream steps. */ @@ -307,24 +306,22 @@ public class DeploymentTrigger { .collect(toList()); } - /** - * Finds the next step to trigger for the given application, if any, and returns these as a list. - */ + /** Finds the next step to trigger for the given application, if any, and returns these as a list. */ private List<Job> computeReadyJobs(DeploymentStatus status) { List<Job> jobs = new ArrayList<>(); status.jobsToRun().forEach((job, versionsList) -> { - for (Versions versions : versionsList) - status.jobSteps().get(job).readyAt(status.application().require(job.application().instance()).change()) - .filter(readyAt -> ! clock.instant().isBefore(readyAt)) - .filter(__ -> ! status.jobs().get(job).get().isRunning()) - .filter(__ -> ! (job.type().isProduction() && isUnhealthyInAnotherZone(status.application(), job))) - .ifPresent(readyAt -> { - jobs.add(deploymentJob(status.application().require(job.application().instance()), - versions, - job.type(), - status.instanceJobs(job.application().instance()).get(job.type()), - readyAt)); - }); + for (Versions versions : versionsList) + status.jobSteps().get(job).readyAt(status.application().require(job.application().instance()).change()) + .filter(readyAt -> ! clock.instant().isBefore(readyAt)) + .filter(__ -> ! (job.type().isProduction() && isUnhealthyInAnotherZone(status.application(), job))) + .filter(__ -> abortIfRunning(versionsList, status.jobs().get(job).get())) // Abort and trigger this later if running with outdated parameters. + .ifPresent(readyAt -> { + jobs.add(deploymentJob(status.application().require(job.application().instance()), + versions, + job.type(), + status.instanceJobs(job.application().instance()).get(job.type()), + readyAt)); + }); }); return Collections.unmodifiableList(jobs); } @@ -339,71 +336,17 @@ public class DeploymentTrigger { return false; } - /** Returns whether the given job can trigger at the given instant */ - public boolean triggerAt(Instant instant, JobType job, JobStatus jobStatus, Versions versions, Instance instance, DeploymentSpec deploymentSpec) { - if (instance.jobPause(job).map(until -> until.isAfter(clock.instant())).orElse(false)) return false; - if (jobStatus.lastTriggered().isEmpty()) return true; - if (jobStatus.isSuccess()) return true; // Success - if (jobStatus.lastCompleted().isEmpty()) return true; // Never completed - if (jobStatus.firstFailing().isEmpty()) return true; // Should not happen as firstFailing should be set for an unsuccessful job - if ( ! versions.targetsMatch(jobStatus.lastCompleted().get().versions())) return true; // Always trigger as targets have changed - if (deploymentSpec.requireInstance(instance.name()).upgradePolicy() == DeploymentSpec.UpgradePolicy.canary) return true; // Don't throttle canaries - - Instant firstFailing = jobStatus.firstFailing().get().end().get(); - Instant lastCompleted = jobStatus.lastCompleted().get().end().get(); - - // Retry all errors immediately for 1 minute - if (firstFailing.isAfter(instant.minus(Duration.ofMinutes(1)))) return true; - - // Retry out of capacity errors in test environments every minute - if (job.environment().isTest() && jobStatus.isOutOfCapacity()) { - return lastCompleted.isBefore(instant.minus(Duration.ofMinutes(1))); - } + /** Returns whether the job is not running, and also aborts it if it's running with outdated versions. */ + private boolean abortIfRunning(List<Versions> versionsList, JobStatus status) { + if ( ! status.isRunning()) + return true; - // Retry other errors - if (firstFailing.isAfter(instant.minus(Duration.ofHours(1)))) { // If we failed within the last hour ... - return lastCompleted.isBefore(instant.minus(Duration.ofMinutes(10))); // ... retry every 10 minutes - } - return lastCompleted.isBefore(instant.minus(Duration.ofHours(2))); // Retry at most every 2 hours - } - - // ---------- Completion logic ---------- + Run last = status.lastTriggered().get(); + if (versionsList.stream().noneMatch(versions -> versions.targetsMatch(last.versions()) + && versions.sourcesMatchIfPresent(last.versions()))) + controller.jobController().abort(last.id()); - /** - * Returns whether the given change is complete for the given application for the given job. - * - * Any job is complete if the given change is already successful on that job. - * A production job is also considered complete if its current change is strictly dominated by what - * is already deployed in its zone, i.e., no parts of the change are upgrades, and the full current - * change for the application downgrades the deployment, which is an acknowledgement that the deployed - * version is broken somehow, such that the job may be locked in failure until a new version is released. - * - * Additionally, if the application is pinned to a Vespa version, and the given change has a (this) platform, - * the deployment for the job must be on the pinned version. - */ - public boolean isComplete(Change change, Change fullChange, Instance instance, JobType jobType, - JobStatus status) { - Optional<Deployment> existingDeployment = deploymentFor(instance, jobType); - if ( change.isPinned() - && change.platform().isPresent() - && ! existingDeployment.map(Deployment::version).equals(change.platform())) - return false; - - return status.lastSuccess() - .map(run -> change.platform().map(run.versions().targetPlatform()::equals).orElse(true) - && change.application().map(run.versions().targetApplication()::equals).orElse(true)) - .orElse(false) - || jobType.isProduction() - && existingDeployment.map(deployment -> ! isUpgrade(change, deployment) && isDowngrade(fullChange, deployment)) - .orElse(false); - } - - private static boolean isUpgrade(Change change, Deployment deployment) { - return change.upgrades(deployment.version()) || change.upgrades(deployment.applicationVersion()); - } - - private static boolean isDowngrade(Change change, Deployment deployment) { - return change.downgrades(deployment.version()) || change.downgrades(deployment.applicationVersion()); + return false; } // ---------- Change management o_O ---------- @@ -411,6 +354,8 @@ public class DeploymentTrigger { private boolean acceptNewApplicationVersion(DeploymentStatus status, InstanceName instance) { if (status.application().require(instance).change().application().isPresent()) return true; // Replacing a previous application change is ok. if (status.hasFailures()) return true; // Allow changes to fix upgrade problems. + if (status.application().deploymentSpec().instance(instance) // Leading upgrade allows app change to join in. + .map(spec -> spec.upgradeRollout() == DeploymentSpec.UpgradeRollout.leading).orElse(false)) return true; return status.application().require(instance).change().platform().isEmpty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index c49e3c88df3..ee955fa8ff8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -1,7 +1,6 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; -import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; @@ -31,6 +30,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; @@ -41,7 +41,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.application.ActivateResult; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -103,6 +103,7 @@ import static java.util.logging.Level.INFO; import static java.util.logging.Level.WARNING; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; /** * Runs steps of a deployment job against its provided controller. @@ -313,10 +314,14 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); } List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), - id.application(), - Set.of(active)); + NodeFilter.all() + .applications(id.application()) + .states(active)); + + Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), - nodes.stream().map(node -> node.parentHostname().get()).collect(toList())); + NodeFilter.all() + .hostnames(parentHostnames)); NodeList nodeList = NodeList.of(nodes, parents, services.get()); boolean firstTick = run.convergenceSummary().isEmpty(); if (firstTick) { // Run the first time (for each convergence step). @@ -419,10 +424,13 @@ public class InternalStepRunner implements StepRunner { : Optional.empty(); } List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(zone, - testerId, - ImmutableSet.of(active, reserved)); + NodeFilter.all() + .applications(testerId) + .states(active, reserved)); + Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(zone, - nodes.stream().map(node -> node.parentHostname().get()).collect(toList())); + NodeFilter.all() + .hostnames(parentHostnames)); NodeList nodeList = NodeList.of(nodes, parents, services.get()); logger.log(nodeList.asList().stream() .flatMap(node -> nodeDetails(node, false)) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index b622fc0bd75..d3615c942b9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -22,13 +21,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import java.net.URI; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; @@ -48,7 +47,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.logging.Level; -import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; @@ -369,7 +367,8 @@ public class JobController { List<Lock> locks = new ArrayList<>(); try { // Ensure no step is still running before we finish the run — report depends transitively on all the other steps. - for (Step step : report.allPrerequisites(run(id).get().steps().keySet())) + Run unlockedRun = run(id).get(); + for (Step step : report.allPrerequisites(unlockedRun.steps().keySet())) locks.add(curator.lock(id.application(), id.type(), step)); locked(id, run -> { // Store the modified run after it has been written to history, in case the latter fails. @@ -400,6 +399,20 @@ public class JobController { metric.jobFinished(run.id().job(), finishedRun.status()); return finishedRun; }); + + DeploymentId deploymentId = new DeploymentId(unlockedRun.id().application(), unlockedRun.id().job().type().zone(controller.system())); + (unlockedRun.versions().targetApplication().isDeployedDirectly() ? + Stream.of(unlockedRun.id().type()) : + JobType.allIn(controller.system()).stream().filter(jobType -> !jobType.environment().isManuallyDeployed())) + .flatMap(jobType -> controller.jobController().runs(unlockedRun.id().application(), jobType).values().stream()) + .mapToLong(run -> run.versions().targetApplication().buildNumber().orElse(Integer.MAX_VALUE)) + .min() + .ifPresent(oldestBuild -> { + if (unlockedRun.versions().targetApplication().isDeployedDirectly()) + controller.applications().applicationStore().pruneDevDiffs(deploymentId, oldestBuild); + else + controller.applications().applicationStore().pruneDiffs(deploymentId.applicationId().tenant(), deploymentId.applicationId().application(), oldestBuild); + }); } finally { for (Lock lock : locks) @@ -425,12 +438,19 @@ public class JobController { applicationPackage.compileVersion(), applicationPackage.buildTime(), sourceUrl, - revision.map(SourceRevision::commit))); + revision.map(SourceRevision::commit), + false)); + byte[] diff = application.get().latestVersion() + .map(v -> v.buildNumber().getAsLong()) + .flatMap(prevBuild -> controller.applications().applicationStore().find(id.tenant(), id.application(), prevBuild)) + .map(prevApplication -> ApplicationPackageDiff.diff(new ApplicationPackage(prevApplication), applicationPackage)) + .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage)); controller.applications().applicationStore().put(id.tenant(), id.application(), version.get(), - applicationPackage.zippedContent()); + applicationPackage.zippedContent(), + diff); controller.applications().applicationStore().putTester(id.tenant(), id.application(), version.get(), @@ -471,8 +491,14 @@ public class JobController { }); } + /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment. */ public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage) { + deploy(id, type, platform, applicationPackage, false); + } + + /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment.*/ + public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage, boolean dryRun) { controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { if ( ! application.get().instances().containsKey(id.instance())) application = controller.applications().withNewInstance(application, id); @@ -480,20 +506,32 @@ public class JobController { controller.applications().store(application); }); - last(id, type).filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); + DeploymentId deploymentId = new DeploymentId(id, type.zone(controller.system())); + Optional<Run> lastRun = last(id, type); + lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); + + long build = 1 + lastRun.map(run -> run.versions().targetApplication().buildNumber().orElse(0)).orElse(0L); + ApplicationVersion version = ApplicationVersion.from(Optional.empty(), build, Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty(), Optional.empty(), true); + + byte[] diff = lastRun.map(run -> run.versions().targetApplication()) + .map(prevVersion -> ApplicationPackageDiff.diff(new ApplicationPackage(controller.applications().applicationStore().get(deploymentId, prevVersion)), applicationPackage)) + .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage)); controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - controller.applications().applicationStore().putDev(id, type.zone(controller.system()), applicationPackage.zippedContent()); + controller.applications().applicationStore().putDev(deploymentId, version, applicationPackage.zippedContent(), diff); start(id, type, new Versions(platform.orElse(applicationPackage.deploymentSpec().majorVersion() .flatMap(controller.applications()::lastCompatibleVersion) + .or(() -> lastRun.map(run -> run.versions().targetPlatform()) + .filter(controller.readVersionStatus()::isActive)) .orElseGet(controller::readSystemVersion)), - ApplicationVersion.unknown, - Optional.empty(), - Optional.empty()), + version, + lastRun.map(run -> run.versions().targetPlatform()), + lastRun.map(run -> run.versions().targetApplication())), false, - JobProfile.development); + dryRun ? JobProfile.developmentDryRun : JobProfile.development); }); locked(id, type, __ -> { @@ -558,7 +596,7 @@ public class JobController { application.get().productionDeployments().values().stream() .flatMap(List::stream) .map(Deployment::applicationVersion) - .filter(version -> ! version.isUnknown()) + .filter(version -> ! version.isUnknown() && ! version.isDeployedDirectly()) .min(Comparator.comparingLong(applicationVersion -> applicationVersion.buildNumber().getAsLong())) .ifPresent(oldestDeployed -> { controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index 1c1d60a2cf0..5f9207953ca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -68,7 +68,9 @@ public enum JobProfile { development(EnumSet.of(deployReal, installReal, - copyVespaLogs)); + copyVespaLogs)), + + developmentDryRun(EnumSet.of(deployReal)); private final Set<Step> steps; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java index 966c9a2dbc3..d46516582be 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.vespa.curator.Lock; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java index 3c3cbfe2b2a..12c226241e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.collections.AbstractFilteringList; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java index 9326035d4c7..bd589af190e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntry.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntry.java index 9c16d80313e..c8e851fc375 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntry.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntry.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java index 6d9206d42b6..6f456d2e217 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index d93a0133c97..1c29bdd397f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableList; @@ -39,11 +39,12 @@ public class Run { private final Optional<Instant> noNodesDownSince; private final Optional<ConvergenceSummary> convergenceSummary; private final Optional<X509Certificate> testerCertificate; + private final boolean dryRun; // For deserialisation only -- do not use! public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, boolean isRedeployment, Instant start, Optional<Instant> end, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince, - Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate) { + Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate, boolean dryRun) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); this.versions = versions; @@ -56,13 +57,14 @@ public class Run { this.noNodesDownSince = noNodesDownSince; this.convergenceSummary = convergenceSummary; this.testerCertificate = testerCertificate; + this.dryRun = dryRun; } public static Run initial(RunId id, Versions versions, boolean isRedeployment, Instant now, JobProfile profile) { EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class); profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step))); return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), running, - -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty()); + -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty(), profile == JobProfile.developmentDryRun); } /** Returns a new Run with the status of the given completed step set accordingly. */ @@ -76,7 +78,7 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(Step.Status.of(status))); return new Run(id, steps, versions, isRedeployment, start, end, this.status == running ? status : this.status, - lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); + lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun); } /** Returns a new Run with a new start time*/ @@ -91,49 +93,49 @@ public class Run { steps.put(step.get(), stepInfo.with(startTime)); return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun); } public Run finished(Instant now) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, Optional.of(now), status == running ? success : status, - lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty()); + lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty(), dryRun); } public Run aborted() { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, aborted, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun); } public Run with(long lastTestRecord) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun); } public Run with(Instant lastVespaLogTimestamp) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun); } public Run noNodesDownSince(Instant noNodesDownSince) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, - Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate); + Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate, dryRun); } public Run withSummary(ConvergenceSummary convergenceSummary) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate); + noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate, dryRun); } public Run with(X509Certificate testerCertificate) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, Optional.of(testerCertificate)); + noNodesDownSince, convergenceSummary, Optional.of(testerCertificate), dryRun); } /** Returns the id of this run. */ @@ -229,6 +231,9 @@ public class Run { return isRedeployment; } + /** Whether this is a dry run deployment. */ + public boolean isDryRun() { return dryRun; } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunLog.java index 44299997b4f..bc27d736ad4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunLog.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunLog.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableList; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index fba3f7ae6e9..7aa685d0698 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index ce34a021218..d58f3dee95f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import java.util.Collection; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java index 526546ddf81..b3964c8e422 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java index ccd624b2a27..779ce6fa7fe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java index 6bcc8a034cc..a1eae23afd3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import java.io.ByteArrayInputStream; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java index e8fb638bc34..b69e8401eb8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package com.yahoo.vespa.hosted.controller.deployment; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java index 1697c05b8fb..d0c901ccb36 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index f8c390edb3b..2c59211e50c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; import com.yahoo.vespa.curator.Lock; @@ -28,7 +28,11 @@ import java.util.stream.Collectors; */ public class NameServiceForwarder { - private static final int QUEUE_CAPACITY = 300; + /** + * The number of {@link NameServiceRequest}s we allow to be queued. When the queue overflows, the first requests + * are dropped in a FIFO order until the queue shrinks below this capacity. + */ + private static final int QUEUE_CAPACITY = 400; private static final Logger log = Logger.getLogger(NameServiceForwarder.class.getName()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java index 9ec8e4d1a2d..09e0fec41d1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index de087737320..308bfbb0408 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.zone.ZoneId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index b096a853541..0ed2e930c57 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -2,16 +2,25 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.Maps; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; +import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; +import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -27,6 +36,8 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { private final ArchiveService archiveService; private final ZoneRegistry zoneRegistry; private final Metric metric; + private final BooleanFlag archiveEnabled; + private final BooleanFlag developerRoleEnabled; public ArchiveAccessMaintainer(Controller controller, Metric metric, Duration interval) { super(controller, interval); @@ -34,6 +45,8 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { this.archiveService = controller.serviceRegistry().archiveService(); this.zoneRegistry = controller().zoneRegistry(); this.metric = metric; + this.archiveEnabled = Flags.ENABLE_ONPREM_TENANT_S3_ARCHIVE.bindTo(controller().flagSource()); + this.developerRoleEnabled = Flags.ENABLE_TENANT_DEVELOPER_ROLE.bindTo(controller().flagSource()); } @Override @@ -43,22 +56,49 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { metric.set(bucketCountMetricName, archiveBucketDb.buckets(zoneId).size(), metric.createContext(Map.of("zone", zoneId.value())))); - var tenantArchiveAccessRoles = controller().tenants().asList().stream() - .filter(t -> t instanceof CloudTenant) - .map(t -> (CloudTenant) t) - .filter(t -> t.archiveAccessRole().isPresent()) - .collect(Collectors.toUnmodifiableMap( - Tenant::name, cloudTenant -> cloudTenant.archiveAccessRole().orElseThrow())); - zoneRegistry.zones().controllerUpgraded().ids().forEach(zoneId -> - archiveBucketDb.buckets(zoneId).forEach(archiveBucket -> - archiveService.updateBucketAndKeyPolicy(zoneId, archiveBucket, - Maps.filterEntries(tenantArchiveAccessRoles, - entry -> archiveBucket.tenants().contains(entry.getKey()))) - ) + zoneRegistry.zones().controllerUpgraded().zones().forEach(z -> { + ZoneId zoneId = z.getId(); + try { + var tenantArchiveAccessRoles = tenantArchiveAccessRoles(z); + archiveBucketDb.buckets(zoneId).forEach(archiveBucket -> + archiveService.updateBucketAndKeyPolicy(zoneId, archiveBucket, + Maps.filterEntries(tenantArchiveAccessRoles, + entry -> archiveBucket.tenants().contains(entry.getKey()))) + ); + } catch (Exception e) { + throw new RuntimeException("Failed to maintain archive access in " + zoneId.value(), e); + } + } ); return 1.0; } + private Map<TenantName, String> tenantArchiveAccessRoles(ZoneApi zone) { + List<Tenant> tenants = controller().tenants().asList(); + if (zoneRegistry.system().isPublic()) { + return tenants.stream() + .filter(t -> t instanceof CloudTenant) + .map(t -> (CloudTenant) t) + .filter(t -> t.archiveAccessRole().isPresent()) + .collect(Collectors.toUnmodifiableMap( + Tenant::name, cloudTenant -> cloudTenant.archiveAccessRole().orElseThrow())); + } else { + return tenants.stream() + .filter(t -> t instanceof AthenzTenant + && enabled(archiveEnabled, t, zone) && enabled(developerRoleEnabled, t, zone)) + .map(Tenant::name) + .collect(Collectors.toUnmodifiableMap( + Function.identity(), t -> zoneRegistry.tenantDeveloperRoleArn(t).orElseThrow())); + + } + } + + private boolean enabled(BooleanFlag flag, Tenant tenant, ZoneApi zone) { + return flag.with(FetchVector.Dimension.TENANT_ID, tenant.name().value()) + .with(FetchVector.Dimension.ZONE_ID, zone.getId().value()) + .value(); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index ab8e5efa0bd..1d71fa66329 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java index d74578d9adc..128d7e6ac71 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java @@ -1,14 +1,12 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; import java.util.Collection; import java.util.List; @@ -16,6 +14,9 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +/** + * @author smorgrav + */ public class ChangeManagementAssessor { private final NodeRepository nodeRepository; @@ -25,31 +26,31 @@ public class ChangeManagementAssessor { } public Assessment assessment(List<String> impactedHostnames, ZoneId zone) { - return assessmentInner(impactedHostnames, nodeRepository.listNodes(zone).nodes(), zone); + return assessmentInner(impactedHostnames, nodeRepository.list(zone, NodeFilter.all()), zone); } - Assessment assessmentInner(List<String> impactedHostnames, List<NodeRepositoryNode> allNodes, ZoneId zone) { + Assessment assessmentInner(List<String> impactedHostnames, List<Node> allNodes, ZoneId zone) { List<String> impactedParentHosts = toParentHosts(impactedHostnames, allNodes); // Group impacted application nodes by parent host - Map<NodeRepositoryNode, List<NodeRepositoryNode>> prParentHost = allNodes.stream() - .filter(nodeRepositoryNode -> nodeRepositoryNode.getState() == NodeState.active) //TODO look at more states? - .filter(node -> impactedParentHosts.contains(node.getParentHostname() == null ? "" : node.getParentHostname())) + Map<Node, List<Node>> prParentHost = allNodes.stream() + .filter(node -> node.state() == Node.State.active) //TODO look at more states? + .filter(node -> impactedParentHosts.contains(node.parentHostname().map(HostName::value).orElse(""))) .collect(Collectors.groupingBy(node -> allNodes.stream() - .filter(parent -> parent.getHostname().equals(node.getParentHostname())) + .filter(parent -> parent.hostname().equals(node.parentHostname().get())) .findFirst().orElseThrow() )); // Group nodes pr cluster - Map<Cluster, List<NodeRepositoryNode>> prCluster = prParentHost.values() + Map<Cluster, List<Node>> prCluster = prParentHost.values() .stream() .flatMap(Collection::stream) .collect(Collectors.groupingBy(ChangeManagementAssessor::clusterKey)); var tenantHosts = prParentHost.keySet().stream() - .filter(node -> node.getType() == NodeType.host) - .map(node -> HostName.from(node.getHostname())) + .filter(node -> node.type() == NodeType.host) + .map(node -> node.hostname()) .collect(Collectors.toList()); boolean allHostsReplacable = tenantHosts.isEmpty() || nodeRepository.isReplaceable( @@ -60,7 +61,7 @@ public class ChangeManagementAssessor { // Report assessment pr cluster var clusterAssessments = prCluster.entrySet().stream().map((entry) -> { Cluster cluster = entry.getKey(); - List<NodeRepositoryNode> nodes = entry.getValue(); + List<Node> nodes = entry.getValue(); long[] totalStats = clusterStats(cluster, allNodes); long[] impactedStats = clusterStats(cluster, nodes); @@ -87,8 +88,8 @@ public class ChangeManagementAssessor { var hostAssessments = prParentHost.entrySet().stream().map((entry) -> { HostAssessment hostAssessment = new HostAssessment(); - hostAssessment.hostName = entry.getKey().getHostname(); - hostAssessment.switchName = entry.getKey().getSwitchHostname(); + hostAssessment.hostName = entry.getKey().hostname().value(); + hostAssessment.switchName = entry.getKey().switchHostname().orElse(null); hostAssessment.numberOfChildren = entry.getValue().size(); //TODO: Some better heuristic for what's considered problematic @@ -103,31 +104,31 @@ public class ChangeManagementAssessor { return new Assessment(clusterAssessments, hostAssessments); } - private List<String> toParentHosts(List<String> impactedHostnames, List<NodeRepositoryNode> allNodes) { + private List<String> toParentHosts(List<String> impactedHostnames, List<Node> allNodes) { return impactedHostnames.stream() .flatMap(hostname -> allNodes.stream() - .filter(node -> List.of(NodeType.config, NodeType.proxy, NodeType.host).contains(node.getType())) - .filter(node -> hostname.equals(node.getHostname()) || hostname.equals(node.getParentHostname())) + .filter(node -> List.of(NodeType.config, NodeType.proxy, NodeType.host).contains(node.type())) + .filter(node -> hostname.equals(node.hostname().value()) || hostname.equals(node.parentHostname().map(HostName::value).orElse(""))) .map(node -> { - if (node.getType() == NodeType.host) - return node.getHostname(); - return node.getParentHostname(); + if (node.type() == NodeType.host) + return node.hostname().value(); + return node.parentHostname().get().value(); }).findFirst().stream() ) .collect(Collectors.toList()); } - private static Cluster clusterKey(NodeRepositoryNode node) { - if (node.getOwner() == null) + private static Cluster clusterKey(Node node) { + if (node.owner().isEmpty()) return Cluster.EMPTY; - String appId = Text.format("%s:%s:%s", node.getOwner().tenant, node.getOwner().application, node.getOwner().instance); - return new Cluster(Node.ClusterType.valueOf(node.getMembership().clustertype), node.getMembership().clusterid, appId, node.getType()); + String appId = node.owner().get().serializedForm(); + return new Cluster(node.clusterType(), node.clusterId(), appId, node.type()); } - private static long[] clusterStats(Cluster cluster, List<NodeRepositoryNode> containerNodes) { - List<NodeRepositoryNode> clusterNodes = containerNodes.stream().filter(nodeRepositoryNode -> cluster.equals(clusterKey(nodeRepositoryNode))).collect(Collectors.toList()); - long groups = clusterNodes.stream().map(nodeRepositoryNode -> nodeRepositoryNode.getMembership() != null ? nodeRepositoryNode.getMembership().group : "").distinct().count(); + private static long[] clusterStats(Cluster cluster, List<Node> containerNodes) { + List<Node> clusterNodes = containerNodes.stream().filter(node -> cluster.equals(clusterKey(node))).collect(Collectors.toList()); + long groups = clusterNodes.stream().map(Node::group).distinct().count(); return new long[] { clusterNodes.size(), groups}; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java index 14e3e685a8a..aa36d204c09 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java @@ -1,10 +1,11 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; @@ -33,12 +34,14 @@ public class ChangeRequestMaintainer extends ControllerMaintainer { private final ChangeRequestClient changeRequestClient; private final CuratorDb curator; private final NodeRepository nodeRepository; + private final SystemName system; public ChangeRequestMaintainer(Controller controller, Duration interval) { super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.changeRequestClient = controller.serviceRegistry().changeRequestClient(); this.curator = controller.curator(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); + this.system = controller.system(); } @@ -63,14 +66,15 @@ public class ChangeRequestMaintainer extends ControllerMaintainer { try (var lock = curator.lockChangeRequests()) { changeRequests.forEach(changeRequest -> { var optionalZone = inferZone(changeRequest, hostsByZone); - optionalZone.ifPresent(zone -> { + optionalZone.ifPresentOrElse(zone -> { var vcmr = existingChangeRequests .getOrDefault(changeRequest.getId(), new VespaChangeRequest(changeRequest, zone)) .withSource(changeRequest.getChangeRequestSource()) .withApproval(changeRequest.getApproval()); logger.fine(() -> "Storing " + vcmr); curator.writeChangeRequest(vcmr); - }); + }, + () -> approveChangeRequest(changeRequest)); }); } } @@ -100,7 +104,7 @@ public class ChangeRequestMaintainer extends ControllerMaintainer { .stream() .collect(Collectors.toMap( zone -> zone, - zone -> nodeRepository.list(zone, false) + zone -> nodeRepository.list(zone, NodeFilter.all()) .stream() .map(node -> node.hostname().value()) .collect(Collectors.toList()) @@ -120,4 +124,12 @@ public class ChangeRequestMaintainer extends ControllerMaintainer { .plus(Duration.ofDays(7)) .isBefore(ZonedDateTime.now()); } + + private void approveChangeRequest(ChangeRequest changeRequest) { + if (system.equals(SystemName.main) && + changeRequest.getApproval() == ChangeRequest.Approval.REQUESTED) { + logger.info("Approving " + changeRequest.getChangeRequestSource().getId()); + changeRequestClient.approveChangeRequest(changeRequest); + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTracker.java index 3c5495a6bfe..47a9bd5e8e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTracker.java @@ -5,9 +5,10 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher; +import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEvent; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import java.time.Duration; @@ -17,21 +18,19 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** - * Automatically fetches and handles scheduled events from AWS: - * 1. Deprovisions the affected hosts if applicable - * 2. Submits an issue detailing the event if some hosts are not processed by 1. + * This tracks maintenance events from cloud providers and deprovisions any affected hosts. * * @author mgimle */ -public class CloudEventReporter extends ControllerMaintainer { +public class CloudEventTracker extends ControllerMaintainer { - private static final Logger log = Logger.getLogger(CloudEventReporter.class.getName()); + private static final Logger log = Logger.getLogger(CloudEventTracker.class.getName()); - private final AwsEventFetcher eventFetcher; + private final CloudEventFetcher eventFetcher; private final Map<String, List<ZoneApi>> zonesByCloudNativeRegion; private final NodeRepository nodeRepository; - CloudEventReporter(Controller controller, Duration interval) { + CloudEventTracker(Controller controller, Duration interval) { super(controller, interval); this.eventFetcher = controller.serviceRegistry().eventFetcherService(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); @@ -55,11 +54,11 @@ public class CloudEventReporter extends ControllerMaintainer { /** Deprovision any host affected by given event */ private void deprovisionAffectedHosts(String region, CloudEvent event) { for (var zone : zonesByCloudNativeRegion.get(region)) { - for (var node : nodeRepository.list(zone.getId(), false)) { + for (var node : nodeRepository.list(zone.getId(), NodeFilter.all())) { if (!affects(node, event)) continue; log.info("Retiring and deprovisioning " + node.hostname().value() + " in " + zone.getId() + ": Affected by maintenance event " + event.instanceEventId); - nodeRepository.retireAndDeprovision(zone.getId(), node.hostname().value()); + nodeRepository.retire(zone.getId(), node.hostname().value(), true, true); } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java index c85383fd7f7..c9310349b9b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index 5ee39f7c8f2..6ecad482cd2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java index f1574381a3d..c87dd262fa3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java index f7c4a95baf1..f1223c7d162 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.concurrent.maintenance.JobMetrics; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 56bf870c7fc..3b7cf313b37 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -8,9 +8,9 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; import java.time.Duration; -import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalUnit; import java.util.Collections; import java.util.List; @@ -36,7 +36,7 @@ public class ControllerMaintenance extends AbstractComponent { @Inject @SuppressWarnings("unused") // instantiated by Dependency Injection - public ControllerMaintenance(Controller controller, Metric metric) { + public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement) { Intervals intervals = new Intervals(controller.system()); upgrader = new Upgrader(controller, intervals.defaultInterval); maintainers.add(upgrader); @@ -58,7 +58,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher)); maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer())); maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().meteringService())); - maintainers.add(new CloudEventReporter(controller, intervals.cloudEventReporter)); + maintainers.add(new CloudEventTracker(controller, intervals.cloudEventReporter)); maintainers.add(new ResourceTagMaintainer(controller, intervals.resourceTagMaintainer, controller.serviceRegistry().resourceTagger())); maintainers.add(new SystemRoutingPolicyMaintainer(controller, intervals.systemRoutingPolicyMaintainer)); maintainers.add(new ApplicationMetaDataGarbageCollector(controller, intervals.applicationMetaDataGarbageCollector)); @@ -71,9 +71,10 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new ArchiveAccessMaintainer(controller, metric, intervals.archiveAccessMaintainer)); maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer)); maintainers.add(new ChangeRequestMaintainer(controller, intervals.changeRequestMaintainer)); - maintainers.add(new VCMRMaintainer(controller, intervals.vcmrMaintainer)); + maintainers.add(new VcmrMaintainer(controller, intervals.vcmrMaintainer)); maintainers.add(new CloudTrialExpirer(controller, intervals.defaultInterval)); maintainers.add(new RetriggerMaintainer(controller, intervals.retriggerMaintainer)); + maintainers.add(new UserManagementMaintainer(controller, intervals.userManagementMaintainer, userManagement)); } public Upgrader upgrader() { return upgrader; } @@ -130,6 +131,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration changeRequestMaintainer; private final Duration vcmrMaintainer; private final Duration retriggerMaintainer; + private final Duration userManagementMaintainer; public Intervals(SystemName system) { this.system = Objects.requireNonNull(system); @@ -163,6 +165,7 @@ public class ControllerMaintenance extends AbstractComponent { this.changeRequestMaintainer = duration(1, HOURS); this.vcmrMaintainer = duration(1, HOURS); this.retriggerMaintainer = duration(1, MINUTES); + this.userManagementMaintainer = duration(12, HOURS); } private Duration duration(long amount, TemporalUnit unit) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 40191190eff..9a8ba9afca2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 4e53e07f5af..bb05840d34f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 9b8dfce849a..aca154a4b5b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index a69af024b96..73ee8527492 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -1,13 +1,14 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.yolean.Exceptions; @@ -32,7 +33,8 @@ public class DeploymentUpgrader extends ControllerMaintainer { protected double maintain() { AtomicInteger attempts = new AtomicInteger(); AtomicInteger failures = new AtomicInteger(); - Versions target = new Versions(controller().readSystemVersion(), ApplicationVersion.unknown, Optional.empty(), Optional.empty()); + Version systemVersion = controller().readSystemVersion(); + for (Application application : controller().applications().readable()) for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) @@ -40,8 +42,10 @@ public class DeploymentUpgrader extends ControllerMaintainer { attempts.incrementAndGet(); JobId job = new JobId(instance.id(), JobType.from(controller().system(), deployment.zone()).get()); if ( ! deployment.zone().environment().isManuallyDeployed()) continue; + + Run last = controller().jobController().last(job).get(); + Versions target = new Versions(systemVersion, last.versions().targetApplication(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetApplication())); if ( ! deployment.version().isBefore(target.targetPlatform())) continue; - if ( controller().clock().instant().isBefore(controller().jobController().last(job).get().start().plus(Duration.ofDays(1)))) continue; if ( ! isLikelyNightFor(job)) continue; log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index cd7ce8c3fa6..40e75cab8aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.Sets; @@ -24,8 +24,10 @@ import java.time.temporal.ChronoUnit; import java.util.HashSet; import java.util.Optional; import java.util.OptionalInt; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates. @@ -60,6 +62,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { deployRefreshedCertificates(); updateRefreshedCertificates(); deleteUnusedCertificates(); + reportUnmanagedCertificates(); } catch (Exception e) { log.log(LogLevel.ERROR, "Exception caught while maintaining endpoint certificates", e); return 0.0; @@ -134,6 +137,16 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { }); } + private void reportUnmanagedCertificates() { + Set<String> managedRequestIds = curator.readAllEndpointCertificateMetadata().values().stream().map(EndpointCertificateMetadata::requestId).collect(Collectors.toSet()); + + for (EndpointCertificateMetadata cameoCertificateMetadata : endpointCertificateProvider.listCertificates()) { + if (!managedRequestIds.contains(cameoCertificateMetadata.requestId())) { + log.info("Certificate metadata exists with provider but is not managed by controller: " + cameoCertificateMetadata.requestId() + ", " + cameoCertificateMetadata.issuer() + ", " + cameoCertificateMetadata.requestedDnsSans()); + } + } + } + private Lock lock(ApplicationId applicationId) { return curator.lock(TenantAndApplicationId.from(applicationId)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java index 10e6f9eb039..1f21c688540 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java @@ -1,13 +1,13 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.api.integration.entity.NodeEntity; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import java.time.Duration; import java.util.EnumSet; @@ -45,16 +45,27 @@ public class HostInfoUpdater extends ControllerMaintainer { int hostsUpdated = 0; try { for (var zone : controller().zoneRegistry().zones().controllerUpgraded().all().ids()) { - for (var node : nodeRepository.list(zone, false)) { + for (var node : nodeRepository.list(zone, NodeFilter.all())) { if (!node.type().isHost()) continue; NodeEntity nodeEntity = nodeEntities.get(registeredHostnameOf(node)); - if (!shouldUpdateSwitch(node, nodeEntity) && !shouldUpdateModel(node, nodeEntity)) continue; + if (nodeEntity == null) continue; - NodeRepositoryNode updatedNode = new NodeRepositoryNode(); - nodeEntity.switchHostname().ifPresent(updatedNode::setSwitchHostname); - buildModelName(nodeEntity).ifPresent(updatedNode::setModelName); - nodeRepository.patchNode(zone, node.hostname().value(), updatedNode); - hostsUpdated++; + boolean updatedHost = false; + Optional<String> modelName = modelNameOf(nodeEntity); + if (modelName.isPresent() && !modelName.equals(node.modelName())) { + nodeRepository.updateModel(zone, node.hostname().value(), modelName.get()); + updatedHost = true; + } + + Optional<String> switchHostname = nodeEntity.switchHostname(); + if (switchHostname.isPresent() && !switchHostname.equals(node.switchHostname())) { + nodeRepository.updateSwitchHostname(zone, node.hostname().value(), switchHostname.get()); + updatedHost = true; + } + + if (updatedHost) { + hostsUpdated++; + } } } } finally { @@ -65,9 +76,8 @@ public class HostInfoUpdater extends ControllerMaintainer { return 1.0; } - private static Optional<String> buildModelName(NodeEntity nodeEntity) { - if(nodeEntity.manufacturer().isEmpty() || nodeEntity.model().isEmpty()) - return Optional.empty(); + private static Optional<String> modelNameOf(NodeEntity nodeEntity) { + if (nodeEntity.manufacturer().isEmpty() || nodeEntity.model().isEmpty()) return Optional.empty(); return Optional.of(nodeEntity.manufacturer().get() + " " + nodeEntity.model().get()); } @@ -80,17 +90,4 @@ public class HostInfoUpdater extends ControllerMaintainer { return matcher.replaceFirst("$1$2"); } - private static boolean shouldUpdateSwitch(Node node, NodeEntity nodeEntity) { - if (nodeEntity == null) return false; - if (nodeEntity.switchHostname().isEmpty()) return false; - return !node.switchHostname().equals(nodeEntity.switchHostname()); - } - - private static boolean shouldUpdateModel(Node node, NodeEntity nodeEntity) { - if (nodeEntity == null) return false; - if (nodeEntity.model().isEmpty()) return false; - if (nodeEntity.manufacturer().isEmpty()) return false; - return !node.modelName().equals(buildModelName(nodeEntity)); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 221a7524ab1..1ea57d3ccb4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.yolean.Exceptions; @@ -112,7 +113,7 @@ public abstract class InfrastructureUpgrader<VERSION> extends ControllerMaintain try { return controller().serviceRegistry().configServer() .nodeRepository() - .list(zone.getVirtualId(), application.id()) + .list(zone.getVirtualId(), NodeFilter.all().applications(application.id())) .stream() .filter(node -> expectUpgradeOf(node, application, zone)) .map(versionField) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 25207b733f0..ba891cdddd8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.concurrent.DaemonThreadFactory; @@ -113,8 +113,10 @@ public class JobRunner extends ControllerMaintainer { jobs.locked(id.application(), id.type(), step, lockedStep -> { jobs.locked(id, run -> run); // Memory visibility. jobs.active(id).ifPresent(run -> { // The run may have become inactive, so we bail out. - if ( ! run.readySteps().contains(step)) + if ( ! run.readySteps().contains(step)) { + changed.set(true); return; // Someone may have updated the run status, making this step obsolete, so we bail out. + } StepInfo stepInfo = run.stepInfo(lockedStep.get()).orElseThrow(); if (stepInfo.startTime().isEmpty()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index d3b05922d26..b2d3e1b780f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; @@ -11,7 +11,6 @@ import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import java.time.Duration; import java.time.Instant; -import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.Objects; @@ -141,18 +140,6 @@ public class OsUpgradeScheduler extends ControllerMaintainer { @Override public Version version(OsVersionTarget currentTarget, Instant now) { Instant scheduledAt = currentTarget.scheduledAt(); - if (currentTarget.scheduledAt().equals(Instant.EPOCH)) { - // TODO(mpolden): Remove this block after 2021-09-01. If we haven't written scheduledAt at least once, - // we need to deduce the scheduled instant from the version. - Version version = currentTarget.osVersion().version(); - String qualifier = version.getQualifier(); - if (!qualifier.matches("^\\d{8,}")) throw new IllegalArgumentException("Could not parse instant from version " + version); - - String dateString = qualifier.substring(0, 8); - scheduledAt = LocalDate.parse(dateString, CALENDAR_VERSION_PATTERN) - .atStartOfDay(ZoneOffset.UTC) - .toInstant(); - } Version currentVersion = currentTarget.osVersion().version(); if (scheduledAt.isBefore(now.minus(SCHEDULING_INTERVAL))) { String calendarVersion = now.minus(AVAILABILITY_INTERVAL) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 8aeede95878..c561a44857b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -50,7 +50,7 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { " with time budget " + zoneUpgradeBudget)); controller().serviceRegistry().configServer().nodeRepository().upgradeOs(zone.getVirtualId(), application.nodeType(), target.osVersion().version(), - Optional.of(zoneUpgradeBudget)); + zoneUpgradeBudget); } @Override @@ -89,15 +89,15 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { /** Returns the available upgrade budget for given zone */ private Duration zoneBudgetOf(Duration totalBudget, ZoneApi zone) { - if (!spendBudget(zone)) return Duration.ZERO; + if (!spendBudgetOn(zone)) return Duration.ZERO; long consecutiveZones = upgradePolicy.asList().stream() - .filter(parallelZones -> parallelZones.stream().anyMatch(this::spendBudget)) + .filter(parallelZones -> parallelZones.stream().anyMatch(this::spendBudgetOn)) .count(); return totalBudget.dividedBy(consecutiveZones); } /** Returns whether to spend upgrade budget on given zone */ - private boolean spendBudget(ZoneApi zone) { + private boolean spendBudgetOn(ZoneApi zone) { if (!zone.getEnvironment().isProduction()) return false; if (controller().zoneRegistry().systemZone().getVirtualId().equals(zone.getVirtualId())) return false; // Controller zone return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java index 271dd277e1c..119540eaa68 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java index 9d93ac719b7..db9dc51ffa9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java index ffe958cb63a..26df8669fb1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java index 0bd74c844ae..34b1ea34227 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 39ad233ce46..854780dd336 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -12,6 +12,7 @@ import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; @@ -142,7 +143,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { return controller().zoneRegistry().zones() .reachable().zones().stream() .map(ZoneApi::getId) - .map(zoneId -> createResourceSnapshotsFromNodes(zoneId, nodeRepository.list(zoneId, false))) + .map(zoneId -> createResourceSnapshotsFromNodes(zoneId, nodeRepository.list(zoneId, NodeFilter.all()))) .flatMap(Collection::stream) .collect(Collectors.toList()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java index e0441df025b..4d2c0a87bc0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -9,10 +9,10 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import java.time.Duration; import java.util.Map; -import java.util.Optional; import java.util.logging.Level; import java.util.stream.Collectors; @@ -47,7 +47,7 @@ public class ResourceTagMaintainer extends ControllerMaintainer { private Map<HostName, ApplicationId> getTenantOfParentHosts(ZoneId zoneId) { return controller().serviceRegistry().configServer().nodeRepository() - .list(zoneId, false) + .list(zoneId, NodeFilter.all()) .stream() .filter(node -> node.type().isHost()) .collect(Collectors.toMap( diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java index 2cc3ac1bd6c..73d28a1eeae 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java index 5f147d6e048..1d5d444a32c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.application.api.DeploymentSpec; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 8f57b3f2239..8b0371e2c1a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java index 637ae10bcc6..7c6decb4a93 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java @@ -1,11 +1,10 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -26,9 +25,15 @@ public class TenantRoleMaintainer extends ControllerMaintainer { protected double maintain() { var roleService = controller().serviceRegistry().roleService(); var tenants = controller().tenants().asList(); + + // Create separate athenz service for all tenants + tenants.forEach(roleService::createTenantRole); + + // Until we have moved to separate athenz service per tenant, make sure we update the shared policy + // to allow ssh logins for hosts in prod/perf with a separate tenant iam role. var tenantsWithRoles = tenants.stream() .map(Tenant::name) - .filter(this::hasProductionDeployment) + .filter(tenant -> hasProductionDeployment(tenant) || hasPerfDeployment(tenant)) .collect(Collectors.toList()); roleService.maintainRoles(tenantsWithRoles); return 1.0; @@ -39,4 +44,13 @@ public class TenantRoleMaintainer extends ControllerMaintainer { .map(Application::productionInstances) .anyMatch(Predicate.not(Map::isEmpty)); } + + private boolean hasPerfDeployment(TenantName tenant) { + List<ZoneId> perfZones = controller().zoneRegistry().zones().controllerUpgraded().in(Environment.perf).ids(); + return controller().applications().asList(tenant).stream() + .map(Application::instances) + .flatMap(instances -> instances.values().stream()) + .flatMap(instance -> instance.deployments().values().stream()) + .anyMatch(x -> perfZones.contains(x.zone())); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdater.java index 8c891339e29..832dbb6b921 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdater.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.ApplicationController; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java new file mode 100644 index 00000000000..870e3af678f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java @@ -0,0 +1,68 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; +import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; +import com.yahoo.vespa.hosted.controller.api.role.ApplicationRole; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.TenantRole; + +import java.time.Duration; +import java.util.List; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Maintains user management resources. + * For now, ensures there's no discrepnacy between expected tenant/application roles and Auth0 roles + * + * @author olaa + */ +public class UserManagementMaintainer extends ControllerMaintainer { + + private final UserManagement userManagement; + + private static final Logger logger = Logger.getLogger(UserManagementMaintainer.class.getName()); + + public UserManagementMaintainer(Controller controller, Duration interval, UserManagement userManagement) { + super(controller, interval, UserManagementMaintainer.class.getSimpleName(), SystemName.allOf(SystemName::isPublic)); + this.userManagement = userManagement; + + } + + @Override + protected double maintain() { + findLeftoverRoles().forEach(role -> { + /* + Log discrepancy now + TODO: userManagement.deleteRole(role); + */ + logger.warning(String.format("Found unexpected role %s - Please investigate", role.toString())); + }); + return 1.0; + } + + // protected for testing + protected List<Role> findLeftoverRoles() { + var tenantRoles = controller().tenants().asList() + .stream() + .flatMap(tenant -> Roles.tenantRoles(tenant.name()).stream()) + .collect(Collectors.toList()); + + var applicationRoles = controller().applications().asList() + .stream() + .map(Application::id) + .flatMap(applicationId -> Roles.applicationRoles(applicationId.tenant(), applicationId.application()).stream()) + .collect(Collectors.toList()); + + return userManagement.listRoles().stream() + .peek(role -> logger.fine(role::toString)) + .filter(role -> role instanceof TenantRole || role instanceof ApplicationRole) + .filter(role -> !tenantRoles.contains(role) && !applicationRoles.contains(role)) + .collect(Collectors.toList()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index 38e10aa6750..471b65dbd42 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.Environment; @@ -8,15 +8,14 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest.Impact; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction.State; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VCMRReport; +import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VcmrReport; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest.Status; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -34,22 +33,25 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** - * @author olaa * - * Maintains status and execution of VCMRs - * For now only retires all affected tenant hosts if zone capacity allows it + * Maintains status and execution of Vespa CMRs. + * + * Currently this retires all affected tenant hosts if zone capacity allows it. + * + * @author olaa */ -public class VCMRMaintainer extends ControllerMaintainer { +public class VcmrMaintainer extends ControllerMaintainer { + + private static final Logger LOG = Logger.getLogger(VcmrMaintainer.class.getName()); + private static final Duration ALLOWED_RETIREMENT_TIME = Duration.ofHours(60); + private static final Duration ALLOWED_POSTPONEMENT_TIME = Duration.ofDays(7); - private final Logger logger = Logger.getLogger(VCMRMaintainer.class.getName()); - private final Duration ALLOWED_RETIREMENT_TIME = Duration.ofHours(60); - private final Duration ALLOWED_POSTPONEMENT_TIME = Duration.ofDays(7); private final CuratorDb curator; private final NodeRepository nodeRepository; private final ChangeRequestClient changeRequestClient; private final SystemName system; - public VCMRMaintainer(Controller controller, Duration interval) { + public VcmrMaintainer(Controller controller, Duration interval) { super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.curator = controller.curator(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); @@ -144,7 +146,7 @@ public class VCMRMaintainer extends ControllerMaintainer { .orElse(new HostAction(node.hostname().value(), State.NONE, Instant.now())); if (changeRequest.getChangeRequestSource().isClosed()) { - logger.fine(() -> changeRequest.getChangeRequestSource().getId() + " is closed, recycling " + node.hostname()); + LOG.fine(() -> changeRequest.getChangeRequestSource().getId() + " is closed, recycling " + node.hostname()); recycleNode(changeRequest.getZoneId(), node, hostAction); removeReport(changeRequest, node); return hostAction.withState(State.COMPLETE); @@ -156,7 +158,7 @@ public class VCMRMaintainer extends ControllerMaintainer { addReport(changeRequest, node); if (isPostponed(changeRequest, hostAction)) { - logger.fine(() -> changeRequest.getChangeRequestSource().getId() + " is postponed, recycling " + node.hostname()); + LOG.fine(() -> changeRequest.getChangeRequestSource().getId() + " is postponed, recycling " + node.hostname()); recycleNode(changeRequest.getZoneId(), node, hostAction); return hostAction.withState(State.PENDING_RETIREMENT); } @@ -167,14 +169,14 @@ public class VCMRMaintainer extends ControllerMaintainer { if (shouldRetire(changeRequest, hostAction)) { if (!node.wantToRetire()) { - logger.info(Text.format("Retiring %s due to %s", node.hostname().value(), changeRequest.getChangeRequestSource().getId())); + LOG.info(Text.format("Retiring %s due to %s", node.hostname().value(), changeRequest.getChangeRequestSource().getId())); // TODO: Remove try/catch once retirement is stabilized try { setWantToRetire(changeRequest.getZoneId(), node, true); } catch (Exception e) { - logger.warning("Failed to retire host " + node.hostname() + ": " + Exceptions.toMessageString(e)); + LOG.warning("Failed to retire host " + node.hostname() + ": " + Exceptions.toMessageString(e)); // Check if retirement actually failed - if (!nodeRepository.getNode(changeRequest.getZoneId(), node.hostname().value()).getWantToRetire()) { + if (!nodeRepository.getNode(changeRequest.getZoneId(), node.hostname().value()).wantToRetire()) { return hostAction; } } @@ -183,12 +185,12 @@ public class VCMRMaintainer extends ControllerMaintainer { } if (hasRetired(node, hostAction)) { - logger.fine(() -> node.hostname() + " has retired"); + LOG.fine(() -> node.hostname() + " has retired"); return hostAction.withState(State.RETIRED); } if (pendingRetirement(node, hostAction)) { - logger.fine(() -> node.hostname() + " is pending retirement"); + LOG.fine(() -> node.hostname() + " is pending retirement"); return hostAction.withState(State.PENDING_RETIREMENT); } @@ -199,8 +201,8 @@ public class VCMRMaintainer extends ControllerMaintainer { private void recycleNode(ZoneId zoneId, Node node, HostAction hostAction) { if (hostAction.getState() == State.RETIRED && node.state() == Node.State.parked) { - logger.info("Setting " + node.hostname() + " to dirty"); - nodeRepository.setState(zoneId, NodeState.dirty, node.hostname().value()); + LOG.info("Setting " + node.hostname() + " to dirty"); + nodeRepository.setState(zoneId, Node.State.dirty, node.hostname().value()); } if (hostAction.getState() == State.RETIRING && node.wantToRetire()) { try { @@ -244,7 +246,7 @@ public class VCMRMaintainer extends ControllerMaintainer { .stream() .collect(Collectors.toMap( zone -> zone, - zone -> nodeRepository.list(zone, false) + zone -> nodeRepository.list(zone, NodeFilter.all()) )); } @@ -274,9 +276,7 @@ public class VCMRMaintainer extends ControllerMaintainer { } private void setWantToRetire(ZoneId zoneId, Node node, boolean wantToRetire) { - var newNode = new NodeRepositoryNode(); - newNode.setWantToRetire(wantToRetire); - nodeRepository.patchNode(zoneId, node.hostname().value(), newNode); + nodeRepository.retire(zoneId, node.hostname().value(), wantToRetire, false); } private void approveChangeRequest(VespaChangeRequest changeRequest) { @@ -287,12 +287,12 @@ public class VCMRMaintainer extends ControllerMaintainer { if (changeRequest.getApproval() != ChangeRequest.Approval.REQUESTED) return; - logger.info("Approving " + changeRequest.getChangeRequestSource().getId()); + LOG.info("Approving " + changeRequest.getChangeRequestSource().getId()); changeRequestClient.approveChangeRequest(changeRequest); } private void removeReport(VespaChangeRequest changeRequest, Node node) { - var report = VCMRReport.fromReports(node.reports()); + var report = VcmrReport.fromReports(node.reports()); if (report.removeVcmr(changeRequest.getChangeRequestSource().getId())) { updateReport(changeRequest.getZoneId(), node, report); @@ -300,7 +300,7 @@ public class VCMRMaintainer extends ControllerMaintainer { } private void addReport(VespaChangeRequest changeRequest, Node node) { - var report = VCMRReport.fromReports(node.reports()); + var report = VcmrReport.fromReports(node.reports()); var source = changeRequest.getChangeRequestSource(); if (report.addVcmr(source.getId(), source.getPlannedStartTime(), source.getPlannedEndTime())) { @@ -308,10 +308,9 @@ public class VCMRMaintainer extends ControllerMaintainer { } } - private void updateReport(ZoneId zoneId, Node node, VCMRReport report) { - logger.info(Text.format("Updating report for %s: %s", node.hostname(), report)); - var newNode = new NodeRepositoryNode(); - newNode.setReports(report.toNodeReports()); - nodeRepository.patchNode(zoneId, node.hostname().value(), newNode); + private void updateReport(ZoneId zoneId, Node node, VcmrReport report) { + LOG.info(Text.format("Updating report for %s: %s", node.hostname(), report)); + nodeRepository.updateReports(zoneId, node.hostname().value(), report.toNodeReports()); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index e4866c43f13..6bf73c45965 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java index 14267807041..6ff3e2d1c52 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package com.yahoo.vespa.hosted.controller.maintenance; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java index b6468464a0b..c3b2d794a71 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -48,7 +49,7 @@ public class CostCalculator { Map<Property, ResourceAllocation> allocationByProperty = new HashMap<>(); var nodes = controller.zoneRegistry().zones() .reachable().in(Environment.prod).ofCloud(cloudName).zones().stream() - .flatMap(zone -> uncheck(() -> nodeRepository.list(zone.getId(), false).stream())) + .flatMap(zone -> uncheck(() -> nodeRepository.list(zone.getId(), NodeFilter.all()).stream())) .filter(node -> node.owner().isPresent() && !node.owner().get().tenant().equals(SystemApplication.TENANT)) .collect(Collectors.toList()); var totalAllocation = ResourceAllocation.ZERO; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java index b65a9290e43..5935a0d51fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationSource.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationSource.java index aeb017e14a3..c414e24a187 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationSource.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationSource.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java index fa1419c6d7d..88c55393b56 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; import com.yahoo.collections.Pair; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java index 112e90e2cd7..26e6a6b89e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * The root package of the controller * diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 26fb4be04af..2f9c8488668 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; @@ -107,6 +107,7 @@ public class ApplicationSerializer { private static final String branchField = "branchField"; private static final String commitField = "commitField"; private static final String authorEmailField = "authorEmailField"; + private static final String deployedDirectlyField = "deployedDirectly"; private static final String compileVersionField = "compileVersion"; private static final String buildTimeField = "buildTime"; private static final String sourceUrlField = "sourceUrl"; @@ -228,6 +229,7 @@ public class ApplicationSerializer { applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli())); applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url)); applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit)); + object.setBool(deployedDirectlyField, applicationVersion.isDeployedDirectly()); } private void toSlime(SourceRevision sourceRevision, Cursor object) { @@ -422,7 +424,11 @@ public class ApplicationSerializer { Optional<String> sourceUrl = SlimeUtils.optionalString(object.field(sourceUrlField)); Optional<String> commit = SlimeUtils.optionalString(object.field(commitField)); - return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit); + // TODO (freva): Simplify once this has rolled out everywhere + Inspector deployedDirectlyInspector = object.field(deployedDirectlyField); + boolean deployedDirectly = deployedDirectlyInspector.valid() && deployedDirectlyInspector.asBool(); + + return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly); } private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java index 3a625c5c42c..269864ad641 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java index 073d1799d3e..5e005c9467c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ChangeRequestSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ChangeRequestSerializer.java index 49da8d7a2a2..dbf14f4df87 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ChangeRequestSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ChangeRequestSerializer.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.zone.ZoneId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java index 2cb981aac03..f9306103e71 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java index 24a6ef72438..8b599b45558 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java index 3ced0d60453..310b3637b84 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.slime.Cursor; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobControlFlags.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobControlFlags.java index 109e761f925..41bf85f021b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobControlFlags.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobControlFlags.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.concurrent.maintenance.JobControlState; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java index 4e3ab293a02..26cd4acdb4c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.slime.ArrayTraverser; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java index ab88e8fe019..f98ecb80dd5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.google.inject.Inject; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java index ba1c5350580..10763e1f22c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java index d68e24a27ea..7551799ec85 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java index e2420993cb2..a4278b76200 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.ImmutableMap; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java index 4eac5a64b0c..9cad402832f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.slime.ArrayTraverser; @@ -45,10 +45,8 @@ public class OsVersionTargetSerializer { Set<OsVersionTarget> osVersionTargets = new TreeSet<>(); array.traverse((ArrayTraverser) (i, inspector) -> { OsVersion osVersion = osVersionSerializer.fromSlime(inspector); - Duration upgradeBudget = Duration.ofMillis(inspector.field(upgradeBudgetField).asLong()); - // TODO(mpolden): Require after 2021-09-01 - Instant scheduledAt = SlimeUtils.optionalInstant(inspector.field(scheduledAtField)) - .orElse(Instant.EPOCH); + Duration upgradeBudget = SlimeUtils.duration(inspector.field(upgradeBudgetField)); + Instant scheduledAt = SlimeUtils.instant(inspector.field(scheduledAtField)); osVersionTargets.add(new OsVersionTarget(osVersion, upgradeBudget, scheduledAt)); }); return Collections.unmodifiableSet(osVersionTargets); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 8ffa4823ead..6af3978d0cd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; @@ -89,6 +89,7 @@ class RunSerializer { private static final String branchField = "branch"; private static final String commitField = "commit"; private static final String authorEmailField = "authorEmail"; + private static final String deployedDirectlyField = "deployedDirectly"; private static final String compileVersionField = "compileVersion"; private static final String buildTimeField = "buildTime"; private static final String sourceUrlField = "sourceUrl"; @@ -99,6 +100,7 @@ class RunSerializer { private static final String noNodesDownSinceField = "noNodesDownSince"; private static final String convergenceSummaryField = "convergenceSummaryV2"; private static final String testerCertificateField = "testerCertificate"; + private static final String isDryRunField = "isDryRun"; Run runFromSlime(Slime slime) { return runFromSlime(slime.get()); @@ -142,7 +144,8 @@ class RunSerializer { convergenceSummaryFrom(runObject.field(convergenceSummaryField)), Optional.of(runObject.field(testerCertificateField)) .filter(Inspector::valid) - .map(certificate -> X509CertificateUtils.fromPem(certificate.asString()))); + .map(certificate -> X509CertificateUtils.fromPem(certificate.asString())), + runObject.field(isDryRunField).valid() && runObject.field(isDryRunField).asBool()); } private Versions versionsFromSlime(Inspector versionsObject) { @@ -175,8 +178,12 @@ class RunSerializer { Optional<String> sourceUrl = SlimeUtils.optionalString(versionObject.field(sourceUrlField)); Optional<String> commit = SlimeUtils.optionalString(versionObject.field(commitField)); + // TODO (freva): Simplify once this has rolled out everywhere + Inspector deployedDirectlyInspector = versionObject.field(deployedDirectlyField); + boolean deployedDirectly = deployedDirectlyInspector.valid() && deployedDirectlyInspector.asBool(); + return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, - compileVersion, buildTime, sourceUrl, commit); + compileVersion, buildTime, sourceUrl, commit, deployedDirectly); } // Don't change this — introduce a separate array instead. @@ -245,6 +252,7 @@ class RunSerializer { .orElseThrow(() -> new IllegalArgumentException("Source versions must be both present or absent.")), versionsObject.setObject(sourceField)); }); + runObject.setBool(isDryRunField, run.isDryRun()); } private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) { @@ -259,6 +267,7 @@ class RunSerializer { applicationVersion.buildTime().ifPresent(time -> versionsObject.setLong(buildTimeField, time.toEpochMilli())); applicationVersion.sourceUrl().ifPresent(url -> versionsObject.setString(sourceUrlField, url)); applicationVersion.commit().ifPresent(commit -> versionsObject.setString(commitField, commit)); + versionsObject.setBool(deployedDirectlyField, applicationVersion.isDeployedDirectly()); } // Don't change this - introduce a separate array with new values if needed. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java index 33596fce2bd..e9e5f8cf032 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java @@ -1,4 +1,4 @@ -// Copyright 2021 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.security.X509CertificateUtils; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 6b167f26314..1b6a0a6a122 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.BiMap; @@ -19,6 +19,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; @@ -56,6 +57,7 @@ public class TenantSerializer { private static final String propertyIdField = "propertyId"; private static final String creatorField = "creator"; private static final String createdAtField = "createdAt"; + private static final String deletedAtField = "deletedAt"; private static final String contactField = "contact"; private static final String contactUrlField = "contactUrl"; private static final String propertyUrlField = "propertyUrl"; @@ -84,9 +86,10 @@ public class TenantSerializer { toSlime(tenant.lastLoginInfo(), tenantObject.setObject(lastLoginInfoField)); switch (tenant.type()) { - case athenz: toSlime((AthenzTenant) tenant, tenantObject); break; - case cloud: toSlime((CloudTenant) tenant, tenantObject); break; - default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); + case athenz: toSlime((AthenzTenant) tenant, tenantObject); break; + case cloud: toSlime((CloudTenant) tenant, tenantObject); break; + case deleted: toSlime((DeletedTenant) tenant, tenantObject); break; + default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } return slime; } @@ -114,6 +117,10 @@ public class TenantSerializer { tenant.archiveAccessRole().ifPresent(role -> root.setString(archiveAccessRoleField, role)); } + private void toSlime(DeletedTenant tenant, Cursor root) { + root.setLong(deletedAtField, tenant.deletedAt().toEpochMilli()); + } + private void developerKeysToSlime(BiMap<PublicKey, Principal> keys, Cursor array) { keys.forEach((key, user) -> { Cursor object = array.addObject(); @@ -139,9 +146,10 @@ public class TenantSerializer { Tenant.Type type = typeOf(tenantObject.field(typeField).asString()); switch (type) { - case athenz: return athenzTenantFrom(tenantObject); - case cloud: return cloudTenantFrom(tenantObject); - default: throw new IllegalArgumentException("Unexpected tenant type '" + type + "'."); + case athenz: return athenzTenantFrom(tenantObject); + case cloud: return cloudTenantFrom(tenantObject); + case deleted: return deletedTenantFrom(tenantObject); + default: throw new IllegalArgumentException("Unexpected tenant type '" + type + "'."); } } @@ -168,6 +176,13 @@ public class TenantSerializer { return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccessRole); } + private DeletedTenant deletedTenantFrom(Inspector tenantObject) { + TenantName name = TenantName.from(tenantObject.field(nameField).asString()); + Instant createdAt = SlimeUtils.instant(tenantObject.field(createdAtField)); + Instant deletedAt = SlimeUtils.instant(tenantObject.field(deletedAtField)); + return new DeletedTenant(name, createdAt, deletedAt); + } + private BiMap<PublicKey, Principal> developerKeysFromSlime(Inspector array) { ImmutableBiMap.Builder<PublicKey, Principal> keys = ImmutableBiMap.builder(); array.traverse((ArrayTraverser) (__, keyObject) -> @@ -321,23 +336,20 @@ public class TenantSerializer { return personLists; } - private BillingInfo billingInfoFrom(Inspector billingInfoObject) { - return new BillingInfo(billingInfoObject.field(customerIdField).asString(), - billingInfoObject.field(productCodeField).asString()); - } - private static Tenant.Type typeOf(String value) { switch (value) { - case "athenz": return Tenant.Type.athenz; - case "cloud": return Tenant.Type.cloud; + case "athenz": return Tenant.Type.athenz; + case "cloud": return Tenant.Type.cloud; + case "deleted": return Tenant.Type.deleted; default: throw new IllegalArgumentException("Unknown tenant type '" + value + "'."); } } private static String valueOf(Tenant.Type type) { switch (type) { - case athenz: return "athenz"; - case cloud: return "cloud"; + case athenz: return "athenz"; + case cloud: return "cloud"; + case deleted: return "deleted"; default: throw new IllegalArgumentException("Unexpected tenant type '" + type + "'."); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java index 87a14660fee..b9c6290f582 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * Persistence layer for the controller. * diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java index 110d994c179..c02cfc2ce65 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.container.jdisc.HttpResponse; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 6cc7446499f..6116b2e27b6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -1,7 +1,6 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import ai.vespa.util.http.hc4.retry.Sleeper; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.HttpRequest.Method; @@ -10,6 +9,7 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.yolean.concurrent.Sleeper; import org.apache.http.Header; import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; @@ -69,7 +69,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C @Inject public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, ServiceIdentityProvider sslContextProvider) { - this(zoneRegistry, sslContextProvider.getIdentitySslContext(), new Sleeper.Default(), + this(zoneRegistry, sslContextProvider.getIdentitySslContext(), Sleeper.DEFAULT, new ConnectionReuseStrategy(zoneRegistry)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java index c3cdd69041f..2a1b8a19475 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.container.jdisc.HttpRequest; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java index b0b4f1a556a..0b629a577d0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.container.jdisc.HttpResponse; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java index c4472fb79e4..ccc874f8f7a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * @author Haakon Dybdahl */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index c1e6c362c83..1774694853b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -25,6 +25,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.io.IOUtils; +import com.yahoo.restapi.ByteArrayResponse; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; @@ -37,6 +38,8 @@ import com.yahoo.slime.JsonParseException; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.text.Text; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -49,6 +52,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbi import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.aws.TenantRoles; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; @@ -57,6 +61,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -71,7 +77,6 @@ import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.application.ActivateResult; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -81,6 +86,7 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; @@ -102,6 +108,7 @@ import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; @@ -145,6 +152,7 @@ import java.util.stream.Stream; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.CONFLICT; +import static com.yahoo.yolean.Exceptions.uncheck; import static java.util.Map.Entry.comparingByKey; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; @@ -165,6 +173,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private final Controller controller; private final AccessControlRequests accessControlRequests; private final TestConfigSerializer testConfigSerializer; + private final ListFlag<String> allowedServiceViewProxy; @Inject public ApplicationApiHandler(LoggingRequestHandler.Context parentCtx, @@ -174,6 +183,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { this.controller = controller; this.accessControlRequests = accessControlRequests; this.testConfigSerializer = new TestConfigSerializer(controller.system()); + allowedServiceViewProxy = Flags.ALLOWED_SERVICE_VIEW_APIS.bindTo(controller.flagSource()); } @Override @@ -237,6 +247,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/compile-version")) return compileVersion(path.get("tenant"), path.get("application")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deployment")) return JobControllerApiHandlerHelper.overviewResponse(controller, TenantAndApplicationId.from(path.get("tenant"), path.get("application")), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/package")) return applicationPackage(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/diff/{number}")) return applicationPackageDiff(path.get("tenant"), path.get("application"), path.get("number")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/metering")) return metering(path.get("tenant"), path.get("application"), request); @@ -245,8 +256,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), Optional.ofNullable(request.getProperty("limit")), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)).descendingMap(), Optional.ofNullable(request.getProperty("limit")), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path)); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/diff/{number}")) return devApplicationPackageDiff(runIdFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -259,6 +271,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/content/{*}")) return content(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/access/support")) return supportAccess(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/node/{node}/service-dump")) return getServiceDump(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("node"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/metrics")) return metrics(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("endpointId"))); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); @@ -309,6 +322,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspend")) return suspend(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/access/support")) return allowSupportAccess(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/node/{node}/service-dump")) return requestServiceDump(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("node"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -358,7 +372,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { Slime slime = new Slime(); Cursor tenantArray = slime.setArray(); List<Application> applications = controller.applications().asList(); - for (Tenant tenant : controller.tenants().asList()) + for (Tenant tenant : controller.tenants().asList(includeDeleted(request))) toSlime(tenantArray.addObject(), tenant, applications.stream().filter(app -> app.id().tenant().equals(tenant.name())).collect(toList()), @@ -375,13 +389,13 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse tenants(HttpRequest request) { Slime slime = new Slime(); Cursor response = slime.setArray(); - for (Tenant tenant : controller.tenants().asList()) + for (Tenant tenant : controller.tenants().asList(includeDeleted(request))) tenantInTenantsListToSlime(tenant, request.getUri(), response.addObject()); return new SlimeJsonResponse(slime); } private HttpResponse tenant(String tenantName, HttpRequest request) { - return controller.tenants().get(TenantName.from(tenantName)) + return controller.tenants().get(TenantName.from(tenantName), includeDeleted(request)) .map(tenant -> tenant(tenant, request)) .orElseGet(() -> ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist")); } @@ -543,8 +557,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse applications(String tenantName, Optional<String> applicationName, HttpRequest request) { TenantName tenant = TenantName.from(tenantName); - if (controller.tenants().get(tenantName).isEmpty()) - return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist"); + getTenantOrThrow(tenantName); List<Application> applications = applicationName.isEmpty() ? controller.applications().asList(tenant) : @@ -582,13 +595,20 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { throw new IllegalArgumentException("Only manually deployed zones have dev packages"); ZoneId zone = type.zone(controller.system()); - byte[] applicationPackage = controller.applications().applicationStore().getDev(id, zone); + ApplicationVersion version = controller.jobController().last(id, type).get().versions().targetApplication(); + byte[] applicationPackage = controller.applications().applicationStore().get(new DeploymentId(id, zone), version); return new ZipResponse(id.toFullString() + "." + zone.value() + ".zip", applicationPackage); } + private HttpResponse devApplicationPackageDiff(RunId runId) { + DeploymentId deploymentId = new DeploymentId(runId.application(), runId.job().type().zone(controller.system())); + return controller.applications().applicationStore().getDevDiff(deploymentId, runId.number()) + .map(ByteArrayResponse::new) + .orElseThrow(() -> new NotExistsException("No application package diff found for " + runId)); + } + private HttpResponse applicationPackage(String tenantName, String applicationName, HttpRequest request) { var tenantAndApplication = TenantAndApplicationId.from(tenantName, applicationName); - var applicationId = ApplicationId.from(tenantName, applicationName, InstanceName.defaultName().value()); long buildNumber; var requestedBuild = Optional.ofNullable(request.getProperty("build")).map(build -> { @@ -618,6 +638,13 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new ZipResponse(filename, applicationPackage.get()); } + private HttpResponse applicationPackageDiff(String tenant, String application, String number) { + TenantAndApplicationId tenantAndApplication = TenantAndApplicationId.from(tenant, application); + return controller.applications().applicationStore().getDiff(tenantAndApplication.tenant(), tenantAndApplication.application(), Long.parseLong(number)) + .map(ByteArrayResponse::new) + .orElseThrow(() -> new NotExistsException("No application package diff found for '" + tenantAndApplication + "' with build number " + number)); + } + private HttpResponse application(String tenantName, String applicationName, HttpRequest request) { Slime slime = new Slime(); toSlime(slime.setObject(), getApplication(tenantName, applicationName), request); @@ -865,7 +892,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse nodes(String tenantName, String applicationName, String instanceName, String environment, String region) { ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); ZoneId zone = requireZone(environment, region); - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(zone, id); + List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(zone, NodeFilter.all().applications(id)); Slime slime = new Slime(); Cursor nodesArray = slime.setObject().setArray("nodes"); @@ -876,11 +903,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { node.reservedTo().ifPresent(tenant -> nodeObject.setString("reservedTo", tenant.value())); nodeObject.setString("orchestration", valueOf(node.serviceState())); nodeObject.setString("version", node.currentVersion().toString()); - nodeObject.setString("flavor", node.flavor()); + node.flavor().ifPresent(flavor -> nodeObject.setString("flavor", flavor)); toSlime(node.resources(), nodeObject); nodeObject.setString("clusterId", node.clusterId()); nodeObject.setString("clusterType", valueOf(node.clusterType())); - nodeObject.setBool("down", node.history().stream().anyMatch(event -> "down".equals(event.getEvent()))); + nodeObject.setBool("down", node.history().stream().anyMatch(event -> "down".equals(event.name()))); nodeObject.setBool("retired", node.retired() || node.wantToRetire()); nodeObject.setBool("restarting", node.wantedRestartGeneration() > node.restartGeneration()); nodeObject.setBool("rebooting", node.wantedRebootGeneration() > node.rebootGeneration()); @@ -1414,6 +1441,13 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { response.setString("status", "pending"); else response.setString("status", "running"); }); + } else { + var deploymentRun = JobType.from(controller.system(), deploymentId.zoneId()) + .flatMap(jobType -> controller.jobController().last(deploymentId.applicationId(), jobType)); + + deploymentRun.ifPresent(run -> { + response.setString("status", run.hasEnded() ? "complete" : "running"); + }); } } @@ -1684,6 +1718,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new HtmlResponse(result); } + String normalizedRestPath = URI.create(restPath).normalize().toString(); + if (allowedServiceViewProxy.value().stream().noneMatch(normalizedRestPath::startsWith)) { + return ErrorResponse.forbidden("Access denied"); + } + Map<?,?> result = controller.serviceRegistry().configServer().getServiceApiResponse(deploymentId, serviceName, restPath); ServiceApiResponse response = new ServiceApiResponse(deploymentId.zoneId(), deploymentId.applicationId(), @@ -1928,7 +1967,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .flatMap(options -> optional("vespaVersion", options)) .map(Version::fromString); - controller.jobController().deploy(id, type, version, applicationPackage); + ensureApplicationExists(TenantAndApplicationId.from(id), request); + + boolean dryRun = Optional.ofNullable(dataParts.get("deployOptions")) + .map(json -> SlimeUtils.jsonToSlime(json).get()) + .flatMap(options -> optional("dryRun", options)) + .map(Boolean::valueOf) + .orElse(false); + + controller.jobController().deploy(id, type, version, applicationPackage, dryRun); RunId runId = controller.jobController().last(id, type).get().id(); Slime slime = new Slime(); Cursor rootObject = slime.setObject(); @@ -1976,17 +2023,17 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private HttpResponse deleteTenant(String tenantName, HttpRequest request) { - Optional<Tenant> tenant = controller.tenants().get(tenantName); - if (tenant.isEmpty()) - return ErrorResponse.notFoundError("Could not delete tenant '" + tenantName + "': Tenant not found"); + boolean forget = request.getBooleanProperty("forget"); + if (forget && !isOperator(request)) + return ErrorResponse.forbidden("Only operators can forget a tenant"); - controller.tenants().delete(tenant.get().name(), - accessControlRequests.credentials(tenant.get().name(), + controller.tenants().delete(TenantName.from(tenantName), + () -> accessControlRequests.credentials(TenantName.from(tenantName), toSlime(request.getData()).get(), - request.getJDiscRequest())); + request.getJDiscRequest()), + forget); - // TODO: Change to a message response saying the tenant was deleted - return tenant(tenant.get(), request); + return new MessageResponse("Deleted tenant " + tenantName); } private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) { @@ -2037,6 +2084,106 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { controller.applications().reachableContentClustersByZone(deployments))); } + private HttpResponse requestServiceDump(String tenant, String application, String instance, String environment, + String region, String hostname, HttpRequest request) { + NodeRepository nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); + ZoneId zone = requireZone(environment, region); + + // Check that no other service dump is in progress + Slime report = getReport(nodeRepository, zone, tenant, application, instance, hostname).orElse(null); + if (report != null) { + Cursor cursor = report.get(); + // Note: same behaviour for both value '0' and missing value. + boolean force = request.getBooleanProperty("force"); + if (!force && cursor.field("failedAt").asLong() == 0 && cursor.field("completedAt").asLong() == 0) { + throw new IllegalArgumentException("Service dump already in progress for " + cursor.field("configId").asString()); + } + } + Slime requestPayload; + try { + requestPayload = SlimeUtils.jsonToSlimeOrThrow(request.getData().readAllBytes()); + } catch (Exception e) { + throw new IllegalArgumentException("Missing or invalid JSON in request content", e); + } + Cursor requestPayloadCursor = requestPayload.get(); + String configId = requestPayloadCursor.field("configId").asString(); + long expiresAt = requestPayloadCursor.field("expiresAt").asLong(); + if (configId.isEmpty()) { + throw new IllegalArgumentException("Missing configId"); + } + Cursor artifactsCursor = requestPayloadCursor.field("artifacts"); + int artifactEntries = artifactsCursor.entries(); + if (artifactEntries == 0) { + throw new IllegalArgumentException("Missing or empty 'artifacts'"); + } + + Slime dumpRequest = new Slime(); + Cursor dumpRequestCursor = dumpRequest.setObject(); + dumpRequestCursor.setLong("createdMillis", controller.clock().millis()); + dumpRequestCursor.setString("configId", configId); + Cursor dumpRequestArtifactsCursor = dumpRequestCursor.setArray("artifacts"); + for (int i = 0; i < artifactEntries; i++) { + dumpRequestArtifactsCursor.addString(artifactsCursor.entry(i).asString()); + } + if (expiresAt > 0) { + dumpRequestCursor.setLong("expiresAt", expiresAt); + } + Cursor dumpOptionsCursor = requestPayloadCursor.field("dumpOptions"); + if (dumpOptionsCursor.children() > 0) { + SlimeUtils.copyObject(dumpOptionsCursor, dumpRequestCursor.setObject("dumpOptions")); + } + var reportsUpdate = Map.of("serviceDump", new String(uncheck(() -> SlimeUtils.toJsonBytes(dumpRequest)))); + nodeRepository.updateReports(zone, hostname, reportsUpdate); + boolean wait = request.getBooleanProperty("wait"); + if (!wait) return new MessageResponse("Request created"); + return waitForServiceDumpResult(nodeRepository, zone, tenant, application, instance, hostname); + } + + private HttpResponse getServiceDump(String tenant, String application, String instance, String environment, + String region, String hostname, HttpRequest request) { + NodeRepository nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); + ZoneId zone = requireZone(environment, region); + Slime report = getReport(nodeRepository, zone, tenant, application, instance, hostname) + .orElseThrow(() -> new NotExistsException("No service dump for node " + hostname)); + return new SlimeJsonResponse(report); + } + + private HttpResponse waitForServiceDumpResult(NodeRepository nodeRepository, ZoneId zone, String tenant, + String application, String instance, String hostname) { + int pollInterval = 2; + Slime report; + while (true) { + report = getReport(nodeRepository, zone, tenant, application, instance, hostname).get(); + Cursor cursor = report.get(); + if (cursor.field("completedAt").asLong() > 0 || cursor.field("failedAt").asLong() > 0) { + break; + } + final Slime copyForLambda = report; + log.fine(() -> uncheck(() -> new String(SlimeUtils.toJsonBytes(copyForLambda)))); + log.fine("Sleeping " + pollInterval + " seconds before checking report status again"); + controller.sleeper().sleep(Duration.ofSeconds(pollInterval)); + } + return new SlimeJsonResponse(report); + } + + private Optional<Slime> getReport(NodeRepository nodeRepository, ZoneId zone, String tenant, + String application, String instance, String hostname) { + Node node; + try { + node = nodeRepository.getNode(zone, hostname); + } catch (IllegalArgumentException e) { + throw new NotExistsException(new Hostname(hostname)); + } + ApplicationId app = ApplicationId.from(tenant, application, instance); + ApplicationId owner = node.owner().orElseThrow(() -> new IllegalArgumentException("Node has no owner")); + if (!app.equals(owner)) { + throw new IllegalArgumentException("Node is not owned by " + app.toFullString()); + } + String json = node.reports().get("serviceDump"); + if (json == null) return Optional.empty(); + return Optional.of(SlimeUtils.jsonToSlimeOrThrow(json)); + } + private static SourceRevision toSourceRevision(Inspector object) { if (!object.field("repository").valid() || !object.field("branch").valid() || @@ -2106,6 +2253,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { break; } + case deleted: break; default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } // TODO jonmv: This should list applications, not instances. @@ -2195,6 +2343,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { metaData.setString("property", athenzTenant.property().id()); break; case cloud: break; + case deleted: break; default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } object.setString("url", withPath("/application/v4/tenant/" + tenant.name().value(), requestURI).toString()); @@ -2218,6 +2367,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .flatMap(app -> app.latestVersion().flatMap(ApplicationVersion::buildTime).stream()) .max(Comparator.naturalOrder()); object.setLong("createdAtMillis", tenant.createdAt().toEpochMilli()); + if (tenant.type() == Tenant.Type.deleted) + object.setLong("deletedAtMillis", ((DeletedTenant) tenant).deletedAt().toEpochMilli()); lastDev.ifPresent(instant -> object.setLong("lastDeploymentToDevMillis", instant.toEpochMilli())); lastSubmission.ifPresent(instant -> object.setLong("lastSubmissionToProdMillis", instant.toEpochMilli())); @@ -2422,10 +2573,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return "true".equals(request.getProperty("activeInstances")); } + private static boolean includeDeleted(HttpRequest request) { + return "true".equals(request.getProperty("includeDeleted")); + } + private static String tenantType(Tenant tenant) { switch (tenant.type()) { case athenz: return "ATHENS"; case cloud: return "CLOUD"; + case deleted: return "DELETED"; default: throw new IllegalArgumentException("Unknown tenant type: " + tenant.getClass().getSimpleName()); } } @@ -2469,6 +2625,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { applicationPackage, Optional.of(requireUserPrincipal(request))); + ensureApplicationExists(TenantAndApplicationId.from(tenant, application), request); + return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application, @@ -2570,5 +2728,13 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .anyMatch(definition -> definition == RoleDefinition.hostedOperator); } + private void ensureApplicationExists(TenantAndApplicationId id, HttpRequest request) { + if (controller.applications().getApplication(id).isEmpty()) { + log.fine("Application does not exist in public, creating: " + id); + var credentials = accessControlRequests.credentials(id.tenant(), null /* not used on public */ , request.getJDiscRequest()); + controller.applications().createApplication(id, credentials); + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java index e343615f066..dbd4250669c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.container.jdisc.HttpResponse; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index eb9b0e03f3d..30bd040a682 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.application.api.DeploymentSpec; @@ -20,7 +20,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; @@ -40,18 +40,14 @@ import java.time.Instant; import java.time.format.TextStyle; import java.util.Arrays; import java.util.Collection; -import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import java.util.stream.Stream; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.canary; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.broken; @@ -92,57 +88,17 @@ class JobControllerApiHandlerHelper { return new SlimeJsonResponse(slime); } - private static void runToSlime(Cursor runObject, Run run, URI baseUriForJobType) { - runObject.setLong("id", run.id().number()); - runObject.setString("status", nameOf(run.status())); - runObject.setLong("start", run.start().toEpochMilli()); - run.end().ifPresent(instant -> runObject.setLong("end", instant.toEpochMilli())); - - versionsToSlime(runObject, run.versions()); - - Cursor stepsObject = runObject.setObject("steps"); - run.steps().forEach((step, info) -> stepsObject.setString(step.name(), info.status().name())); - Cursor tasksObject = runObject.setObject("tasks"); - taskStatus(deployReal, run).ifPresent(status -> tasksObject.setString("deploy", status)); - taskStatus(Step.installReal, run).ifPresent(status -> tasksObject.setString("install", status)); - taskStatus(Step.endTests, run).ifPresent(status -> tasksObject.setString("test", status)); - - runObject.setString("log", baseUriForJobType.resolve(baseUriForJobType.getPath() + "/run/" + run.id().number()).normalize().toString()); - } - - /** Returns the status of the task represented by the given step, if it has started. */ - private static Optional<String> taskStatus(Step step, Run run) { - return run.readySteps().contains(step) ? Optional.of("running") - : Optional.ofNullable(run.steps().get(step)) - .filter(info -> info.status() != unfinished) - .map(info -> info.status().name()); - } - /** Returns a response with the runs for the given job type. */ static HttpResponse runResponse(Map<RunId, Run> runs, Optional<String> limitStr, URI baseUriForJobType) { Slime slime = new Slime(); Cursor cursor = slime.setObject(); - // TODO (freva): Remove after console migrated to use new format - if (limitStr.isEmpty()) - runs.forEach((runid, run) -> runToSlime(cursor.setObject(Long.toString(runid.number())), run, baseUriForJobType)); - else { - int limit = limitStr.map(Integer::parseInt).orElse(Integer.MAX_VALUE); - toSlime(cursor.setArray("runs"), runs.values().stream() - .sorted(Comparator.comparing((Run run) -> run.id().number()).reversed()) - .collect(Collectors.toUnmodifiableList()), limit, baseUriForJobType); - } + int limit = limitStr.map(Integer::parseInt).orElse(Integer.MAX_VALUE); + toSlime(cursor.setArray("runs"), runs.values(), limit, baseUriForJobType); return new SlimeJsonResponse(slime); } - private static void versionsToSlime(Cursor runObject, Versions versions) { - runObject.setString("wantedPlatform", versions.targetPlatform().toString()); - applicationVersionToSlime(runObject.setObject("wantedApplication"), versions.targetApplication()); - versions.sourcePlatform().ifPresent(version -> runObject.setString("currentPlatform", version.toString())); - versions.sourceApplication().ifPresent(version -> applicationVersionToSlime(runObject.setObject("currentApplication"), version)); - } - static void applicationVersionToSlime(Cursor versionObject, ApplicationVersion version) { versionObject.setString("hash", version.id()); if (version.isUnknown()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java index b49b47e0cc7..8d03ca74500 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.container.jdisc.HttpRequest; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java index 9255cbfd7b2..888c402b6ec 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java index 9ac94d0208c..621ce9189e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java @@ -1,19 +1,14 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.athenz; +import com.google.inject.Inject; import com.yahoo.config.provision.SystemName; -import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; -import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.restapi.RestApi; +import com.yahoo.restapi.RestApiRequestHandler; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.text.Text; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; @@ -22,20 +17,19 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; -import com.yahoo.yolean.Exceptions; import java.util.Map; -import java.util.Optional; -import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.restapi.RestApi.route; + /** * This API proxies requests to an Athenz server. * * @author jonmv */ @SuppressWarnings("unused") // Handler -public class AthenzApiHandler extends LoggingRequestHandler { +public class AthenzApiHandler extends RestApiRequestHandler<AthenzApiHandler> { private final static Logger log = Logger.getLogger(AthenzApiHandler.class.getName()); @@ -43,55 +37,32 @@ public class AthenzApiHandler extends LoggingRequestHandler { private final AthenzDomain sandboxDomain; private final EntityService properties; + @Inject public AthenzApiHandler(Context parentCtx, AthenzFacade athenz, Controller controller) { - super(parentCtx); + super(parentCtx, AthenzApiHandler::createRestApi); this.athenz = athenz; this.sandboxDomain = new AthenzDomain(sandboxDomainIn(controller.system())); this.properties = controller.serviceRegistry().entityService(); } - @Override - public HttpResponse handle(HttpRequest request) { - Method method = request.getMethod(); - try { - switch (method) { - case GET: return get(request); - case POST: return post(request); - default: return ErrorResponse.methodNotAllowed("Method '" + method + "' is unsupported"); - } - } - catch (IllegalArgumentException|IllegalStateException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); - } - } - - private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/athenz/v1")) return root(request); - if (path.matches("/athenz/v1/domains")) return domainList(request); - if (path.matches("/athenz/v1/properties")) return properties(); - - return ErrorResponse.notFoundError(Text.format("No '%s' handler at '%s'", request.getMethod(), - request.getUri().getPath())); + private static RestApi createRestApi(AthenzApiHandler self) { + return RestApi.builder() + .addRoute(route("/athenz/v1") + .get(self::root)) + .addRoute(route("/athenz/v1/domains") + .get(self::domainList)) + .addRoute(route("/athenz/v1/properties") + .get(self::properties)) + .addRoute(route("/athenz/v1/user") + .post(self::signup)) + .build(); } - private HttpResponse post(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/athenz/v1/user")) return signup(request); - return ErrorResponse.notFoundError(Text.format("No '%s' handler at '%s'", request.getMethod(), - request.getUri().getPath())); + private HttpResponse root(RestApi.RequestContext ctx) { + return new ResourceResponse(ctx.request(), "domains", "properties"); } - private HttpResponse root(HttpRequest request) { - return new ResourceResponse(request, "domains", "properties"); - } - - - private HttpResponse properties() { + private Slime properties(RestApi.RequestContext ctx) { Slime slime = new Slime(); Cursor response = slime.setObject(); Cursor array = response.setArray("properties"); @@ -100,26 +71,27 @@ public class AthenzApiHandler extends LoggingRequestHandler { propertyObject.setString("propertyid", entry.getKey().id()); propertyObject.setString("property", entry.getValue().id()); } - return new SlimeJsonResponse(slime); + return slime; } - private HttpResponse domainList(HttpRequest request) { + private Slime domainList(RestApi.RequestContext ctx) { Slime slime = new Slime(); Cursor array = slime.setObject().setArray("data"); - for (AthenzDomain athenzDomain : athenz.getDomainList(request.getProperty("prefix"))) + for (AthenzDomain athenzDomain : athenz.getDomainList(ctx.queryParameters().getString("prefix").orElse(null))) array.addString(athenzDomain.getName()); - return new SlimeJsonResponse(slime); + return slime; } - private HttpResponse signup(HttpRequest request) { - AthenzUser user = athenzUser(request); + private String signup(RestApi.RequestContext ctx) { + AthenzUser user = athenzUser(ctx); athenz.addTenantAdmin(sandboxDomain, user); - return new MessageResponse("User '" + user.getName() + "' added to admin role of '" + sandboxDomain.getName() + "'"); + return "User '" + user.getName() + "' added to admin role of '" + sandboxDomain.getName() + "'"; } - private static AthenzUser athenzUser(HttpRequest request) { - return Optional.ofNullable(request.getJDiscRequest().getUserPrincipal()).filter(AthenzPrincipal.class::isInstance) + private static AthenzUser athenzUser(RestApi.RequestContext ctx) { + return ctx.userPrincipal() + .filter(AthenzPrincipal.class::isInstance) .map(AthenzPrincipal.class::cast) .map(AthenzPrincipal::getIdentity) .filter(AthenzUser.class::isInstance) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index a890776381c..b8c8ce4e63a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.billing; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index 1d2132af6a1..630305b0ab4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -11,7 +11,6 @@ import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -26,7 +25,6 @@ import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import javax.ws.rs.BadRequestException; -import java.io.IOException; import java.math.BigDecimal; import java.time.Clock; import java.time.Instant; @@ -34,8 +32,8 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.Comparator; -import java.util.Optional; import java.util.List; +import java.util.Optional; /** * @author ogronnesby @@ -78,11 +76,6 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler .addRoute(RestApi.route("/billing/v2/accountant/preview/tenant/{tenant}") .get(self::previewBill) .post(Slime.class, self::createBill)) - /* - * Utility - map Slime.class => SlimeJsonResponse - */ - .addRequestMapper(Slime.class, BillingApiHandlerV2::slimeRequestMapper) - .addResponseMapper(Slime.class, BillingApiHandlerV2::slimeResponseMapper) .build(); } @@ -337,16 +330,4 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return inspector.field(field).asString(); } - private static Optional<Slime> slimeRequestMapper(RestApi.RequestContext requestContext) { - try { - return Optional.of(SlimeUtils.jsonToSlime(requestContext.requestContentOrThrow().content().readAllBytes())); - } catch (IOException e) { - throw new IllegalArgumentException("Could not parse JSON input"); - } - } - - private static HttpResponse slimeResponseMapper(RestApi.RequestContext ctx, Slime slime) { - return new SlimeJsonResponse(slime); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java index cffdd9fc928..0ea8d09ba09 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.changemanagement; import com.yahoo.config.provision.Environment; @@ -16,6 +16,7 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; @@ -278,10 +279,9 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { } private Optional<ZoneId> affectedZone(List<String> hosts) { - var affectedHosts = hosts.stream() - .map(HostName::from) - .collect(Collectors.toList()); - + NodeFilter affectedHosts = NodeFilter.all().hostnames(hosts.stream() + .map(HostName::from) + .collect(Collectors.toSet())); for (var zone : getProdZones()) { var affectedHostsInZone = controller.serviceRegistry().configServer().nodeRepository().list(zone, affectedHosts); if (!affectedHostsInZone.isEmpty()) @@ -293,7 +293,7 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { private List<String> hostsOnSwitch(List<String> switches) { return getProdZones().stream() - .flatMap(zone -> controller.serviceRegistry().configServer().nodeRepository().list(zone, false).stream()) + .flatMap(zone -> controller.serviceRegistry().configServer().nodeRepository().list(zone, NodeFilter.all()).stream()) .filter(node -> node.switchHostname().map(switches::contains).orElse(false)) .map(node -> node.hostname().value()) .collect(Collectors.toList()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AccessRequestResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AccessRequestResponse.java index e17421764e5..30e103048cf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AccessRequestResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AccessRequestResponse.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 8f083f4e988..7c1f049548a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.controller; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java index 0ac90349eaf..05768410891 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.controller; import com.yahoo.concurrent.maintenance.JobControl; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java index ff5ad00912f..19f1ac5449f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.controller; import com.yahoo.config.provision.zone.ZoneId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java index 8968b5ecbe0..e68517f7134 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.controller; import com.yahoo.slime.Cursor; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java index bd71a663328..3047d1ed234 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.deployment; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java index c13a9b42382..1fe5ebfa9a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.deployment; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index 2b9fe708df6..08ec3caa829 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; import com.google.inject.Inject; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java index 9b1ccc09499..4762d2c4612 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; import com.google.inject.Inject; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java index 6f5b1f30592..96400e50b9a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java @@ -1,24 +1,35 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.horizon; import com.google.inject.Inject; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.horizon.HorizonClient; import com.yahoo.vespa.hosted.controller.api.integration.horizon.HorizonResponse; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; +import com.yahoo.vespa.hosted.controller.api.role.TenantRole; import com.yahoo.yolean.Exceptions; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.EnumSet; import java.util.Optional; +import java.util.Set; import java.util.logging.Level; +import java.util.stream.Collectors; /** * Proxies metrics requests from Horizon UI @@ -29,20 +40,32 @@ public class HorizonApiHandler extends LoggingRequestHandler { private final SystemName systemName; private final HorizonClient client; + private final BooleanFlag enabledHorizonDashboard; + + private static final EnumSet<RoleDefinition> operatorRoleDefinitions = + EnumSet.of(RoleDefinition.hostedOperator, RoleDefinition.hostedSupporter); @Inject - public HorizonApiHandler(LoggingRequestHandler.Context parentCtx, Controller controller) { + public HorizonApiHandler(LoggingRequestHandler.Context parentCtx, Controller controller, FlagSource flagSource) { super(parentCtx); this.systemName = controller.system(); this.client = controller.serviceRegistry().horizonClient(); + this.enabledHorizonDashboard = Flags.ENABLED_HORIZON_DASHBOARD.bindTo(flagSource); } @Override public HttpResponse handle(HttpRequest request) { + var roles = getRoles(request); + var operator = roles.stream().map(Role::definition).anyMatch(operatorRoleDefinitions::contains); + var authorizedTenants = getAuthorizedTenants(roles); + + if (!operator && authorizedTenants.isEmpty()) + return ErrorResponse.forbidden("No tenant with enabled metrics view"); + try { switch (request.getMethod()) { case GET: return get(request); - case POST: return post(request); + case POST: return post(request, authorizedTenants, operator); case PUT: return put(request); default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); } @@ -59,16 +82,13 @@ public class HorizonApiHandler extends LoggingRequestHandler { private HttpResponse get(HttpRequest request) { Path path = new Path(request.getUri()); if (path.matches("/horizon/v1/config/dashboard/topFolders")) return new JsonInputStreamResponse(client.getTopFolders()); - if (path.matches("/horizon/v1/config/dashboard/file/{id}")) return new JsonInputStreamResponse(client.getDashboard(path.get("id"))); - if (path.matches("/horizon/v1/config/dashboard/favorite")) return new JsonInputStreamResponse(client.getFavorite(request.getProperty("user"))); - if (path.matches("/horizon/v1/config/dashboard/recent")) return new JsonInputStreamResponse(client.getRecent(request.getProperty("user"))); + if (path.matches("/horizon/v1/config/dashboard/file/{id}")) return new JsonInputStreamResponse(client.getDashboard()); return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse post(HttpRequest request) { + private HttpResponse post(HttpRequest request, Set<TenantName> authorizedTenants, boolean operator) { Path path = new Path(request.getUri()); - if (path.matches("/horizon/v1/tsdb/api/query/graph")) return tsdbQuery(request, true); - if (path.matches("/horizon/v1/meta/search/timeseries")) return tsdbQuery(request, false); + if (path.matches("/horizon/v1/tsdb/api/query/graph")) return tsdbQuery(request, authorizedTenants, operator); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -78,11 +98,10 @@ public class HorizonApiHandler extends LoggingRequestHandler { return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse tsdbQuery(HttpRequest request, boolean isMetricQuery) { - SecurityContext securityContext = getAttribute(request, SecurityContext.ATTRIBUTE_NAME, SecurityContext.class); + private HttpResponse tsdbQuery(HttpRequest request, Set<TenantName> authorizedTenants, boolean operator) { try { - byte[] data = TsdbQueryRewriter.rewrite(request.getData().readAllBytes(), securityContext.roles(), systemName); - return new JsonInputStreamResponse(isMetricQuery ? client.getMetrics(data) : client.getMetaData(data)); + byte[] data = TsdbQueryRewriter.rewrite(request.getData().readAllBytes(), authorizedTenants, operator, systemName); + return new JsonInputStreamResponse(client.getMetrics(data)); } catch (TsdbQueryRewriter.UnauthorizedException e) { return ErrorResponse.forbidden("Access denied"); } catch (IOException e) { @@ -90,11 +109,20 @@ public class HorizonApiHandler extends LoggingRequestHandler { } } - private static <T> T getAttribute(HttpRequest request, String attributeName, Class<T> clazz) { - return Optional.ofNullable(request.getJDiscRequest().context().get(attributeName)) - .filter(clazz::isInstance) - .map(clazz::cast) - .orElseThrow(() -> new IllegalArgumentException("Attribute '" + attributeName + "' was not set on request")); + private static Set<Role> getRoles(HttpRequest request) { + return Optional.ofNullable(request.getJDiscRequest().context().get(SecurityContext.ATTRIBUTE_NAME)) + .filter(SecurityContext.class::isInstance) + .map(SecurityContext.class::cast) + .map(SecurityContext::roles) + .orElseThrow(() -> new IllegalArgumentException("Attribute '" + SecurityContext.ATTRIBUTE_NAME + "' was not set on request")); + } + + private Set<TenantName> getAuthorizedTenants(Set<Role> roles) { + return roles.stream() + .filter(TenantRole.class::isInstance) + .map(role -> ((TenantRole) role).tenant()) + .filter(tenant -> enabledHorizonDashboard.with(FetchVector.Dimension.TENANT_ID, tenant.value()).value()) + .collect(Collectors.toSet()); } private static class JsonInputStreamResponse extends HttpResponse { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java index e034be46063..d6a0e785b43 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.horizon; import com.fasterxml.jackson.databind.JsonNode; @@ -7,12 +7,8 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; -import com.yahoo.vespa.hosted.controller.api.role.TenantRole; import java.io.IOException; -import java.util.EnumSet; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -23,20 +19,8 @@ import java.util.stream.Collectors; public class TsdbQueryRewriter { private static final ObjectMapper mapper = new ObjectMapper(); - private static final EnumSet<RoleDefinition> operatorRoleDefinitions = - EnumSet.of(RoleDefinition.hostedOperator, RoleDefinition.hostedSupporter); - - public static byte[] rewrite(byte[] data, Set<Role> roles, SystemName systemName) throws IOException { - boolean operator = roles.stream().map(Role::definition).anyMatch(operatorRoleDefinitions::contains); - - // Anyone with any tenant relation can view metrics for apps within those tenants - Set<TenantName> authorizedTenants = roles.stream() - .filter(TenantRole.class::isInstance) - .map(role -> ((TenantRole) role).tenant()) - .collect(Collectors.toUnmodifiableSet()); - if (!operator && authorizedTenants.isEmpty()) - throw new UnauthorizedException(); + public static byte[] rewrite(byte[] data, Set<TenantName> authorizedTenants, boolean operator, SystemName systemName) throws IOException { JsonNode root = mapper.readTree(data); requireLegalType(root); getField(root, "executionGraph", ArrayNode.class) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 828e7e63483..ff8a97611f1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; @@ -31,8 +32,10 @@ import java.time.Instant; import java.util.Comparator; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.logging.Level; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * This implements the /routing/v1 API, which provides operator with global routing control at both zone- and @@ -85,11 +88,52 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}")) return application(path, request); if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}")) return instance(path, request); if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path); + if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}/endpoint")) return endpoints(path); if (path.matches("/routing/v1/status/environment")) return environment(request); if (path.matches("/routing/v1/status/environment/{environment}/region/{region}")) return zone(path); return ErrorResponse.notFoundError("Nothing at " + path); } + private HttpResponse endpoints(Path path) { + var instanceId = instanceFrom(path); + var endpoints = controller.routing().endpointsOf(instanceId) + .sortedBy(Comparator.comparing(Endpoint::name)) + .asList(); + + var deployments = endpoints.stream() + .flatMap(e -> e.zones().stream()) + .distinct() + .map(zoneId -> new DeploymentId(instanceId, zoneId)) + .sorted(Comparator.comparing(DeploymentId::dottedString)) + .collect(Collectors.toList()); + + var deploymentsStatus = deployments.stream() + .collect(Collectors.toMap( + deploymentId -> deploymentId, + deploymentId -> Stream.concat( + directGlobalRoutingStatus(deploymentId).stream(), + sharedGlobalRoutingStatus(deploymentId).stream() + ).collect(Collectors.toList()) + )); + + var slime = new Slime(); + var root = slime.setObject(); + var endpointsRoot = root.setArray("endpoints"); + endpoints.forEach(endpoint -> { + var endpointRoot = endpointsRoot.addObject(); + endpointToSlime(endpointRoot, endpoint); + var zonesRoot = endpointRoot.setArray("zones"); + endpoint.zones().stream().sorted(Comparator.comparing(ZoneId::value)).forEach(zoneId -> { + var deploymentId = new DeploymentId(instanceId, zoneId); + deploymentsStatus.getOrDefault(deploymentId, List.of()).forEach(status -> { + deploymentStatusToSlime(zonesRoot.addObject(), deploymentId, status, endpoint.routingMethod()); + }); + }); + }); + + return new SlimeJsonResponse(slime); + } + private HttpResponse environment(HttpRequest request) { var zones = controller.zoneRegistry().zones().all().ids(); if (isRecursive(request)) { @@ -241,42 +285,50 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { for (var zone : zones) { var deploymentId = requireDeployment(new DeploymentId(instance.id(), zone), instance); // Include status from rotation - if (rotationCanRouteTo(zone)) { - var rotationStatus = controller.routing().globalRotationStatus(deploymentId); - // Status is equal across all global endpoints, as the status is per deployment, not per endpoint. - var endpointStatus = rotationStatus.values().stream().findFirst(); - if (endpointStatus.isPresent()) { - var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch()); - GlobalRouting.Agent agent; - try { - agent = GlobalRouting.Agent.valueOf(endpointStatus.get().getAgent()); - } catch (IllegalArgumentException e) { - agent = GlobalRouting.Agent.unknown; - } - var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in - ? GlobalRouting.Status.in - : GlobalRouting.Status.out; - deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId, - new GlobalRouting(status, agent, changedAt), - RoutingMethod.shared); - } - } + sharedGlobalRoutingStatus(deploymentId).ifPresent(status -> { + deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId, status, RoutingMethod.shared); + }); // Include status from routing policies - var routingPolicies = controller.routing().policies().get(deploymentId); - for (var policy : routingPolicies.values()) { - if (policy.endpoints().isEmpty()) continue; // This policy does not apply to a global endpoint - if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(RoutingMethod.exclusive)) continue; - deploymentStatusToSlime(deploymentsArray.addObject(), new DeploymentId(policy.id().owner(), - policy.id().zone()), - policy.status().globalRouting(), RoutingMethod.exclusive); - } + directGlobalRoutingStatus(deploymentId).forEach(status -> { + deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId, status, RoutingMethod.exclusive); + }); } } } } + private Optional<GlobalRouting> sharedGlobalRoutingStatus(DeploymentId deploymentId) { + if (rotationCanRouteTo(deploymentId.zoneId())) { + var rotationStatus = controller.routing().globalRotationStatus(deploymentId); + // Status is equal across all global endpoints, as the status is per deployment, not per endpoint. + var endpointStatus = rotationStatus.values().stream().findFirst(); + if (endpointStatus.isPresent()) { + var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch()); + GlobalRouting.Agent agent; + try { + agent = GlobalRouting.Agent.valueOf(endpointStatus.get().getAgent()); + } catch (IllegalArgumentException e) { + agent = GlobalRouting.Agent.unknown; + } + var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in + ? GlobalRouting.Status.in + : GlobalRouting.Status.out; + return Optional.of(new GlobalRouting(status, agent, changedAt)); + } + } + return Optional.empty(); + } + + private List<GlobalRouting> directGlobalRoutingStatus(DeploymentId deploymentId) { + return controller.routing().policies().get(deploymentId).values().stream() + .filter(p -> ! p.endpoints().isEmpty()) // This policy does not apply to a global endpoint + .filter(p -> controller.zoneRegistry().routingMethods(p.id().zone()).contains(RoutingMethod.exclusive)) + .map(p -> p.status().globalRouting()) + .collect(Collectors.toList()); + } + /** Returns whether a rotation can route traffic to given zone */ private boolean rotationCanRouteTo(ZoneId zone) { // A system may support multiple routing methods, i.e. it has both exclusively routed zones and zones using @@ -304,6 +356,14 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { object.setLong("changedAt", globalRouting.changedAt().toEpochMilli()); } + private static void endpointToSlime(Cursor object, Endpoint endpoint) { + object.setString("name", endpoint.name()); + object.setString("dnsName", endpoint.dnsName()); + object.setString("routingMethod", endpoint.routingMethod().name()); + object.setString("cluster", endpoint.cluster().value()); + object.setString("scope", endpoint.scope().name()); + } + private TenantName tenantFrom(Path path) { return TenantName.from(path.get("tenant")); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java index d169cd97df7..529e892ced9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java @@ -251,6 +251,13 @@ class SystemFlagsDeployResult { return new OperationError(message, Set.of(), OperationType.VALIDATE_ARCHIVE, null, null); } + static OperationError dataForUndefinedFlag(FlagsTarget target, FlagId id) { + return new OperationError("Flag data present for undefined flag. Remove flag data files if flag's definition " + + "is already removed from Flags / PermanentFlags. Consult ModelContext.FeatureFlags " + + "for safe removal of flag used by config-model.", + Set.of(), OperationType.DATA_FOR_UNDEFINED_FLAG, id, null); + } + String message() { return message; } Set<FlagsTarget> targets() { return targets; } OperationType operation() { return operation; } @@ -284,7 +291,8 @@ class SystemFlagsDeployResult { } enum OperationType { - CREATE("create"), DELETE("delete"), UPDATE("update"), LIST("list"), VALIDATE_ARCHIVE("validate-archive"); + CREATE("create"), DELETE("delete"), UPDATE("update"), LIST("list"), VALIDATE_ARCHIVE("validate-archive"), + DATA_FOR_UNDEFINED_FLAG("data-for-undefined-flag"); private final String stringValue; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java index 21a429b59ad..e0b65b0834d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java @@ -78,7 +78,7 @@ class SystemFlagsDeployer { return SystemFlagsDeployResult.merge(results); } - private SystemFlagsDeployResult deployFlags(FlagsTarget target, Set<FlagData> flagData, boolean dryRun) { + private SystemFlagsDeployResult deployFlags(FlagsTarget target, List<FlagData> flagData, boolean dryRun) { Map<FlagId, FlagData> wantedFlagData = lookupTable(flagData); Map<FlagId, FlagData> currentFlagData; List<FlagId> definedFlags; @@ -98,7 +98,7 @@ class SystemFlagsDeployer { updateExistingFlagData(target, dryRun, wantedFlagData, currentFlagData, results, errors); removeOldFlagData(target, dryRun, wantedFlagData, currentFlagData, results, errors); failOnNewFlagDataForUndefinedFlags(target, wantedFlagData, currentFlagData, definedFlags, errors); - warnOnExistingFlagDataForUndefinedFlags(target, wantedFlagData, currentFlagData, definedFlags, warnings); + failOnFlagDataForUndefinedFlags(target, wantedFlagData, currentFlagData, definedFlags, errors); return new SystemFlagsDeployResult(results, errors, warnings); } @@ -191,14 +191,14 @@ class SystemFlagsDeployer { } } - private static void warnOnExistingFlagDataForUndefinedFlags(FlagsTarget target, - Map<FlagId, FlagData> wantedFlagData, - Map<FlagId, FlagData> currentFlagData, - List<FlagId> definedFlags, - List<Warning> warnings) { + private static void failOnFlagDataForUndefinedFlags(FlagsTarget target, + Map<FlagId, FlagData> wantedFlagData, + Map<FlagId, FlagData> currentFlagData, + List<FlagId> definedFlags, + List<OperationError> errors) { for (FlagId flagId : currentFlagData.keySet()) { if (wantedFlagData.containsKey(flagId) && !definedFlags.contains(flagId)) { - warnings.add(Warning.dataForUndefinedFlag(target, flagId)); + errors.add(OperationError.dataForUndefinedFlag(target, flagId)); } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 7b0a2c9d6d6..7e88f127026 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -19,8 +19,7 @@ import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeStream; import com.yahoo.slime.SlimeUtils; import com.yahoo.text.Text; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.configserver.flags.FlagsDb; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; @@ -41,7 +40,6 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; import java.security.PublicKey; -import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; @@ -69,15 +67,15 @@ public class UserApiHandler extends LoggingRequestHandler { private final UserManagement users; private final Controller controller; - private final BooleanFlag enable_public_signup_flow; + private final FlagsDb flagsDb; private final IntFlag maxTrialTenants; @Inject - public UserApiHandler(Context parentCtx, UserManagement users, Controller controller, FlagSource flagSource) { + public UserApiHandler(Context parentCtx, UserManagement users, Controller controller, FlagSource flagSource, FlagsDb flagsDb) { super(parentCtx); this.users = users; this.controller = controller; - this.enable_public_signup_flow = PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource); + this.flagsDb = flagsDb; this.maxTrialTenants = PermanentFlags.MAX_TRIAL_TENANTS.bindTo(flagSource); } @@ -168,8 +166,6 @@ public class UserApiHandler extends LoggingRequestHandler { root.setBool("isPublic", controller.system().isPublic()); root.setBool("isCd", controller.system().isCd()); - root.setBool(enable_public_signup_flow.id().toString(), - enable_public_signup_flow.with(FetchVector.Dimension.CONSOLE_USER_EMAIL, user.email()).value()); root.setBool("hasTrialCapacity", hasTrialCapacity()); toSlime(root.setObject("user"), user); @@ -191,6 +187,8 @@ public class UserApiHandler extends LoggingRequestHandler { operatorRoles.forEach(role -> operator.addString(role.definition().name())); } + UserFlagsSerializer.toSlime(root, flagsDb.getAllFlagData(), tenantRolesByTenantName.keySet(), !operatorRoles.isEmpty(), user.email()); + return new SlimeJsonResponse(slime); } @@ -243,7 +241,7 @@ public class UserApiHandler extends LoggingRequestHandler { }); } - private void toSlime(Cursor userObject, User user) { + private static void toSlime(Cursor userObject, User user) { if (user.name() != null) userObject.setString("name", user.name()); userObject.setString("email", user.email()); if (user.nickname() != null) userObject.setString("nickname", user.nickname()); @@ -370,7 +368,7 @@ public class UserApiHandler extends LoggingRequestHandler { return Exceptions.uncheck(() -> SlimeUtils.jsonToSlime(IOUtils.readBytes(request.getData(), 1 << 10)).get()); } - private <Type> Type require(String name, Function<Inspector, Type> mapper, Inspector object) { + private static <Type> Type require(String name, Function<Inspector, Type> mapper, Inspector object) { if ( ! object.field(name).valid()) throw new IllegalArgumentException("Missing field '" + name + "'."); return mapper.apply(object.field(name)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializer.java new file mode 100644 index 00000000000..44d537883f9 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializer.java @@ -0,0 +1,86 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.user; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.lang.MutableBoolean; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagDefinition; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.RawFlag; +import com.yahoo.vespa.flags.UnboundFlag; +import com.yahoo.vespa.flags.json.Condition; +import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.flags.json.Rule; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @author freva + */ +public class UserFlagsSerializer { + static void toSlime(Cursor cursor, Map<FlagId, FlagData> rawFlagData, + Set<TenantName> authorizedForTenantNames, boolean isOperator, String userEmail) { + FetchVector resolveVector = FetchVector.fromMap(Map.of(FetchVector.Dimension.CONSOLE_USER_EMAIL, userEmail)); + List<FlagData> filteredFlagData = Flags.getAllFlags().stream() + // Only include flags that have CONSOLE_USER_EMAIL dimension, this should be replaced with more explicit + // 'target' annotation if/when that is added to flag definition + .filter(fd -> fd.getDimensions().contains(FetchVector.Dimension.CONSOLE_USER_EMAIL)) + .map(FlagDefinition::getUnboundFlag) + .map(flag -> filteredFlagData(flag, Optional.ofNullable(rawFlagData.get(flag.id())), authorizedForTenantNames, isOperator, resolveVector)) + .collect(Collectors.toUnmodifiableList()); + + byte[] bytes = FlagData.serializeListToUtf8Json(filteredFlagData); + SlimeUtils.copyObject(SlimeUtils.jsonToSlime(bytes).get(), cursor); + } + + private static <T> FlagData filteredFlagData(UnboundFlag<T, ?, ?> definition, Optional<FlagData> original, + Set<TenantName> authorizedForTenantNames, boolean isOperator, FetchVector resolveVector) { + MutableBoolean encounteredEmpty = new MutableBoolean(false); + Optional<RawFlag> defaultValue = Optional.of(definition.serializer().serialize(definition.defaultValue())); + // Include the original rules from flag DB and the default value from code if there is no default rule in DB + List<Rule> rules = Stream.concat(original.stream().flatMap(fd -> fd.rules().stream()), Stream.of(new Rule(defaultValue))) + // Exclude rules that do not match the resolveVector + .filter(rule -> rule.partialMatch(resolveVector)) + // Re-create each rule with value explicitly set, either from DB or default from code and + // a filtered set of conditions + .map(rule -> new Rule(rule.getValueToApply().or(() -> defaultValue), + rule.conditions().stream() + .flatMap(condition -> filteredCondition(condition, authorizedForTenantNames, isOperator, resolveVector).stream()) + .collect(Collectors.toUnmodifiableList()))) + // We can stop as soon as we hit the first rule that has no conditions + .takeWhile(rule -> !encounteredEmpty.getAndSet(rule.conditions().isEmpty())) + .collect(Collectors.toUnmodifiableList()); + + return new FlagData(definition.id(), new FetchVector(), rules); + } + + private static Optional<Condition> filteredCondition(Condition condition, Set<TenantName> authorizedForTenantNames, + boolean isOperator, FetchVector resolveVector) { + // If the condition is one of the conditions that we resolve on the server, e.g. email, we do not need to + // propagate it back to the user + if (resolveVector.hasDimension(condition.dimension())) return Optional.empty(); + + // For the other dimensions, filter the values down to an allowed subset + switch (condition.dimension()) { + case TENANT_ID: return valueSubset(condition, tenant -> isOperator || authorizedForTenantNames.contains(TenantName.from(tenant))); + case APPLICATION_ID: return valueSubset(condition, appId -> isOperator || authorizedForTenantNames.stream().anyMatch(tenant -> appId.startsWith(tenant.value() + ":"))); + default: throw new IllegalArgumentException("Dimension " + condition.dimension() + " is not supported for user flags"); + } + } + + private static Optional<Condition> valueSubset(Condition condition, Predicate<String> predicate) { + Condition.CreateParams createParams = condition.toCreateParams(); + return Optional.of(createParams + .withValues(createParams.values().stream().filter(predicate).collect(Collectors.toUnmodifiableList())) + .createAs(condition.type())); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index fd6dfa61180..0ac4380f560 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.zone.v1; import com.yahoo.config.provision.Environment; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/package-info.java index 7793548766e..6c27f12954a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/package-info.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/package-info.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * @author mpolden */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 37c81f8c2c1..510635b005b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.zone.v2; import com.yahoo.config.provision.zone.ZoneId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/package-info.java index 95dfed8b7b2..9cb62748b63 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/package-info.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/package-info.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * @author mpolden */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java index 4f608736c45..ca5d2d5915f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; import com.yahoo.text.Text; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java index 10b15488f6e..2b75777fbbd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; import java.util.Objects; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java index 508df263837..fe9280b1193 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; import com.yahoo.vespa.curator.Lock; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java index 80bee1c5199..e15c8fa0ac4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; import com.yahoo.config.application.api.DeploymentInstanceSpec; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java index 70e554cd30d..032f01433b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index e2a8be15361..eca763ebc33 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.application.api.DeploymentSpec; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java index a908b341039..d8f4370ebcf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; import com.yahoo.vespa.hosted.controller.api.role.Role; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index 48f8d3e43cb..b8a44a682e7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -72,7 +72,7 @@ public class CloudAccessControl implements AccessControl { private void requireTenantTrialLimitNotReached(List<Tenant> existing) { var trialPlanId = PlanId.from("trial"); - var tenantNames = existing.stream().map(Tenant::name).collect(Collectors.toList()); + var tenantNames = existing.stream().filter(tenant -> tenant.type() == Tenant.Type.cloud).map(Tenant::name).collect(Collectors.toList()); var trialTenants = billingController.tenantsWithPlan(tenantNames, trialPlanId).size(); if (maxTrialTenants.value() >= 0 && maxTrialTenants.value() <= trialTenants) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccess.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccess.java index a8024a2ced3..27cf61564aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccess.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccess.java @@ -1,4 +1,4 @@ -// Copyright 2021 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.support.access; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessChange.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessChange.java index 8cb502db6ab..93659742538 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessChange.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessChange.java @@ -1,4 +1,4 @@ -// Copyright 2021 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.support.access; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java index 6bbec918ba9..a4af9f8e268 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java @@ -1,4 +1,4 @@ -// Copyright 2021 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.support.access; import com.yahoo.vespa.athenz.api.AthenzIdentity; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessGrant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessGrant.java index cdbb58675a5..76ceb6400bb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessGrant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessGrant.java @@ -1,4 +1,4 @@ -// Copyright 2021 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.support.access; import java.security.cert.X509Certificate; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java deleted file mode 100644 index 7fa46031c98..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.tenant; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.hosted.controller.api.identifiers.Property; -import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; - -import java.time.Instant; -import java.util.Objects; -import java.util.Optional; - -/** - * Represents an Athenz tenant in hosted Vespa. - * - * @author mpolden - */ -public class AthenzTenant extends Tenant { - - private final AthenzDomain domain; - private final Property property; - private final Optional<PropertyId> propertyId; - - /** - * This should only be used by serialization. - * Use {@link #create(TenantName, AthenzDomain, Property, Optional, Instant)}. - * */ - public AthenzTenant(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, - Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo) { - super(name, createdAt, lastLoginInfo, contact); - this.domain = Objects.requireNonNull(domain, "domain must be non-null"); - this.property = Objects.requireNonNull(property, "property must be non-null"); - this.propertyId = Objects.requireNonNull(propertyId, "propertyId must be non-null"); - } - - /** Property name of this tenant */ - public Property property() { - return property; - } - - /** Property ID of the tenant, if any */ - public Optional<PropertyId> propertyId() { - return propertyId; - } - - /** Athenz domain of this tenant */ - public AthenzDomain domain() { - return domain; - } - - /** Returns true if tenant is in given domain */ - public boolean in(AthenzDomain domain) { - return this.domain.equals(domain); - } - - @Override - public String toString() { - return "athenz tenant '" + name() + "'"; - } - - /** Create a new Athenz tenant */ - public static AthenzTenant create(TenantName name, AthenzDomain domain, Property property, - Optional<PropertyId> propertyId, Instant createdAt) { - return new AthenzTenant(requireName(name), domain, property, propertyId, Optional.empty(), createdAt, LastLoginInfo.EMPTY); - } - - @Override - public Type type() { - return Type.athenz; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java deleted file mode 100644 index 1060b118beb..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.tenant; - -import com.google.common.collect.BiMap; -import com.google.common.collect.ImmutableBiMap; -import com.yahoo.config.provision.TenantName; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; - -import java.security.Principal; -import java.security.PublicKey; -import java.time.Instant; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.regex.Pattern; - -/** - * A paying tenant in a Vespa cloud service. - * - * @author jonmv - */ -public class CloudTenant extends Tenant { - - private static final Pattern VALID_ARCHIVE_ACCESS_ROLE_PATTERN = Pattern.compile("arn:aws:iam::\\d{12}:.+"); - - private final Optional<Principal> creator; - private final BiMap<PublicKey, Principal> developerKeys; - private final TenantInfo info; - private final List<TenantSecretStore> tenantSecretStores; - private final Optional<String> archiveAccessRole; - - /** Public for the serialization layer — do not use! */ - public CloudTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator, - BiMap<PublicKey, Principal> developerKeys, TenantInfo info, - List<TenantSecretStore> tenantSecretStores, Optional<String> archiveAccessRole) { - super(name, createdAt, lastLoginInfo, Optional.empty()); - this.creator = creator; - this.developerKeys = developerKeys; - this.info = Objects.requireNonNull(info); - this.tenantSecretStores = tenantSecretStores; - this.archiveAccessRole = archiveAccessRole; - if (!archiveAccessRole.map(role -> VALID_ARCHIVE_ACCESS_ROLE_PATTERN.matcher(role).matches()).orElse(true)) - throw new IllegalArgumentException(Text.format("Invalid archive access role '%s': Must match expected pattern: '%s'", - archiveAccessRole.get(), VALID_ARCHIVE_ACCESS_ROLE_PATTERN.pattern())); - if (archiveAccessRole.map(role -> role.length() > 100).orElse(false)) - throw new IllegalArgumentException("Invalid archive access role too long, must be 100 or less characters"); - } - - /** Creates a tenant with the given name, provided it passes validation. */ - public static CloudTenant create(TenantName tenantName, Instant createdAt, Principal creator) { - return new CloudTenant(requireName(tenantName), - createdAt, - LastLoginInfo.EMPTY, - Optional.ofNullable(creator), - ImmutableBiMap.of(), TenantInfo.EMPTY, List.of(), Optional.empty()); - } - - /** The user that created the tenant */ - public Optional<Principal> creator() { - return creator; - } - - /** Legal name, addresses etc */ - public TenantInfo info() { - return info; - } - - /** An iam role which is allowed to access the S3 (log, dump) archive) */ - public Optional<String> archiveAccessRole() { - return archiveAccessRole; - } - - /** Returns the set of developer keys and their corresponding developers for this tenant. */ - public BiMap<PublicKey, Principal> developerKeys() { return developerKeys; } - - /** List of configured secret stores */ - public List<TenantSecretStore> tenantSecretStores() { - return tenantSecretStores; - } - - @Override - public Type type() { - return Type.cloud; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/LastLoginInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/LastLoginInfo.java deleted file mode 100644 index 15f2f97e7d1..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/LastLoginInfo.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.tenant; - -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -/** - * @author freva - */ -public class LastLoginInfo { - - public static final LastLoginInfo EMPTY = new LastLoginInfo(Map.of()); - - private final Map<UserLevel, Instant> lastLoginByUserLevel; - - public LastLoginInfo(Map<UserLevel, Instant> lastLoginByUserLevel) { - this.lastLoginByUserLevel = Map.copyOf(lastLoginByUserLevel); - } - - public Optional<Instant> get(UserLevel userLevel) { - return Optional.ofNullable(lastLoginByUserLevel.get(userLevel)); - } - - /** - * Returns new instance with updated last login time if the given {@code loginAt} timestamp is after the current - * for the given {@code userLevel}, otherwise returns this - */ - public LastLoginInfo withLastLoginIfLater(UserLevel userLevel, Instant loginAt) { - Instant lastLogin = lastLoginByUserLevel.getOrDefault(userLevel, Instant.EPOCH); - if (loginAt.isAfter(lastLogin)) { - Map<UserLevel, Instant> lastLoginByUserLevel = new HashMap<>(this.lastLoginByUserLevel); - lastLoginByUserLevel.put(userLevel, loginAt); - return new LastLoginInfo(lastLoginByUserLevel); - } - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - LastLoginInfo lastLoginInfo = (LastLoginInfo) o; - return lastLoginByUserLevel.equals(lastLoginInfo.lastLoginByUserLevel); - } - - @Override - public int hashCode() { - return lastLoginByUserLevel.hashCode(); - } - - public enum UserLevel { user, developer, administrator }; -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java deleted file mode 100644 index f8b54e7eff3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.tenant; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; - -import java.time.Instant; -import java.util.Objects; -import java.util.Optional; - -/** - * A tenant in hosted Vespa. - * - * @author mpolden - */ -public abstract class Tenant { - - private final TenantName name; - private final Instant createdAt; - private final LastLoginInfo lastLoginInfo; - private final Optional<Contact> contact; - - Tenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Contact> contact) { - this.name = name; - this.createdAt = createdAt; - this.lastLoginInfo = lastLoginInfo; - this.contact = contact; - } - - /** Name of this tenant */ - public TenantName name() { - return name; - } - - /** Instant when the tenant was created */ - public Instant createdAt() { - return createdAt; - } - - /** Returns login information for this tenant */ - public LastLoginInfo lastLoginInfo() { - return lastLoginInfo; - } - - /** Contact information for this tenant */ - public Optional<Contact> contact() { - return contact; - } - - public abstract Type type(); - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Tenant tenant = (Tenant) o; - return Objects.equals(name, tenant.name); - } - - @Override - public int hashCode() { - return Objects.hash(name); - } - - public static TenantName requireName(TenantName name) { - if ( ! name.value().matches("^(?=.{1,20}$)[a-z](-?[a-z0-9]+)*$")) { - throw new IllegalArgumentException("New tenant or application names must start with a letter, may " + - "contain no more than 20 characters, and may only contain lowercase " + - "letters, digits or dashes, but no double-dashes."); - } - return name; - } - - - public enum Type { - - /** Tenant authenticated through Athenz. */ - athenz, - - /** Tenant authenticated through some cloud identity provider. */ - cloud - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java deleted file mode 100644 index a20477d7aab..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.tenant; - -import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -/** - * Tenant information beyond technical tenant id and user authorizations. - * - * This info is used to capture generic support information and invoiced billing information. - * - * All fields are non null but strings can be empty - * - * @author smorgrav - */ -public class TenantInfo { - private final String name; - private final String email; - private final String website; - private final String contactName; - private final String contactEmail; - private final String invoiceEmail; - private final TenantInfoAddress address; - private final TenantInfoBillingContact billingContact; - - TenantInfo(String name, String email, String website, String contactName, String contactEmail, - String invoiceEmail, TenantInfoAddress address, TenantInfoBillingContact billingContact) { - this.name = Objects.requireNonNull(name); - this.email = Objects.requireNonNull(email); - this.website = Objects.requireNonNull(website); - this.contactName = Objects.requireNonNull(contactName); - this.contactEmail = Objects.requireNonNull(contactEmail); - this.invoiceEmail = Objects.requireNonNull(invoiceEmail); - this.address = Objects.requireNonNull(address); - this.billingContact = Objects.requireNonNull(billingContact); - } - - public static final TenantInfo EMPTY = new TenantInfo("","","", "", "", "", - TenantInfoAddress.EMPTY, TenantInfoBillingContact.EMPTY); - - public String name() { - return name; - } - - public String email() { - return email; - } - - public String website() { - return website; - } - - public String contactName() { - return contactName; - } - - public String contactEmail() { - return contactEmail; - } - - public String invoiceEmail() { - return invoiceEmail; - } - - public TenantInfoAddress address() { - return address; - } - - public TenantInfoBillingContact billingContact() { - return billingContact; - } - - public TenantInfo withName(String newName) { - return new TenantInfo(newName, email, website, contactName, contactEmail, invoiceEmail, address, billingContact); - } - - public TenantInfo withEmail(String newEmail) { - return new TenantInfo(name, newEmail, website, contactName, contactEmail, invoiceEmail, address, billingContact); - } - - public TenantInfo withWebsite(String newWebsite) { - return new TenantInfo(name, email, newWebsite, contactName, contactEmail, invoiceEmail, address, billingContact); - } - - public TenantInfo withContactName(String newContactName) { - return new TenantInfo(name, email, website, newContactName, contactEmail, invoiceEmail, address, billingContact); - } - - public TenantInfo withContactEmail(String newContactEmail) { - return new TenantInfo(name, email, website, contactName, newContactEmail, invoiceEmail, address, billingContact); - } - - public TenantInfo withInvoiceEmail(String newInvoiceEmail) { - return new TenantInfo(name, email, website, contactName, contactEmail, newInvoiceEmail, address, billingContact); - } - - public TenantInfo withAddress(TenantInfoAddress newAddress) { - return new TenantInfo(name, email, website, contactName, contactEmail, invoiceEmail, newAddress, billingContact); - } - - public TenantInfo withBillingContact(TenantInfoBillingContact newBillingContact) { - return new TenantInfo(name, email, website, contactName, contactEmail, invoiceEmail, address, newBillingContact); - } - - public boolean isEmpty() { - return this.equals(EMPTY); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TenantInfo that = (TenantInfo) o; - return name.equals(that.name) && - email.equals(that.email) && - website.equals(that.website) && - contactName.equals(that.contactName) && - contactEmail.equals(that.contactEmail) && - invoiceEmail.equals(that.invoiceEmail) && - address.equals(that.address) && - billingContact.equals(that.billingContact); - } - - @Override - public int hashCode() { - return Objects.hash(name, email, website, contactName, contactEmail, invoiceEmail, address, billingContact); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java deleted file mode 100644 index a12f351abd6..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.tenant; - -import java.util.Objects; - -/** - * Address formats are quite diverse across the world both in therms of what fields are used, named and - * the order of them. - * - * To be generic a little future proof the address fields here are a mix of free text (address lines) and fixed fields. - * The address lines can be street address, P.O box, c/o name, apartment, suite, unit, building floor etc etc. - * - * All fields are mandatory but can be an empty string (ie. not null) - * - * @author smorgrav - */ -public class TenantInfoAddress { - - private final String addressLines; - private final String postalCodeOrZip; - private final String city; - private final String stateRegionProvince; - private final String country; - - TenantInfoAddress(String addressLines, String postalCodeOrZip, String city, String country, String stateRegionProvince) { - this.addressLines = Objects.requireNonNull(addressLines);; - this.city = Objects.requireNonNull(city); - this.postalCodeOrZip = Objects.requireNonNull(postalCodeOrZip); - this.country = Objects.requireNonNull(country); - this.stateRegionProvince = Objects.requireNonNull(stateRegionProvince); - } - - public static final TenantInfoAddress EMPTY = new TenantInfoAddress("","","", "", ""); - - public String addressLines() { - return addressLines; - } - - public String postalCodeOrZip() { - return postalCodeOrZip; - } - - public String city() { - return city; - } - - public String country() { - return country; - } - - public String stateRegionProvince() { - return stateRegionProvince; - } - - public TenantInfoAddress withAddressLines(String newAddressLines) { - return new TenantInfoAddress(newAddressLines, postalCodeOrZip, city, country, stateRegionProvince); - } - - public TenantInfoAddress withPostalCodeOrZip(String newPostalCodeOrZip) { - return new TenantInfoAddress(addressLines, newPostalCodeOrZip, city, country, stateRegionProvince); - } - - public TenantInfoAddress withCity(String newCity) { - return new TenantInfoAddress(addressLines, postalCodeOrZip, newCity, country, stateRegionProvince); - } - - public TenantInfoAddress withCountry(String newCountry) { - return new TenantInfoAddress(addressLines, postalCodeOrZip, city, newCountry, stateRegionProvince); - } - - public TenantInfoAddress withStateRegionProvince(String newStateRegionProvince) { - return new TenantInfoAddress(addressLines, postalCodeOrZip, city, country, newStateRegionProvince); - } - - public boolean isEmpty() { - return this.equals(EMPTY); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TenantInfoAddress that = (TenantInfoAddress) o; - return addressLines.equals(that.addressLines) && - postalCodeOrZip.equals(that.postalCodeOrZip) && - city.equals(that.city) && - stateRegionProvince.equals(that.stateRegionProvince) && - country.equals(that.country); - } - - @Override - public int hashCode() { - return Objects.hash(addressLines, postalCodeOrZip, city, stateRegionProvince, country); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java deleted file mode 100644 index a00dd626f0a..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.tenant; - -import java.util.Objects; - -/** - * @author smorgrav - */ -public class TenantInfoBillingContact { - private final String name; - private final String email; - private final String phone; - private final TenantInfoAddress address; - - TenantInfoBillingContact(String name, String email, String phone, TenantInfoAddress address) { - this.name = Objects.requireNonNull(name); - this.email = Objects.requireNonNull(email); - this.phone = Objects.requireNonNull(phone); - this.address = Objects.requireNonNull(address); - } - - public static final TenantInfoBillingContact EMPTY = - new TenantInfoBillingContact("","", "", TenantInfoAddress.EMPTY); - - public String name() { - return name; - } - - public String email() { return email; } - - public String phone() { - return phone; - } - - public TenantInfoAddress address() { - return address; - } - - public TenantInfoBillingContact withName(String newName) { - return new TenantInfoBillingContact(newName, email, phone, address); - } - - public TenantInfoBillingContact withEmail(String newEmail) { - return new TenantInfoBillingContact(name, newEmail, phone, address); - } - - public TenantInfoBillingContact withPhone(String newPhone) { - return new TenantInfoBillingContact(name, email, newPhone, address); - } - - public TenantInfoBillingContact withAddress(TenantInfoAddress newAddress) { - return new TenantInfoBillingContact(name, email, phone, newAddress); - } - - public boolean isEmpty() { - return this.equals(EMPTY); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TenantInfoBillingContact that = (TenantInfoBillingContact) o; - return name.equals(that.name) && - email.equals(that.email) && - phone.equals(that.phone) && - address.equals(that.address); - } - - @Override - public int hashCode() { - return Objects.hash(name, email, phone, address); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/package-info.java deleted file mode 100644 index 9218bfcd850..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author mpolden - */ -@ExportPackage -package com.yahoo.vespa.hosted.controller.tenant; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java index 1ac82317695..a08ec77bbb9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.tls; import com.google.inject.Inject; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/package-info.java index 508b8cc9423..e0edf2c2100 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/package-info.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/package-info.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * @author mpolden */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java index df018d64748..5d244875ae5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java index 2671f30255e..dadca43c5a0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index 78c7b36181e..92351df9430 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.maintenance.OsUpgrader; @@ -71,7 +72,7 @@ public class OsVersionStatus { .osVersion(application.nodeType()) .orElse(Version.emptyVersion); - for (var node : controller.serviceRegistry().configServer().nodeRepository().list(zone.getVirtualId(), application.id())) { + for (var node : controller.serviceRegistry().configServer().nodeRepository().list(zone.getVirtualId(), NodeFilter.all().applications(application.id()))) { if (!OsUpgrader.canUpgrade(node)) continue; Optional<Instant> suspendedAt = node.suspendedSince(); NodeVersion nodeVersion = new NodeVersion(node.hostname(), zone.getVirtualId(), node.currentOsVersion(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java index 5dd9f4d0685..badb3014ddf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; import org.jetbrains.annotations.NotNull; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index beefaadb2a1..1833e7b3b35 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.maintenance.SystemUpgrader; @@ -156,7 +157,7 @@ public class VersionStatus { for (var zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) { for (var application : SystemApplication.notController()) { var nodes = controller.serviceRegistry().configServer().nodeRepository() - .list(zone.getId(), application.id()).stream() + .list(zone.getId(), NodeFilter.all().applications(application.id())).stream() .filter(SystemUpgrader::eligibleForUpgrade) .collect(Collectors.toList()); if (nodes.isEmpty()) continue; diff --git a/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.athenz.config.athenz.def b/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.athenz.config.athenz.def index dc1a2337aaf..672e773591e 100644 --- a/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.athenz.config.athenz.def +++ b/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.athenz.config.athenz.def @@ -1,4 +1,4 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=vespa.hosted.controller.athenz.config # URL to ZMS API endpoint diff --git a/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.config.controller.def b/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.config.controller.def index 069deaf276d..c63e429abcd 100644 --- a/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.config.controller.def +++ b/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.config.controller.def @@ -1,4 +1,4 @@ -# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Generic config for controller namespace=vespa.hosted.controller.config diff --git a/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.tls.config.tls.def b/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.tls.config.tls.def index 1a163ba1fff..7130a5c5dc7 100644 --- a/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.tls.config.tls.def +++ b/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.tls.config.tls.def @@ -1,4 +1,4 @@ -# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=vespa.hosted.controller.tls.config # Path to the CA trust store diff --git a/controller-server/src/main/resources/configdefinitions/vespa.hosted.rotation.config.rotations.def b/controller-server/src/main/resources/configdefinitions/vespa.hosted.rotation.config.rotations.def index d4f3636d0d8..77863e65a55 100644 --- a/controller-server/src/main/resources/configdefinitions/vespa.hosted.rotation.config.rotations.def +++ b/controller-server/src/main/resources/configdefinitions/vespa.hosted.rotation.config.rotations.def @@ -1,4 +1,4 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=vespa.hosted.rotation.config rotations{} string diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index e52d1900a9d..887e51367dc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.google.common.collect.Sets; @@ -26,7 +26,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.Endpoint; @@ -114,7 +114,7 @@ public class ControllerTest { context.runJob(stagingTest); // production job succeeding now - context.jobAborted(productionUsWest1); + context.triggerJobs().jobAborted(productionUsWest1); context.runJob(productionUsWest1); // causes triggering of next production job diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index bae0867736a..445a1c73297 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -57,6 +57,7 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import com.yahoo.yolean.concurrent.Sleeper; import java.time.Duration; import java.time.Instant; @@ -387,7 +388,8 @@ public final class ControllerTester { new MockMavenRepository(), serviceRegistry, new MetricsMock(), new SecretStoreMock(), - new ControllerConfig.Builder().build()); + new ControllerConfig.Builder().build(), + Sleeper.NOOP); // Calculate initial versions controller.updateVersionStatus(VersionStatus.compute(controller)); return controller; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java index 91d71a1aa25..2751b1697db 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiffTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiffTest.java new file mode 100644 index 00000000000..b2aba721a6f --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiffTest.java @@ -0,0 +1,128 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Map; +import java.util.zip.Deflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff.diff; + +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff.diffAgainstEmpty; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class ApplicationPackageDiffTest { + private static final ApplicationPackage app1 = applicationPackage(Map.of("file1", "contents of the\nfirst file", "dir/myfile", "Second file", "dir/binary", "øøøø")); + private static final ApplicationPackage app2 = applicationPackage(Map.of("file1", "updated contents\nof the\nfirst file\nafter some changes", "dir/myfile2", "Second file", "dir/binary", "øøøø")); + + @Test + public void no_diff() { + assertEquals("No diff\n", new String(diff(app1, app1))); + } + + @Test + public void diff_against_empty() { + assertEquals("--- dir/binary\n" + + "Diff skipped: File is binary (new file -> 8B)\n" + + "\n" + + "--- dir/myfile\n" + + "@@ -1,0 +1,1 @@\n" + + "+ Second file\n" + + "\n" + + "--- file1\n" + + "@@ -1,0 +1,2 @@\n" + + "+ contents of the\n" + + "+ first file\n" + + "\n", new String(diffAgainstEmpty(app1))); + } + + @Test + public void full_diff() { + // Even though dir/binary is binary file, we can see they are identical, so it should not print "Diff skipped" + assertEquals("--- dir/myfile\n" + + "@@ -1,1 +1,0 @@\n" + + "- Second file\n" + + "\n" + + "--- dir/myfile2\n" + + "@@ -1,0 +1,1 @@\n" + + "+ Second file\n" + + "\n" + + "--- file1\n" + + "@@ -1,2 +1,4 @@\n" + + "+ updated contents\n" + + "+ of the\n" + + "- contents of the\n" + + " first file\n" + + "+ after some changes\n" + + "\n", new String(diff(app1, app2))); + } + + @Test + public void skips_diff_for_too_large_files() { + assertEquals("--- dir/myfile\n" + + "@@ -1,1 +1,0 @@\n" + + "- Second file\n" + + "\n" + + "--- dir/myfile2\n" + + "@@ -1,0 +1,1 @@\n" + + "+ Second file\n" + + "\n" + + "--- file1\n" + + "Diff skipped: File too large (26B -> 53B)\n" + + "\n", new String(diff(app1, app2, 12, 1000, 1000))); + } + + @Test + public void skips_diff_if_file_diff_is_too_large() { + assertEquals("--- dir/myfile\n" + + "@@ -1,1 +1,0 @@\n" + + "- Second file\n" + + "\n" + + "--- dir/myfile2\n" + + "@@ -1,0 +1,1 @@\n" + + "+ Second file\n" + + "\n" + + "--- file1\n" + + "Diff skipped: Diff too large (96B)\n" + + "\n", new String(diff(app1, app2, 1000, 50, 1000))); + } + + @Test + public void skips_diff_if_total_diff_is_too_large() { + assertEquals("--- dir/myfile\n" + + "@@ -1,1 +1,0 @@\n" + + "- Second file\n" + + "\n" + + "--- dir/myfile2\n" + + "Diff skipped: Total diff size >20B)\n" + + "\n" + + "--- file1\n" + + "Diff skipped: Total diff size >20B)\n" + + "\n", new String(diff(app1, app2, 1000, 1000, 20))); + } + + private static ApplicationPackage applicationPackage(Map<String, String> files) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream out = new ZipOutputStream(baos)) { + out.setLevel(Deflater.NO_COMPRESSION); // This is for testing purposes so we skip compression for performance + for (Map.Entry<String, String> file : files.entrySet()) { + ZipEntry entry = new ZipEntry(file.getKey()); + out.putNextEntry(entry); + out.write(file.getValue().getBytes(UTF_8)); + out.closeEntry(); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return new ApplicationPackage(baos.toByteArray()); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index 1849be9b6bd..2ee03457046 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -1,5 +1,5 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; @@ -109,10 +109,10 @@ public class ApplicationPackageTest { } private static Map<String, String> unzip(byte[] zip) { - return new ZipStreamReader(new ByteArrayInputStream(zip), __ -> true, 1 << 10) + return new ZipStreamReader(new ByteArrayInputStream(zip), __ -> true, 1 << 10, true) .entries().stream() .collect(Collectors.toMap(entry -> entry.zipEntry().getName(), - entry -> new String(entry.content(), UTF_8))); + entry -> new String(entry.contentOrThrow(), UTF_8))); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/LinesComparatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/LinesComparatorTest.java new file mode 100644 index 00000000000..92137094f62 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/LinesComparatorTest.java @@ -0,0 +1,112 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; + +import org.junit.Test; + +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +public class LinesComparatorTest { + private static final String text1 = "This part of the\n" + + "document has stayed the\n" + + "same from version to\n" + + "version. It shouldn't\n" + + "be shown if it doesn't\n" + + "change. Otherwise, that\n" + + "would not be helping to\n" + + "compress the size of the\n" + + "changes.\n" + + "\n" + + "This paragraph contains\n" + + "text that is outdated.\n" + + "It will be deleted in the\n" + + "near future.\n" + + "\n" + + "It is important to spell\n" + + "check this dokument. On\n" + + "the other hand, a\n" + + "misspelled word isn't\n" + + "the end of the world.\n" + + "Nothing in the rest of\n" + + "this paragraph needs to\n" + + "be changed. Things can\n" + + "be added after it."; + private static final String text2 = "This is an important\n" + + "notice! It should\n" + + "therefore be located at\n" + + "the beginning of this\n" + + "document!\n" + + "\n" + + "This part of the\n" + + "document has stayed the\n" + + "same from version to\n" + + "version. It shouldn't\n" + + "be shown if it doesn't\n" + + "change. Otherwise, that\n" + + "would not be helping to\n" + + "compress the size of the\n" + + "changes.\n" + + "\n" + + "It is important to spell\n" + + "check this document. On\n" + + "the other hand, a\n" + + "misspelled word isn't\n" + + "the end of the world.\n" + + "Nothing in the rest of\n" + + "this paragraph needs to\n" + + "be changed. Things can\n" + + "be added after it.\n" + + "\n" + + "This paragraph contains\n" + + "important new additions\n" + + "to this document."; + + @Test + public void diff_test() { + assertDiff(null, "", ""); + assertDiff(null, text1, text1); + assertDiff(text1.lines().map(line -> "- " + line).collect(Collectors.joining("\n", "@@ -1,24 +1,0 @@\n", "\n")), text1, ""); + assertDiff(text1.lines().map(line -> "+ " + line).collect(Collectors.joining("\n", "@@ -1,0 +1,24 @@\n", "\n")), "", text1); + assertDiff("@@ -1,3 +1,9 @@\n" + + "+ This is an important\n" + + "+ notice! It should\n" + + "+ therefore be located at\n" + + "+ the beginning of this\n" + + "+ document!\n" + + "+ \n" + + " This part of the\n" + + " document has stayed the\n" + + " same from version to\n" + + "@@ -7,14 +13,9 @@\n" + + " would not be helping to\n" + + " compress the size of the\n" + + " changes.\n" + + "- \n" + + "- This paragraph contains\n" + + "- text that is outdated.\n" + + "- It will be deleted in the\n" + + "- near future.\n" + + " \n" + + " It is important to spell\n" + + "+ check this document. On\n" + + "- check this dokument. On\n" + + " the other hand, a\n" + + " misspelled word isn't\n" + + " the end of the world.\n" + + "@@ -22,3 +23,7 @@\n" + + " this paragraph needs to\n" + + " be changed. Things can\n" + + " be added after it.\n" + + "+ \n" + + "+ This paragraph contains\n" + + "+ important new additions\n" + + "+ to this document.\n", text1, text2); + } + + private static void assertDiff(String expected, String left, String right) { + assertEquals(Optional.ofNullable(expected), + LinesComparator.diff(left.lines().collect(Collectors.toList()), right.lines().collect(Collectors.toList()))); + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReaderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java index abd234f0fa4..afbd232f01c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReaderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java @@ -1,5 +1,5 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +package com.yahoo.vespa.hosted.controller.application.pkg; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; @@ -38,15 +38,15 @@ public class ZipStreamReaderTest { public void test_size_limit() { Map<String, String> entries = Map.of("foo.xml", "foobar"); try { - new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals, 1); + new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals, 1, true); fail("Expected exception"); } catch (IllegalArgumentException ignored) {} entries = Map.of("foo.xml", "foobar", "foo.jar", "0".repeat(100) // File not extracted and thus not subject to size limit ); - ZipStreamReader reader = new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals,10); - byte[] extracted = reader.entries().get(0).content(); + ZipStreamReader reader = new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals, 10, true); + byte[] extracted = reader.entries().get(0).contentOrThrow(); assertEquals("foobar", new String(extracted, StandardCharsets.UTF_8)); } @@ -65,7 +65,7 @@ public class ZipStreamReaderTest { ); tests.forEach((name, expectException) -> { try { - new ZipStreamReader(new ByteArrayInputStream(zip(Map.of(name, "foo"))), name::equals, 1024); + new ZipStreamReader(new ByteArrayInputStream(zip(Map.of(name, "foo"))), name::equals, 1024, true); assertFalse("Expected exception for '" + name + "'", expectException); } catch (IllegalArgumentException ignored) { assertTrue("Unexpected exception for '" + name + "'", expectException); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index a3580a9fda3..a509c457111 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.certificate; import com.yahoo.config.application.api.DeploymentSpec; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/concurrent/OnceTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/concurrent/OnceTest.java index e11fdcba7c6..e9426e96a80 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/concurrent/OnceTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/concurrent/OnceTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.concurrent; import org.junit.Test; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 808409cf793..64821756105 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; @@ -10,7 +10,7 @@ import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import javax.security.auth.x500.X500Principal; import java.io.ByteArrayOutputStream; @@ -53,6 +53,7 @@ public class ApplicationPackageBuilder { private OptionalInt majorVersion = OptionalInt.empty(); private String instances = "default"; private String upgradePolicy = null; + private String upgradeRollout = null; private String globalServiceId = null; private String athenzIdentityAttributes = null; private String searchDefinition = "search test { }"; @@ -75,6 +76,11 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder upgradeRollout(String upgradeRollout) { + this.upgradeRollout = upgradeRollout; + return this; + } + public ApplicationPackageBuilder globalServiceId(String globalServiceId) { this.globalServiceId = globalServiceId; return this; @@ -221,10 +227,11 @@ public class ApplicationPackageBuilder { } xml.append(">\n"); xml.append(" <instance id='").append(instances).append("'>\n"); - if (upgradePolicy != null) { - xml.append(" <upgrade policy='"); - xml.append(upgradePolicy); - xml.append("'/>\n"); + if (upgradePolicy != null || upgradeRollout != null) { + xml.append(" <upgrade "); + if (upgradePolicy != null) xml.append("policy='").append(upgradePolicy).append("' "); + if (upgradeRollout != null) xml.append("rollout='").append(upgradeRollout).append("' "); + xml.append("/>\n"); } xml.append(notifications); if (explicitSystemTest) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 5d6f1965009..83e1a019109 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.base.Supplier; @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -27,7 +28,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -192,7 +193,8 @@ public class DeploymentContext { for (var spec : application().deploymentSpec().instances()) for (JobType type : new DeploymentSteps(spec, tester.controller()::system).productionJobs()) assertTrue(tester.configServer().nodeRepository() - .list(type.zone(tester.controller().system()), applicationId.defaultInstance()).stream() // TODO jonmv: support more + .list(type.zone(tester.controller().system()), + NodeFilter.all().applications(applicationId.defaultInstance())).stream() // TODO jonmv: support more .allMatch(node -> node.currentVersion().equals(version))); assertFalse(instance().change().hasTargets()); @@ -321,7 +323,7 @@ public class DeploymentContext { /** Runs a deployment of the given package to the given dev/perf job, on the given version. */ public DeploymentContext runJob(JobType type, ApplicationPackage applicationPackage, Version vespaVersion) { - jobs.deploy(instanceId, type, Optional.ofNullable(vespaVersion), applicationPackage); + jobs.deploy(instanceId, type, Optional.ofNullable(vespaVersion), applicationPackage, false); return runJob(type); } @@ -490,8 +492,8 @@ public class DeploymentContext { Run run = jobs.last(job) .filter(r -> r.id().type() == job.type()) .orElseThrow(() -> new AssertionError(job.type() + " is not among the active: " + jobs.active())); - assertFalse(run.id() + " should not have failed yet", run.hasFailed()); - assertFalse(run.id() + " should not have ended yet", run.hasEnded()); + assertFalse(run.id() + " should not have failed yet: " + run, run.hasFailed()); + assertFalse(run.id() + " should not have ended yet: " + run, run.hasEnded()); return run; } @@ -543,7 +545,7 @@ public class DeploymentContext { assertTrue(jobs.run(id).get().hasEnded()); assertFalse(jobs.run(id).get().hasFailed()); assertEquals(job.type().isProduction(), instance().deployments().containsKey(zone)); - assertTrue(configServer().nodeRepository().list(zone, TesterId.of(id.application()).id()).isEmpty()); + assertTrue(configServer().nodeRepository().list(zone, NodeFilter.all().applications(TesterId.of(id.application()).id())).isEmpty()); } private JobId jobId(JobType type) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index b939598c704..0f32d01ffe3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 7077e14a648..353756b3a4f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; @@ -8,7 +8,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Assert; @@ -30,7 +30,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApSoutheast1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionAwsUsEast1a; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdAwsUsEast1a; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdUsCentral1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdUsEast1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionEuWest1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; @@ -61,7 +61,7 @@ import static org.junit.Assert.assertTrue; */ public class DeploymentTriggerTest { - private DeploymentTester tester = new DeploymentTester(); + private final DeploymentTester tester = new DeploymentTester(); @Test public void testTriggerFailing() { @@ -107,6 +107,27 @@ public class DeploymentTriggerTest { } @Test + public void leadingUpgradeAllowsApplicationChangeWhileUpgrading() { + var applicationPackage = new ApplicationPackageBuilder().region("us-east-3") + .upgradeRollout("leading") + .build(); + var app = tester.newDeploymentContext(); + + app.submit(applicationPackage).deploy(); + + Change upgrade = Change.of(new Version("7.8.9")); + tester.controllerTester().upgradeSystem(upgrade.platform().get()); + tester.upgrader().maintain(); + app.runJob(systemTest).runJob(stagingTest); + tester.triggerJobs(); + app.assertRunning(productionUsEast3); + assertEquals(upgrade, app.instance().change()); + + app.submit(applicationPackage); + assertEquals(upgrade.with(app.lastSubmission().get()), app.instance().change()); + } + + @Test public void abortsJobsOnNewApplicationChange() { var app = tester.newDeploymentContext(); app.submit() @@ -120,15 +141,15 @@ public class DeploymentTriggerTest { app.submit(); assertTrue(tester.jobs().active(id).isPresent()); + tester.triggerJobs(); tester.runner().run(); - assertFalse(tester.jobs().active(id).isPresent()); + assertTrue(tester.jobs().active(id).isPresent()); // old run + app.runJob(systemTest).runJob(stagingTest).runJob(stagingTest); // outdated run is aborted when otherwise blocking a new run tester.triggerJobs(); - assertEquals(EnumSet.of(systemTest, stagingTest), tester.jobs().active().stream() - .map(run -> run.id().type()) - .collect(Collectors.toCollection(() -> EnumSet.noneOf(JobType.class)))); + app.jobAborted(productionUsCentral1); - app.deploy(); + app.runJob(productionUsCentral1).runJob(productionUsWest1).runJob(productionUsEast3); assertEquals(Change.empty(), app.instance().change()); tester.controllerTester().upgradeSystem(new Version("8.9")); @@ -138,6 +159,8 @@ public class DeploymentTriggerTest { // Jobs are not aborted when the new submission remains outstanding. app.submit(); + app.runJob(systemTest).runJob(stagingTest); + tester.triggerJobs(); tester.runner().run(); assertEquals(EnumSet.of(productionUsCentral1), tester.jobs().active().stream() .map(run -> run.id().type()) @@ -468,8 +491,9 @@ public class DeploymentTriggerTest { Version version2 = Version.fromString("7.2"); tester.controllerTester().upgradeSystem(version2); tester.upgrader().maintain(); - app1.runJob(systemTest).runJob(stagingTest) // tests for previous version — these are "reused" later. - .runJob(systemTest).runJob(stagingTest).timeOutConvergence(productionUsCentral1); + tester.triggerJobs(); + app1.jobAborted(systemTest).jobAborted(stagingTest); + app1.runJob(systemTest).runJob(stagingTest).timeOutConvergence(productionUsCentral1); assertEquals(version2, app1.deployment(productionUsCentral1.zone(main)).version()); Instant triggered = app1.instanceJobs().get(productionUsCentral1).lastTriggered().get().start(); tester.clock().advance(Duration.ofHours(1)); @@ -481,7 +505,8 @@ public class DeploymentTriggerTest { assertEquals("Change becomes latest non-broken version", Change.of(version1), app1.instance().change()); // version1 proceeds 'til the last job, where it fails; us-central-1 is skipped, as current change is strictly dominated by what's deployed there. - app1.failDeployment(productionEuWest1); + app1.runJob(systemTest).runJob(stagingTest) + .failDeployment(productionEuWest1); assertEquals(triggered, app1.instanceJobs().get(productionUsCentral1).lastTriggered().get().start()); // Roll out a new application version, which gives a dual change -- this should trigger us-central-1, but only as long as it hasn't yet deployed there. @@ -548,7 +573,7 @@ public class DeploymentTriggerTest { app.runJob(systemTest).runJob(stagingTest); // Finish old run of the aborted production job. - app.jobAborted(productionUsEast3); + app.triggerJobs().jobAborted(productionUsEast3); // New upgrade is already tested for both jobs. @@ -701,16 +726,15 @@ public class DeploymentTriggerTest { // Finish deployment for apps 2 and 3, then release a new version, leaving only app1 with an application upgrade. app2.deploy(); app3.deploy(); - app1.assertRunning(stagingTest); - assertEquals(1, tester.jobs().active().size()); + app1.runJob(stagingTest); + assertEquals(0, tester.jobs().active().size()); tester.controllerTester().upgradeSystem(new Version("6.2")); tester.upgrader().maintain(); app1.submit(applicationPackage); - app1.jobAborted(stagingTest); // Tests for app1 trigger before the others since it carries an application upgrade. - tester.readyJobsTrigger().maintain(); + tester.readyJobsTrigger().run(); app1.assertRunning(systemTest); app1.assertRunning(stagingTest); assertEquals(2, tester.jobs().active().size()); @@ -1055,12 +1079,12 @@ public class DeploymentTriggerTest { @Test public void mixedDirectAndPipelineJobsInProduction() { - ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("cd-us-central-1") + ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("cd-us-east-1") .region("cd-aws-us-east-1a") .build(); - var zones = List.of(ZoneId.from("test.cd-us-central-1"), - ZoneId.from("staging.cd-us-central-1"), - ZoneId.from("prod.cd-us-central-1"), + var zones = List.of(ZoneId.from("test.cd-us-west-1"), + ZoneId.from("staging.cd-us-west-1"), + ZoneId.from("prod.cd-us-east-1"), ZoneId.from("prod.cd-aws-us-east-1a")); tester.controllerTester() .setZones(zones, SystemName.cd) @@ -1069,35 +1093,36 @@ public class DeploymentTriggerTest { tester.controllerTester().computeVersionStatus(); var app = tester.newDeploymentContext(); - app.runJob(productionCdUsCentral1, cdPackage); + app.runJob(productionCdUsEast1, cdPackage); app.submit(cdPackage); app.runJob(systemTest); // Staging test requires unknown initial version, and is broken. - tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); - app.runJob(productionCdUsCentral1) + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false); + app.runJob(productionCdUsEast1) .abortJob(stagingTest) // Complete failing run. .runJob(stagingTest) .runJob(productionCdAwsUsEast1a); - app.runJob(productionCdUsCentral1, cdPackage); + app.runJob(productionCdUsEast1, cdPackage); var version = new Version("7.1"); tester.controllerTester().upgradeSystem(version); tester.upgrader().maintain(); // System and staging tests both require unknown versions, and are broken. - tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); - app.runJob(productionCdUsCentral1) - .abortJob(systemTest) - .abortJob(stagingTest) + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false); + app.runJob(productionCdUsEast1) + .jobAborted(systemTest) + .jobAborted(stagingTest) .runJob(systemTest) .runJob(stagingTest) .runJob(productionCdAwsUsEast1a); - app.runJob(productionCdUsCentral1, cdPackage); + app.runJob(productionCdUsEast1, cdPackage); app.submit(cdPackage); - app.runJob(systemTest); + app.jobAborted(systemTest) + .runJob(systemTest); // Staging test requires unknown initial version, and is broken. - tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); - app.runJob(productionCdUsCentral1) + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false); + app.runJob(productionCdUsEast1) .jobAborted(stagingTest) .runJob(stagingTest) .runJob(productionCdAwsUsEast1a); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index d8bddac4187..780d2d226f3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -20,11 +20,12 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbi import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; @@ -194,13 +195,13 @@ public class InternalStepRunnerTest { assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester)); Node systemTestNode = tester.configServer().nodeRepository().list(JobType.systemTest.zone(system()), - app.instanceId()).iterator().next(); + NodeFilter.all().applications(app.instanceId())).iterator().next(); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().minus(Duration.ofSeconds(1))); tester.configServer().nodeRepository().putNodes(JobType.systemTest.zone(system()), - new Node.Builder(systemTestNode) - .serviceState(Node.ServiceState.allowedDown) - .suspendedSince(tester.clock().instant()) - .build()); + Node.builder(systemTestNode) + .serviceState(Node.ServiceState.allowedDown) + .suspendedSince(tester.clock().instant()) + .build()); tester.runner().run(); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/QuotaUsageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/QuotaUsageTest.java index d2901aeac97..5ce6c95ee82 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/QuotaUsageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/QuotaUsageTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.zone.ZoneId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java index 6fb6004c60f..fd2de2e1c1d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import org.junit.Test; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java index c187552b0b3..44bcbcfb4fa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; import com.yahoo.config.provision.HostName; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java index 59e2b6c04d8..a1e7dd4efe3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java @@ -1,15 +1,15 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import java.time.Instant; import java.util.Map; @@ -30,7 +30,9 @@ public class ApplicationStoreMock implements ApplicationStore { private static final byte[] tombstone = new byte[0]; private final Map<ApplicationId, Map<ApplicationVersion, byte[]>> store = new ConcurrentHashMap<>(); - private final Map<ApplicationId, Map<ZoneId, byte[]>> devStore = new ConcurrentHashMap<>(); + private final Map<DeploymentId, byte[]> devStore = new ConcurrentHashMap<>(); + private final Map<ApplicationId, Map<Long, byte[]>> diffs = new ConcurrentHashMap<>(); + private final Map<DeploymentId, Map<Long, byte[]>> devDiffs = new ConcurrentHashMap<>(); private final Map<ApplicationId, NavigableMap<Instant, byte[]>> meta = new ConcurrentHashMap<>(); private final Map<DeploymentId, NavigableMap<Instant, byte[]>> metaManual = new ConcurrentHashMap<>(); @@ -43,15 +45,30 @@ public class ApplicationStoreMock implements ApplicationStore { } @Override - public byte[] get(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion) { - byte[] bytes = store.get(appId(tenant, application)).get(applicationVersion); + public byte[] get(DeploymentId deploymentId, ApplicationVersion applicationVersion) { + if (applicationVersion.isDeployedDirectly()) + return requireNonNull(devStore.get(deploymentId)); + + TenantAndApplicationId tenantAndApplicationId = TenantAndApplicationId.from(deploymentId.applicationId()); + byte[] bytes = store.get(appId(tenantAndApplicationId.tenant(), tenantAndApplicationId.application())).get(applicationVersion); if (bytes == null) - throw new IllegalArgumentException("No application package found for " + tenant + "." + application + + throw new IllegalArgumentException("No application package found for " + tenantAndApplicationId + " with version " + applicationVersion.id()); return bytes; } @Override + public Optional<byte[]> getDiff(TenantName tenantName, ApplicationName applicationName, long buildNumber) { + return Optional.ofNullable(diffs.get(appId(tenantName, applicationName))).map(map -> map.get(buildNumber)); + } + + @Override + public void pruneDiffs(TenantName tenantName, ApplicationName applicationName, long beforeBuildNumber) { + Optional.ofNullable(diffs.get(appId(tenantName, applicationName))) + .ifPresent(map -> map.keySet().removeIf(buildNumber -> buildNumber < beforeBuildNumber)); + } + + @Override public Optional<byte[]> find(TenantName tenant, ApplicationName application, long buildNumber) { return store.getOrDefault(appId(tenant, application), Map.of()).entrySet().stream() .filter(kv -> kv.getKey().buildNumber().orElse(Long.MIN_VALUE) == buildNumber) @@ -60,9 +77,10 @@ public class ApplicationStoreMock implements ApplicationStore { } @Override - public void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage) { - store.putIfAbsent(appId(tenant, application), new ConcurrentHashMap<>()); - store.get(appId(tenant, application)).put(applicationVersion, applicationPackage); + public void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage, byte[] diff) { + store.computeIfAbsent(appId(tenant, application), __ -> new ConcurrentHashMap<>()).put(applicationVersion, applicationPackage); + applicationVersion.buildNumber().ifPresent(buildNumber -> + diffs.computeIfAbsent(appId(tenant, application), __ -> new ConcurrentHashMap<>()).put(buildNumber, diff)); } @Override @@ -83,8 +101,8 @@ public class ApplicationStoreMock implements ApplicationStore { @Override public void putTester(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] testerPackage) { - store.putIfAbsent(testerId(tenant, application), new ConcurrentHashMap<>()); - store.get(testerId(tenant, application)).put(applicationVersion, testerPackage); + store.computeIfAbsent(testerId(tenant, application), key -> new ConcurrentHashMap<>()) + .put(applicationVersion, testerPackage); } @Override @@ -99,14 +117,21 @@ public class ApplicationStoreMock implements ApplicationStore { } @Override - public void putDev(ApplicationId application, ZoneId zone, byte[] applicationPackage) { - devStore.putIfAbsent(application, new ConcurrentHashMap<>()); - devStore.get(application).put(zone, applicationPackage); + public Optional<byte[]> getDevDiff(DeploymentId deploymentId, long buildNumber) { + return Optional.ofNullable(devDiffs.get(deploymentId)).map(map -> map.get(buildNumber)); + } + + @Override + public void pruneDevDiffs(DeploymentId deploymentId, long beforeBuildNumber) { + Optional.ofNullable(devDiffs.get(deploymentId)) + .ifPresent(map -> map.keySet().removeIf(buildNumber -> buildNumber < beforeBuildNumber)); } @Override - public byte[] getDev(ApplicationId application, ZoneId zone) { - return requireNonNull(devStore.get(application).get(zone)); + public void putDev(DeploymentId deploymentId, ApplicationVersion applicationVersion, byte[] applicationPackage, byte[] diff) { + devStore.put(deploymentId, applicationPackage); + applicationVersion.buildNumber().ifPresent(buildNumber -> + devDiffs.computeIfAbsent(deploymentId, __ -> new ConcurrentHashMap<>()).put(buildNumber, diff)); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java index 1cbda9c165f..aa4cabb4fe8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.component.AbstractComponent; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/AthenzFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/AthenzFilterMock.java index 6cc6ca012c7..ea7521e8250 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/AthenzFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/AthenzFilterMock.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.fasterxml.jackson.core.JsonProcessingException; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 92c8cbc4889..cbdf5dcb075 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -31,6 +31,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerE import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ProxyResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.QuotaUsage; @@ -39,7 +40,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.ClusterView; @@ -63,10 +64,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BooleanSupplier; import java.util.function.Consumer; -import java.util.function.Function; import java.util.function.Supplier; import java.util.logging.Level; import java.util.stream.Collectors; @@ -136,25 +134,25 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer new com.yahoo.vespa.hosted.controller.api.integration.configserver.Application(application, List.of(cluster))); - Node parent = nodeRepository().list(zone, SystemApplication.tenantHost.id()).stream().findAny() + Node parent = nodeRepository().list(zone, NodeFilter.all().applications(SystemApplication.tenantHost.id())).stream().findAny() .orElseThrow(() -> new IllegalStateException("No parent hosts in " + zone)); - nodeRepository().putNodes(zone, new Node.Builder().hostname(hostFor(application, zone)) - .state(Node.State.reserved) - .type(NodeType.tenant) - .owner(application) - .parentHostname(parent.hostname()) - .currentVersion(initialVersion) - .wantedVersion(initialVersion) - .currentDockerImage(initialDockerImage) - .wantedDockerImage(initialDockerImage) - .currentOsVersion(Version.emptyVersion) - .wantedOsVersion(Version.emptyVersion) - .resources(new NodeResources(2, 8, 50, 1, slow, remote)) - .serviceState(Node.ServiceState.unorchestrated) - .flavor("d-2-8-50") - .clusterId(clusterId.value()) - .clusterType(Node.ClusterType.container) - .build()); + nodeRepository().putNodes(zone, Node.builder().hostname(hostFor(application, zone)) + .state(Node.State.reserved) + .type(NodeType.tenant) + .owner(application) + .parentHostname(parent.hostname()) + .currentVersion(initialVersion) + .wantedVersion(initialVersion) + .currentDockerImage(initialDockerImage) + .wantedDockerImage(initialDockerImage) + .currentOsVersion(Version.emptyVersion) + .wantedOsVersion(Version.emptyVersion) + .resources(new NodeResources(2, 8, 50, 1, slow, remote)) + .serviceState(Node.ServiceState.unorchestrated) + .flavor("d-2-8-50") + .clusterId(clusterId.value()) + .clusterType(Node.ClusterType.container) + .build()); } public HostName hostFor(ApplicationId application, ZoneId zone) { @@ -174,16 +172,16 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer for (ZoneId zone : zones) { for (SystemApplication application : applications) { for (int i = 1; i <= 3; i++) { - Node node = new Node.Builder() - .hostname(HostName.from("node-" + i + "-" + application.id().application() + Node node = Node.builder() + .hostname(HostName.from("node-" + i + "-" + application.id().application() .value() + "-" + zone.value())) - .state(Node.State.active) - .type(application.nodeType()) - .owner(application.id()) - .currentVersion(initialVersion).wantedVersion(initialVersion) - .currentOsVersion(Version.emptyVersion).wantedOsVersion(Version.emptyVersion) - .build(); - nodeRepository().putNode(zone, node); + .state(Node.State.active) + .type(application.nodeType()) + .owner(application.id()) + .currentVersion(initialVersion).wantedVersion(initialVersion) + .currentOsVersion(Version.emptyVersion).wantedOsVersion(Version.emptyVersion) + .build(); + nodeRepository().putNodes(zone, node); } convergeServices(application.id(), zone); } @@ -192,7 +190,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer /** Converge all services belonging to the given application */ public void convergeServices(ApplicationId application, ZoneId zone) { - List<Node> nodes = nodeRepository.list(zone, application); + List<Node> nodes = nodeRepository.list(zone, NodeFilter.all().applications(application)); serviceStatus.put(new DeploymentId(application, zone), new ServiceConvergence(application, zone, true, @@ -222,7 +220,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer /** Set version for an application in a given zone */ public void setVersion(Version version, ApplicationId application, ZoneId zone) { - setVersion(zone, nodeRepository.list(zone, application), version, false); + setVersion(zone, nodeRepository.list(zone, NodeFilter.all().applications(application)), version, false); } /** Set version for nodeCount number of nodes in application in a given zone */ @@ -232,7 +230,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer /** Set OS version for an application in a given zone */ public void setOsVersion(Version version, ApplicationId application, ZoneId zone) { - setVersion(zone, nodeRepository.list(zone, application), version, true); + setVersion(zone, nodeRepository.list(zone, NodeFilter.all().applications(application)), version, true); } /** Set OS version for an application in a given zone */ @@ -244,9 +242,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer for (var node : nodes) { Node newNode; if (osVersion) { - newNode = new Node.Builder(node).currentOsVersion(version).wantedOsVersion(version).build(); + newNode = Node.builder(node).currentOsVersion(version).wantedOsVersion(version).build(); } else { - newNode = new Node.Builder(node).currentVersion(version).wantedVersion(version).build(); + newNode = Node.builder(node).currentVersion(version).wantedVersion(version).build(); } nodeRepository().putNodes(zone, newNode); } @@ -385,7 +383,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer applications.put(id, new Application(id.applicationId(), lastPrepareVersion, new ApplicationPackage(deployment.applicationPackage()))); ClusterSpec.Id cluster = ClusterSpec.Id.from("default"); - if (nodeRepository().list(id.zoneId(), id.applicationId()).isEmpty()) + if (nodeRepository().list(id.zoneId(), NodeFilter.all().applications(id.applicationId())).isEmpty()) provision(id.zoneId(), id.applicationId(), cluster); this.containerEndpoints.put( @@ -408,12 +406,12 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return () -> { Application application = applications.get(id); application.activate(); - List<Node> nodes = nodeRepository.list(id.zoneId(), id.applicationId()); + List<Node> nodes = nodeRepository.list(id.zoneId(), NodeFilter.all().applications(id.applicationId())); for (Node node : nodes) { - nodeRepository.putNodes(id.zoneId(), new Node.Builder(node) - .state(Node.State.active) - .wantedVersion(application.version().get()) - .build()); + nodeRepository.putNodes(id.zoneId(), Node.builder(node) + .state(Node.State.active) + .wantedVersion(application.version().get()) + .build()); } serviceStatus.put(id, new ServiceConvergence(id.applicationId(), id.zoneId(), @@ -474,7 +472,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer public void deactivate(DeploymentId deployment) { ApplicationId applicationId = deployment.applicationId(); nodeRepository().removeNodes(deployment.zoneId(), - nodeRepository().list(deployment.zoneId(), applicationId)); + nodeRepository().list(deployment.zoneId(), NodeFilter.all().applications(applicationId))); if ( ! applications.containsKey(deployment)) return; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java index d481aaa2c77..ef2b6f6f7e2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.component.AbstractComponent; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ContainerRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ContainerRegistryMock.java index 9fa2867631a..4791017548d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ContainerRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ContainerRegistryMock.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.vespa.hosted.controller.api.integration.container.ContainerImage; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsMock.java index 61c98258549..dcea323a8e8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsMock.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 4079591730d..ef99183ecde 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -1,7 +1,6 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; -import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; @@ -15,25 +14,18 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Applicatio import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationStats; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Load; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepoStats; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.configserver.TargetVersions; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; import java.net.URI; import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; -import java.util.function.Function; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -50,159 +42,47 @@ public class NodeRepositoryMock implements NodeRepository { private final Map<DeploymentId, Pair<Double, Double>> trafficFractions = new HashMap<>(); private final Map<ZoneId, Map<TenantName, URI>> archiveUris = new HashMap<>(); - // A separate/alternative list of NodeRepositoryNode nodes. - // Methods operating with Node and NodeRepositoryNode lives separate lives. - private final Map<ZoneId, List<NodeRepositoryNode>> nodeRepoNodes = new HashMap<>(); - - private boolean allowPatching = false; + private boolean allowPatching = true; private boolean hasSpareCapacity = false; - /** Add or update given nodes in zone */ - public void putNodes(ZoneId zone, List<Node> nodes) { - Map<HostName, Node> zoneNodes = nodeRepository.computeIfAbsent(zone, (k) -> new HashMap<>()); + @Override + public void addNodes(ZoneId zone, List<Node> nodes) { + Map<HostName, Node> existingNodes = nodeRepository.getOrDefault(zone, Map.of()); for (var node : nodes) { - zoneNodes.put(node.hostname(), node); + if (existingNodes.containsKey(node.hostname())) { + throw new IllegalArgumentException("Node " + node.hostname() + " already added in zone " + zone); + } } - } - - public void putNode(ZoneId zone, Node node) { - nodeRepository.computeIfAbsent(zone, (k) -> new HashMap<>()).put(node.hostname(), node); - } - - public void putApplication(ZoneId zone, Application application) { - applications.putIfAbsent(zone, new HashMap<>()); - applications.get(zone).put(application.id(), application); - } - - @Override - public NodeRepoStats getStats(ZoneId zone) { - List<ApplicationStats> applicationStats = - applications.containsKey(zone) - ? applications.get(zone).keySet().stream() - .map(id -> new ApplicationStats(id, Load.zero(), 0, 0)) - .collect(Collectors.toList()) - : List.of(); - - return new NodeRepoStats(Load.zero(), Load.zero(), applicationStats); - } - - public Pair<Double, Double> getTrafficFraction(ApplicationId application, ZoneId zone) { - return trafficFractions.get(new DeploymentId(application, zone)); - } - - /** Add or update given node in zone */ - public void putNodes(ZoneId zone, Node node) { - putNodes(zone, Collections.singletonList(node)); - } - - /** Remove given nodes from zone */ - public void removeNodes(ZoneId zone, List<Node> nodes) { - nodes.forEach(node -> nodeRepository.get(zone).remove(node.hostname())); - } - - /** Remove all nodes in all zones */ - public void clear() { - nodeRepository.clear(); - nodeRepoNodes.clear(); - } - - /** Replace nodes in zone with given nodes */ - public void setNodes(ZoneId zone, List<Node> nodes) { - nodeRepository.put(zone, nodes.stream().collect(Collectors.toMap(Node::hostname, Function.identity()))); - } - - public Node require(HostName hostName) { - return nodeRepository.values().stream() - .map(zoneNodes -> zoneNodes.get(hostName)) - .filter(Objects::nonNull) - .findFirst() - .orElseThrow(() -> new NoSuchElementException("No node with the hostname " + hostName + " is known.")); - } - - /** Replace nodes in zone with a fixed set of nodes */ - public void setFixedNodes(ZoneId zone) { - var nodeA = new Node.Builder() - .hostname(HostName.from("hostA")) - .parentHostname(HostName.from("parentHostA")) - .state(Node.State.active) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(24, 24, 500, 1)) - .clusterId("clusterA") - .clusterType(Node.ClusterType.container) - .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) - .build(); - var nodeB = new Node.Builder() - .hostname(HostName.from("hostB")) - .parentHostname(HostName.from("parentHostB")) - .state(Node.State.active) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant2", "app2", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(40, 24, 500, 1)) - .cost(20) - .clusterId("clusterB") - .clusterType(Node.ClusterType.container) - .build(); - setNodes(zone, List.of(nodeA, nodeB)); - } - - @Override - public void addNodes(ZoneId zone, Collection<NodeRepositoryNode> nodes) { - nodeRepoNodes.put(zone, new ArrayList<>(nodes)); + putNodes(zone, nodes); } @Override public void deleteNode(ZoneId zone, String hostname) { - throw new UnsupportedOperationException(); + require(zone, hostname); + nodeRepository.get(zone).remove(HostName.from(hostname)); } @Override - public void setState(ZoneId zone, NodeState nodeState, String hostName) { - var existing = list(zone, List.of(HostName.from(hostName))); - if (existing.size() != 1) throw new IllegalArgumentException("Node " + hostName + " not found in " + zone); - - var node = new Node.Builder(existing.get(0)) - .state(Node.State.valueOf(nodeState.name())) - .build(); + public void setState(ZoneId zone, Node.State state, String hostname) { + Node node = Node.builder(require(zone, hostname)) + .state(Node.State.valueOf(state.name())) + .build(); putNodes(zone, node); } @Override - public NodeRepositoryNode getNode(ZoneId zone, String hostname) { - throw new UnsupportedOperationException(); - } - - @Override - public NodeList listNodes(ZoneId zone) { - return new NodeList(nodeRepoNodes.get(zone)); - } - - @Override - public List<Node> list(ZoneId zone, boolean includeDeprovisioned) { - return List.copyOf(nodeRepository.getOrDefault(zone, Map.of()).values()); + public Node getNode(ZoneId zone, String hostname) { + return require(zone, hostname); } @Override - public List<Node> list(ZoneId zone, ApplicationId application) { - return nodeRepository.getOrDefault(zone, Collections.emptyMap()).values().stream() - .filter(node -> node.owner().map(application::equals).orElse(false)) - .collect(Collectors.toList()); - } - - @Override - public List<Node> list(ZoneId zone, List<HostName> hostnames) { - return nodeRepository.getOrDefault(zone, Collections.emptyMap()).values().stream() - .filter(node -> hostnames.contains(node.hostname())) + public List<Node> list(ZoneId zone, NodeFilter filter) { + return nodeRepository.getOrDefault(zone, Map.of()).values().stream() + .filter(node -> filter.includeDeprovisioned() || node.state() != Node.State.deprovisioned) + .filter(node -> filter.applications().isEmpty() || + (node.owner().isPresent() && filter.applications().contains(node.owner().get()))) + .filter(node -> filter.hostnames().isEmpty() || filter.hostnames().contains(node.hostname())) + .filter(node -> filter.states().isEmpty() || filter.states().contains(node.state())) .collect(Collectors.toList()); } @@ -218,6 +98,18 @@ public class NodeRepositoryMock implements NodeRepository { } @Override + public NodeRepoStats getStats(ZoneId zone) { + List<ApplicationStats> applicationStats = + applications.containsKey(zone) + ? applications.get(zone).keySet().stream() + .map(id -> new ApplicationStats(id, Load.zero(), 0, 0)) + .collect(Collectors.toList()) + : List.of(); + + return new NodeRepoStats(Load.zero(), Load.zero(), applicationStats); + } + + @Override public Map<TenantName, URI> getArchiveUris(ZoneId zone) { return Map.copyOf(archiveUris.getOrDefault(zone, Map.of())); } @@ -244,13 +136,13 @@ public class NodeRepositoryMock implements NodeRepository { nodeRepository.getOrDefault(zone, Map.of()).values() .stream() .filter(node -> node.type() == type) - .map(node -> new Node.Builder(node).wantedVersion(version).build()) + .map(node -> Node.builder(node).wantedVersion(version).build()) .forEach(node -> putNodes(zone, node)); } @Override - public void upgradeOs(ZoneId zone, NodeType type, Version version, Optional<Duration> upgradeBudget) { - upgradeBudget.ifPresent(d -> this.osUpgradeBudgets.put(Objects.hash(zone, type, version), d)); + public void upgradeOs(ZoneId zone, NodeType type, Version version, Duration upgradeBudget) { + this.osUpgradeBudgets.put(Objects.hash(zone, type, version), upgradeBudget); this.targetVersions.compute(zone, (ignored, targetVersions) -> { if (targetVersions == null) { targetVersions = TargetVersions.EMPTY; @@ -261,7 +153,7 @@ public class NodeRepositoryMock implements NodeRepository { nodeRepository.getOrDefault(zone, Map.of()).values() .stream() .filter(node -> node.type() == type) - .map(node -> new Node.Builder(node).wantedOsVersion(version).build()) + .map(node -> Node.builder(node).wantedOsVersion(version).build()) .forEach(node -> putNodes(zone, node)); } @@ -279,80 +171,135 @@ public class NodeRepositoryMock implements NodeRepository { } @Override - public void retireAndDeprovision(ZoneId zoneId, String hostName) { - nodeRepository.get(zoneId).remove(HostName.from(hostName)); + public void retire(ZoneId zone, String hostname, boolean wantToRetire, boolean wantToDeprovision) { + patchNodes(zone, hostname, (node) -> Node.builder(node).wantToRetire(wantToRetire).wantToDeprovision(wantToDeprovision).build()); } @Override - public void patchNode(ZoneId zoneId, String hostName, NodeRepositoryNode node) { - if (!allowPatching) throw new UnsupportedOperationException(); - List<Node> existing = list(zoneId, List.of(HostName.from(hostName))); - if (existing.size() != 1) throw new IllegalArgumentException("Node " + hostName + " not found in " + zoneId); + public void updateReports(ZoneId zone, String hostname, Map<String, String> reports) { + Map<String, String> trimmedReports = reports.entrySet().stream() + // Null value clears a report + .filter(kv -> kv.getValue() != null) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + patchNodes(zone, hostname, (node) -> Node.builder(node).reports(trimmedReports).build()); + } - // Note: Only supports switchHostname, modelName and wantToRetire - Node.Builder newNode = new Node.Builder(existing.get(0)); - if (node.getSwitchHostname() != null) - newNode.switchHostname(node.getSwitchHostname()); - if (node.getModelName() != null) - newNode.modelName(node.getModelName()); - if (node.getWantToRetire() != null) - newNode.wantToRetire(node.getWantToRetire()); - if (!node.getReports().isEmpty()) - newNode.reports(node.getReports()); + @Override + public void updateModel(ZoneId zone, String hostname, String modelName) { + patchNodes(zone, hostname, (node) -> Node.builder(node).modelName(modelName).build()); + } - putNodes(zoneId, newNode.build()); + @Override + public void updateSwitchHostname(ZoneId zone, String hostname, String switchHostname) { + patchNodes(zone, hostname, (node) -> Node.builder(node).switchHostname(switchHostname).build()); } @Override - public void reboot(ZoneId zoneId, String hostName) { + public void reboot(ZoneId zone, String hostname) { throw new UnsupportedOperationException(); } @Override - public boolean isReplaceable(ZoneId zoneId, List<HostName> hostNames) { + public boolean isReplaceable(ZoneId zone, List<HostName> hostnames) { return hasSpareCapacity; } + /** Add or update given nodes in zone */ + public void putNodes(ZoneId zone, List<Node> nodes) { + Map<HostName, Node> zoneNodes = nodeRepository.computeIfAbsent(zone, (k) -> new HashMap<>()); + for (var node : nodes) { + zoneNodes.put(node.hostname(), node); + } + } + + /** Add or update given node in zone */ + public void putNodes(ZoneId zone, Node node) { + putNodes(zone, List.of(node)); + } + + public void putApplication(ZoneId zone, Application application) { + applications.computeIfAbsent(zone, (k) -> new HashMap<>()) + .put(application.id(), application); + } + + public Pair<Double, Double> getTrafficFraction(ApplicationId application, ZoneId zone) { + return trafficFractions.get(new DeploymentId(application, zone)); + } + + /** Remove given nodes from zone */ + public void removeNodes(ZoneId zone, List<Node> nodes) { + nodes.forEach(node -> nodeRepository.get(zone).remove(node.hostname())); + } + + /** Remove all nodes in all zones */ + public void clear() { + nodeRepository.clear(); + } + + /** Add a fixed set of nodes to given zone */ + public void addFixedNodes(ZoneId zone) { + var nodeA = Node.builder() + .hostname(HostName.from("hostA")) + .parentHostname(HostName.from("parentHostA")) + .state(Node.State.active) + .type(NodeType.tenant) + .owner(ApplicationId.from("tenant1", "app1", "default")) + .currentVersion(Version.fromString("7.42")) + .wantedVersion(Version.fromString("7.42")) + .currentOsVersion(Version.fromString("7.6")) + .wantedOsVersion(Version.fromString("7.6")) + .serviceState(Node.ServiceState.expectedUp) + .resources(new NodeResources(24, 24, 500, 1)) + .clusterId("clusterA") + .clusterType(Node.ClusterType.container) + .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) + .build(); + var nodeB = Node.builder() + .hostname(HostName.from("hostB")) + .parentHostname(HostName.from("parentHostB")) + .state(Node.State.active) + .type(NodeType.tenant) + .owner(ApplicationId.from("tenant2", "app2", "default")) + .currentVersion(Version.fromString("7.42")) + .wantedVersion(Version.fromString("7.42")) + .currentOsVersion(Version.fromString("7.6")) + .wantedOsVersion(Version.fromString("7.6")) + .serviceState(Node.ServiceState.expectedUp) + .resources(new NodeResources(40, 24, 500, 1)) + .cost(20) + .clusterId("clusterB") + .clusterType(Node.ClusterType.container) + .build(); + putNodes(zone, List.of(nodeA, nodeB)); + } + public Optional<Duration> osUpgradeBudget(ZoneId zone, NodeType type, Version version) { return Optional.ofNullable(osUpgradeBudgets.get(Objects.hash(zone, type, version))); } public void doUpgrade(DeploymentId deployment, Optional<HostName> hostName, Version version) { - modifyNodes(deployment, hostName, node -> { - assert node.wantedVersion().equals(version); - return new Node.Builder(node) - .currentVersion(version) - .currentDockerImage(node.wantedDockerImage()) - .build(); + patchNodes(deployment, hostName, node -> { + return Node.builder(node) + .currentVersion(version) + .currentDockerImage(node.wantedDockerImage()) + .build(); }); } - private void modifyNodes(DeploymentId deployment, Optional<HostName> hostname, UnaryOperator<Node> modification) { - List<Node> nodes = hostname.map(this::require) - .map(Collections::singletonList) - .orElse(list(deployment.zoneId(), deployment.applicationId())); - putNodes(deployment.zoneId(), - nodes.stream().map(modification).collect(Collectors.toList())); - } - public void requestRestart(DeploymentId deployment, Optional<HostName> hostname) { - modifyNodes(deployment, hostname, node -> new Node.Builder(node).wantedRestartGeneration(node.wantedRestartGeneration() + 1).build()); + patchNodes(deployment, hostname, node -> Node.builder(node).wantedRestartGeneration(node.wantedRestartGeneration() + 1).build()); } public void doRestart(DeploymentId deployment, Optional<HostName> hostname) { - modifyNodes(deployment, hostname, node -> new Node.Builder(node).restartGeneration(node.restartGeneration() + 1).build()); + patchNodes(deployment, hostname, node -> Node.builder(node).restartGeneration(node.restartGeneration() + 1).build()); } public void requestReboot(DeploymentId deployment, Optional<HostName> hostname) { - modifyNodes(deployment, hostname, node -> new Node.Builder(node).wantedRebootGeneration(node.wantedRebootGeneration() + 1).build()); + patchNodes(deployment, hostname, node -> Node.builder(node).wantedRebootGeneration(node.wantedRebootGeneration() + 1).build()); } public void doReboot(DeploymentId deployment, Optional<HostName> hostname) { - modifyNodes(deployment, hostname, node -> new Node.Builder(node).rebootGeneration(node.rebootGeneration() + 1).build()); - } - - public void addReport(ZoneId zoneId, HostName hostName, String reportId, JsonNode report) { - nodeRepository.get(zoneId).get(hostName).reports().put(reportId, report); + patchNodes(deployment, hostname, node -> Node.builder(node).rebootGeneration(node.rebootGeneration() + 1).build()); } public NodeRepositoryMock allowPatching(boolean allowPatching) { @@ -364,4 +311,33 @@ public class NodeRepositoryMock implements NodeRepository { this.hasSpareCapacity = hasSpareCapacity; } + private Node require(ZoneId zone, String hostname) { + return require(zone, HostName.from(hostname)); + } + + private Node require(ZoneId zone, HostName hostname) { + Node node = nodeRepository.getOrDefault(zone, Map.of()).get(hostname); + if (node == null) throw new IllegalArgumentException("Node not found in " + zone + ": " + hostname); + return node; + } + + private void patchNodes(ZoneId zone, String hostname, UnaryOperator<Node> patcher) { + patchNodes(zone, Optional.of(HostName.from(hostname)), patcher); + } + + private void patchNodes(DeploymentId deployment, Optional<HostName> hostname, UnaryOperator<Node> patcher) { + patchNodes(deployment.zoneId(), hostname, patcher); + } + + private void patchNodes(ZoneId zone, Optional<HostName> hostname, UnaryOperator<Node> patcher) { + if (!allowPatching) throw new UnsupportedOperationException("Patching is disabled in this mock"); + List<Node> nodes; + if (hostname.isPresent()) { + nodes = List.of(require(zone, hostname.get())); + } else { + nodes = list(zone, NodeFilter.all()); + } + putNodes(zone, nodes.stream().map(patcher).collect(Collectors.toList())); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java index c088965c2ad..37c050079e0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.component.AbstractComponent; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 9002933b767..f64c54ff24d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AccessControlSer import com.yahoo.vespa.hosted.controller.api.integration.athenz.MockAccessControlService; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; -import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher; +import com.yahoo.vespa.hosted.controller.api.integration.aws.MockCloudEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; @@ -69,7 +69,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final MemoryEntityService memoryEntityService = new MemoryEntityService(); private final DummySystemMonitor systemMonitor = new DummySystemMonitor(); private final CostReportConsumerMock costReportConsumerMock = new CostReportConsumerMock(); - private final MockAwsEventFetcher mockAwsEventFetcher = new MockAwsEventFetcher(); + private final MockCloudEventFetcher mockAwsEventFetcher = new MockCloudEventFetcher(); private final ArtifactRepositoryMock artifactRepositoryMock = new ArtifactRepositoryMock(); private final MockTesterCloud mockTesterCloud; private final ApplicationStoreMock applicationStoreMock = new ApplicationStoreMock(); @@ -168,7 +168,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override - public MockAwsEventFetcher eventFetcherService() { + public MockCloudEventFetcher eventFetcherService() { return mockAwsEventFetcher; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index fe03b69a3fe..7ce17aff782 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; @@ -216,6 +217,8 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return URI.create("https://api.tld:4443/"); } + @Override public Optional<String> tenantDeveloperRoleArn(TenantName tenant) { return Optional.empty(); } + @Override public boolean hasZone(ZoneId zoneId) { return zones.stream().anyMatch(zone -> zone.getId().equals(zoneId)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java index f79d47766ba..09ee1f8fe51 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.InstanceName; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java index d7934f08fee..451991f9604 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; @@ -9,7 +9,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java index 476d2465202..add1a319384 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java @@ -1,33 +1,33 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +/** + * @author smorgrav + */ public class ChangeManagementAssessorTest { - private ChangeManagementAssessor changeManagementAssessor = new ChangeManagementAssessor(new NodeRepositoryMock()); + private final ChangeManagementAssessor changeManagementAssessor = new ChangeManagementAssessor(new NodeRepositoryMock()); @Test public void empty_input_variations() { ZoneId zone = ZoneId.from("prod", "eu-trd"); List<String> hostNames = new ArrayList<>(); - List<NodeRepositoryNode> allNodesInZone = new ArrayList<>(); + List<Node> allNodesInZone = new ArrayList<>(); // Both zone and hostnames are empty ChangeManagementAssessor.Assessment assessment @@ -39,7 +39,7 @@ public class ChangeManagementAssessorTest { public void one_host_one_cluster_no_groups() { ZoneId zone = ZoneId.from("prod", "eu-trd"); List<String> hostNames = Collections.singletonList("host1"); - List<NodeRepositoryNode> allNodesInZone = new ArrayList<>(); + List<Node> allNodesInZone = new ArrayList<>(); allNodesInZone.add(createNode("node1", "host1", "default", 0 )); allNodesInZone.add(createNode("node2", "host1", "default", 0 )); allNodesInZone.add(createNode("node3", "host1", "default", 0 )); @@ -69,8 +69,8 @@ public class ChangeManagementAssessorTest { @Test public void one_of_two_groups_in_one_of_two_clusters() { ZoneId zone = ZoneId.from("prod", "eu-trd"); - List<String> hostNames = Arrays.asList("host1", "host2", "host5"); - List<NodeRepositoryNode> allNodesInZone = new ArrayList<>(); + List<String> hostNames = List.of("host1", "host2", "host5"); + List<Node> allNodesInZone = new ArrayList<>(); // Two impacted nodes on host1 allNodesInZone.add(createNode("node1", "host1", "default", 0 )); @@ -123,8 +123,8 @@ public class ChangeManagementAssessorTest { @Test public void two_config_nodes() { var zone = ZoneId.from("prod", "eu-trd"); - var hostNames = Arrays.asList("config1", "config2"); - var allNodesInZone = new ArrayList<NodeRepositoryNode>(); + var hostNames = List.of("config1", "config2"); + var allNodesInZone = new ArrayList<Node>(); // Add config nodes and parents allNodesInZone.add(createNode("config1", "confighost1", "config", 0, NodeType.config)); @@ -141,8 +141,8 @@ public class ChangeManagementAssessorTest { @Test public void one_of_three_proxy_nodes() { var zone = ZoneId.from("prod", "eu-trd"); - var hostNames = Arrays.asList("routing1"); - var allNodesInZone = new ArrayList<NodeRepositoryNode>(); + var hostNames = List.of("routing1"); + var allNodesInZone = new ArrayList<Node>(); // Add routing nodes and parents allNodesInZone.add(createNode("routing1", "parentrouting1", "routing", 0, NodeType.proxy)); @@ -156,47 +156,33 @@ public class ChangeManagementAssessorTest { assertEquals("33% of routing nodes impacted. Consider reprovisioning if too many", assessment.get(0).impact); } - private NodeOwner createOwner() { - NodeOwner owner = new NodeOwner(); - owner.tenant = "mytenant"; - owner.application = "myapp"; - owner.instance = "default"; - return owner; - } - - private NodeMembership createMembership(String clusterId, int group) { - NodeMembership membership = new NodeMembership(); - membership.group = "" + group; - membership.clusterid = clusterId; - membership.clustertype = "content"; - membership.index = 2; - membership.retired = false; - return membership; - } - - private NodeRepositoryNode createNode(String nodename, String hostname, String clusterId, int group) { + private Node createNode(String nodename, String hostname, String clusterId, int group) { return createNode(nodename, hostname, clusterId, group, NodeType.tenant); } - private NodeRepositoryNode createNode(String nodename, String hostname, String clusterId, int group, NodeType nodeType) { - NodeRepositoryNode node = new NodeRepositoryNode(); - node.setHostname(nodename); - node.setParentHostname(hostname); - node.setState(NodeState.active); - node.setOwner(createOwner()); - node.setMembership(createMembership(clusterId, group)); - node.setType(nodeType); - - return node; + private Node createNode(String nodename, String hostname, String clusterId, int group, NodeType nodeType) { + return Node.builder().hostname(nodename) + .parentHostname(hostname) + .state(Node.State.active) + .owner(ApplicationId.from("mytenant", "myapp", "default")) + .group(String.valueOf(group)) + .clusterId(clusterId) + .clusterType(Node.ClusterType.content) + .type(nodeType) + .build(); } - private NodeRepositoryNode createHost(String hostname, NodeType nodeType) { - NodeRepositoryNode node = new NodeRepositoryNode(); - node.setHostname(hostname); - node.setSwitchHostname("switch1"); - node.setType(nodeType); - node.setOwner(createOwner()); - node.setMembership(createMembership(nodeType.name(), 0)); - return node; + private Node createHost(String hostname, NodeType nodeType) { + return Node.builder() + .hostname(hostname) + .switchHostname("switch1") + .state(Node.State.active) + .owner(ApplicationId.from("mytenant", "myapp", "default")) + .group(String.valueOf(0)) + .clusterId(nodeType.name()) + .clusterType(Node.ClusterType.content) + .type(nodeType) + .build(); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java index 15a2cf3063d..0e76c0375f2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.zone.ZoneId; @@ -39,6 +39,7 @@ public class ChangeRequestMaintainerTest { assertEquals(Status.CANCELED, persistedChangeRequest.getChangeRequestSource().getStatus()); assertEquals(ChangeRequest.Approval.APPROVED, persistedChangeRequest.getApproval()); assertEquals(time, persistedChangeRequest.getChangeRequestSource().getPlannedStartTime()); + assertEquals(0, changeRequestClient.getApprovedChangeRequests().size()); } @Test @@ -58,17 +59,39 @@ public class ChangeRequestMaintainerTest { assertEquals(newChangeRequest, persistedChangeRequests.get(0)); } - private ChangeRequest newChangeRequest(String id, ChangeRequest.Approval approval) { - return newChangeRequest(id, approval, ZonedDateTime.now(), Status.CLOSED); + @Test + public void approves_change_request_if_non_prod() { + var time = ZonedDateTime.now(); + var prodChangeRequest = newChangeRequest("id1", ChangeRequest.Approval.REQUESTED, time, Status.WAITING_FOR_APPROVAL); + var nonProdApprovalRequested = newChangeRequest("id2", "unknown-node", ChangeRequest.Approval.REQUESTED, time, Status.WAITING_FOR_APPROVAL); + var nonProdApproved = newChangeRequest("id3", "unknown-node", ChangeRequest.Approval.APPROVED, time, Status.WAITING_FOR_APPROVAL); + + changeRequestClient.setUpcomingChangeRequests(List.of( + prodChangeRequest, + nonProdApprovalRequested, + nonProdApproved + )); + changeRequestMaintainer.maintain(); + + var persistedChangeRequests = tester.curator().readChangeRequests(); + assertEquals(1, persistedChangeRequests.size()); + assertEquals(prodChangeRequest.getId(), persistedChangeRequests.get(0).getId()); + + assertEquals(1, changeRequestClient.getApprovedChangeRequests().size()); + assertEquals(nonProdApprovalRequested.getId(), changeRequestClient.getApprovedChangeRequests().get(0).getId()); } private ChangeRequest newChangeRequest(String id, ChangeRequest.Approval approval, ZonedDateTime time, Status status) { + return newChangeRequest(id, "node-1-tenant-host-prod.us-east-3", approval, time, status); + } + + private ChangeRequest newChangeRequest(String id, String hostname, ChangeRequest.Approval approval, ZonedDateTime time, Status status) { return new ChangeRequest.Builder() .id(id) .approval(approval) .impact(ChangeRequest.Impact.VERY_HIGH) .impactedSwitches(List.of()) - .impactedHosts(List.of("node-1-tenant-host-prod.us-east-3")) + .impactedHosts(List.of(hostname)) .changeRequestSource(new ChangeRequestSource.Builder() .plannedStartTime(time) .plannedEndTime(time) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java index 680743055c9..f0aef75bb11 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; @@ -6,8 +6,9 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEvent; -import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher; +import com.yahoo.vespa.hosted.controller.api.integration.aws.MockCloudEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import org.junit.Test; @@ -23,7 +24,7 @@ import static org.junit.Assert.assertEquals; /** * @author olaa */ -public class CloudEventReporterTest { +public class CloudEventTrackerTest { private final ControllerTester tester = new ControllerTester(); private final ZoneApiMock unsupportedZone = createZone("prod.zone3", "region-1", "other"); @@ -41,20 +42,20 @@ public class CloudEventReporterTest { @Test public void maintain() { setUpZones(); - CloudEventReporter cloudEventReporter = new CloudEventReporter(tester.controller(), Duration.ofMinutes(15)); - assertEquals(Set.of("host1.com", "host2.com", "host3.com"), getHostnames(unsupportedZone.getId())); - assertEquals(Set.of("host1.com", "host2.com", "host3.com"), getHostnames(zone1.getId())); - assertEquals(Set.of("host4.com", "host5.com", "confighost.com"), getHostnames(zone2.getId())); + CloudEventTracker cloudEventTracker = new CloudEventTracker(tester.controller(), Duration.ofMinutes(15)); + assertEquals(Set.of("host1.com", "host2.com", "host3.com"), hostsNotDeprovisioning(unsupportedZone.getId())); + assertEquals(Set.of("host1.com", "host2.com", "host3.com"), hostsNotDeprovisioning(zone1.getId())); + assertEquals(Set.of("host4.com", "host5.com", "confighost.com"), hostsNotDeprovisioning(zone2.getId())); mockEvents(); - cloudEventReporter.maintain(); - assertEquals(Set.of("host1.com", "host2.com", "host3.com"), getHostnames(unsupportedZone.getId())); - assertEquals(Set.of("host3.com"), getHostnames(zone1.getId())); - assertEquals(Set.of("host4.com"), getHostnames(zone2.getId())); + cloudEventTracker.maintain(); + assertEquals(Set.of("host1.com", "host2.com", "host3.com"), hostsNotDeprovisioning(unsupportedZone.getId())); + assertEquals(Set.of("host3.com"), hostsNotDeprovisioning(zone1.getId())); + assertEquals(Set.of("host4.com"), hostsNotDeprovisioning(zone2.getId())); } private void mockEvents() { - MockAwsEventFetcher eventFetcher = (MockAwsEventFetcher) tester.controller().serviceRegistry().eventFetcherService(); + MockCloudEventFetcher eventFetcher = (MockCloudEventFetcher) tester.controller().serviceRegistry().eventFetcherService(); Date date = new Date(); CloudEvent event1 = new CloudEvent("event 1", @@ -121,17 +122,18 @@ public class CloudEventReporterTest { } private Node createNode(String hostname, NodeType nodeType) { - return new Node.Builder() - .hostname(HostName.from(hostname)) - .type(nodeType) - .build(); + return Node.builder() + .hostname(HostName.from(hostname)) + .type(nodeType) + .build(); } - private Set<String> getHostnames(ZoneId zoneId) { - return tester.configServer().nodeRepository().list(zoneId, false) - .stream() - .map(node -> node.hostname().value()) - .collect(Collectors.toSet()); + private Set<String> hostsNotDeprovisioning(ZoneId zoneId) { + return tester.configServer().nodeRepository().list(zoneId, NodeFilter.all()) + .stream() + .filter(node -> !node.wantToDeprovision()) + .map(node -> node.hostname().value()) + .collect(Collectors.toSet()); } private ZoneApiMock createZone(String zoneId, String cloudNativeRegionName, String cloud) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java index f3c4f9f7438..c772ca8b8f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java index a75bada7b6f..37b138527fe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirerTest.java index 36a0c57f716..b52d9de2fe8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java index 4bdb657d3af..a452d550488 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java index cbffd6d610f..d78b48c362a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java @@ -54,7 +54,7 @@ public class CostReportMaintainerTest { private void addNodes() { for (var zone : tester.zoneRegistry().zones().all().zones()) { - tester.configServer().nodeRepository().setFixedNodes(zone.getId()); + tester.configServer().nodeRepository().addFixedNodes(zone.getId()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index 31f8aaf9e2d..d55b5cea4ee 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.Environment; @@ -6,7 +6,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index af2c0c6b08d..0e365ffd135 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; @@ -7,7 +7,7 @@ import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index 8083b847c0b..dc50b32b338 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; @@ -9,7 +9,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java index 7a8f775e8b1..cc428ebc94f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java @@ -1,25 +1,17 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import org.junit.Test; import java.time.Duration; import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalUnit; -import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; @@ -63,19 +55,13 @@ public class DeploymentUpgraderTest { assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); - // 14 hours pass, but not upgraded before a day has passed since last deployment - tester.clock().advance(Duration.ofHours(14)); + // 11 hours pass, but not upgraded since it's not likely in the middle of the night + tester.clock().advance(Duration.ofHours(11)); upgrader.maintain(); assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); - // 35 hours pass, but not upgraded since it's not likely in the middle of the night - tester.clock().advance(Duration.ofHours(21)); - upgrader.maintain(); - assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); - assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); - - // 38 hours pass, and the dev deployment, only, is upgraded + // 14 hours pass, and the dev deployment, only, is upgraded tester.clock().advance(Duration.ofHours(3)); upgrader.maintain(); assertEquals(tester.clock().instant().truncatedTo(MILLIS), tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 3ed26b4fb6e..5698aefaa46 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java index 0baee28143c..a06e15de3c7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; @@ -6,6 +6,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.entity.NodeEntity; import org.junit.Test; @@ -83,12 +84,12 @@ public class HostInfoUpdaterTest { // Updates node registered under a different hostname ZoneId zone = tester.zoneRegistry().zones().controllerUpgraded().all().ids().get(0); String hostnameSuffix = ".prod." + zone.value(); - Node configNode = new Node.Builder().hostname(HostName.from("cfg3" + hostnameSuffix)) - .type(NodeType.config) - .build(); - Node configHost = new Node.Builder().hostname(HostName.from("cfghost3" + hostnameSuffix)) - .type(NodeType.confighost) - .build(); + Node configNode = Node.builder().hostname(HostName.from("cfg3" + hostnameSuffix)) + .type(NodeType.config) + .build(); + Node configHost = Node.builder().hostname(HostName.from("cfghost3" + hostnameSuffix)) + .type(NodeType.confighost) + .build(); tester.serviceRegistry().configServer().nodeRepository().putNodes(zone, List.of(configNode, configHost)); String switchHostname = switchHostname(configHost); NodeEntity configNodeEntity = new NodeEntity("cfg3" + hostnameSuffix, "RD350G", "Lenovo", switchHostname); @@ -108,7 +109,7 @@ public class HostInfoUpdaterTest { private static List<Node> allNodes(ControllerTester tester) { List<Node> nodes = new ArrayList<>(); for (var zone : tester.zoneRegistry().zones().controllerUpgraded().all().ids()) { - nodes.addAll(tester.serviceRegistry().configServer().nodeRepository().list(zone, false)); + nodes.addAll(tester.serviceRegistry().configServer().nodeRepository().list(zone, NodeFilter.all())); } return nodes; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 023c5671b60..7db52b6121b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; @@ -7,7 +7,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.JobController; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 29c5573a1f5..2fb5aee354b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -14,7 +14,8 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -28,7 +29,6 @@ import org.junit.Test; import java.time.Duration; import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -301,7 +301,7 @@ public class MetricsReporterTest { var version0 = Version.fromString("7.0"); tester.upgradeSystem(version0); reporter.maintain(); - var hosts = tester.configServer().nodeRepository().list(zone, SystemApplication.configServer.id()); + var hosts = tester.configServer().nodeRepository().list(zone, NodeFilter.all().applications(SystemApplication.configServer.id())); assertPlatformChangeDuration(Duration.ZERO, hosts); var targets = List.of(Version.fromString("7.1"), Version.fromString("7.2")); @@ -362,7 +362,7 @@ public class MetricsReporterTest { tester.configServer().setOsVersion(version0, SystemApplication.tenantHost.id(), zone); tester.configServer().setOsVersion(version0, SystemApplication.configServerHost.id(), zone); runAll(statusUpdater, reporter); - List<Node> hosts = tester.configServer().nodeRepository().list(zone, false); + List<Node> hosts = tester.configServer().nodeRepository().list(zone, NodeFilter.all()); assertOsChangeDuration(Duration.ZERO, hosts); var targets = List.of(Version.fromString("8.1"), Version.fromString("8.2")); @@ -383,10 +383,10 @@ public class MetricsReporterTest { // Nodes are told to upgrade, but do not suspend yet assertEquals("Wanted OS version is raised for all nodes", nextVersion, - tester.configServer().nodeRepository().list(zone, SystemApplication.tenantHost.id()).stream() + tester.configServer().nodeRepository().list(zone, NodeFilter.all().applications(SystemApplication.tenantHost.id())).stream() .map(Node::wantedOsVersion).min(Comparator.naturalOrder()).get()); assertTrue("No nodes are suspended", tester.controller().serviceRegistry().configServer() - .nodeRepository().list(zone, false).stream() + .nodeRepository().list(zone, NodeFilter.all()).stream() .noneMatch(node -> node.serviceState() == Node.ServiceState.allowedDown)); // Another 30 minutes pass @@ -537,16 +537,16 @@ public class MetricsReporterTest { } private List<Node> getNodes(ZoneId zone, List<Node> nodes, ControllerTester tester) { - return tester.configServer().nodeRepository().list(zone, nodes.stream() - .map(Node::hostname) - .collect(Collectors.toList())); + return tester.configServer().nodeRepository().list(zone, NodeFilter.all().hostnames(nodes.stream() + .map(Node::hostname) + .collect(Collectors.toSet()))); } private void updateNodes(List<Node> nodes, UnaryOperator<Node.Builder> builderOps, ZoneId zone, ControllerTester tester) { var currentNodes = getNodes(zone, nodes, tester); var updatedNodes = currentNodes.stream() - .map(node -> builderOps.apply(new Node.Builder(node)).build()) + .map(node -> builderOps.apply(Node.builder(node)).build()) .collect(Collectors.toList()); tester.configServer().nodeRepository().putNodes(zone, updatedNodes); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java index 770d0a898fe..51bda73025d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; @@ -7,14 +7,12 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.deployment.StableOsVersion; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.util.List; -import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -67,35 +65,6 @@ public class OsUpgradeSchedulerTest { assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); } - @Test // TODO(mpolden): Remove this after 2021-09-01 - public void schedule_calendar_versioned_without_scheduled_time() { - ControllerTester tester = new ControllerTester(); - OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); - Instant t0 = Instant.parse("2021-01-23T07:00:00.00Z"); // Inside trigger period - tester.clock().setInstant(t0); - - CloudName cloud = CloudName.from("cloud"); - ZoneApi zone = zone("prod.us-west-1", cloud); - tester.zoneRegistry().setZones(zone).reprovisionToUpgradeOsIn(zone); - - // Initial run does nothing as the cloud does not have a target - scheduler.maintain(); - assertTrue("No target set", tester.controller().osVersionTarget(cloud).isEmpty()); - - // Target is set - Version version0 = Version.fromString("7.0.0.20210123190005"); - // Simulate setting target without scheduledAt, to force parsing scheduled time from version number - tester.curator().writeOsVersionTargets(Set.of(new OsVersionTarget(new OsVersion(version0, cloud), - Duration.ofDays(1), Instant.EPOCH))); - - // Just over 45 days pass, and a new target replaces the expired one - Version version1 = Version.fromString("7.0.0.20210302"); - tester.clock().advance(Duration.ofDays(45).plus(Duration.ofSeconds(1))); - scheduler.maintain(); - assertEquals("New target set", version1, - tester.controller().osVersionTarget(cloud).get().osVersion().version()); - } - @Test public void schedule_stable_release() { ControllerTester tester = new ControllerTester(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index c29e10ab643..ec1a5455413 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; @@ -8,6 +8,7 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; @@ -261,19 +262,19 @@ public class OsUpgraderTest { } private List<Node> nodesRequiredToUpgrade(ZoneApi zone, SystemApplication application) { - return nodeRepository().list(zone.getVirtualId(), application.id()) + return nodeRepository().list(zone.getVirtualId(), NodeFilter.all().applications(application.id())) .stream() .filter(OsUpgrader::canUpgrade) .collect(Collectors.toList()); } private void failNodeIn(ZoneApi zone, SystemApplication application) { - List<Node> nodes = nodeRepository().list(zone.getVirtualId(), application.id()); + List<Node> nodes = nodeRepository().list(zone.getVirtualId(), NodeFilter.all().applications(application.id())); if (nodes.isEmpty()) { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); - nodeRepository().putNodes(zone.getVirtualId(), new Node.Builder(node).state(Node.State.failed).build()); + nodeRepository().putNodes(zone.getVirtualId(), Node.builder(node).state(Node.State.failed).build()); } /** Simulate OS upgrade of nodes allocated to application. In a real system this is done by the node itself */ @@ -285,9 +286,9 @@ public class OsUpgraderTest { assertWanted(wantedVersion, application, zones); for (ZoneApi zone : zones) { for (Node node : nodesRequiredToUpgrade(zone, application)) { - nodeRepository().putNodes(zone.getVirtualId(), new Node.Builder(node).wantedOsVersion(version) - .currentOsVersion(version) - .build()); + nodeRepository().putNodes(zone.getVirtualId(), Node.builder(node).wantedOsVersion(version) + .currentOsVersion(version) + .build()); } assertCurrent(version, application, zone); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java index 307540a9694..f5ae6bafc65 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 00d39788e38..8e90cfc69f3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -1,10 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggererTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggererTest.java index 3c22ee3f4c3..8b5a12c8879 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggererTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggererTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; @@ -43,7 +43,8 @@ public class ReindexingTriggererTest { } now = now.plus(interval); } - assertEquals("Should be in window of opportunity exactly four times each period", 4, triggered); + // Summer/winter time :'( + assertTrue("Should be in window of opportunity three to five times each period", 3 <= triggered && triggered <= 5); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index e61516cbb1a..a255a6c37d8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; @@ -114,8 +114,8 @@ public class ResourceMeterMaintainerTest { ZoneApiMock zone1 = ZoneApiMock.newBuilder().withId("prod.region-2").build(); ZoneApiMock zone2 = ZoneApiMock.newBuilder().withId("test.region-3").build(); tester.zoneRegistry().setZones(zone1, zone2); - tester.configServer().nodeRepository().setFixedNodes(zone1.getId()); - tester.configServer().nodeRepository().setFixedNodes(zone2.getId()); + tester.configServer().nodeRepository().addFixedNodes(zone1.getId()); + tester.configServer().nodeRepository().addFixedNodes(zone2.getId()); tester.configServer().nodeRepository().putNodes(zone1.getId(), createNodes()); } @@ -126,21 +126,21 @@ public class ResourceMeterMaintainerTest { Node.State.failed, Node.State.parked, Node.State.active) - .map(state -> new Node.Builder() - .hostname(HostName.from("host" + state)) - .parentHostname(HostName.from("parenthost" + state)) - .state(state) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(24, 24, 500, 1)) - .clusterId("clusterA") - .clusterType(state == Node.State.active ? Node.ClusterType.admin : Node.ClusterType.container) - .build()) + .map(state -> Node.builder() + .hostname(HostName.from("host" + state)) + .parentHostname(HostName.from("parenthost" + state)) + .state(state) + .type(NodeType.tenant) + .owner(ApplicationId.from("tenant1", "app1", "default")) + .currentVersion(Version.fromString("7.42")) + .wantedVersion(Version.fromString("7.42")) + .currentOsVersion(Version.fromString("7.6")) + .wantedOsVersion(Version.fromString("7.6")) + .serviceState(Node.ServiceState.expectedUp) + .resources(new NodeResources(24, 24, 500, 1)) + .clusterId("clusterA") + .clusterType(state == Node.State.active ? Node.ClusterType.admin : Node.ClusterType.container) + .build()) .collect(Collectors.toUnmodifiableList()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java index 516c28ab5cd..a90c8a9593b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -15,7 +15,6 @@ import org.junit.Test; import java.time.Duration; import java.util.List; import java.util.Map; -import java.util.Optional; import static com.yahoo.vespa.hosted.controller.maintenance.ResourceTagMaintainer.SHARED_HOST_APPLICATION; import static org.junit.Assert.assertEquals; @@ -25,7 +24,7 @@ import static org.junit.Assert.assertEquals; */ public class ResourceTagMaintainerTest { - final ControllerTester tester = new ControllerTester(); + private final ControllerTester tester = new ControllerTester(); @Test public void maintain() { @@ -51,24 +50,24 @@ public class ResourceTagMaintainerTest { } public void setNodes(ZoneId zone) { - var hostA = new Node.Builder() - .hostname(HostName.from("parentHostA." + zone.value())) - .type(NodeType.host) - .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) - .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) - .build(); - var nodeA = new Node.Builder() - .hostname(HostName.from("hostA." + zone.value())) - .type(NodeType.tenant) - .parentHostname(HostName.from("parentHostA." + zone.value())) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .build(); - var hostB = new Node.Builder() - .hostname(HostName.from("parentHostB." + zone.value())) - .type(NodeType.host) - .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) - .build(); - tester.configServer().nodeRepository().setNodes(zone, List.of(hostA, nodeA, hostB)); + var hostA = Node.builder() + .hostname(HostName.from("parentHostA." + zone.value())) + .type(NodeType.host) + .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) + .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) + .build(); + var nodeA = Node.builder() + .hostname(HostName.from("hostA." + zone.value())) + .type(NodeType.tenant) + .parentHostname(HostName.from("parentHostA." + zone.value())) + .owner(ApplicationId.from("tenant1", "app1", "default")) + .build(); + var hostB = Node.builder() + .hostname(HostName.from("parentHostB." + zone.value())) + .type(NodeType.host) + .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) + .build(); + tester.configServer().nodeRepository().putNodes(zone, List.of(hostA, nodeA, hostB)); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java index df93efab893..0c1b3b39b65 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java @@ -1,11 +1,11 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.RetriggerEntry; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java index 1eadae18668..db4b881dabc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ClusterSpec; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index db2353860ae..7d6da68440e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; @@ -305,7 +306,7 @@ public class SystemUpgraderTest { for (Node node : listNodes(zone, application)) { nodeRepository().putNodes( zone.getId(), - new Node.Builder(node).currentVersion(node.wantedVersion()).build()); + Node.builder(node).currentVersion(node.wantedVersion()).build()); } assertCurrentVersion(application, version, zone); }); @@ -322,14 +323,14 @@ public class SystemUpgraderTest { } private void failNodeIn(ZoneApi zone, SystemApplication application) { - List<Node> nodes = nodeRepository().list(zone.getId(), application.id()); + List<Node> nodes = nodeRepository().list(zone.getId(), NodeFilter.all().applications(application.id())); if (nodes.isEmpty()) { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); nodeRepository().putNodes( zone.getId(), - new Node.Builder(node).state(Node.State.failed).build()); + Node.builder(node).state(Node.State.failed).build()); } private void assertSystemVersion(Version version) { @@ -372,7 +373,7 @@ public class SystemUpgraderTest { } private List<Node> listNodes(ZoneApi zone, SystemApplication application) { - return nodeRepository().list(zone.getId(), application.id()).stream() + return nodeRepository().list(zone.getId(), NodeFilter.all().applications(application.id())).stream() .filter(SystemUpgrader::eligibleForUpgrade) .collect(Collectors.toList()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java index 050610905f3..c9cd7f41d2b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; @@ -6,15 +6,17 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import org.hamcrest.Matchers; import org.junit.Test; import java.time.Duration; import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; /** * @author mortent @@ -28,6 +30,7 @@ public class TenantRoleMaintainerTest { var devAppTenant1 = tester.newDeploymentContext("tenant1", "app1", "default"); var prodAppTenant2 = tester.newDeploymentContext("tenant2", "app2", "default"); var devAppTenant2 = tester.newDeploymentContext("tenant2","app3","default"); + var perfAppTenant1 = tester.newDeploymentContext("tenant3","app1","default"); ApplicationPackage appPackage = new ApplicationPackageBuilder() .region("us-west-1") .build(); @@ -36,6 +39,9 @@ public class TenantRoleMaintainerTest { devAppTenant1.runJob(JobType.devUsEast1, appPackage); devAppTenant2.runJob(JobType.devUsEast1, appPackage); + // Deploy perf apps + perfAppTenant1.runJob(JobType.perfUsEast3, appPackage); + // Deploy prod prodAppTenant2.submit(appPackage).deploy(); assertEquals(1, permanentDeployments(devAppTenant1.instance())); @@ -48,8 +54,7 @@ public class TenantRoleMaintainerTest { var roleService = tester.controller().serviceRegistry().roleService(); List<TenantName> tenantNames = ((MockRoleService) roleService).maintainedTenants(); - assertEquals(1, tenantNames.size()); - assertEquals(prodAppTenant2.application().id().tenant(), tenantNames.get(0)); + assertThat(tenantNames, Matchers.containsInAnyOrder(prodAppTenant2.application().id().tenant(), perfAppTenant1.application().id().tenant())); } private long permanentDeployments(Instance instance) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java index 29d77c38b1a..08af46d8d33 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; @@ -7,7 +7,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import org.junit.Test; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 326f4bf311e..1dd4ef24c9e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -167,7 +167,7 @@ public class UpgraderTest { // --- Failing application is repaired by changing the application, causing confidence to move above 'high' threshold // Deploy application change default0.submit(applicationPackage("default")); - default0.jobAborted(stagingTest); + default0.triggerJobs().jobAborted(stagingTest); default0.deploy(); tester.controllerTester().computeVersionStatus(); @@ -232,7 +232,7 @@ public class UpgraderTest { // State: Default applications started upgrading to version5 tester.clock().advance(Duration.ofHours(1)); tester.upgrader().maintain(); - default3.failDeployment(stagingTest); + default3.triggerJobs().jobAborted(stagingTest); default0.runJob(systemTest) .failDeployment(stagingTest); default1.runJob(systemTest) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainerTest.java new file mode 100644 index 00000000000..b658e86a575 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainerTest.java @@ -0,0 +1,46 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockUserManagement; +import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; +import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import org.junit.Test; + +import java.time.Duration; + +import static org.junit.Assert.*; + +/** + * @author olaa + */ +public class UserManagementMaintainerTest { + + private final ControllerTester tester = new ControllerTester(); + private final UserManagement userManagement = new MockUserManagement(); + private final UserManagementMaintainer userManagementMaintainer = new UserManagementMaintainer(tester.controller(), Duration.ofMinutes(1), userManagement); + + private final TenantName tenant = TenantName.from("tenant1"); + private final ApplicationName app = ApplicationName.from("app1"); + private final TenantName deletedTenant = TenantName.from("deleted-tenant"); + + @Test + public void finds_superfluous_roles() { + tester.createTenant(tenant.value()); + tester.createApplication(tenant.value(), app.value()); + + Roles.tenantRoles(tenant).forEach(userManagement::createRole); + Roles.applicationRoles(tenant, app).forEach(userManagement::createRole); + Roles.tenantRoles(deletedTenant).forEach(userManagement::createRole); + userManagement.createRole(Role.hostedSupporter()); + + var expectedRoles = Roles.tenantRoles(deletedTenant); + var actualRoles = userManagementMaintainer.findLeftoverRoles(); + + assertEquals(expectedRoles.size(), actualRoles.size()); + assertTrue(expectedRoles.containsAll(actualRoles) && actualRoles.containsAll(expectedRoles)); + } + +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java index 16ed6b7ef98..a2da6e357b6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java @@ -5,11 +5,12 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestSource; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction.State; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VCMRReport; +import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VcmrReport; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest.Status; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; @@ -20,16 +21,20 @@ import java.time.Duration; import java.time.Instant; import java.time.ZonedDateTime; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author olaa */ -public class VCMRMaintainerTest { +public class VcmrMaintainerTest { private ControllerTester tester; - private VCMRMaintainer maintainer; + private VcmrMaintainer maintainer; private NodeRepositoryMock nodeRepo; private final ZoneId zoneId = ZoneId.from("prod.us-east-3"); private final HostName host1 = HostName.from("host1"); @@ -39,19 +44,22 @@ public class VCMRMaintainerTest { @Before public void setup() { tester = new ControllerTester(); - maintainer = new VCMRMaintainer(tester.controller(), Duration.ofMinutes(1)); + maintainer = new VcmrMaintainer(tester.controller(), Duration.ofMinutes(1)); nodeRepo = tester.serviceRegistry().configServer().nodeRepository().allowPatching(true); } @Test public void recycle_hosts_after_completion() { - var vcmrReport = new VCMRReport(); + var vcmrReport = new VcmrReport(); vcmrReport.addVcmr("id123", ZonedDateTime.now(), ZonedDateTime.now()); var parkedNode = createNode(host1, NodeType.host, Node.State.parked, true); var failedNode = createNode(host2, NodeType.host, Node.State.failed, false); - parkedNode = new Node.Builder(parkedNode) - .reports(vcmrReport.toNodeReports()) - .build(); + Map<String, String> reports = vcmrReport.toNodeReports().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, + kv -> kv.getValue().toString())); + parkedNode = Node.builder(parkedNode) + .reports(reports) + .build(); nodeRepo.putNodes(zoneId, List.of(parkedNode, failedNode)); @@ -59,12 +67,11 @@ public class VCMRMaintainerTest { maintainer.maintain(); // Only the parked node is recycled, VCMR report is cleared - var nodeList = nodeRepo.list(zoneId, List.of(host1, host2)); + var nodeList = nodeRepo.list(zoneId, NodeFilter.all().hostnames(host1, host2)); assertEquals(Node.State.dirty, nodeList.get(0).state()); assertEquals(Node.State.failed, nodeList.get(1).state()); - var report = nodeList.get(0).reports(); - assertNull(report.get(VCMRReport.getReportId())); + assertTrue(nodeList.get(0).reports().isEmpty()); var writtenChangeRequest = tester.curator().readChangeRequest(changeRequestId).get(); assertEquals(Status.COMPLETED, writtenChangeRequest.getStatus()); @@ -105,7 +112,7 @@ public class VCMRMaintainerTest { assertEquals(State.NONE, failedNodeAction.getState()); assertEquals(Status.IN_PROGRESS, writtenChangeRequest.getStatus()); - activeNode = nodeRepo.list(zoneId, List.of(activeNode.hostname())).get(0); + activeNode = nodeRepo.list(zoneId, NodeFilter.all().hostnames(activeNode.hostname())).get(0); assertTrue(activeNode.wantToRetire()); } @@ -162,8 +169,8 @@ public class VCMRMaintainerTest { var approvedChangeRequests = tester.serviceRegistry().changeRequestClient().getApprovedChangeRequests(); assertEquals(1, approvedChangeRequests.size()); - activeNode = nodeRepo.list(zoneId, List.of(host2)).get(0); - var report = VCMRReport.fromReports(activeNode.reports()); + activeNode = nodeRepo.list(zoneId, NodeFilter.all().hostnames(host2)).get(0); + var report = VcmrReport.fromReports(activeNode.reports()); var reportAdded = report.getVcmrs().stream() .filter(vcmr -> vcmr.getId().equals(changeRequestId)) .count() == 1; @@ -184,11 +191,11 @@ public class VCMRMaintainerTest { var hostAction = writtenChangeRequest.getHostActionPlan().get(0); assertEquals(State.PENDING_RETIREMENT, hostAction.getState()); - parkedNode = nodeRepo.list(zoneId, List.of(parkedNode.hostname())).get(0); + parkedNode = nodeRepo.list(zoneId, NodeFilter.all().hostnames(parkedNode.hostname())).get(0); assertEquals(Node.State.dirty, parkedNode.state()); assertFalse(parkedNode.wantToRetire()); - retiringNode = nodeRepo.list(zoneId, List.of(retiringNode.hostname())).get(0); + retiringNode = nodeRepo.list(zoneId, NodeFilter.all().hostnames(retiringNode.hostname())).get(0); assertEquals(Node.State.active, retiringNode.state()); assertFalse(retiringNode.wantToRetire()); } @@ -235,11 +242,11 @@ public class VCMRMaintainerTest { } private Node createNode(HostName hostname, NodeType nodeType, Node.State state, boolean wantToRetire) { - return new Node.Builder() - .hostname(hostname) - .type(nodeType) - .state(state) - .wantToRetire(wantToRetire) - .build(); + return Node.builder() + .hostname(hostname) + .type(nodeType) + .state(state) + .wantToRetire(wantToRetire) + .build(); } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java index 0d2390f9d92..d85ec868d56 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.ControllerTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index 786809fb7b1..3e1d4ab14d7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 37a173ffc37..f1421b5affd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; @@ -88,7 +88,8 @@ public class ApplicationSerializerTest { Optional.of(Version.fromString("1.2.3")), Optional.of(Instant.ofEpochMilli(666)), Optional.empty(), - Optional.of("best commit")); + Optional.of("best commit"), + true); assertEquals("https://github/org/repo/tree/commit1", applicationVersion1.sourceUrl().get()); ApplicationVersion applicationVersion2 = ApplicationVersion diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java index 76940cf83e0..a9baeb9589d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java index ff59e14947b..f2171032e98 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializerTest.java index e8029e0bdbd..1e63b87e7c6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java index a2c995eaaf0..4ef09bbfce0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.HostName; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java index f13f92dee85..1fec1dbce02 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializerTest.java index c625d9f7f64..c722b3bbb96 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.ImmutableSet; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java index cff845c9dbe..482303a0e49 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java index 8feaf32d9aa..1af4465528d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.ImmutableSet; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 03a050db74e..eca2b17a5f1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.ImmutableMap; @@ -82,13 +82,16 @@ public class RunSerializerTest { assertEquals(running, run.status()); assertEquals(3, run.lastTestLogEntry()); assertEquals(new Version(1, 2, 3), run.versions().targetPlatform()); - ApplicationVersion applicationVersion = ApplicationVersion.from(new SourceRevision("git@github.com:user/repo.git", - "master", - "f00bad"), + ApplicationVersion applicationVersion = ApplicationVersion.from(Optional.of(new SourceRevision("git@github.com:user/repo.git", + "master", + "f00bad")), 123, - "a@b", - Version.fromString("6.3.1"), - Instant.ofEpochMilli(100)); + Optional.of("a@b"), + Optional.of(Version.fromString("6.3.1")), + Optional.of(Instant.ofEpochMilli(100)), + Optional.empty(), + Optional.empty(), + true); assertEquals(applicationVersion, run.versions().targetApplication()); assertEquals(applicationVersion.authorEmail(), run.versions().targetApplication().authorEmail()); assertEquals(applicationVersion.buildTime(), run.versions().targetApplication().buildTime()); @@ -148,6 +151,7 @@ public class RunSerializerTest { assertEquals(run.testerCertificate(), phoenix.testerCertificate()); assertEquals(run.versions(), phoenix.versions()); assertEquals(run.steps(), phoenix.steps()); + assertEquals(run.isDryRun(), phoenix.isDryRun()); Run initial = Run.initial(id, run.versions(), run.isRedeployment(), run.start(), JobProfile.production); assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializerTest.java index 97cf53d7b89..127f6b0dcbf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializerTest.java @@ -1,4 +1,4 @@ -// Copyright 2021 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.security.KeyAlgorithm; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index e123c4cca62..0b986667911 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -1,5 +1,5 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence;// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence;// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretSto import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import com.yahoo.vespa.hosted.controller.tenant.TenantInfoAddress; @@ -172,6 +173,16 @@ public class TenantSerializerTest { assertEquals(fullInfo, roundTripInfo); } + @Test + public void deleted_tenant() { + DeletedTenant tenant = new DeletedTenant( + TenantName.from("tenant1"), Instant.ofEpochMilli(1234L), Instant.ofEpochMilli(2345L)); + DeletedTenant serialized = (DeletedTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + assertEquals(tenant.name(), serialized.name()); + assertEquals(tenant.createdAt(), serialized.createdAt()); + assertEquals(tenant.deletedAt(), serialized.deletedAt()); + } + private static Contact contact() { return new Contact( URI.create("http://contact1.test"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 0f40dd27664..7b9131a38dd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -40,6 +40,7 @@ "branch": "master", "commit": "f00bad", "build": 123, + "deployedDirectly": true, "authorEmail": "a@b", "compileVersion": "6.3.1", "buildTime": 100, @@ -48,7 +49,8 @@ "repository": "git@github.com:user/repo.git", "branch": "master", "commit": "badb17", - "build": 122 + "build": 122, + "deployedDirectly": false } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java index 1fce7ba5695..0b28d94386d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; import com.github.tomakehurst.wiremock.junit.WireMockRule; @@ -7,6 +7,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; +import com.yahoo.yolean.concurrent.Sleeper; import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpCoreContext; import org.junit.Rule; @@ -38,7 +39,7 @@ public class ConfigServerRestExecutorImplTest { public void proxy_with_retries() throws Exception { var connectionReuseStrategy = new CountingConnectionReuseStrategy(Set.of("127.0.0.1")); var proxy = new ConfigServerRestExecutorImpl(new ZoneRegistryMock(SystemName.cd), SSLContext.getDefault(), - (duration) -> {}, connectionReuseStrategy); + Sleeper.NOOP, connectionReuseStrategy); URI url = url(); String path = url.getPath(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java index 15ec138354d..9d80dd25ec0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.jdisc.http.HttpRequest; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java index 539d4a6dd75..4cc2c1cc79e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.jdisc.http.HttpRequest; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ApplicationRequestToDiscFilterRequestWrapper.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ApplicationRequestToDiscFilterRequestWrapper.java index 7cc00f1ad52..b25cb913c83 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ApplicationRequestToDiscFilterRequestWrapper.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ApplicationRequestToDiscFilterRequestWrapper.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi; import com.yahoo.application.container.handler.Request; @@ -114,6 +114,7 @@ public class ApplicationRequestToDiscFilterRequestWrapper extends DiscFilterRequ } @Override + @Deprecated public void setUri(URI uri) { throw new UnsupportedOperationException(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index 10f143a8e96..1d844859c37 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -26,7 +26,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Optional; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.regex.Pattern; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ResponseHandlerToApplicationResponseWrapper.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ResponseHandlerToApplicationResponseWrapper.java index 1af5de3e4ea..d57920e8b3c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ResponseHandlerToApplicationResponseWrapper.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ResponseHandlerToApplicationResponseWrapper.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi; import com.yahoo.jdisc.Response; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 5f76a30bf45..a4de6ab7700 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -1,6 +1,7 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; +import ai.vespa.hosted.api.MultiPartStreamer; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -10,10 +11,12 @@ import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; @@ -232,6 +235,42 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { 200); } + @Test + public void create_application_on_deploy() { + var application = ApplicationName.from("unique"); + var applicationPackage = new ApplicationPackageBuilder().build(); + + assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isEmpty()); + + tester.assertResponse( + request("/application/v4/tenant/scoober/application/unique/instance/default/deploy/dev-aws-us-east-1c", POST) + .data(createApplicationDeployData(Optional.of(applicationPackage), Optional.empty(), true)) + .roles(Set.of(Role.developer(tenantName))), + "{\"message\":\"Deployment started in run 1 of dev-aws-us-east-1c for scoober.unique. This may take about 15 minutes the first time.\",\"run\":1}"); + + assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isPresent()); + } + + @Test + public void create_application_on_submit() { + var application = ApplicationName.from("unique"); + var applicationPackage = new ApplicationPackageBuilder() + .trustDefaultCertificate() + .build(); + + assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isEmpty()); + + var data = ApplicationApiTest.createApplicationSubmissionData(applicationPackage, 123); + + tester.assertResponse( + request("/application/v4/tenant/scoober/application/unique/submit", POST) + .data(data) + .roles(Set.of(Role.developer(tenantName))), + "{\"message\":\"Application package version: 1.0.1-commit1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + + assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isPresent()); + } + private ApplicationPackageBuilder prodBuilder() { return new ApplicationPackageBuilder() .instances("default") @@ -264,8 +303,31 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { JobType.productionAwsUsEast1c, Optional.empty(), applicationPackage); + } + private MultiPartStreamer createApplicationDeployData(Optional<ApplicationPackage> applicationPackage, + Optional<ApplicationVersion> applicationVersion, boolean deployDirectly) { + MultiPartStreamer streamer = new MultiPartStreamer(); + streamer.addJson("deployOptions", deployOptions(deployDirectly, applicationVersion)); + applicationPackage.ifPresent(ap -> streamer.addBytes("applicationZip", ap.zippedContent())); + return streamer; + } + + private String deployOptions(boolean deployDirectly, Optional<ApplicationVersion> applicationVersion) { + return "{\"vespaVersion\":null," + + "\"ignoreValidationErrors\":false," + + "\"deployDirectly\":" + deployDirectly + + applicationVersion.map(version -> + "," + + "\"buildNumber\":" + version.buildNumber().getAsLong() + "," + + "\"sourceRevision\":{" + + "\"repository\":\"" + version.source().get().repository() + "\"," + + "\"branch\":\"" + version.source().get().branch() + "\"," + + "\"commit\":\"" + version.source().get().commit() + "\"" + + "}" + ).orElse("") + + "}"; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index a01097cfcb6..d2eb43d31f8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.MultiPartStreamer; @@ -52,7 +52,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; @@ -84,7 +84,6 @@ import java.io.File; import java.math.BigInteger; import java.net.URI; import java.security.cert.X509Certificate; -import java.time.Duration; import java.time.Instant; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; @@ -251,7 +250,7 @@ public class ApplicationApiTest extends ControllerContainerTest { var app1 = deploymentTester.newDeploymentContext(id); // POST (deploy) an application to start a manual deployment in prod is not allowed - MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST) .data(entity) .userIdentity(USER_ID), @@ -272,6 +271,12 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); app1.runJob(JobType.devUsEast1); + // POST (deploy) a job to restart a manual deployment to dev + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1", POST) + .userIdentity(USER_ID), + "{\"message\":\"Triggered dev-us-east-1 for tenant1.application1.instance1\"}"); + app1.runJob(JobType.devUsEast1); + // GET dev application package tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1/package", GET) .userIdentity(USER_ID), @@ -284,7 +289,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST an application package is not generally allowed under user instance tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/otheruser/deploy/dev-us-east-1", POST) .userIdentity(OTHER_USER_ID) - .data(createApplicationDeployData(applicationPackageInstance1, false)), + .data(createApplicationDeployData(applicationPackageInstance1)), accessDenied, 403); @@ -298,9 +303,16 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST an application package is not allowed under user instance for tenant admins tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/myuser/deploy/dev-us-east-1", POST) .userIdentity(USER_ID) - .data(createApplicationDeployData(applicationPackageInstance1, false)), + .data(createApplicationDeployData(applicationPackageInstance1)), new File("deployment-job-accepted-2.json")); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/myuser/job/dev-us-east-1/diff/1", GET).userIdentity(HOSTED_VESPA_OPERATOR), + (response) -> assertTrue(response.getBodyAsString(), + response.getBodyAsString().contains("--- search-definitions/test.sd\n" + + "@@ -1,0 +1,1 @@\n" + + "+ search test { }\n")), + 200); + // DELETE a dev deployment is allowed under user instance for tenant admins tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/myuser/environment/dev/region/us-east-1", DELETE) .userIdentity(USER_ID), @@ -612,6 +624,18 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), "{\"enabled\":true,\"clusters\":[{\"name\":\"cluster\",\"pending\":[{\"type\":\"type\",\"requiredGeneration\":100}],\"ready\":[{\"type\":\"type\",\"readyAtMillis\":345,\"startedAtMillis\":456,\"endedAtMillis\":567,\"state\":\"failed\",\"message\":\"(#`д´)ノ\",\"progress\":0.1}]}]}"); + // POST to request a service dump + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/node/host-tenant1:application1:instance1-prod.us-central-1/service-dump", POST) + .userIdentity(HOSTED_VESPA_OPERATOR) + .data("{\"configId\":\"default/container.1\",\"artifacts\":[\"jvm-dump\"],\"dumpOptions\":{\"duration\":30}}"), + "{\"message\":\"Request created\"}"); + + // GET to get status of service dump + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/node/host-tenant1:application1:instance1-prod.us-central-1/service-dump", GET) + .userIdentity(HOSTED_VESPA_OPERATOR), + "{\"createdMillis\":" + tester.controller().clock().millis() + ",\"configId\":\"default/container.1\"" + + ",\"artifacts\":[\"jvm-dump\"],\"dumpOptions\":{\"duration\":30}}"); + // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) .userIdentity(USER_ID), @@ -746,6 +770,12 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(createApplicationSubmissionData(packageWithService, 123)), "{\"message\":\"Application package version: 1.0.2-commit1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/diff/2", GET).userIdentity(HOSTED_VESPA_OPERATOR), + (response) -> assertTrue(response.getBodyAsString(), + response.getBodyAsString().contains("+ <deployment version='1.0' athenz-domain='domain1' athenz-service='service'>\n" + + "- <deployment version='1.0' >\n")), + 200); + // GET last submitted application package tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET).userIdentity(HOSTED_VESPA_OPERATOR), (response) -> { @@ -805,7 +835,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET system test job overview. tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", GET) - .userIdentity(USER_ID).properties(Map.of("limit", "100")), + .userIdentity(USER_ID), new File("system-test-job.json")); // GET system test run 1 details. @@ -839,7 +869,32 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE an empty tenant tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE).userIdentity(USER_ID) .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - new File("tenant-without-applications.json")); + "{\"message\":\"Deleted tenant tenant1\"}"); + + // The tenant is not found + tester.assertResponse(request("/application/v4/tenant/tenant1", GET).userIdentity(USER_ID) + .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}", 404); + + // ... unless we specify to show deleted tenants + tester.assertResponse(request("/application/v4/tenant/tenant1", GET).properties(Map.of("includeDeleted", "true")) + .userIdentity(HOSTED_VESPA_OPERATOR), + new File("tenant1-deleted.json")); + + // Tenant cannot be recreated + tester.assertResponse(request("/application/v4/tenant/tenant1", POST).userIdentity(USER_ID) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Tenant 'tenant1' already exists\"}", 400); + + + // Forget a deleted tenant + tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE).properties(Map.of("forget", "true")) + .userIdentity(HOSTED_VESPA_OPERATOR), + "{\"message\":\"Deleted tenant tenant1\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1", GET).properties(Map.of("includeDeleted", "true")) + .userIdentity(HOSTED_VESPA_OPERATOR), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}", 404); } private void addIssues(DeploymentTester tester, TenantAndApplicationId id) { @@ -971,38 +1026,24 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void testDeployDirectly() { + public void testDeployWithApplicationPackage() { // Setup - createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); - // Create tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", POST).userIdentity(USER_ID) - .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") - .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - new File("tenant-without-applications.json")); - - // Create application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) - .userIdentity(USER_ID) - .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - new File("instance-reference.json")); - - // Add build service to operator role - addUserToHostedOperatorRole(HostedAthenzIdentities.from(SCREWDRIVER_ID)); - // POST (deploy) a system application with an application package - MultiPartStreamer noAppEntity = createApplicationDeployData(Optional.empty(), true); + MultiPartStreamer noAppEntity = createApplicationDeployData(Optional.empty()); tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/routing/environment/prod/region/us-central-1/instance/default/deploy", POST) .data(noAppEntity) .userIdentity(HOSTED_VESPA_OPERATOR), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Deployment of system applications during a system upgrade is not allowed\"}", 400); - deploymentTester.controllerTester().upgradeSystem(deploymentTester.controller().readVersionStatus().controllerVersion().get().versionNumber()); + deploymentTester.controllerTester() + .upgradeSystem(deploymentTester.controller().readVersionStatus().controllerVersion().get() + .versionNumber()); tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/routing/environment/prod/region/us-central-1/instance/default/deploy", POST) - .data(noAppEntity) - .userIdentity(HOSTED_VESPA_OPERATOR), - new File("deploy-result.json")); + .data(noAppEntity) + .userIdentity(HOSTED_VESPA_OPERATOR), + new File("deploy-result.json")); } @Test @@ -1058,7 +1099,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void testErrorResponses() throws Exception { + public void testErrorResponses() { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); // PUT (update) non-existing tenant returns 403 as tenant access cannot be determined when the tenant does not exist @@ -1168,7 +1209,7 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // POST (deploy) an application to legacy deploy path - MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), @@ -1193,11 +1234,18 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete instance 'tenant1.application1.instance1': Instance not found\"}", 404); + // DELETE and forget an application as non-operator + tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE).properties(Map.of("forget", "true")) + .userIdentity(USER_ID) + .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"Only operators can forget a tenant\"}", + 403); + // DELETE tenant tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - new File("tenant-without-applications.json")); + "{\"message\":\"Deleted tenant tenant1\"}"); // DELETE tenant again returns 403 as tenant access cannot be determined when the tenant does not exist tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE) .userIdentity(USER_ID), @@ -1268,7 +1316,7 @@ public class ApplicationApiTest extends ControllerContainerTest { 200); // Deploy to an authorized zone by a user tenant is disallowed - MultiPartStreamer entity = createApplicationDeployData(applicationPackageDefault, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageDefault); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) .data(entity) .userIdentity(USER_ID), @@ -1375,7 +1423,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .build(); createTenantAndApplication(); - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackage); // POST (deploy) an application to dev through a deployment job, with user instance and a proper tenant tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST) .data(entity) @@ -1422,7 +1470,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .build(); // deploy the application to a dev zone. Should fail since the developer is not authorized to launch the service - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackage); tester.assertResponse(request("/application/v4/tenant/sandbox/application/myapp/instance/default/deploy/dev-us-east-1", POST) .data(entity) .userIdentity(developer), @@ -1431,7 +1479,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Allow developer launch privilege to domain1.service. Deployment now completes. AthenzDbMock.Domain domainMock = tester.athenzClientFactory().getSetup().getOrCreateDomain(ATHENZ_TENANT_DOMAIN); - domainMock.withPolicy("user." + developer.id(), "launch", "service.service"); + domainMock.withPolicy("launch-" +developer.id(), "user." + developer.id(), "launch", "service.service"); tester.assertResponse(request("/application/v4/tenant/sandbox/application/myapp/instance/default/deploy/dev-us-east-1", POST) @@ -1583,6 +1631,62 @@ public class ApplicationApiTest extends ControllerContainerTest { assertEquals(0, activeGrants.size()); } + @Test + public void testServiceView() { + createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); + String serviceApi="/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service"; + // Not allowed to request apis not listed in feature flag allowed-service-view-apis. e.g /document/v1 + tester.assertResponse(request(serviceApi + "/storagenode-awe3slno6mmq2fye191y324jl/document/v1/", GET) + .userIdentity(USER_ID) + .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"Access denied\"}", + 403); + + // Test path traversal + tester.assertResponse(request(serviceApi + "/storagenode-awe3slno6mmq2fye191y324jl/state/v1/../../document/v1/", GET) + .userIdentity(USER_ID) + .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"Access denied\"}", + 403); + + // Test urlencoded path traversal + tester.assertResponse(request(serviceApi + "/storagenode-awe3slno6mmq2fye191y324jl/state%2Fv1%2F..%2F..%2Fdocument%2Fv1%2F", GET) + .userIdentity(USER_ID) + .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"Access denied\"}", + 403); + } + + @Test + public void create_application_on_deploy() { + // Setup + createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); + addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); + + // Create tenant + tester.assertResponse(request("/application/v4/tenant/tenant1", POST).userIdentity(USER_ID) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), + new File("tenant-without-applications.json")); + + // Deploy application + var id = ApplicationId.from("tenant1", "application1", "instance1"); + var appId = TenantAndApplicationId.from(id); + var entity = createApplicationDeployData(applicationPackageInstance1); + + assertTrue(tester.controller().applications().getApplication(appId).isEmpty()); + + // POST (deploy) an application to start a manual deployment to dev + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1/", POST) + .data(entity) + .oktaIdentityToken(OKTA_IT) + .oktaAccessToken(OKTA_AT) + .userIdentity(USER_ID), + "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); + + assertTrue(tester.controller().applications().getApplication(appId).isPresent()); + } + private static String serializeInstant(Instant i) { return DateTimeFormatter.ISO_INSTANT.format(i.truncatedTo(ChronoUnit.SECONDS)); } @@ -1595,18 +1699,18 @@ public class ApplicationApiTest extends ControllerContainerTest { .build(); } - private MultiPartStreamer createApplicationDeployData(ApplicationPackage applicationPackage, boolean deployDirectly) { - return createApplicationDeployData(Optional.of(applicationPackage), deployDirectly); + private MultiPartStreamer createApplicationDeployData(ApplicationPackage applicationPackage) { + return createApplicationDeployData(Optional.of(applicationPackage)); } - private MultiPartStreamer createApplicationDeployData(Optional<ApplicationPackage> applicationPackage, boolean deployDirectly) { - return createApplicationDeployData(applicationPackage, Optional.empty(), deployDirectly); + private MultiPartStreamer createApplicationDeployData(Optional<ApplicationPackage> applicationPackage) { + return createApplicationDeployData(applicationPackage, Optional.empty()); } private MultiPartStreamer createApplicationDeployData(Optional<ApplicationPackage> applicationPackage, - Optional<ApplicationVersion> applicationVersion, boolean deployDirectly) { + Optional<ApplicationVersion> applicationVersion) { MultiPartStreamer streamer = new MultiPartStreamer(); - streamer.addJson("deployOptions", deployOptions(deployDirectly, applicationVersion)); + streamer.addJson("deployOptions", deployOptions(applicationVersion)); applicationPackage.ifPresent(ap -> streamer.addBytes("applicationZip", ap.zippedContent())); return streamer; } @@ -1618,10 +1722,9 @@ public class ApplicationApiTest extends ControllerContainerTest { .addBytes(EnvironmentResource.APPLICATION_TEST_ZIP, "content".getBytes()); } - private String deployOptions(boolean deployDirectly, Optional<ApplicationVersion> applicationVersion) { + private String deployOptions(Optional<ApplicationVersion> applicationVersion) { return "{\"vespaVersion\":null," + - "\"ignoreValidationErrors\":false," + - "\"deployDirectly\":" + deployDirectly + + "\"ignoreValidationErrors\":false" + applicationVersion.map(version -> "," + "\"buildNumber\":" + version.buildNumber().getAsLong() + "," + @@ -1654,7 +1757,8 @@ public class ApplicationApiTest extends ControllerContainerTest { */ private void allowLaunchOfService(com.yahoo.vespa.athenz.api.AthenzService service) { AthenzDbMock.Domain domainMock = tester.athenzClientFactory().getSetup().getOrCreateDomain(service.getDomain()); - domainMock.withPolicy(tester.controller().zoneRegistry().accessControlDomain().value()+".provider.*","launch", "service." + service.getName()); + String principalRegex = tester.controller().zoneRegistry().accessControlDomain().value() + ".provider.*"; + domainMock.withPolicy("provider-launch", principalRegex,"launch", "service." + service.getName()); } /** diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 72295497c03..2ae755ac8fe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.component.Version; @@ -9,7 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.Test; @@ -157,9 +157,10 @@ public class JobControllerApiHandlerHelperTest { tester.configServer().setLogStream(() -> "Nope, this won't be logged"); tester.configServer().convergeServices(app.instanceId(), zone); tester.runner().run(); + assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), devUsEast1).get().id(), "8"), "dev-us-east-1-log-second-part.json"); + tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage()); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root")), "dev-overview.json"); - assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), devUsEast1).get().id(), "8"), "dev-us-east-1-log-second-part.json"); } @Test @@ -175,6 +176,19 @@ public class JobControllerApiHandlerHelperTest { "jobs-direct-deployment.json"); } + @Test + public void testResponsesWithDryRunDeployment() { + var tester = new DeploymentTester(); + var app = tester.newDeploymentContext(); + tester.clock().setInstant(Instant.EPOCH); + var region = "us-west-1"; + var applicationPackage = new ApplicationPackageBuilder().region(region).build(); + // Deploy directly to production zone, like integration tests, with dryRun. + tester.controller().jobController().deploy(tester.instance().id(), productionUsWest1, Optional.empty(), applicationPackage, true); + assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root/")), + "jobs-direct-deployment.json"); + } + private void compare(HttpResponse response, String expected) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java index 1a623c4e3eb..5bf22ede19d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import com.google.inject.Key; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java index 57774c3f412..c69cd51e20d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 55be0881ec2..abe3d4100d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -413,7 +413,7 @@ "id": 1, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/run/1", "start": "(ignore)", - "status": "aborted", + "status": "running", "versions": { "targetPlatform": "6.1.0", "targetApplication": { @@ -469,7 +469,7 @@ "id": 2, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/2", "start": "(ignore)", - "status": "aborted", + "status": "running", "versions": { "targetPlatform": "6.1.0", "targetApplication": { @@ -502,7 +502,9 @@ "status": "success", "versions": { "targetPlatform": "6.1.0", - "targetApplication": {} + "targetApplication": { + "build": 1 + } }, "steps": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json index 3a78f8c44a0..dce73ad56cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json @@ -1,22 +1,31 @@ { - "1": { - "id": 1, - "status": "success", - "start": 14503000, - "end": 14503000, - "wantedPlatform": "7.1", - "wantedApplication": { - "hash": "unknown" - }, - "steps": { - "deployReal": "succeeded", - "installReal": "succeeded", - "copyVespaLogs": "succeeded" - }, - "tasks": { - "deploy": "succeeded", - "install": "succeeded" - }, - "log": "https://some.url:43/root/run/1" - } + "runs": [ + { + "id": 1, + "url": "https://some.url:43/root/run/1", + "start": 14503000, + "end": 14503000, + "status": "success", + "versions": { + "targetPlatform": "7.1.0", + "targetApplication": { + "build": 1 + } + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json index e3beb371acd..63fd0845d1b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -5,7 +5,39 @@ "runs": [ { "versions": { - "targetApplication": {}, + "targetApplication": { + "build": 2 + }, + "targetPlatform": "6.1.0", + "sourceApplication": { + "build": 1 + }, + "sourcePlatform": "6.1.0" + }, + "start": 0, + "id": 2, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "unfinished" + }, + { + "name": "copyVespaLogs", + "status": "unfinished" + } + ], + "url": "https://some.url:43/root/run/2", + "status": "running" + }, + { + "versions": { + "targetApplication": { + "build": 1 + }, "targetPlatform": "6.1.0" }, "start": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json index 72411d155c7..3ef993c6589 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json @@ -6,7 +6,7 @@ { "at": 0, "type": "info", - "message": "Deploying platform version 6.1 and application version unknown ..." + "message": "Deploying platform version 6.1 and application version 1.0.1 ..." }, { "at": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index d61bebc81d1..f2f8e14f093 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -21,6 +21,7 @@ "revision": "(ignore)", "deployTimeEpochMs": "(ignore)", "screwdriverId": "123", + "status": "complete", "quota": "(ignore)", "activity": { "lastQueried": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 9a742a9b176..736e1fe082c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -4,6 +4,37 @@ "jobName": "dev-us-east-1", "runs": [ { + "id": 2, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/run/2", + "start": "(ignore)", + "end": "(ignore)", + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": { + "build": 1 + }, + "sourcePlatform":"6.1.0", + "sourceApplication": { + "build": 1 + } + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] + }, + { "id": 1, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/run/1", "start": "(ignore)", @@ -11,7 +42,9 @@ "status": "success", "versions": { "targetPlatform": "6.1.0", - "targetApplication": {} + "targetApplication": { + "build": 1 + } }, "steps": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index 2601937faee..f8aba54356b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -5,7 +5,9 @@ "runs": [ { "versions": { - "targetApplication": {}, + "targetApplication": { + "build": 1 + }, "targetPlatform": "7.1.0" }, "start": 14503000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json index 37ae9e4b56b..a98ae5c678d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json @@ -1,242 +1,402 @@ { - "1": { - "id": 1, - "status": "success", - "start": 0, - "end": 0, - "wantedPlatform": "6.1", - "wantedApplication": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" + "runs": [ + { + "id": 1, + "url": "https://some.url:43/root/run/1", + "start": 0, + "end": 0, + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" + "steps": [ + { + "name": "deployTester", + "status": "succeeded" + }, + { + "name": "installTester", + "status": "succeeded" + }, + { + "name": "deployInitialReal", + "status": "succeeded" + }, + { + "name": "installInitialReal", + "status": "succeeded" + }, + { + "name": "startStagingSetup", + "status": "succeeded" + }, + { + "name": "endStagingSetup", + "status": "succeeded" + }, + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "startTests", + "status": "succeeded" + }, + { + "name": "endTests", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + }, + { + "name": "deactivateReal", + "status": "succeeded" + }, + { + "name": "deactivateTester", + "status": "succeeded" + }, + { + "name": "report", + "status": "succeeded" + } + ] }, - "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", - "deployInitialReal": "succeeded", - "installInitialReal": "succeeded", - "startStagingSetup": "succeeded", - "endStagingSetup": "succeeded", - "deployReal": "succeeded", - "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "copyVespaLogs": "succeeded", - "deactivateReal": "succeeded", - "deactivateTester": "succeeded", - "report": "succeeded" - }, - "tasks": { - "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" - }, - "log": "https://some.url:43/root/run/1" - }, - "2": { - "id": 2, - "status": "success", - "start": 1000, - "end": 1000, - "wantedPlatform": "6.1", - "wantedApplication": { - "hash": "1.0.2-commit1", - "build": 2, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" + { + "id": 2, + "url": "https://some.url:43/root/run/2", + "start": 1000, + "end": 1000, + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": { + "build": 2, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "sourcePlatform": "6.1.0", + "sourceApplication": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" + "steps": [ + { + "name": "deployTester", + "status": "succeeded" + }, + { + "name": "installTester", + "status": "succeeded" + }, + { + "name": "deployInitialReal", + "status": "succeeded" + }, + { + "name": "installInitialReal", + "status": "succeeded" + }, + { + "name": "startStagingSetup", + "status": "succeeded" + }, + { + "name": "endStagingSetup", + "status": "succeeded" + }, + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "startTests", + "status": "succeeded" + }, + { + "name": "endTests", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + }, + { + "name": "deactivateReal", + "status": "succeeded" + }, + { + "name": "deactivateTester", + "status": "succeeded" + }, + { + "name": "report", + "status": "succeeded" + } + ] }, - "currentPlatform": "6.1", - "currentApplication": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" + { + "id": 3, + "url": "https://some.url:43/root/run/3", + "start": 14403000, + "end": 14403000, + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": { + "build": 3, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "sourcePlatform": "6.1.0", + "sourceApplication": { + "build": 2, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", - "deployInitialReal": "succeeded", - "installInitialReal": "succeeded", - "startStagingSetup": "succeeded", - "endStagingSetup": "succeeded", - "deployReal": "succeeded", - "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "copyVespaLogs": "succeeded", - "deactivateReal": "succeeded", - "deactivateTester": "succeeded", - "report": "succeeded" + "steps": [ + { + "name": "deployTester", + "status": "succeeded" + }, + { + "name": "installTester", + "status": "succeeded" + }, + { + "name": "deployInitialReal", + "status": "succeeded" + }, + { + "name": "installInitialReal", + "status": "succeeded" + }, + { + "name": "startStagingSetup", + "status": "succeeded" + }, + { + "name": "endStagingSetup", + "status": "succeeded" + }, + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "startTests", + "status": "succeeded" + }, + { + "name": "endTests", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + }, + { + "name": "deactivateReal", + "status": "succeeded" + }, + { + "name": "deactivateTester", + "status": "succeeded" + }, + { + "name": "report", + "status": "succeeded" + } + ] }, - "tasks": { - "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" - }, - "log": "https://some.url:43/root/run/2" - }, - "3": { - "id": 3, - "status": "success", - "start": 14403000, - "end": 14403000, - "wantedPlatform": "6.1", - "wantedApplication": { - "hash": "1.0.3-commit1", - "build": 3, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" + { + "id": 4, + "url": "https://some.url:43/root/run/4", + "start": 14403000, + "end": 14403000, + "status": "installationFailed", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": { + "build": 3, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "sourcePlatform": "6.1.0", + "sourceApplication": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" + "steps": [ + { + "name": "deployTester", + "status": "succeeded" + }, + { + "name": "installTester", + "status": "unfinished" + }, + { + "name": "deployInitialReal", + "status": "succeeded" + }, + { + "name": "installInitialReal", + "status": "failed" + }, + { + "name": "startStagingSetup", + "status": "unfinished" + }, + { + "name": "endStagingSetup", + "status": "unfinished" + }, + { + "name": "deployReal", + "status": "unfinished" + }, + { + "name": "installReal", + "status": "unfinished" + }, + { + "name": "startTests", + "status": "unfinished" + }, + { + "name": "endTests", + "status": "unfinished" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + }, + { + "name": "deactivateReal", + "status": "succeeded" + }, + { + "name": "deactivateTester", + "status": "succeeded" + }, + { + "name": "report", + "status": "succeeded" + } + ] }, - "currentPlatform": "6.1", - "currentApplication": { - "hash": "1.0.2-commit1", - "build": 2, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" + { + "id": 5, + "url": "https://some.url:43/root/run/5", + "start": 14503000, + "end": 14503000, + "status": "installationFailed", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": { + "build": 3, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "sourcePlatform": "6.1.0", + "sourceApplication": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", - "deployInitialReal": "succeeded", - "installInitialReal": "succeeded", - "startStagingSetup": "succeeded", - "endStagingSetup": "succeeded", - "deployReal": "succeeded", - "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "copyVespaLogs": "succeeded", - "deactivateReal": "succeeded", - "deactivateTester": "succeeded", - "report": "succeeded" - }, - "tasks": { - "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" - }, - "log": "https://some.url:43/root/run/3" - }, - "4": { - "id": 4, - "status": "installationFailed", - "start": 14403000, - "end": 14403000, - "wantedPlatform": "6.1", - "wantedApplication": { - "hash": "1.0.3-commit1", - "build": 3, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "currentPlatform": "6.1", - "currentApplication": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "steps": { - "deployTester": "succeeded", - "installTester": "unfinished", - "deployInitialReal": "succeeded", - "installInitialReal": "failed", - "startStagingSetup": "unfinished", - "endStagingSetup": "unfinished", - "deployReal": "unfinished", - "installReal": "unfinished", - "startTests": "unfinished", - "endTests": "unfinished", - "copyVespaLogs": "succeeded", - "deactivateReal": "succeeded", - "deactivateTester": "succeeded", - "report": "succeeded" - }, - "tasks": {}, - "log": "https://some.url:43/root/run/4" - }, - "5": { - "id": 5, - "status": "installationFailed", - "start": 14503000, - "end": 14503000, - "wantedPlatform": "6.1", - "wantedApplication": { - "hash": "1.0.3-commit1", - "build": 3, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "currentPlatform": "6.1", - "currentApplication": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "steps": { - "deployTester": "succeeded", - "installTester": "unfinished", - "deployInitialReal": "succeeded", - "installInitialReal": "failed", - "startStagingSetup": "unfinished", - "endStagingSetup": "unfinished", - "deployReal": "unfinished", - "installReal": "unfinished", - "startTests": "unfinished", - "endTests": "unfinished", - "copyVespaLogs": "succeeded", - "deactivateReal": "succeeded", - "deactivateTester": "succeeded", - "report": "succeeded" - }, - "tasks": {}, - "log": "https://some.url:43/root/run/5" - } + "steps": [ + { + "name": "deployTester", + "status": "succeeded" + }, + { + "name": "installTester", + "status": "unfinished" + }, + { + "name": "deployInitialReal", + "status": "succeeded" + }, + { + "name": "installInitialReal", + "status": "failed" + }, + { + "name": "startStagingSetup", + "status": "unfinished" + }, + { + "name": "endStagingSetup", + "status": "unfinished" + }, + { + "name": "deployReal", + "status": "unfinished" + }, + { + "name": "installReal", + "status": "unfinished" + }, + { + "name": "startTests", + "status": "unfinished" + }, + { + "name": "endTests", + "status": "unfinished" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + }, + { + "name": "deactivateReal", + "status": "succeeded" + }, + { + "name": "deactivateTester", + "status": "succeeded" + }, + { + "name": "report", + "status": "succeeded" + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index 2cf846ab6bb..0525f059dd0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -59,6 +59,21 @@ "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": 14503000, + "type": "info", + "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" + }, + { + "at": 14503000, + "type": "info", + "message": "--- platform vespa/vespa:6.1" + }, + { + "at": 14503000, + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" } ], "deployInitialReal": [ @@ -102,6 +117,21 @@ { "at": 14503000, "type": "info", + "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" + }, + { + "at": 14503000, + "type": "info", + "message": "--- platform vespa/vespa:6.1" + }, + { + "at": 14503000, + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": 14503000, + "type": "info", "message": "Deployment expired before installation was successful." } ], @@ -120,7 +150,7 @@ } ] }, - "lastId": 21, + "lastId": 27, "steps": { "deployTester": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json index 342db367807..7ee3952a8b5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json @@ -78,6 +78,21 @@ { "at": "(ignore)", "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": "(ignore)", + "type": "info", + "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "info", + "message": "--- platform vespa/vespa:6.1" + }, + { + "at": "(ignore)", + "type": "info", "message": "Tester container successfully installed!" } ], @@ -197,6 +212,21 @@ { "at": "(ignore)", "type": "info", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "info", + "message": "--- platform vespa/vespa:6.1" + }, + { + "at": "(ignore)", + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": "(ignore)", + "type": "info", "message": "Found endpoints:" }, { @@ -264,7 +294,7 @@ } ] }, - "lastId": 48, + "lastId": 54, "steps": { "deployTester": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json index 2f5e93ea3a0..7f59eaf75c2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json @@ -73,6 +73,21 @@ { "at": 0, "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": 0, + "type": "info", + "message": "host-tenant:application:default-t-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "info", + "message": "--- platform vespa/vespa:6.1" + }, + { + "at": 0, + "type": "info", "message": "Tester container successfully installed!" } ], @@ -192,6 +207,21 @@ { "at": 0, "type": "info", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "info", + "message": "--- platform vespa/vespa:6.1" + }, + { + "at": 0, + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": 0, + "type": "info", "message": "Found endpoints:" }, { @@ -259,7 +289,7 @@ } ] }, - "lastId": 48, + "lastId": 54, "steps": { "deployTester": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-deleted.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-deleted.json new file mode 100644 index 00000000000..1c4b76932ac --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-deleted.json @@ -0,0 +1,9 @@ +{ + "tenant": "tenant1", + "type": "DELETED", + "applications": [], + "metaData": { + "createdAtMillis": "(ignore)", + "deletedAtMillis": "(ignore)" + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java index 3d159422396..3c4ed8bc8df 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.billing; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java index 80cee3af58b..ea2c303d3f6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java @@ -1,24 +1,18 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.changemanagement; import com.yahoo.application.container.handler.Request; -import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestSource; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; -import org.intellij.lang.annotations.Language; import org.junit.Before; import org.junit.Test; @@ -42,8 +36,7 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { public void before() { tester = new ContainerTester(container, responses); addUserToHostedOperatorRole(operator); - tester.serviceRegistry().configServer().nodeRepository().addNodes(ZoneId.from("prod.us-east-3"), createNodes()); - tester.serviceRegistry().configServer().nodeRepository().putNodes(ZoneId.from("prod.us-east-3"), createNode()); + tester.serviceRegistry().configServer().nodeRepository().putNodes(ZoneId.from("prod.us-east-3"), createNodes()); tester.controller().curator().writeChangeRequest(createChangeRequest()); } @@ -85,23 +78,11 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { assertEquals(VespaChangeRequest.Status.COMPLETED, changeRequest.getStatus()); } - private void assertResponse(Request request, @Language("JSON") String body, int statusCode) { - addIdentityToRequest(request, operator); - tester.assertResponse(request, body, statusCode); - } - private void assertFile(Request request, String filename) { addIdentityToRequest(request, operator); tester.assertResponse(request, new File(filename)); } - private Node createNode() { - return new Node.Builder() - .hostname(HostName.from("host1")) - .switchHostname("switch1") - .build(); - } - private VespaChangeRequest createChangeRequest() { var instant = Instant.ofEpochMilli(9001); var date = ZonedDateTime.ofInstant(instant, java.time.ZoneId.of("UTC")); @@ -124,8 +105,8 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { ); } - private List<NodeRepositoryNode> createNodes() { - List<NodeRepositoryNode> nodes = new ArrayList<>(); + private List<Node> createNodes() { + List<Node> nodes = new ArrayList<>(); nodes.add(createNode("node1", "host1", "default", 0 )); nodes.add(createNode("node2", "host1", "default", 0 )); nodes.add(createNode("node3", "host1", "default", 0 )); @@ -135,44 +116,27 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { return nodes; } - private NodeOwner createOwner() { - NodeOwner owner = new NodeOwner(); - owner.tenant = "mytenant"; - owner.application = "myapp"; - owner.instance = "default"; - return owner; - } - - private NodeMembership createMembership(String clusterId, int group) { - NodeMembership membership = new NodeMembership(); - membership.group = "" + group; - membership.clusterid = clusterId; - membership.clustertype = "content"; - membership.index = 2; - membership.retired = false; - return membership; - } - - private NodeRepositoryNode createNode(String nodename, String hostname, String clusterId, int group) { - NodeRepositoryNode node = new NodeRepositoryNode(); - node.setHostname(nodename); - node.setParentHostname(hostname); - node.setState(NodeState.active); - node.setOwner(createOwner()); - node.setMembership(createMembership(clusterId, group)); - node.setType(NodeType.tenant); - - return node; + private Node createNode(String nodename, String hostname, String clusterId, int group) { + return Node.builder() + .hostname(nodename) + .parentHostname(hostname).state(Node.State.active) + .owner(ApplicationId.from("mytenant", "myapp", "default")) + .type(com.yahoo.config.provision.NodeType.tenant) + .clusterId(clusterId) + .group(String.valueOf(group)) + .clusterType(Node.ClusterType.content) + .build(); } - private NodeRepositoryNode createHost(String hostname, String switchName) { - NodeRepositoryNode node = new NodeRepositoryNode(); - node.setHostname(hostname); - node.setSwitchHostname(switchName); - node.setOwner(createOwner()); - node.setType(NodeType.host); - node.setMembership(createMembership("host", 0)); - return node; + private Node createHost(String hostname, String switchName) { + return Node.builder() + .hostname(hostname) + .switchHostname(switchName) + .owner(ApplicationId.from("mytenant", "myapp", "default")) + .type(com.yahoo.config.provision.NodeType.host) + .clusterId("host") + .group("0") + .build(); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index fc83c58cc67..7bc01de2053 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.controller; import com.yahoo.application.container.handler.Request; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 668baa50cc1..2edf1867fd3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -16,7 +16,7 @@ "name": "ChangeRequestMaintainer" }, { - "name": "CloudEventReporter" + "name": "CloudEventTracker" }, { "name": "CloudTrialExpirer" @@ -100,7 +100,10 @@ "name": "Upgrader" }, { - "name": "VCMRMaintainer" + "name": "UserManagementMaintainer" + }, + { + "name": "VcmrMaintainer" }, { "name": "VersionStatusUpdater" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index 63474ebb7c9..438da66e6e8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.deployment; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index cd24ec170c5..460afb102d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java index c95691fc120..9e17b44c9a6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java index 5bf7b03295f..407d9f8765c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; import com.yahoo.application.container.handler.Request; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiTest.java new file mode 100644 index 00000000000..b2b5b2286f7 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiTest.java @@ -0,0 +1,64 @@ +package com.yahoo.vespa.hosted.controller.restapi.horizon; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; +import org.junit.Test; + +import java.util.Set; + +/** + * @author olaa + */ +public class HorizonApiTest extends ControllerContainerCloudTest { + + @Test + public void only_operators_and_flag_enabled_tenants_allowed() { + ContainerTester tester = new ContainerTester(container, ""); + TenantName tenantName = TenantName.defaultName(); + + tester.assertResponse(request("/horizon/v1/config/dashboard/topFolders") + .roles(Set.of(Role.hostedOperator())), + "", 200); + + tester.assertResponse(request("/horizon/v1/config/dashboard/topFolders") + .roles(Set.of(Role.reader(tenantName))), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"No tenant with enabled metrics view\"}", 403); + + ((InMemoryFlagSource) tester.controller().flagSource()) + .withBooleanFlag(Flags.ENABLED_HORIZON_DASHBOARD.id(), true); + + tester.assertResponse(request("/horizon/v1/config/dashboard/topFolders") + .roles(Set.of(Role.reader(tenantName))), + "", 200); + } + + @Override + protected SystemName system() { + return SystemName.PublicCd; + } + + @Override + protected String variablePartXml() { + return " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControlRequests'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControl'/>\n" + + + " <handler id=\"com.yahoo.vespa.hosted.controller.restapi.horizon.HorizonApiHandler\" bundle=\"controller-server\">\n" + + " <binding>http://*/horizon/v1/*</binding>\n" + + " </handler>\n" + + + " <http>\n" + + " <server id='default' port='8080' />\n" + + " <filtering>\n" + + " <request-chain id='default'>\n" + + " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/>\n" + + " <binding>http://*/*</binding>\n" + + " </request-chain>\n" + + " </filtering>\n" + + " </http>\n"; + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java index ab9d50f8eae..d31d9c28c6c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java @@ -22,33 +22,28 @@ public class TsdbQueryRewriterTest { @Test public void rewrites_query() throws IOException { - assertRewrite("filters-complex.json", "filters-complex.expected.json", Role.reader(TenantName.from("tenant2"))); + assertRewrite("filters-complex.json", "filters-complex.expected.json", Set.of(TenantName.from("tenant2")), false); assertRewrite("filter-in-execution-graph.json", "filter-in-execution-graph.expected.json", - Role.reader(TenantName.from("tenant2")), Role.athenzTenantAdmin(TenantName.from("tenant3"))); + Set.of(TenantName.from("tenant2"), TenantName.from("tenant3")), false); assertRewrite("filter-in-execution-graph.json", "filter-in-execution-graph.expected.operator.json", - Role.reader(TenantName.from("tenant2")), Role.athenzTenantAdmin(TenantName.from("tenant3")), Role.hostedOperator()); + Set.of(TenantName.from("tenant2"), TenantName.from("tenant3")), true); assertRewrite("no-filters.json", "no-filters.expected.json", - Role.reader(TenantName.from("tenant2")), Role.athenzTenantAdmin(TenantName.from("tenant3"))); + Set.of(TenantName.from("tenant2"), TenantName.from("tenant3")), false); assertRewrite("filters-meta-query.json", "filters-meta-query.expected.json", - Role.reader(TenantName.from("tenant2")), Role.athenzTenantAdmin(TenantName.from("tenant3"))); + Set.of(TenantName.from("tenant2"), TenantName.from("tenant3")), false); } - @Test(expected = TsdbQueryRewriter.UnauthorizedException.class) - public void throws_if_no_roles() throws IOException { - assertRewrite("filters-complex.json", "filters-complex.expected.json"); - } - - private static void assertRewrite(String initialFilename, String expectedFilename, Role... roles) throws IOException { + private static void assertRewrite(String initialFilename, String expectedFilename, Set<TenantName> tenants, boolean operator) throws IOException { byte[] data = Files.readAllBytes(Paths.get("src/test/resources/horizon", initialFilename)); - data = TsdbQueryRewriter.rewrite(data, Set.of(roles), SystemName.Public); + data = TsdbQueryRewriter.rewrite(data, tenants, operator, SystemName.Public); ByteArrayOutputStream baos = new ByteArrayOutputStream(); new JsonFormat(false).encode(baos, SlimeUtils.jsonToSlime(data)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index 7364723f5f0..90685980835 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; @@ -140,9 +141,9 @@ public class OsApiTest extends ControllerContainerTest { for (ZoneId zone : zones) { for (SystemApplication application : SystemApplication.all()) { var targetVersion = nodeRepository().targetVersionsOf(zone).osVersion(application.nodeType()); - for (Node node : nodeRepository().list(zone, application.id())) { + for (Node node : nodeRepository().list(zone, NodeFilter.all().applications(application.id()))) { var version = targetVersion.orElse(node.wantedOsVersion()); - nodeRepository().putNodes(zone, new Node.Builder(node).currentOsVersion(version).wantedOsVersion(version).build()); + nodeRepository().putNodes(zone, Node.builder(node).currentOsVersion(version).wantedOsVersion(version).build()); } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index b1bd7df059c..42c9b5bdf4f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -362,4 +362,19 @@ public class RoutingApiTest extends ControllerContainerTest { 400); } + @Test + public void endpoints_list() { + var context = deploymentTester.newDeploymentContext("t1", "a1", "default"); + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + var applicationPackage = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context.submit(applicationPackage).deploy(); + + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/endpoint", "", Request.Method.GET), + new File("endpoint/endpoints.json")); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/endpoint/endpoints.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/endpoint/endpoints.json new file mode 100644 index 00000000000..f78f913cb7e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/endpoint/endpoints.json @@ -0,0 +1,31 @@ +{ + "endpoints": [ + { + "name": "default", + "dnsName": "a1--t1.global.vespa.oath.cloud", + "routingMethod": "shared", + "cluster": "default", + "scope": "global", + "zones": [ + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": 1497618757000 + }, + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": 1497618757000 + } + ] + } + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java index 35a13cdeeec..549dd1ed253 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java @@ -166,8 +166,8 @@ public class SystemFlagsDeployerTest { .build(); SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(prodUsEast3Target)); SystemFlagsDeployResult result = deployer.deployFlags(archive, true); - assertThat(result.warnings()) - .containsOnly(SystemFlagsDeployResult.Warning.dataForUndefinedFlag(prodUsEast3Target, new FlagId("my-flag"))); + assertThat(result.errors()) + .containsOnly(OperationError.dataForUndefinedFlag(prodUsEast3Target, new FlagId("my-flag"))); } private static FlagData flagData(String filename) throws IOException { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java index acd481030e2..c884eae8afc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java @@ -5,6 +5,8 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.utils.AthenzIdentities; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.user.User; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; @@ -25,40 +27,42 @@ public class UserApiOnPremTest extends ControllerContainerTest { @Test public void userMetadataOnPremTest() { - ContainerTester tester = new ContainerTester(container, responseFiles); - ControllerTester controller = new ControllerTester(tester); - User user = new User("dev@domail", "Joe Developer", "dev", null); + try (Flags.Replacer ignored = Flags.clearFlagsForTesting(PermanentFlags.MAX_TRIAL_TENANTS.id(), PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id())) { + ContainerTester tester = new ContainerTester(container, responseFiles); + ControllerTester controller = new ControllerTester(tester); + User user = new User("dev@domail", "Joe Developer", "dev", null); - controller.createTenant("tenant1", "domain1", 1L); - controller.createApplication("tenant1", "app1", "default"); - controller.createApplication("tenant1", "app2", "default"); - controller.createApplication("tenant1", "app2", "myinstance"); - controller.createApplication("tenant1", "app3"); + controller.createTenant("tenant1", "domain1", 1L); + controller.createApplication("tenant1", "app1", "default"); + controller.createApplication("tenant1", "app2", "default"); + controller.createApplication("tenant1", "app2", "myinstance"); + controller.createApplication("tenant1", "app3"); - controller.createTenant("tenant2", "domain2", 2L); - controller.createApplication("tenant2", "app2", "test"); + controller.createTenant("tenant2", "domain2", 2L); + controller.createApplication("tenant2", "app2", "test"); - controller.createTenant("tenant3", "domain3", 3L); - controller.createApplication("tenant3", "app1"); + controller.createTenant("tenant3", "domain3", 3L); + controller.createApplication("tenant3", "app1"); - controller.createTenant("sandbox", "domain4", 4L); - controller.createApplication("sandbox", "app1", "default"); - controller.createApplication("sandbox", "app2", "default"); - controller.createApplication("sandbox", "app2", "dev"); + controller.createTenant("sandbox", "domain4", 4L); + controller.createApplication("sandbox", "app1", "default"); + controller.createApplication("sandbox", "app2", "default"); + controller.createApplication("sandbox", "app2", "dev"); - AthenzIdentity operator = AthenzIdentities.from("vespa.alice"); - controller.athenzDb().addHostedOperator(operator); - AthenzIdentity tenantAdmin = AthenzIdentities.from("domain1.bob"); - Stream.of("domain1", "domain2", "domain4") - .map(AthenzDomain::new) - .map(controller.athenzDb()::getOrCreateDomain) - .forEach(d -> d.admin(AthenzIdentities.from("domain1.bob"))); + AthenzIdentity operator = AthenzIdentities.from("vespa.alice"); + controller.athenzDb().addHostedOperator(operator); + AthenzIdentity tenantAdmin = AthenzIdentities.from("domain1.bob"); + Stream.of("domain1", "domain2", "domain4") + .map(AthenzDomain::new) + .map(controller.athenzDb()::getOrCreateDomain) + .forEach(d -> d.admin(AthenzIdentities.from("domain1.bob"))); - tester.assertResponse(createUserRequest(user, operator), - new File("user-without-applications.json")); + tester.assertResponse(createUserRequest(user, operator), + new File("user-without-applications.json")); - tester.assertResponse(createUserRequest(user, tenantAdmin), - new File("user-with-applications-athenz.json")); + tester.assertResponse(createUserRequest(user, tenantAdmin), + new File("user-with-applications-athenz.json")); + } } private Request createUserRequest(User user, AthenzIdentity identity) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 03f1d75a50b..9198369a3ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.user; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; @@ -205,65 +206,68 @@ public class UserApiTest extends ControllerContainerCloudTest { @Test public void userMetadataTest() { - ContainerTester tester = new ContainerTester(container, responseFiles); - ((InMemoryFlagSource) tester.controller().flagSource()) - .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); - ControllerTester controller = new ControllerTester(tester); - Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter(), Role.hostedAccountant()); - User user = new User("dev@domail", "Joe Developer", "dev", null); - - tester.assertResponse(request("/user/v1/user") - .roles(operator) - .user(user), - new File("user-without-applications.json")); - - controller.createTenant("tenant1", Tenant.Type.cloud); - controller.createApplication("tenant1", "app1", "default"); - controller.createApplication("tenant1", "app2", "default"); - controller.createApplication("tenant1", "app2", "myinstance"); - controller.createApplication("tenant1", "app3"); - - controller.createTenant("tenant2", Tenant.Type.cloud); - controller.createApplication("tenant2", "app2", "test"); - - controller.createTenant("tenant3", Tenant.Type.cloud); - controller.createApplication("tenant3", "app1"); - - controller.createTenant("sandbox", Tenant.Type.cloud); - controller.createApplication("sandbox", "app1", "default"); - controller.createApplication("sandbox", "app2", "default"); - controller.createApplication("sandbox", "app2", "dev"); - - // Should still be empty because none of the roles explicitly refer to any of the applications - tester.assertResponse(request("/user/v1/user") - .roles(operator) - .user(user), - new File("user-without-applications.json")); - - // Empty applications because tenant dummy does not exist - tester.assertResponse(request("/user/v1/user") - .roles(Set.of(Role.administrator(TenantName.from("tenant1")), - Role.developer(TenantName.from("tenant2")), - Role.developer(TenantName.from("sandbox")), - Role.reader(TenantName.from("sandbox")))) - .user(user), - new File("user-with-applications-cloud.json")); + try (Flags.Replacer ignored = Flags.clearFlagsForTesting(PermanentFlags.MAX_TRIAL_TENANTS.id(), PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id())) { + ContainerTester tester = new ContainerTester(container, responseFiles); + ((InMemoryFlagSource) tester.controller().flagSource()) + .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); + ControllerTester controller = new ControllerTester(tester); + Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter(), Role.hostedAccountant()); + User user = new User("dev@domail", "Joe Developer", "dev", null); + + tester.assertResponse(request("/user/v1/user") + .roles(operator) + .user(user), + new File("user-without-applications.json")); + + controller.createTenant("tenant1", Tenant.Type.cloud); + controller.createApplication("tenant1", "app1", "default"); + controller.createApplication("tenant1", "app2", "default"); + controller.createApplication("tenant1", "app2", "myinstance"); + controller.createApplication("tenant1", "app3"); + + controller.createTenant("tenant2", Tenant.Type.cloud); + controller.createApplication("tenant2", "app2", "test"); + + controller.createTenant("tenant3", Tenant.Type.cloud); + controller.createApplication("tenant3", "app1"); + + controller.createTenant("sandbox", Tenant.Type.cloud); + controller.createApplication("sandbox", "app1", "default"); + controller.createApplication("sandbox", "app2", "default"); + controller.createApplication("sandbox", "app2", "dev"); + + // Should still be empty because none of the roles explicitly refer to any of the applications + tester.assertResponse(request("/user/v1/user") + .roles(operator) + .user(user), + new File("user-without-applications.json")); + + // Empty applications because tenant dummy does not exist + tester.assertResponse(request("/user/v1/user") + .roles(Set.of(Role.administrator(TenantName.from("tenant1")), + Role.developer(TenantName.from("tenant2")), + Role.developer(TenantName.from("sandbox")), + Role.reader(TenantName.from("sandbox")))) + .user(user), + new File("user-with-applications-cloud.json")); + } } @Test public void maxTrialTenants() { - ContainerTester tester = new ContainerTester(container, responseFiles); - ((InMemoryFlagSource) tester.controller().flagSource()) - .withIntFlag(PermanentFlags.MAX_TRIAL_TENANTS.id(), 1) - .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); - ControllerTester controller = new ControllerTester(tester); - Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter(), Role.hostedAccountant()); - User user = new User("dev@domail", "Joe Developer", "dev", null); - - controller.createTenant("tenant1", Tenant.Type.cloud); - - tester.assertResponse( - request("/user/v1/user").user(user), - new File("user-without-trial-capacity-cloud.json")); + try (Flags.Replacer ignored = Flags.clearFlagsForTesting(PermanentFlags.MAX_TRIAL_TENANTS.id(), PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id())) { + ContainerTester tester = new ContainerTester(container, responseFiles); + ((InMemoryFlagSource) tester.controller().flagSource()) + .withIntFlag(PermanentFlags.MAX_TRIAL_TENANTS.id(), 1) + .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); + ControllerTester controller = new ControllerTester(tester); + User user = new User("dev@domail", "Joe Developer", "dev", null); + + controller.createTenant("tenant1", Tenant.Type.cloud); + + tester.assertResponse( + request("/user/v1/user").user(user), + new File("user-without-trial-capacity-cloud.json")); + } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java new file mode 100644 index 00000000000..8625628b74e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java @@ -0,0 +1,133 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.user; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.config.provision.TenantName; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.test.json.JsonTestHelper; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.JsonNodeRawFlag; +import com.yahoo.vespa.flags.json.Condition; +import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.flags.json.Rule; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.CONSOLE_USER_EMAIL; +import static com.yahoo.vespa.flags.FetchVector.Dimension.TENANT_ID; + +/** + * @author freva + */ +public class UserFlagsSerializerTest { + + @Test + public void user_flag_test() throws IOException { + String email1 = "alice@domain.tld"; + String email2 = "bob@domain.tld"; + + try (Flags.Replacer ignored = Flags.clearFlagsForTesting()) { + Flags.defineStringFlag("string-id", "default value", List.of("owner"), "1970-01-01", "2100-01-01", "desc", "mod", CONSOLE_USER_EMAIL); + Flags.defineIntFlag("int-id", 123, List.of("owner"), "1970-01-01", "2100-01-01", "desc", "mod", CONSOLE_USER_EMAIL, TENANT_ID, APPLICATION_ID); + Flags.defineDoubleFlag("double-id", 3.14d, List.of("owner"), "1970-01-01", "2100-01-01", "desc", "mod"); + Flags.defineListFlag("list-id", List.of("a"), String.class, List.of("owner"), "1970-01-01", "2100-01-01", "desc", "mod", CONSOLE_USER_EMAIL); + Flags.defineJacksonFlag("jackson-id", new ExampleJacksonClass(123, "abc"), ExampleJacksonClass.class, + List.of("owner"), "1970-01-01", "2100-01-01", "desc", "mod", CONSOLE_USER_EMAIL, TENANT_ID); + + Map<FlagId, FlagData> flagData = Stream.of( + flagData("string-id", rule("\"value1\"", condition(CONSOLE_USER_EMAIL, Condition.Type.WHITELIST, email1))), + flagData("int-id", rule("456")), + flagData("list-id", + rule("[\"value1\"]", condition(CONSOLE_USER_EMAIL, Condition.Type.WHITELIST, email1), condition(APPLICATION_ID, Condition.Type.BLACKLIST, "tenant1:video:default", "tenant1:video:default", "tenant2:music:default")), + rule("[\"value2\"]", condition(CONSOLE_USER_EMAIL, Condition.Type.WHITELIST, email2)), + rule("[\"value1\",\"value3\"]", condition(APPLICATION_ID, Condition.Type.BLACKLIST, "tenant1:video:default", "tenant1:video:default", "tenant2:music:default"))), + flagData("jackson-id", rule("{\"integer\":456,\"string\":\"xyz\"}", condition(CONSOLE_USER_EMAIL, Condition.Type.WHITELIST, email1), condition(TENANT_ID, Condition.Type.WHITELIST, "tenant1", "tenant3"))) + ).collect(Collectors.toMap(FlagData::id, fd -> fd)); + + // double-id is not here as it does not have CONSOLE_USER_EMAIL dimension + assertUserFlags("{\"flags\":[" + + "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB + "{\"id\":\"jackson-id\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"tenant\"}],\"value\":{\"integer\":456,\"string\":\"xyz\"}},{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Resolved for email + // Resolved for email, but conditions are empty since this user is not authorized for any tenants + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\"}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\"}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"value1\"}]}]}", // resolved for email + flagData, Set.of(), false, email1); + + // Same as the first one, but user is authorized for tenant1 + assertUserFlags("{\"flags\":[" + + "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB + "{\"id\":\"jackson-id\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"tenant\",\"values\":[\"tenant1\"]}],\"value\":{\"integer\":456,\"string\":\"xyz\"}},{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Resolved for email + // Resolved for email, but conditions have filtered out tenant2 + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"value1\"}]}]}", // resolved for email + flagData, Set.of("tenant1"), false, email1); + + // As operator no conditions are filtered, but the email precondition is applied + assertUserFlags("{\"flags\":[" + + "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB + "{\"id\":\"jackson-id\",\"rules\":[{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Default from code, no DB values match + // Includes last value from DB which is not conditioned on email and the default from code + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\",\"tenant2:music:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"default value\"}]}]}", // Default from code + flagData, Set.of(), true, "operator@domain.tld"); + } + } + + private static FlagData flagData(String id, Rule... rules) { + return new FlagData(new FlagId(id), new FetchVector(), rules); + } + + private static Rule rule(String data, Condition... conditions) { + return new Rule(Optional.ofNullable(data).map(JsonNodeRawFlag::fromJson), conditions); + } + + private static Condition condition(FetchVector.Dimension dimension, Condition.Type type, String... values) { + return new Condition.CreateParams(dimension).withValues(values).createAs(type); + } + + private static void assertUserFlags(String expected, Map<FlagId, FlagData> rawFlagData, + Set<String> authorizedForTenantNames, boolean isOperator, String userEmail) throws IOException { + Slime slime = new Slime(); + UserFlagsSerializer.toSlime(slime.setObject(), rawFlagData, authorizedForTenantNames.stream().map(TenantName::from).collect(Collectors.toSet()), isOperator, userEmail); + JsonTestHelper.assertJsonEquals(expected, + new String(SlimeUtils.toJsonBytes(slime), StandardCharsets.UTF_8)); + } + + @JsonIgnoreProperties(ignoreUnknown = true) + private static class ExampleJacksonClass { + @JsonProperty("integer") public final int integer; + @JsonProperty("string") public final String string; + private ExampleJacksonClass(@JsonProperty("integer") int integer, @JsonProperty("string") String string) { + this.integer = integer; + this.string = string; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ExampleJacksonClass that = (ExampleJacksonClass) o; + return integer == that.integer && + Objects.equals(string, that.string); + } + + @Override + public int hashCode() { + return Objects.hash(integer, string); + } + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json index 5d3a38334ad..006c3b98a4d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json @@ -1,7 +1,6 @@ { "isPublic": false, "isCd": false, - "enable-public-signup-flow": (ignore), "hasTrialCapacity": (ignore), "user": { "name": "Joe Developer", @@ -31,5 +30,6 @@ "reader" ] } - } + }, + "flags": [{"id":"enable-public-signup-flow","rules":[{"value":false}]}] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json index ae3dc68d9e3..4ae55e97baa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json @@ -1,7 +1,6 @@ { "isPublic": true, "isCd": false, - "enable-public-signup-flow": (ignore), "hasTrialCapacity": true, "user": { "name": "Joe Developer", @@ -26,5 +25,6 @@ "developer" ] } - } + }, + "flags": [{"id":"enable-public-signup-flow","rules":[{"value":false}]}] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json index 3bf999b490b..9f9578e6ed8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json @@ -1,7 +1,6 @@ { "isPublic": (ignore), "isCd": (ignore), - "enable-public-signup-flow": (ignore), "hasTrialCapacity": (ignore), "user": { "name": "Joe Developer", @@ -14,5 +13,6 @@ "hostedOperator", "hostedSupporter", "hostedAccountant" - ] + ], + "flags": [{"id":"enable-public-signup-flow","rules":[{"value":false}]}] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-trial-capacity-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-trial-capacity-cloud.json index 27242424579..2b98a75068a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-trial-capacity-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-trial-capacity-cloud.json @@ -1,7 +1,6 @@ { "isPublic": true, "isCd": false, - "enable-public-signup-flow": true, "hasTrialCapacity": false, "user": { "name": "Joe Developer", @@ -9,5 +8,6 @@ "nickname": "dev", "verified":false }, - "tenants": {} + "tenants": {}, + "flags": [{"id":"enable-public-signup-flow","rules":[{"value":false}]}] }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java index a8845fa92a3..fcf8402d23c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.zone.v1; import com.yahoo.config.provision.Environment; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java index a260f4a138b..fd39c13707b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.zone.v2; import com.yahoo.application.container.handler.Request.Method; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index aa9775f1d43..910e5943989 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -1,11 +1,11 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; @@ -146,10 +146,10 @@ public class RotationRepositoryTest { public void prefixes_system_when_not_main() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .globalServiceId("foo") - .region("cd-us-central-1") + .region("cd-us-east-1") .region("cd-us-west-1") .build(); - var zones = List.of(ZoneApiMock.fromId("prod.cd-us-central-1"), ZoneApiMock.fromId("prod.cd-us-west-1")); + var zones = List.of(ZoneApiMock.fromId("prod.cd-us-east-1"), ZoneApiMock.fromId("prod.cd-us-west-1")); tester.controllerTester().zoneRegistry() .setZones(zones) .setRoutingMethod(zones, RoutingMethod.shared) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 79b564eee52..2002b59dc1e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; import com.google.common.collect.ImmutableMap; @@ -24,7 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/Keys.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/Keys.java index 7d1e540b20d..7a293e661c9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/Keys.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/Keys.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.tls; import com.yahoo.security.KeyAlgorithm; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecretStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecretStoreMock.java index 6b5fa6503b3..8b5cdefddf8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecretStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecretStoreMock.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.tls; import com.google.inject.Inject; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecureContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecureContainerTest.java index c1188778292..de268f34093 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecureContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecureContainerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.tls; import com.yahoo.application.Networking; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 4dd283cf5d7..a1108d5f03c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -9,9 +9,10 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -69,8 +70,8 @@ public class VersionStatusTest { Version version1 = Version.fromString("6.5"); // Upgrade some config servers for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) { - for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { - Node upgradedNode = new Node.Builder(node).currentVersion(version1).build(); + for (Node node : tester.configServer().nodeRepository().list(zone.getId(), NodeFilter.all().applications(SystemApplication.configServer.id()))) { + Node upgradedNode = Node.builder(node).currentVersion(version1).build(); tester.configServer().nodeRepository().putNodes(zone.getId(), upgradedNode); break; } @@ -113,8 +114,8 @@ public class VersionStatusTest { // Downgrade one config server in each zone Version ancientVersion = Version.fromString("5.1"); for (ZoneApi zone : tester.controller().zoneRegistry().zones().all().zones()) { - for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { - Node downgradedNode = new Node.Builder(node).currentVersion(ancientVersion).build(); + for (Node node : tester.configServer().nodeRepository().list(zone.getId(), NodeFilter.all().applications(SystemApplication.configServer.id()))) { + Node downgradedNode = Node.builder(node).currentVersion(ancientVersion).build(); tester.configServer().nodeRepository().putNodes(zone.getId(), downgradedNode); break; } @@ -303,7 +304,7 @@ public class VersionStatusTest { Confidence.low, confidence(tester.controller(), version2)); // Remaining canary upgrades to version2 which raises confidence to normal and more apps upgrade - canary2.failDeployment(systemTest); + canary2.triggerJobs().jobAborted(systemTest).jobAborted(stagingTest); canary2.runJob(stagingTest); canary2.deployPlatform(version2); tester.controllerTester().computeVersionStatus(); |