diff options
author | gjoranv <gjoranv@gmail.com> | 2023-11-03 18:08:32 +0100 |
---|---|---|
committer | gjoranv <gjoranv@gmail.com> | 2023-11-06 00:31:08 +0100 |
commit | 596421557e3165ef25dd478edf64b2812d5b4777 (patch) | |
tree | 36ed938c7fe0519caf83cbb798d64bd98aa8aa0e /controller-server/src/main/java/com | |
parent | c5d8e300da1bee0cff8e83a3c0a4b9a9a4fa8375 (diff) |
More controller code to internal repo.
Diffstat (limited to 'controller-server/src/main/java/com')
279 files changed, 0 insertions, 42344 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java deleted file mode 100644 index 0e6f29c760d..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller; - -import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.AccountId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.application.ApplicationActivity; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.QuotaUsage; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; -import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.security.PublicKey; -import java.time.Instant; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.TreeMap; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * An application. Belongs to a {@link Tenant}, and may have multiple {@link Instance}s. - * - * This is immutable. - * - * @author jonmv - */ -public class Application { - - private final TenantAndApplicationId id; - private final Instant createdAt; - private final DeploymentSpec deploymentSpec; - private final ValidationOverrides validationOverrides; - private final RevisionHistory revisions; - private final OptionalLong projectId; - private final Optional<IssueId> deploymentIssueId; - private final Optional<IssueId> ownershipIssueId; - private final Optional<User> userOwner; - private final Optional<AccountId> issueOwner; - private final OptionalInt majorVersion; - private final ApplicationMetrics metrics; - private final Set<PublicKey> deployKeys; - private final Map<InstanceName, Instance> instances; - - /** Creates an empty application. */ - public Application(TenantAndApplicationId id, Instant now) { - this(id, now, DeploymentSpec.empty, ValidationOverrides.empty, Optional.empty(), Optional.empty(), - Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(0, 0), - Set.of(), OptionalLong.empty(), RevisionHistory.empty(), List.of()); - } - - // Do not use directly - edit through LockedApplication. - public Application(TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, - Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> userOwner, Optional<AccountId> issueOwner, - OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, OptionalLong projectId, - RevisionHistory revisions, Collection<Instance> instances) { - this.id = Objects.requireNonNull(id, "id cannot be null"); - this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null"); - this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); - this.validationOverrides = Objects.requireNonNull(validationOverrides, "validationOverrides cannot be null"); - this.deploymentIssueId = Objects.requireNonNull(deploymentIssueId, "deploymentIssueId cannot be null"); - this.ownershipIssueId = Objects.requireNonNull(ownershipIssueId, "ownershipIssueId cannot be null"); - this.userOwner = Objects.requireNonNull(userOwner, "owner cannot be null"); - this.issueOwner = Objects.requireNonNull(issueOwner, "issueOwner cannot be null"); - this.majorVersion = Objects.requireNonNull(majorVersion, "majorVersion cannot be null"); - this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null"); - this.deployKeys = Objects.requireNonNull(deployKeys, "deployKeys cannot be null"); - this.projectId = Objects.requireNonNull(projectId, "projectId cannot be null"); - this.revisions = revisions; - this.instances = instances.stream().collect( - Collectors.collectingAndThen(Collectors.toMap(Instance::name, - Function.identity(), - (i1, i2) -> { - throw new IllegalArgumentException("Duplicate instance " + i1.id()); - }, - TreeMap::new), - Collections::unmodifiableMap) - ); - } - - public TenantAndApplicationId id() { return id; } - - public Instant createdAt() { return createdAt; } - - /** - * Returns the last deployed deployment spec of this application, - * or the empty deployment spec if it has never been deployed - */ - public DeploymentSpec deploymentSpec() { return deploymentSpec; } - - /** Returns the project id of this application, if it has any. */ - public OptionalLong projectId() { return projectId; } - - /** Returns the known revisions for this application. */ - public RevisionHistory revisions() { return revisions; } - - /** - * Returns the last deployed validation overrides of this application, - * or the empty validation overrides if it has never been deployed - * (or was deployed with an empty/missing validation overrides) - */ - public ValidationOverrides validationOverrides() { return validationOverrides; } - - /** Returns the instances of this application */ - public Map<InstanceName, Instance> instances() { return instances; } - - /** Returns the instances of this application which are defined in its deployment spec. */ - public Map<InstanceName, Instance> productionInstances() { - return deploymentSpec.instanceNames().stream() - .collect(Collectors.toUnmodifiableMap(Function.identity(), instances::get)); - } - - /** Returns the instance with the given name, if it exists. */ - public Optional<Instance> get(InstanceName instance) { return Optional.ofNullable(instances.get(instance)); } - - /** Returns the instance with the given name, or throws. */ - public Instance require(InstanceName instance) { - return get(instance).orElseThrow(() -> new IllegalArgumentException("Unknown instance '" + instance + "' in '" + id + "'")); - } - - /** Returns ID of any open deployment issue filed for this */ - public Optional<IssueId> deploymentIssueId() { - return deploymentIssueId; - } - - /** Returns ID of the last ownership issue filed for this */ - public Optional<IssueId> ownershipIssueId() { - return ownershipIssueId; - } - - public Optional<User> userOwner() { - return userOwner; - } - - public Optional<AccountId> issueOwner() { - return issueOwner; - } - - /** - * Overrides the system major version for this application. This override takes effect if the deployment - * spec does not specify a major version. - */ - public OptionalInt majorVersion() { return majorVersion; } - - /** Returns metrics for this */ - public ApplicationMetrics metrics() { - return metrics; - } - - /** Returns activity for this */ - public ApplicationActivity activity() { - return ApplicationActivity.from(instances.values().stream() - .flatMap(instance -> instance.deployments().values().stream()) - .toList()); - } - - public Map<InstanceName, List<Deployment>> productionDeployments() { - return instances.values().stream() - .collect(Collectors.toUnmodifiableMap(Instance::name, - instance -> List.copyOf(instance.productionDeployments().values()))); - } - /** - * Returns the oldest platform version this has deployed in a permanent zone (not test or staging). - * - * This is unfortunately quite similar to {@link ApplicationController#oldestInstalledPlatform(Application)}, - * but this checks only what the controller has deployed to the production zones, while that checks the node repository - * to see what's actually installed on each node. Thus, this is the right choice for, e.g., target Vespa versions for - * new deployments, while that is the right choice for version to compile against. - */ - public Optional<Version> oldestDeployedPlatform() { - return productionDeployments().values().stream().flatMap(List::stream) - .map(Deployment::version) - .min(Comparator.naturalOrder()); - } - - /** Returns the oldest application version this has deployed in a permanent zone (not test or staging) */ - public Optional<RevisionId> oldestDeployedRevision() { - return productionRevisions().min(Comparator.naturalOrder()); - } - - /** Returns the latest application version this has deployed in a permanent zone (not test or staging) */ - public Optional<RevisionId> latestDeployedRevision() { - return productionRevisions().max(Comparator.naturalOrder()); - } - - private Stream<RevisionId> productionRevisions() { - return productionDeployments().values().stream().flatMap(List::stream) - .map(Deployment::revision) - .filter(RevisionId::isProduction); - } - - /** Returns the total quota usage for this application, excluding temporary deployments */ - public QuotaUsage quotaUsage() { - return instances().values().stream() - .map(Instance::quotaUsage) - .reduce(QuotaUsage::add) - .orElse(QuotaUsage.none); - } - - /** Returns the total quota usage for manual deployments for this application */ - public QuotaUsage manualQuotaUsage() { - return instances().values().stream() - .map(Instance::manualQuotaUsage) - .reduce(QuotaUsage::add) - .orElse(QuotaUsage.none); - } - - /** Returns the total quota usage for this application, excluding one specific deployment (and temporary deployments) */ - public QuotaUsage quotaUsage(ApplicationId application, ZoneId zone) { - return instances().values().stream() - .map(instance -> instance.quotaUsageExcluding(application, zone)) - .reduce(QuotaUsage::add) - .orElse(QuotaUsage.none); - } - - /** Returns the set of deploy keys for this application. */ - public Set<PublicKey> deployKeys() { return deployKeys; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (! (o instanceof Application other)) return false; - return id.equals(other.id); - } - - @Override - public int hashCode() { - return id.hashCode(); - } - - @Override - public String toString() { - return "application '" + id + "'"; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java deleted file mode 100644 index d7a3d4fb9e5..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ /dev/null @@ -1,1111 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller; - -import com.yahoo.component.Version; -import com.yahoo.component.VersionCompatibility; -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; -import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.Tags; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.text.Text; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.ListFlag; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.StringFlag; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentEndpoints; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; -import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult.LogEntry; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneTokenVersions; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; -import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics.Warning; -import com.yahoo.vespa.hosted.controller.application.DeploymentQuotaCalculator; -import com.yahoo.vespa.hosted.controller.application.EndpointList; -import com.yahoo.vespa.hosted.controller.application.QuotaUsage; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageValidator; -import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml; -import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; -import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates; -import com.yahoo.vespa.hosted.controller.concurrent.Once; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; -import com.yahoo.vespa.hosted.controller.deployment.JobStatus; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.notification.Notification; -import com.yahoo.vespa.hosted.controller.notification.NotificationSource; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.routing.PreparedEndpoints; -import com.yahoo.vespa.hosted.controller.security.AccessControl; -import com.yahoo.vespa.hosted.controller.security.Credentials; -import com.yahoo.vespa.hosted.controller.support.access.SupportAccessGrant; -import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; -import com.yahoo.yolean.Exceptions; - -import java.io.ByteArrayInputStream; -import java.security.Principal; -import java.security.cert.X509Certificate; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.function.UnaryOperator; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.vespa.flags.FetchVector.Dimension.INSTANCE_ID; -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; -import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.broken; -import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.high; -import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.low; -import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.normal; -import static java.util.Comparator.naturalOrder; -import static java.util.stream.Collectors.collectingAndThen; -import static java.util.stream.Collectors.counting; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; - -/** - * A singleton owned by {@link Controller} which contains the methods and state for controlling applications. - * - * @author bratseth - */ -public class ApplicationController { - - private static final Logger log = Logger.getLogger(ApplicationController.class.getName()); - - /** The controller owning this */ - private final Controller controller; - - /** For persistence */ - private final CuratorDb curator; - - private final ArtifactRepository artifactRepository; - private final ApplicationStore applicationStore; - private final AccessControl accessControl; - private final ConfigServer configServer; - private final Clock clock; - private final DeploymentTrigger deploymentTrigger; - private final ApplicationPackageValidator applicationPackageValidator; - private final EndpointCertificates endpointCertificates; - private final StringFlag dockerImageRepoFlag; - private final ListFlag<String> incompatibleVersions; - private final BillingController billingController; - private final ListFlag<String> cloudAccountsFlag; - - private final Map<DeploymentId, com.yahoo.vespa.hosted.controller.api.integration.configserver.Application> deploymentInfo = new ConcurrentHashMap<>(); - - ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, Clock clock, - FlagSource flagSource, BillingController billingController) { - this.controller = Objects.requireNonNull(controller); - this.curator = Objects.requireNonNull(curator); - this.accessControl = Objects.requireNonNull(accessControl); - this.configServer = controller.serviceRegistry().configServer(); - this.clock = Objects.requireNonNull(clock); - this.billingController = Objects.requireNonNull(billingController); - - artifactRepository = controller.serviceRegistry().artifactRepository(); - applicationStore = controller.serviceRegistry().applicationStore(); - dockerImageRepoFlag = PermanentFlags.DOCKER_IMAGE_REPO.bindTo(flagSource); - incompatibleVersions = PermanentFlags.INCOMPATIBLE_VERSIONS.bindTo(flagSource); - cloudAccountsFlag = PermanentFlags.CLOUD_ACCOUNTS.bindTo(flagSource); - deploymentTrigger = new DeploymentTrigger(controller, clock); - applicationPackageValidator = new ApplicationPackageValidator(controller); - endpointCertificates = new EndpointCertificates(controller, - controller.serviceRegistry().endpointCertificateProvider(), - controller.serviceRegistry().endpointCertificateValidator()); - - // Update serialization format of all applications - Once.after(Duration.ofMinutes(1), () -> { - Instant start = clock.instant(); - int count = 0; - for (TenantAndApplicationId id : curator.readApplicationIds()) { - lockApplicationIfPresent(id, application -> { - for (var declaredInstance : application.get().deploymentSpec().instances()) - if ( ! application.get().instances().containsKey(declaredInstance.name())) - application = withNewInstance(application, id.instance(declaredInstance.name())); - store(application); - }); - count++; - } - log.log(Level.INFO, Text.format("Wrote %d applications in %s", count, - Duration.between(start, clock.instant()))); - }); - } - - /** Validate the given application package */ - public void validatePackage(ApplicationPackage applicationPackage, Application application) { - applicationPackageValidator.validate(application, applicationPackage, clock.instant()); - } - - public Set<CloudAccount> accountsOf(TenantName tenant) { - return cloudAccountsFlag.with(FetchVector.Dimension.TENANT_ID, tenant.value()) - .value().stream() - .map(CloudAccount::from) - .collect(Collectors.toSet()); - } - - /** Returns the application with the given id, or null if it is not present */ - public Optional<Application> getApplication(TenantAndApplicationId id) { - return curator.readApplication(id); - } - - /** Returns the instance with the given id, or null if it is not present */ - public Optional<Instance> getInstance(ApplicationId id) { - return getApplication(TenantAndApplicationId.from(id)).flatMap(application -> application.get(id.instance())); - } - - /** - * Returns in-memory info for the given deployment pulled from the node repo. - * Info on any existing deployment can be missing if it has not yet been fetched since this instance was started. - * This is kept up to date by DeploymentInfoMaintainer. - * Accessing this is thread safe. - */ - // TODO: Replace the wire level Application by a DeploymentInfo class in the model - public Map<DeploymentId, com.yahoo.vespa.hosted.controller.api.integration.configserver.Application> deploymentInfo() { return deploymentInfo; } - - /** - * Triggers reindexing for the given document types in the given clusters, for the given application. - * <p> - * If no clusters are given, reindexing is triggered for the entire application; otherwise - * if no documents types are given, reindexing is triggered for all given clusters; otherwise - * reindexing is triggered for the cartesian product of the given clusters and document types. - */ - public void reindex(ApplicationId id, ZoneId zoneId, List<String> clusterNames, List<String> documentTypes, boolean indexedOnly, Double speed, String cause) { - configServer.reindex(new DeploymentId(id, zoneId), clusterNames, documentTypes, indexedOnly, speed, cause); - } - - /** Returns the reindexing status for the given application in the given zone. */ - public ApplicationReindexing applicationReindexing(ApplicationId id, ZoneId zoneId) { - return configServer.getReindexing(new DeploymentId(id, zoneId)); - } - - /** Enables reindexing for the given application in the given zone. */ - public void enableReindexing(ApplicationId id, ZoneId zoneId) { - configServer.enableReindexing(new DeploymentId(id, zoneId)); - } - - /** Disables reindexing for the given application in the given zone. */ - public void disableReindexing(ApplicationId id, ZoneId zoneId) { - configServer.disableReindexing(new DeploymentId(id, zoneId)); - } - - /** - * Returns the application with the given id - * - * @throws IllegalArgumentException if it does not exist - */ - public Application requireApplication(TenantAndApplicationId id) { - return getApplication(id).orElseThrow(() -> new IllegalArgumentException(id + " not found")); - } - - /** - * Returns the instance with the given id - * - * @throws IllegalArgumentException if it does not exist - */ - // TODO jonvm: remove or inline - public Instance requireInstance(ApplicationId id) { - return getInstance(id).orElseThrow(() -> new IllegalArgumentException(id + " not found")); - } - - /** Returns a snapshot of all applications */ - public List<Application> asList() { - return curator.readApplications(false); - } - - /** - * Returns a snapshot of all readable applications. Unlike {@link ApplicationController#asList()} this ignores - * applications that cannot currently be read (e.g. due to serialization issues) and may return an incomplete - * snapshot. - * - * This should only be used in cases where acting on a subset of applications is better than none. - */ - public List<Application> readable() { - return curator.readApplications(true); - } - - /** Returns the ID of all known applications. */ - public List<TenantAndApplicationId> idList() { - return curator.readApplicationIds(); - } - - /** Returns a snapshot of all applications of a tenant */ - public List<Application> asList(TenantName tenant) { - return curator.readApplications(tenant); - } - - public ArtifactRepository artifacts() { return artifactRepository; } - - public ApplicationStore applicationStore() { return applicationStore; } - - /** Returns all currently reachable content clusters among the given deployments. */ - public Map<ZoneId, List<String>> reachableContentClustersByZone(Collection<DeploymentId> ids) { - Map<ZoneId, List<String>> clusters = new TreeMap<>(Comparator.comparing(ZoneId::value)); - for (DeploymentId id : ids) - if (isHealthy(id)) - clusters.put(id.zoneId(), List.copyOf(configServer.getContentClusters(id))); - - return Collections.unmodifiableMap(clusters); - } - - /** Reads the oldest installed platform for the given application and zone from job history, or a node repo. */ - private Optional<Version> oldestInstalledPlatform(JobStatus job) { - Version oldest = null; - for (Run run : job.runs().descendingMap().values()) { - Version version = run.versions().targetPlatform(); - if (oldest == null || version.isBefore(oldest)) - oldest = version; - - if (run.hasSucceeded()) - return Optional.of(oldest); - } - // If no successful run was found, ask the node repository in the relevant zone. - return oldestInstalledPlatform(job.id()); - } - - /** Reads the oldest installed platform for the given application and zone from the node repo of that zone. */ - private Optional<Version> oldestInstalledPlatform(JobId job) { - return configServer.nodeRepository().list(job.type().zone(), - NodeFilter.all() - .applications(job.application()) - .states(active, reserved)) - .stream() - .map(Node::currentVersion) - .filter(version -> ! version.isEmpty()) - .min(naturalOrder()); - } - - /** Returns the oldest Vespa version installed on any active or reserved production node for the given application. */ - public Optional<Version> oldestInstalledPlatform(Application application) { - return controller.jobController().deploymentStatus(application).jobs() - .production() - .not().test() - .asList().stream() - .map(this::oldestInstalledPlatform) - .flatMap(Optional::stream) - .min(naturalOrder()); - } - - /** - * Returns the preferred Vespa version to compile against, for - * <p> - * The returned version is not newer than the oldest deployed platform for the application, unless - * the target major differs from the oldest deployed platform, in which case it is not newer than - * the oldest available platform version on that major instead. - * <p> - * The returned version is compatible with a platform version available in the system. - * <p> - * A candidate is sought first among versions with non-broken confidence, then among those with forgotten confidence. - * <p> - * The returned version is the latest in the relevant candidate set. - * <p> - * If no such version exists, an {@link IllegalArgumentException} is thrown. - */ - public Version compileVersion(TenantAndApplicationId id, OptionalInt wantedMajor) { - - // Read version status, and pick out target platforms we could run the compiled package on. - Optional<Application> application = getApplication(id); - Optional<Version> oldestInstalledPlatform = application.flatMap(this::oldestInstalledPlatform); - VersionStatus versionStatus = controller.readVersionStatus(); - UpgradePolicy policy = application.flatMap(app -> app.deploymentSpec().instances().stream() - .map(DeploymentInstanceSpec::upgradePolicy) - .max(naturalOrder())) - .orElse(UpgradePolicy.defaultPolicy); - Confidence targetConfidence = switch (policy) { - case canary -> broken; - case defaultPolicy -> normal; - case conservative -> high; - }; - - // Target platforms are all versions not older than the oldest installed platform, unless forcing a major version change. - // Only platforms not older than the system version, and with appropriate confidence, are considered targets. - Predicate<Version> isTargetPlatform = wantedMajor.isEmpty() && oldestInstalledPlatform.isEmpty() - ? __ -> true // No preferences for version: any platform version is ok. - : wantedMajor.isEmpty() || (oldestInstalledPlatform.isPresent() && wantedMajor.getAsInt() == oldestInstalledPlatform.get().getMajor()) - ? version -> ! version.isBefore(oldestInstalledPlatform.get()) // Major empty, or on same as oldest: ensure not a platform downgrade. - : version -> wantedMajor.getAsInt() == version.getMajor(); // Major specified, and not on same as oldest (possibly empty): any on that major. - Set<Version> platformVersions = versionStatus.deployableVersions().stream() - .filter(version -> version.confidence().equalOrHigherThan(targetConfidence)) - .map(VespaVersion::versionNumber) - .filter(isTargetPlatform) - .collect(toSet()); - oldestInstalledPlatform.ifPresent(fallback -> { - if (wantedMajor.isEmpty() || wantedMajor.getAsInt() == fallback.getMajor()) - platformVersions.add(fallback); - }); - - if (platformVersions.isEmpty()) - throw new IllegalArgumentException("this system has no available versions" + - (wantedMajor.isPresent() ? " on specified major: " + wantedMajor.getAsInt() : "")); - - // The returned compile version must be compatible with at least one target platform. - // If it is incompatible with any of the current platforms, the system will trigger a platform change. - // The returned compile version should also be at least as old as both the oldest target platform version, - // and the oldest current platform, unless the two are incompatible, in which case only the target matters, - // or there are no installed platforms, in which case we prefer the newest target platform. - VersionCompatibility compatibility = versionCompatibility(id.defaultInstance()); // Wrong id level >_< - Version oldestTargetPlatform = platformVersions.stream().min(naturalOrder()).get(); - Version newestVersion = oldestInstalledPlatform.isEmpty() - ? platformVersions.stream().max(naturalOrder()).get() - : compatibility.accept(oldestInstalledPlatform.get(), oldestTargetPlatform) - && oldestInstalledPlatform.get().isBefore(oldestTargetPlatform) - ? oldestInstalledPlatform.get() - : oldestTargetPlatform; - Predicate<Version> systemCompatible = version -> ! version.isAfter(newestVersion) - && platformVersions.stream().anyMatch(platform -> compatibility.accept(platform, version)); - - // Find the newest, system-compatible version with non-broken confidence. - Optional<Version> nonBroken = versionStatus.versions().stream() - .filter(VespaVersion::isReleased) - .filter(version -> version.confidence().equalOrHigherThan(low)) - .map(VespaVersion::versionNumber) - .filter(systemCompatible) - .max(naturalOrder()); - - // Fall back to the newest, system-compatible version with unknown confidence. For public systems, this implies high confidence. - Set<Version> knownVersions = versionStatus.versions().stream().map(VespaVersion::versionNumber).collect(toSet()); - Optional<Version> unknown = controller.mavenRepository().metadata().versions(clock.instant()).stream() - .filter(version -> ! knownVersions.contains(version)) - .filter(systemCompatible) - .max(naturalOrder()); - - if (nonBroken.isPresent()) { - if (controller.system().isPublic() && unknown.isPresent() && unknown.get().isAfter(nonBroken.get())) - return unknown.get(); - - return nonBroken.get(); - } - - if (unknown.isPresent()) - return unknown.get(); - - throw new IllegalArgumentException("no suitable, released compile version exists" + - (wantedMajor.isPresent() ? " for specified major: " + wantedMajor.getAsInt() : "")); - } - - /** - * Creates a new application for an existing tenant. - * - * @throws IllegalArgumentException if the application already exists - */ - public Application createApplication(TenantAndApplicationId id, Credentials credentials) { - try (Mutex lock = lock(id)) { - if (getApplication(id).isPresent()) - throw new IllegalArgumentException("Could not create '" + id + "': Application already exists"); - if (getApplication(dashToUnderscore(id)).isPresent()) // VESPA-1945 - throw new IllegalArgumentException("Could not create '" + id + "': Application " + dashToUnderscore(id) + " already exists"); - - com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId.validate(id.application().value()); - - if (controller.tenants().get(id.tenant()).isEmpty()) - throw new IllegalArgumentException("Could not create '" + id + "': This tenant does not exist"); - accessControl.createApplication(id, credentials); - - LockedApplication locked = new LockedApplication(new Application(id, clock.instant()), lock); - store(locked); - log.info("Created " + locked); - return locked.get(); - } - } - - /** - * Creates a new instance for an existing application. - * - * @throws IllegalArgumentException if the instance already exists, or has an invalid instance name. - */ - public void createInstance(ApplicationId id) { - lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - store(withNewInstance(application, id)); - }); - } - - /** Returns given application with a new instance */ - public LockedApplication withNewInstance(LockedApplication application, ApplicationId instance) { - if (instance.instance().isTester()) - throw new IllegalArgumentException("'" + instance + "' is a tester application!"); - InstanceId.validate(instance.instance().value()); - - if (getInstance(instance).isPresent()) - throw new IllegalArgumentException("Could not create '" + instance + "': Instance already exists"); - if (getInstance(dashToUnderscore(instance)).isPresent()) // VESPA-1945 - throw new IllegalArgumentException("Could not create '" + instance + "': Instance " + dashToUnderscore(instance) + " already exists"); - - log.info("Created " + instance); - return application.withNewInstance(instance.instance()); - } - - /** Deploys an application package for an existing application instance. */ - public DeploymentResult deploy(JobId job, boolean deploySourceVersions, Consumer<String> deployLogger, UnaryOperator<Optional<CloudAccount>> cloudAccountOverride) { - if (job.application().instance().isTester()) - throw new IllegalArgumentException("'" + job.application() + "' is a tester application!"); - - TenantAndApplicationId applicationId = TenantAndApplicationId.from(job.application()); - ZoneId zone = job.type().zone(); - DeploymentId deployment = new DeploymentId(job.application(), zone); - - try (Mutex deploymentLock = lockForDeployment(job.application(), zone)) { - Run run = controller.jobController().last(job) - .orElseThrow(() -> new IllegalStateException("No known run of '" + job + "'")); - - if (run.hasEnded()) - throw new IllegalStateException("No deployment expected for " + job + " now, as no job is running"); - - Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform()); - RevisionId revision = run.versions().sourceRevision().filter(__ -> deploySourceVersions).orElse(run.versions().targetRevision()); - ApplicationPackageStream applicationPackage = new ApplicationPackageStream(() -> applicationStore.stream(deployment, revision)); - AtomicReference<RevisionId> lastRevision = new AtomicReference<>(); - // Prepare endpoints lazily - Supplier<PreparedEndpoints> preparedEndpoints = () -> { - try (Mutex lock = lock(applicationId)) { - LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); - application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set); - return prepareEndpoints(deployment, job, application, applicationPackage, deployLogger, lock); - } - }; - - // Carry out deployment without holding the application lock. - DeploymentDataAndResult dataAndResult = deploy(job.application(), applicationPackage, zone, platform, preparedEndpoints, - run.isDryRun(), run.testerCertificate(), cloudAccountOverride); - - // Record the quota usage for this application - var quotaUsage = deploymentQuotaUsage(zone, job.application()); - - // For direct deployments use the full deployment ID, but otherwise use just the tenant and application as - // the source since it's the same application, so it should have the same warnings. - // These notifications are only updated when the last submitted revision is deployed here. - NotificationSource source = zone.environment().isManuallyDeployed() - ? NotificationSource.from(deployment) - : revision.equals(lastRevision.get()) ? NotificationSource.from(applicationId) : null; - if (source != null) { - List<String> warnings = Optional.ofNullable(dataAndResult.result().log()) - .map(logs -> logs.stream() - .filter(LogEntry::concernsPackage) - .filter(log -> log.level().intValue() >= Level.WARNING.intValue()) - .map(LogEntry::message) - .sorted() - .distinct() - .toList()) - .orElseGet(List::of); - if (warnings.isEmpty()) - controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage); - else - controller.notificationsDb().setApplicationPackageNotification(source, warnings); - } - - lockApplicationOrThrow(applicationId, application -> - store(application.with(job.application().instance(), - i -> i.withNewDeployment(zone, revision, platform, - clock.instant(), warningsFrom(dataAndResult.result().log()), - quotaUsage, dataAndResult.data().cloudAccount().orElse(CloudAccount.empty), - dataAndResult.data.dataPlaneTokens())))); - return dataAndResult.result(); - } - } - - private PreparedEndpoints prepareEndpoints(DeploymentId deployment, JobId job, LockedApplication application, - ApplicationPackageStream applicationPackage, - Consumer<String> deployLogger, - Mutex applicationLock) { - Instance instance = application.get().require(job.application().instance()); - Tags tags = applicationPackage.truncatedPackage().deploymentSpec().instance(instance.name()) - .map(DeploymentInstanceSpec::tags) - .orElseGet(Tags::empty); - EndpointCertificate certificate = endpointCertificates.get(deployment, - applicationPackage.truncatedPackage().deploymentSpec(), - applicationLock); - deployLogger.accept("Using CA signed certificate version %s".formatted(certificate.version())); - BasicServicesXml services = applicationPackage.truncatedPackage().services(deployment, tags); - return controller.routing().of(deployment).prepare(services, certificate, application); - } - - /** Stores the deployment spec and validation overrides from the application package, and runs cleanup. Returns new instances. */ - public List<InstanceName> storeWithUpdatedConfig(LockedApplication application, ApplicationPackage applicationPackage) { - validatePackage(applicationPackage, application.get()); - - application = application.with(applicationPackage.deploymentSpec()); - application = application.with(applicationPackage.validationOverrides()); - - var existingInstances = application.get().instances(); - var declaredInstances = applicationPackage.deploymentSpec().instances(); - for (var declaredInstance : declaredInstances) { - if ( ! existingInstances.containsKey(declaredInstance.name())) - application = withNewInstance(application, application.get().id().instance(declaredInstance.name())); - } - - // Delete zones not listed in DeploymentSpec, if allowed - // We do this at deployment time for externally built applications, and at submission time - // for internally built ones, to be able to return a validation failure message when necessary - for (InstanceName name : existingInstances.keySet()) { - application = withoutDeletedDeployments(application, name); - } - - // Validate new deployment spec thoroughly before storing it. - DeploymentStatus status = controller.jobController().deploymentStatus(application.get()); - Change dummyChange = Change.of(RevisionId.forProduction(Long.MAX_VALUE)); // Should always run everywhere. - for (var jobs : status.jobsToRun(applicationPackage.deploymentSpec().instanceNames().stream() - .collect(toMap(name -> name, __ -> dummyChange))) - .entrySet()) { - for (var job : jobs.getValue()) { - decideCloudAccountOf(new DeploymentId(jobs.getKey().application(), job.type().zone()), - applicationPackage.deploymentSpec()); - } - } - - for (Notification notification : controller.notificationsDb().listNotifications(NotificationSource.from(application.get().id()), true)) { - if ( notification.source().instance().isPresent() - && ( ! declaredInstances.contains(notification.source().instance().get()) - || ! notification.source().zoneId().map(application.get().require(notification.source().instance().get()).deployments()::containsKey).orElse(false))) - controller.notificationsDb().removeNotifications(notification.source()); - } - - store(application); - return declaredInstances.stream() - .map(DeploymentInstanceSpec::name) - .filter(instance -> ! existingInstances.containsKey(instance)) - .toList(); - } - - /** Deploy a system application to given zone */ - public void deploy(SystemApplication application, ZoneId zone, Version version, boolean allowDowngrade) { - if (application.hasApplicationPackage()) { - deploySystemApplicationPackage(application, zone, version); - } else { - // Deploy by calling node repository directly - configServer.nodeRepository().upgrade(zone, application.nodeType(), version, allowDowngrade); - } - } - - /** Deploy a system application to given zone */ - public DeploymentResult deploySystemApplicationPackage(SystemApplication application, ZoneId zone, Version version) { - if (application.hasApplicationPackage()) { - ApplicationPackageStream applicationPackage = new ApplicationPackageStream( - () -> new ByteArrayInputStream(artifactRepository.getSystemApplicationPackage(application.id(), zone, version)) - ); - return deploy(application.id(), applicationPackage, zone, version, null, false, Optional.empty(), UnaryOperator.identity()).result(); - } else { - throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); - } - } - - /** Deploys the given tester application to the given zone. */ - public DeploymentResult deployTester(TesterId tester, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform, UnaryOperator<Optional<CloudAccount>> cloudAccountOverride) { - return deploy(tester.id(), applicationPackage, zone, platform, null, false, Optional.empty(), cloudAccountOverride).result(); - } - - private record DeploymentDataAndResult(DeploymentData data, DeploymentResult result) {} - - private DeploymentDataAndResult deploy(ApplicationId application, ApplicationPackageStream applicationPackage, - ZoneId zone, Version platform, Supplier<PreparedEndpoints> preparedEndpoints, - boolean dryRun, Optional<X509Certificate> testerCertificate, - UnaryOperator<Optional<CloudAccount>> cloudAccountOverride) { - DeploymentId deployment = new DeploymentId(application, zone); - // Routing and metadata may have changed, so we need to refresh state after deployment, even if deployment fails. - interface CleanCloseable extends AutoCloseable { void close(); } - AtomicReference<EndpointList> generatedEndpoints = new AtomicReference<>(EndpointList.EMPTY); - try (CleanCloseable postDeployment = () -> updateRoutingAndMeta(deployment, applicationPackage, generatedEndpoints)) { - Optional<DockerImage> dockerImageRepo = Optional.ofNullable( - dockerImageRepoFlag - .with(FetchVector.Dimension.ZONE_ID, zone.value()) - .with(INSTANCE_ID, application.serializedForm()) - .value()) - .filter(s -> !s.isBlank()) - .map(DockerImage::fromString); - - Optional<AthenzDomain> domain = controller.tenants().get(application.tenant()) - .filter(tenant-> tenant instanceof AthenzTenant) - .map(tenant -> ((AthenzTenant)tenant).domain()); - - Supplier<Quota> deploymentQuota = () -> DeploymentQuotaCalculator.calculate(billingController.getQuota(application.tenant()), - asList(application.tenant()), application, zone, applicationPackage.truncatedPackage().deploymentSpec()); - - List<TenantSecretStore> tenantSecretStores = controller.tenants() - .get(application.tenant()) - .filter(tenant-> tenant instanceof CloudTenant) - .map(tenant -> ((CloudTenant) tenant).tenantSecretStores()) - .orElse(List.of()); - List<X509Certificate> operatorCertificates = controller.supportAccess().activeGrantsFor(deployment).stream() - .map(SupportAccessGrant::certificate) - .toList(); - if (testerCertificate.isPresent()) { - operatorCertificates = Stream.concat(operatorCertificates.stream(), testerCertificate.stream()).toList(); - } - Supplier<Optional<CloudAccount>> cloudAccount = () -> cloudAccountOverride.apply(decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec())); - Supplier<DeploymentEndpoints> endpoints = () -> { - if (preparedEndpoints == null) return DeploymentEndpoints.none; - PreparedEndpoints prepared = preparedEndpoints.get(); - generatedEndpoints.set(prepared.endpoints().generated()); - return new DeploymentEndpoints(prepared.containerEndpoints(), Optional.of(prepared.certificate())); - }; - Supplier<List<DataplaneTokenVersions>> dataplaneTokenVersions = () -> { - Tags tags = applicationPackage.truncatedPackage().deploymentSpec() - .instance(application.instance()) - .map(DeploymentInstanceSpec::tags) - .orElse(Tags.empty()); - BasicServicesXml services = applicationPackage.truncatedPackage().services(deployment, tags); - Set<TokenId> referencedTokens = services.containers().stream() - .flatMap(container -> container.dataPlaneTokens().stream()) - .collect(toSet()); - List<DataplaneTokenVersions> currentTokens = controller.dataplaneTokenService().listTokens(application.tenant()).stream() - .filter(token -> referencedTokens.contains(token.tokenId())) - .toList(); - return Stream.concat(currentTokens.stream(), - referencedTokens.stream() - .filter(token -> currentTokens.stream().noneMatch(t -> t.tokenId().equals(token))) - .map(token -> new DataplaneTokenVersions(token, List.of(), Instant.EPOCH))) - .toList(); - }; - DeploymentData deploymentData = new DeploymentData(application, zone, applicationPackage::zipStream, platform, - endpoints, dockerImageRepo, domain, deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dataplaneTokenVersions, dryRun); - ConfigServer.PreparedApplication preparedApplication = configServer.deploy(deploymentData); - - return new DeploymentDataAndResult(deploymentData, preparedApplication.deploymentResult()); - } - } - - private void updateRoutingAndMeta(DeploymentId id, ApplicationPackageStream data, AtomicReference<EndpointList> generatedEndpoints) { - if (id.applicationId().instance().isTester()) return; - controller.routing().of(id).activate(data.truncatedPackage().deploymentSpec(), generatedEndpoints.get()); - if ( ! id.zoneId().environment().isManuallyDeployed()) return; - controller.applications().applicationStore().putMeta(id, clock.instant(), data.truncatedPackage().metaDataZip()); - } - - public Optional<CloudAccount> decideCloudAccountOf(DeploymentId deployment, DeploymentSpec spec) { - ZoneId zoneId = deployment.zoneId(); - CloudName cloud = controller.zoneRegistry().get(zoneId).getCloudName(); - CloudAccount requestedAccount = spec.cloudAccount(cloud, deployment.applicationId().instance(), deployment.zoneId()); - if (requestedAccount.isUnspecified()) - return Optional.empty(); - - TenantName tenant = deployment.applicationId().tenant(); - Set<CloudAccount> tenantAccounts = accountsOf(tenant); - if ( ! tenantAccounts.contains(requestedAccount)) { - throw new IllegalArgumentException("Requested cloud account '" + requestedAccount.value() + - "' is not valid for tenant '" + tenant + "'"); - } - if ( ! controller.zoneRegistry().hasZone(zoneId, requestedAccount)) { - throw new IllegalArgumentException("Zone " + zoneId + " is not configured in requested cloud account '" + - requestedAccount.value() + "'"); - } - return Optional.of(requestedAccount); - } - - private LockedApplication withoutDeletedDeployments(LockedApplication application, InstanceName instance) { - DeploymentSpec deploymentSpec = application.get().deploymentSpec(); - List<ZoneId> deploymentsToRemove = application.get().require(instance).productionDeployments().values().stream() - .map(Deployment::zone) - .filter(zone -> deploymentSpec.instance(instance).isEmpty() - || ! deploymentSpec.requireInstance(instance).deploysTo(zone.environment(), - zone.region())) - .toList(); - - if (deploymentsToRemove.isEmpty()) - return application; - - if ( ! application.get().validationOverrides().allows(ValidationId.deploymentRemoval, clock.instant())) - throw new IllegalArgumentException(ValidationId.deploymentRemoval.value() + ": " + application.get().require(instance) + - " is deployed in " + - deploymentsToRemove.stream() - .map(zone -> zone.region().value()) - .collect(joining(", ")) + - ", but " + (deploymentsToRemove.size() > 1 ? "these " : "this ") + - "instance and region combination" + - (deploymentsToRemove.size() > 1 ? "s are" : " is") + - " removed from deployment.xml. " + - ValidationOverrides.toAllowMessage(ValidationId.deploymentRemoval)); - // Remove the instance as well, if it is no longer referenced, and contains only production deployments that are removed now. - boolean removeInstance = ! deploymentSpec.instanceNames().contains(instance) - && application.get().require(instance).deployments().size() == deploymentsToRemove.size(); - for (ZoneId zone : deploymentsToRemove) { - application = deactivate(application.get().id().instance(instance), zone, Optional.of(application)).get(); - } - if (removeInstance) { - application = application.without(instance); - } - return application; - } - - /** - * Deletes the given application. All known instances of the applications will be deleted. - * - * @throws IllegalArgumentException if the application has deployments or the caller is not authorized - */ - public void deleteApplication(TenantAndApplicationId id, Credentials credentials) { - deleteApplication(id, Optional.of(credentials)); - } - - public void deleteApplication(TenantAndApplicationId id, Optional<Credentials> credentials) { - lockApplicationOrThrow(id, application -> { - var deployments = application.get().instances().values().stream() - .filter(instance -> ! instance.deployments().isEmpty()) - .collect(toMap(instance -> instance.name(), - instance -> instance.deployments().keySet().stream() - .map(ZoneId::toString) - .collect(joining(", ")))); - if ( ! deployments.isEmpty()) - throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments: " + deployments); - - for (Instance instance : application.get().instances().values()) { - controller.routing().removeRotationEndpointsFromDns(application.get(), instance.name()); - application = application.without(instance.name()); - } - - applicationStore.removeAll(id.tenant(), id.application()); - applicationStore.putMetaTombstone(id.tenant(), id.application(), clock.instant()); - - credentials.ifPresent(creds -> accessControl.deleteApplication(id, creds)); - curator.removeApplication(id); - - controller.jobController().collectGarbage(); - controller.notificationsDb().removeNotifications(NotificationSource.from(id)); - log.info("Deleted " + id); - }); - } - - /** - * Deletes the the given application instance. - * - * @throws IllegalArgumentException if the application has deployments or the caller is not authorized - * @throws NotExistsException if the instance does not exist - */ - public void deleteInstance(ApplicationId instanceId) { - if (getInstance(instanceId).isEmpty()) - throw new NotExistsException("Could not delete instance '" + instanceId + "': Instance not found"); - - lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> { - if ( ! application.get().require(instanceId.instance()).deployments().isEmpty()) - throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments in: " + - application.get().require(instanceId.instance()).deployments().keySet().stream().map(ZoneId::toString) - .sorted().collect(joining(", "))); - - if ( ! application.get().deploymentSpec().equals(DeploymentSpec.empty) - && application.get().deploymentSpec().instanceNames().contains(instanceId.instance())) - throw new IllegalArgumentException("Can not delete '" + instanceId + "', which is specified in 'deployment.xml'; remove it there first"); - - controller.routing().removeRotationEndpointsFromDns(application.get(), instanceId.instance()); - curator.writeApplication(application.without(instanceId.instance()).get()); - controller.jobController().collectGarbage(); - controller.notificationsDb().removeNotifications(NotificationSource.from(instanceId)); - log.info("Deleted " + instanceId); - }); - } - - /** - * Replace any previous version of this application by this instance - * - * @param application a locked application to store - */ - public void store(LockedApplication application) { - curator.writeApplication(application.get()); - } - - /** - * Acquire a locked application to modify and store, if there is an application with the given id. - * - * @param applicationId ID of the application to lock and get. - * @param action Function which acts on the locked application. - */ - public void lockApplicationIfPresent(TenantAndApplicationId applicationId, Consumer<LockedApplication> action) { - try (Mutex lock = lock(applicationId)) { - getApplication(applicationId).map(application -> new LockedApplication(application, lock)).ifPresent(action); - } - } - - /** - * Acquire a locked application to modify and store, or throw an exception if no application has the given id. - * - * @param applicationId ID of the application to lock and require. - * @param action Function which acts on the locked application. - * @throws IllegalArgumentException when application does not exist. - */ - public void lockApplicationOrThrow(TenantAndApplicationId applicationId, Consumer<LockedApplication> action) { - try (Mutex lock = lock(applicationId)) { - action.accept(new LockedApplication(requireApplication(applicationId), lock)); - } - } - - /** - * Tells config server to schedule a restart of all nodes in this deployment - * - * @param restartFilter Variables to filter which nodes to restart. - */ - public void restart(DeploymentId deploymentId, RestartFilter restartFilter) { - configServer.restart(deploymentId, restartFilter); - } - - /** - * Asks the config server whether this deployment is currently healthy, i.e., serving traffic as usual. - * If this cannot be ascertained, we must assume it is not. - */ - public boolean isHealthy(DeploymentId deploymentId) { - try { - return ! isSuspended(deploymentId); // consider adding checks again global routing status, etc.? - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Failed getting suspension status of " + deploymentId + ": " + Exceptions.toMessageString(e)); - return false; - } - } - - /** - * Asks the config server whether this deployment is currently <i>suspended</i>: - * Not in a state where it should receive traffic. - */ - public boolean isSuspended(DeploymentId deploymentId) { - return configServer.isSuspended(deploymentId); - } - - /** Sets suspension status of the given deployment in its zone. */ - public void setSuspension(DeploymentId deploymentId, boolean suspend) { - configServer.setSuspension(deploymentId, suspend); - } - - /** Deactivate application in the given zone. Even if the application itself does not exist, deactivation of the deployment will still be attempted */ - public void deactivate(ApplicationId instanceId, ZoneId zone) { - TenantAndApplicationId applicationId = TenantAndApplicationId.from(instanceId); - try (Mutex deploymentLock = lockForDeployment(instanceId, zone)) { - try (Mutex lock = lock(applicationId)) { - Optional<LockedApplication> application = getApplication(applicationId).map(app -> new LockedApplication(app, lock)); - deactivate(instanceId, zone, application).ifPresent(this::store); - } - } - } - - /** - * Deactivates a locked application without storing it - * - * @return the application with the deployment in the given zone removed - */ - private Optional<LockedApplication> deactivate(ApplicationId instanceId, ZoneId zone, Optional<LockedApplication> application) { - DeploymentId id = new DeploymentId(instanceId, zone); - interface CleanCloseable extends AutoCloseable { void close(); } - try (CleanCloseable postDeactivation = () -> { - application.ifPresent(app -> controller.routing().of(id).deactivate(app.get().deploymentSpec())); - if (id.zoneId().environment().isManuallyDeployed()) - applicationStore.putMetaTombstone(id, clock.instant()); - if ( ! id.zoneId().environment().isTest()) - controller.notificationsDb().removeNotifications(NotificationSource.from(id)); - }) { - configServer.deactivate(id); - return application.map(app -> app.with(instanceId.instance(), instance -> instance.withoutDeploymentIn(id.zoneId()))); - } - } - - public DeploymentTrigger deploymentTrigger() { return deploymentTrigger; } - - /** - * Returns a lock which provides exclusive rights to changing this application. - * Any operation which stores an application need to first acquire this lock, then read, modify - * and store the application, and finally release (close) the lock. - */ - Mutex lock(TenantAndApplicationId application) { - return curator.lock(application); - } - - /** - * Returns a lock which provides exclusive rights to deploying this application to the given zone. - */ - private Mutex lockForDeployment(ApplicationId application, ZoneId zone) { - return curator.lockForDeployment(application, zone); - } - - public VersionCompatibility versionCompatibility(ApplicationId id) { - return VersionCompatibility.fromVersionList(incompatibleVersions.with(INSTANCE_ID, id.serializedForm()).value()); - } - - /** - * Verifies that the application can be deployed to the tenant, following these rules: - * - * 1. Verify that the Athenz service can be launched by the config server - * 2. If the principal is given, verify that the principal is tenant admin or admin of the tenant domain - * 3. If the principal is not given, verify that the Athenz domain of the tenant equals Athenz domain given in deployment.xml - * - * @param tenantName tenant where application should be deployed - * @param applicationPackage application package - * @param deployer principal initiating the deployment, possibly empty - */ - public void verifyApplicationIdentityConfiguration(TenantName tenantName, Optional<DeploymentId> deployment, ApplicationPackage applicationPackage, Optional<Principal> deployer) { - Optional<AthenzDomain> identityDomain = applicationPackage.deploymentSpec().athenzDomain() - .map(domain -> new AthenzDomain(domain.value())); - if (identityDomain.isEmpty()) { - // If there is no domain configured in deployment.xml there is nothing to do. - return; - } - - // Verify that the system supports launching services. - // Consider adding a capability to the system. - if ( ! (accessControl instanceof AthenzFacade)) { - throw new IllegalArgumentException("Athenz domain and service specified in deployment.xml, but not supported by system."); - } - - // Verify that the config server is allowed to launch the service specified - verifyAllowedLaunchAthenzService(applicationPackage.deploymentSpec()); - - // If a user principal is initiating the request, verify that the user is allowed to launch the service. - // Either the user is member of the domain admin role, or is given the "launch" privilege on the service. - Optional<AthenzUser> athenzUser = getUser(deployer); - if (athenzUser.isPresent()) { - // This is a direct deployment, and we need only validate what the configserver will actually launch. - DeploymentId id = deployment.orElseThrow(() -> new IllegalArgumentException("Unable to evaluate access, no zone provided in deployment")); - var serviceToLaunch = applicationPackage.deploymentSpec().athenzService(id.applicationId().instance(), - id.zoneId().environment(), - id.zoneId().region()) - .map(service -> new AthenzService(identityDomain.get(), service.value())); - if (serviceToLaunch.isPresent()) { - if ( - ! ((AthenzFacade) accessControl).canLaunch(athenzUser.get(), serviceToLaunch.get()) && // launch privilege - ! ((AthenzFacade) accessControl).hasTenantAdminAccess(athenzUser.get(), identityDomain.get()) // tenant admin - ) { - throw new IllegalArgumentException("User " + athenzUser.get().getFullName() + " is not allowed to launch " + - "service " + serviceToLaunch.get().getFullName() + ". " + - "Please reach out to the domain admin."); - } - } else { - // This is a rare edge case where deployment.xml specifies athenz-service on each step, but not on the root. - // It is undefined which service should be launched, so handle this as an error. - throw new IllegalArgumentException("Athenz domain configured, but no service defined for deployment to " + id.zoneId().value()); - } - } else { - // If this is a deployment pipeline, verify that the domain in deployment.xml is the same as the tenant domain. Access control is already validated before this step. - Tenant tenant = controller.tenants().require(tenantName); - AthenzDomain tenantDomain = ((AthenzTenant) tenant).domain(); - if ( ! Objects.equals(tenantDomain, identityDomain.get())) - throw new IllegalArgumentException("Athenz domain in deployment.xml: [" + identityDomain.get().getName() + "] " + - "must match tenant domain: [" + tenantDomain.getName() + "]"); - } - } - - private TenantAndApplicationId dashToUnderscore(TenantAndApplicationId id) { - return TenantAndApplicationId.from(id.tenant().value(), id.application().value().replaceAll("-", "_")); - } - - private ApplicationId dashToUnderscore(ApplicationId id) { - return dashToUnderscore(TenantAndApplicationId.from(id)).instance(id.instance()); - } - - private QuotaUsage deploymentQuotaUsage(ZoneId zoneId, ApplicationId applicationId) { - var application = configServer.nodeRepository().getApplication(zoneId, applicationId); - return DeploymentQuotaCalculator.calculateQuotaUsage(application); - } - - /* - * Get the AthenzUser from this principal or Optional.empty if this does not represent a user. - */ - private Optional<AthenzUser> getUser(Optional<Principal> deployer) { - return deployer - .filter(AthenzPrincipal.class::isInstance) - .map(AthenzPrincipal.class::cast) - .map(AthenzPrincipal::getIdentity) - .filter(AthenzUser.class::isInstance) - .map(AthenzUser.class::cast); - } - - /* - * Verifies that the configured athenz service (if any) can be launched. - */ - private void verifyAllowedLaunchAthenzService(DeploymentSpec deploymentSpec) { - deploymentSpec.athenzDomain().ifPresent(domain -> { - controller.zoneRegistry().zones().reachable().ids().forEach(zone -> { - AthenzIdentity configServerAthenzIdentity = controller.zoneRegistry().getConfigServerHttpsIdentity(zone); - deploymentSpec.athenzService().ifPresent(service -> { - verifyAthenzServiceCanBeLaunchedBy(configServerAthenzIdentity, new AthenzService(domain.value(), service.value())); - }); - deploymentSpec.instances().forEach(spec -> { - spec.athenzService(zone.environment(), zone.region()).ifPresent(service -> { - verifyAthenzServiceCanBeLaunchedBy(configServerAthenzIdentity, new AthenzService(domain.value(), service.value())); - }); - }); - }); - }); - } - - private void verifyAthenzServiceCanBeLaunchedBy(AthenzIdentity configServerAthenzIdentity, AthenzService athenzService) { - if ( ! ((AthenzFacade) accessControl).canLaunch(configServerAthenzIdentity, athenzService)) - throw new IllegalArgumentException("Not allowed to launch Athenz service " + athenzService.getFullName()); - } - - /** Extract deployment warnings metric from deployment result */ - private static Map<DeploymentMetrics.Warning, Integer> warningsFrom(List<DeploymentResult.LogEntry> log) { - return log.stream() - .filter(entry -> entry.level().intValue() >= Level.WARNING.intValue()) - // TODO: Categorize warnings. Response from config server should be updated to include the appropriate - // category and typed log level - .collect(groupingBy(__ -> Warning.all, - collectingAndThen(counting(), Long::intValue))); - } - - public void verifyPlan(TenantName tenantName) { - var planId = controller.serviceRegistry().billingController().getPlan(tenantName); - Optional<Plan> plan = controller.serviceRegistry().planRegistry().plan(planId); - if (plan.isEmpty()) - throw new IllegalArgumentException("Tenant '" + tenantName.value() + "' has no plan, not allowed to deploy. See https://cloud.vespa.ai/support"); - if (plan.get().quota().calculate().equals(Quota.zero())) - throw new IllegalArgumentException("Tenant '" + tenantName.value() + "' has a plan '" + - plan.get().displayName() + "' with zero quota, not allowed to deploy. See https://cloud.vespa.ai/support"); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java deleted file mode 100644 index 0b693bb9894..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.component.Version; -import com.yahoo.component.Vtag; -import com.yahoo.component.annotation.Inject; -import com.yahoo.concurrent.maintenance.JobControl; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.jdisc.Metric; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; -import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.application.MailVerifier; -import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger; -import com.yahoo.vespa.hosted.controller.config.ControllerConfig; -import com.yahoo.vespa.hosted.controller.deployment.JobController; -import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; -import com.yahoo.vespa.hosted.controller.notification.NotificationsDb; -import com.yahoo.vespa.hosted.controller.notification.Notifier; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.persistence.JobControlFlags; -import com.yahoo.vespa.hosted.controller.restapi.dataplanetoken.DataplaneTokenService; -import com.yahoo.vespa.hosted.controller.security.AccessControl; -import com.yahoo.vespa.hosted.controller.support.access.SupportAccessControl; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import com.yahoo.yolean.concurrent.Sleeper; - -import java.security.SecureRandom; -import java.time.Clock; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Random; -import java.util.Set; -import java.util.function.Predicate; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.toSet; - -/** - * API to the controller. This contains the object model of everything the controller cares about, mainly tenants and - * applications. The object model is persisted to curator. - * - * All the individual model objects reachable from the Controller are immutable. - * - * Access to the controller is multi-thread safe, provided the locking methods are - * used when accessing, modifying and storing objects provided by the controller. - * - * @author bratseth - */ -public class Controller extends AbstractComponent { - - private static final Logger log = Logger.getLogger(Controller.class.getName()); - - private final CuratorDb curator; - private final JobControl jobControl; - private final ApplicationController applicationController; - private final TenantController tenantController; - private final JobController jobController; - private final Clock clock; - private final Sleeper sleeper; - private final ZoneRegistry zoneRegistry; - private final ServiceRegistry serviceRegistry; - private final AuditLogger auditLogger; - private final FlagSource flagSource; - private final NameServiceForwarder nameServiceForwarder; - private final MavenRepository mavenRepository; - private final Metric metric; - private final RoutingController routingController; - private final OsController osController; - private final ControllerConfig controllerConfig; - private final SecretStore secretStore; - private final CuratorArchiveBucketDb archiveBucketDb; - private final NotificationsDb notificationsDb; - private final SupportAccessControl supportAccessControl; - private final Notifier notifier; - private final MailVerifier mailVerifier; - private final DataplaneTokenService dataplaneTokenService; - private final Random random; - private final Random secureRandom; // Type is Random to allow for test determinism - - /** - * Creates a controller - * - * @param curator the curator instance storing the persistent state of the controller. - */ - @Inject - public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, FlagSource flagSource, - MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, - ControllerConfig controllerConfig) { - this(curator, rotationsConfig, accessControl, flagSource, - mavenRepository, serviceRegistry, metric, secretStore, controllerConfig, Sleeper.DEFAULT, new Random(), - new SecureRandom()); - } - - public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, - FlagSource flagSource, MavenRepository mavenRepository, - ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, - ControllerConfig controllerConfig, Sleeper sleeper, Random random, Random secureRandom) { - this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); - this.serviceRegistry = Objects.requireNonNull(serviceRegistry, "ServiceRegistry cannot be null"); - this.zoneRegistry = Objects.requireNonNull(serviceRegistry.zoneRegistry(), "ZoneRegistry cannot be null"); - this.clock = Objects.requireNonNull(serviceRegistry.clock(), "Clock cannot be null"); - this.sleeper = Objects.requireNonNull(sleeper, "Sleeper cannot be null"); - this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); - this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null"); - this.metric = Objects.requireNonNull(metric, "Metric cannot be null"); - this.controllerConfig = Objects.requireNonNull(controllerConfig, "ControllerConfig cannot be null"); - this.secretStore = Objects.requireNonNull(secretStore, "SecretStore cannot be null"); - this.random = Objects.requireNonNull(random, "Random cannot be null"); - this.secureRandom = Objects.requireNonNull(secureRandom, "SecureRandom cannot be null"); - - nameServiceForwarder = new NameServiceForwarder(curator); - jobController = new JobController(this); - applicationController = new ApplicationController(this, curator, accessControl, clock, flagSource, serviceRegistry.billingController()); - tenantController = new TenantController(this, curator, accessControl); - routingController = new RoutingController(this, rotationsConfig); - osController = new OsController(this); - auditLogger = new AuditLogger(curator, clock); - jobControl = new JobControl(new JobControlFlags(curator, flagSource)); - archiveBucketDb = new CuratorArchiveBucketDb(this); - notifier = new Notifier(curator, serviceRegistry.consoleUrls(), serviceRegistry.mailer(), flagSource); - notificationsDb = new NotificationsDb(this); - supportAccessControl = new SupportAccessControl(this); - mailVerifier = new MailVerifier(serviceRegistry.consoleUrls(), tenantController, serviceRegistry.mailer(), curator, clock); - dataplaneTokenService = new DataplaneTokenService(this); - - // Record the version of this controller - curator().writeControllerVersion(this.hostname(), serviceRegistry.controllerVersion()); - - jobController.updateStorage(); - } - - /** Returns the instance controlling tenants */ - public TenantController tenants() { return tenantController; } - - /** Returns the instance controlling applications */ - public ApplicationController applications() { return applicationController; } - - /** Returns the instance controlling deployment jobs. */ - public JobController jobController() { return jobController; } - - /** Returns the instance controlling routing */ - public RoutingController routing() { - return routingController; - } - - /** Returns the instance controlling OS upgrades */ - public OsController os() { - return osController; - } - - /** Returns the service registry of this */ - public ServiceRegistry serviceRegistry() { - return serviceRegistry; - } - - /** Provides access to the feature flags of this */ - public FlagSource flagSource() { - return flagSource; - } - - public Clock clock() { return clock; } - - public Sleeper sleeper() { return sleeper; } - - public ZoneRegistry zoneRegistry() { return zoneRegistry; } - - public NameServiceForwarder nameServiceForwarder() { return nameServiceForwarder; } - - public MavenRepository mavenRepository() { return mavenRepository; } - - public ControllerConfig controllerConfig() { return controllerConfig; } - - /** Replace the current version status by a new one */ - public void updateVersionStatus(VersionStatus newStatus) { - VersionStatus currentStatus = readVersionStatus(); - if (newStatus.systemVersion().isPresent() && - ! newStatus.systemVersion().equals(currentStatus.systemVersion())) { - log.info("Changing system version from " + printableVersion(currentStatus.systemVersion()) + - " to " + printableVersion(newStatus.systemVersion())); - } - Set<Version> obsoleteVersions = currentStatus.versions().stream().map(VespaVersion::versionNumber).collect(toSet()); - for (VespaVersion version : newStatus.versions()) { - obsoleteVersions.remove(version.versionNumber()); - VespaVersion current = currentStatus.version(version.versionNumber()); - if (current == null) - log.info("New version " + version.versionNumber().toFullString() + " added"); - else if ( ! current.confidence().equals(version.confidence())) - log.info("Confidence for version " + version.versionNumber().toFullString() + - " changed from " + current.confidence() + " to " + version.confidence()); - } - for (Version version : obsoleteVersions) - log.info("Version " + version.toFullString() + " is obsolete, and will be forgotten"); - - curator.writeVersionStatus(newStatus); - removeConfidenceOverride(obsoleteVersions::contains); - } - - /** Returns the latest known version status. Calling this is free but the status may be slightly out of date. */ - public VersionStatus readVersionStatus() { return curator.readVersionStatus(); } - - /** Remove confidence override for versions matching given filter */ - public void removeConfidenceOverride(Predicate<Version> filter) { - try (Mutex lock = curator.lockConfidenceOverrides()) { - Map<Version, VespaVersion.Confidence> overrides = new LinkedHashMap<>(curator.readConfidenceOverrides()); - overrides.keySet().removeIf(filter); - curator.writeConfidenceOverrides(overrides); - } - } - - /** Returns the current system version: The controller should drive towards running all applications on this version */ - public Version readSystemVersion() { - return systemVersion(readVersionStatus()); - } - - /** Returns the current system version from given status: The controller should drive towards running all applications on this version */ - public Version systemVersion(VersionStatus versionStatus) { - return versionStatus.systemVersion() - .map(VespaVersion::versionNumber) - .orElse(Vtag.currentVersion); - } - - /** Returns the hostname of this controller */ - public HostName hostname() { - return serviceRegistry.getHostname(); - } - - public SystemName system() { - return zoneRegistry.system(); - } - - public CuratorDb curator() { - return curator; - } - - public AuditLogger auditLogger() { - return auditLogger; - } - - public Metric metric() { - return metric; - } - - public SecretStore secretStore() { - return secretStore; - } - - /** Clouds present in this system */ - public Set<CloudName> clouds() { - return zoneRegistry.zones().all().zones().stream() - .map(ZoneApi::getCloudName) - .collect(Collectors.toUnmodifiableSet()); - } - - private static String printableVersion(Optional<VespaVersion> vespaVersion) { - return vespaVersion.map(v -> v.versionNumber().toFullString()).orElse("unknown"); - } - - public JobControl jobControl() { - return jobControl; - } - - public CuratorArchiveBucketDb archiveBucketDb() { - return archiveBucketDb; - } - - public NotificationsDb notificationsDb() { - return notificationsDb; - } - - public SupportAccessControl supportAccess() { - return supportAccessControl; - } - - public Notifier notifier() { - return notifier; - } - - public MailVerifier mailVerifier() { - return mailVerifier; - } - - public DataplaneTokenService dataplaneTokenService() { - return dataplaneTokenService; - } - - /** Returns a random number generator. If secure is true, this returns a {@link SecureRandom} suitable for - * cryptographic purposes */ - public Random random(boolean secure) { - return secure ? secureRandom : random; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java deleted file mode 100644 index 0a9c680251c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneTokenVersions; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.application.AssignedRotation; -import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; -import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.QuotaUsage; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; - -import java.time.Instant; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalDouble; -import java.util.OptionalLong; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static java.util.Comparator.naturalOrder; - -/** - * An instance of an application. - * - * This is immutable. - * - * @author bratseth - */ -public class Instance { - - private final ApplicationId id; - private final Map<ZoneId, Deployment> deployments; - private final List<AssignedRotation> rotations; - private final RotationStatus rotationStatus; - private final Map<JobType, Instant> jobPauses; - private final Change change; - - /** Creates an empty instance */ - public Instance(ApplicationId id) { - this(id, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty()); - } - - /** Creates an empty instance*/ - public Instance(ApplicationId id, Collection<Deployment> deployments, Map<JobType, Instant> jobPauses, - List<AssignedRotation> rotations, RotationStatus rotationStatus, Change change) { - this.id = Objects.requireNonNull(id, "id cannot be null"); - this.deployments = Objects.requireNonNull(deployments, "deployments cannot be null").stream() - .collect(Collectors.toUnmodifiableMap(Deployment::zone, Function.identity())); - this.jobPauses = Map.copyOf(Objects.requireNonNull(jobPauses, "deploymentJobs cannot be null")); - this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null")); - this.rotationStatus = Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null"); - this.change = Objects.requireNonNull(change, "change cannot be null"); - } - - public Instance withNewDeployment(ZoneId zone, RevisionId revision, Version version, Instant instant, - Map<DeploymentMetrics.Warning, Integer> warnings, QuotaUsage quotaUsage, CloudAccount cloudAccount, - List<DataplaneTokenVersions> dataPlaneTokens) { - Map<TokenId, Instant> dataPlaneTokenIds = dataPlaneTokens.stream().collect(Collectors.toMap(token -> token.tokenId(), - token -> token.lastUpdated())); - // Use info from previous deployment if available, otherwise create a new one. - Deployment previousDeployment = deployments.getOrDefault(zone, new Deployment(zone, cloudAccount, revision, - version, instant, - DeploymentMetrics.none, - DeploymentActivity.none, - QuotaUsage.none, - OptionalDouble.empty(), - dataPlaneTokenIds)); - Deployment newDeployment = new Deployment(zone, cloudAccount, revision, version, instant, - previousDeployment.metrics().with(warnings), - previousDeployment.activity(), - quotaUsage, - previousDeployment.cost(), - dataPlaneTokenIds); - return with(newDeployment); - } - - public Instance withJobPause(JobType jobType, OptionalLong pausedUntil) { - Map<JobType, Instant> jobPauses = new HashMap<>(this.jobPauses); - if (pausedUntil.isPresent()) - jobPauses.put(jobType, Instant.ofEpochMilli(pausedUntil.getAsLong())); - else - jobPauses.remove(jobType); - - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); - } - - public Instance recordActivityAt(Instant instant, ZoneId zone) { - Deployment deployment = deployments.get(zone); - if (deployment == null) return this; - return with(deployment.recordActivityAt(instant)); - } - - public Instance with(ZoneId zone, DeploymentMetrics deploymentMetrics) { - Deployment deployment = deployments.get(zone); - if (deployment == null) return this; // No longer deployed in this zone. - return with(deployment.withMetrics(deploymentMetrics)); - } - - public Instance withDeploymentCosts(Map<ZoneId, Double> costByZone) { - Map<ZoneId, Deployment> deployments = this.deployments.entrySet().stream() - .map(entry -> Optional.ofNullable(costByZone.get(entry.getKey())) - .map(entry.getValue()::withCost) - .orElseGet(entry.getValue()::withoutCost)) - .collect(Collectors.toUnmodifiableMap(Deployment::zone, deployment -> deployment)); - return with(deployments); - } - - public Instance withoutDeploymentIn(ZoneId zone) { - Map<ZoneId, Deployment> deployments = new LinkedHashMap<>(this.deployments); - deployments.remove(zone); - return with(deployments); - } - - public Instance with(List<AssignedRotation> assignedRotations) { - return new Instance(id, deployments.values(), jobPauses, assignedRotations, rotationStatus, change); - } - - public Instance with(RotationStatus rotationStatus) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); - } - - public Instance withChange(Change change) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); - } - - private Instance with(Deployment deployment) { - Map<ZoneId, Deployment> deployments = new LinkedHashMap<>(this.deployments); - deployments.put(deployment.zone(), deployment); - return with(deployments); - } - - private Instance with(Map<ZoneId, Deployment> deployments) { - return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); - } - - public ApplicationId id() { return id; } - - public InstanceName name() { return id.instance(); } - - /** Returns an immutable map of the current deployments of this */ - public Map<ZoneId, Deployment> deployments() { return deployments; } - - /** - * Returns an immutable map of the current *production* deployments of this - * (deployments also includes manually deployed environments) - */ - public Map<ZoneId, Deployment> productionDeployments() { - return deployments.values().stream() - .filter(deployment -> deployment.zone().environment() == Environment.prod) - .collect(Collectors.toUnmodifiableMap(Deployment::zone, Function.identity())); - } - - /** Returns the instant until which the given job is paused, or empty. */ - public Optional<Instant> jobPause(JobType jobType) { - return Optional.ofNullable(jobPauses.get(jobType)); - } - - /** Returns the set of instants until which any paused jobs of this instance should remain paused, indexed by job type. */ - public Map<JobType, Instant> jobPauses() { - return jobPauses; - } - - /** Returns all rotations assigned to this */ - public List<AssignedRotation> rotations() { - return rotations; - } - - /** Returns the status of the global rotation(s) assigned to this */ - public RotationStatus rotationStatus() { - return rotationStatus; - } - - /** Returns the currently deploying change for this instance. */ - public Change change() { - return change; - } - - /** Returns the total quota usage for this instance, excluding temporary deployments **/ - public QuotaUsage quotaUsage() { - return deployments.values().stream() - .filter(d -> !d.zone().environment().isTest()) // Exclude temporary deployments - .map(Deployment::quota).reduce(QuotaUsage::add).orElse(QuotaUsage.none); - } - - /** Returns the total quota usage for manual deployments for this instance **/ - public QuotaUsage manualQuotaUsage() { - return deployments.values().stream() - .filter(d -> d.zone().environment().isManuallyDeployed()) - .map(Deployment::quota).reduce(QuotaUsage::add).orElse(QuotaUsage.none); - } - - /** Returns the total quota usage for this instance, excluding one specific deployment (and temporary deployments) */ - public QuotaUsage quotaUsageExcluding(ApplicationId application, ZoneId zone) { - return deployments.values().stream() - .filter(d -> !d.zone().environment().isTest()) // Exclude temporary deployments - .filter(d -> !(application.equals(id) && d.zone().equals(zone))) - .map(Deployment::quota).reduce(QuotaUsage::add).orElse(QuotaUsage.none); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if ( ! (o instanceof Instance)) return false; - - Instance that = (Instance) o; - - return id.equals(that.id); - } - - @Override - public int hashCode() { - return id.hashCode(); - } - - @Override - public String toString() { - return "application instance '" + id.toFullString() + "'"; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java deleted file mode 100644 index 830e40bd638..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.hosted.controller.api.integration.organization.AccountId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; -import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; - -import java.security.PublicKey; -import java.time.Instant; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.function.UnaryOperator; - -/** - * An application that has been locked for modification. Provides methods for modifying an application's fields. - * - * @author jonmv - */ -public class LockedApplication { - - private final Mutex lock; - private final TenantAndApplicationId id; - private final Instant createdAt; - private final DeploymentSpec deploymentSpec; - private final ValidationOverrides validationOverrides; - private final Optional<IssueId> deploymentIssueId; - private final Optional<IssueId> ownershipIssueId; - private final Optional<User> userOwner; - private final Optional<AccountId> issueOwner; - private final OptionalInt majorVersion; - private final ApplicationMetrics metrics; - private final Set<PublicKey> deployKeys; - private final OptionalLong projectId; - private final RevisionHistory revisions; - private final Map<InstanceName, Instance> instances; - - /** - * Used to create a locked application - * - * @param application The application to lock. - * @param lock The lock for the application. - */ - LockedApplication(Application application, Mutex lock) { - this(Objects.requireNonNull(lock, "lock cannot be null"), application.id(), application.createdAt(), - application.deploymentSpec(), application.validationOverrides(), application.deploymentIssueId(), application.ownershipIssueId(), - application.userOwner(), application.issueOwner(), application.majorVersion(), application.metrics(), application.deployKeys(), - application.projectId(), application.instances(), application.revisions()); - } - - private LockedApplication(Mutex lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, - ValidationOverrides validationOverrides, Optional<IssueId> deploymentIssueId, - Optional<IssueId> ownershipIssueId, Optional<User> userOwner, Optional<AccountId> issueOwner, - OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, - OptionalLong projectId, Map<InstanceName, Instance> instances, RevisionHistory revisions) { - this.lock = lock; - this.id = id; - this.createdAt = createdAt; - this.deploymentSpec = deploymentSpec; - this.validationOverrides = validationOverrides; - this.deploymentIssueId = deploymentIssueId; - this.ownershipIssueId = ownershipIssueId; - this.userOwner = userOwner; - this.issueOwner = issueOwner; - this.majorVersion = majorVersion; - this.metrics = metrics; - this.deployKeys = deployKeys; - this.projectId = projectId; - this.revisions = revisions; - this.instances = Map.copyOf(instances); - } - - /** Returns a read-only copy of this */ - public Application get() { - return new Application(id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, deployKeys, - projectId, revisions, instances.values()); - } - - LockedApplication withNewInstance(InstanceName instance) { - var instances = new HashMap<>(this.instances); - instances.put(instance, new Instance(id.instance(instance))); - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, deployKeys, - projectId, instances, revisions); - } - - public LockedApplication with(InstanceName instance, UnaryOperator<Instance> modification) { - var instances = new HashMap<>(this.instances); - instances.put(instance, modification.apply(instances.get(instance))); - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, deployKeys, - projectId, instances, revisions); - } - - public LockedApplication without(InstanceName instance) { - var instances = new HashMap<>(this.instances); - instances.remove(instance); - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, deployKeys, - projectId, instances, revisions); - } - - public LockedApplication withProjectId(OptionalLong projectId) { - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, deployKeys, - projectId, instances, revisions); - } - - public LockedApplication withDeploymentIssueId(IssueId issueId) { - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - Optional.ofNullable(issueId), ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, deployKeys, - projectId, instances, revisions); - } - - public LockedApplication with(DeploymentSpec deploymentSpec) { - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, deployKeys, - projectId, instances, revisions); - } - - public LockedApplication with(ValidationOverrides validationOverrides) { - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, deployKeys, - projectId, instances, revisions); - } - - public LockedApplication withOwnershipIssueId(IssueId issueId) { - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, Optional.of(issueId), userOwner, issueOwner, majorVersion, metrics, deployKeys, - projectId, instances, revisions); - } - - public LockedApplication withOwner(AccountId issueOwner) { - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, Optional.of(issueOwner), majorVersion, metrics, deployKeys, - projectId, instances, revisions); - } - - /** Set a major version for this, or set to null to remove any major version override */ - public LockedApplication withMajorVersion(Integer majorVersion) { - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, - issueOwner, majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion), - metrics, deployKeys, projectId, instances, revisions); - } - - public LockedApplication with(ApplicationMetrics metrics) { - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, deployKeys, - projectId, instances, revisions); - } - - public LockedApplication withDeployKey(PublicKey pemDeployKey) { - Set<PublicKey> keys = new LinkedHashSet<>(deployKeys); - keys.add(pemDeployKey); - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, keys, - projectId, instances, revisions); - } - - public LockedApplication withoutDeployKey(PublicKey pemDeployKey) { - Set<PublicKey> keys = new LinkedHashSet<>(deployKeys); - keys.remove(pemDeployKey); - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, keys, - projectId, instances, revisions); - } - - public LockedApplication withRevisions(UnaryOperator<RevisionHistory> change) { - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, - deployKeys, projectId, instances, change.apply(revisions)); - } - - @Override - public String toString() { - return "application '" + id + "'"; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java deleted file mode 100644 index bfba17bef22..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller; - -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; -import com.google.common.collect.ImmutableBiMap; -import com.yahoo.config.provision.TenantName; -import com.yahoo.security.KeyUtils; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.hosted.controller.api.identifiers.Property; -import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; -import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; -import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; -import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; -import com.yahoo.vespa.hosted.controller.tenant.BillingReference; -import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; -import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; - -import java.security.Principal; -import java.security.PublicKey; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -/** - * A tenant that has been locked for modification. Provides methods for modifying a tenant's fields. - * - * @author mpolden - * @author jonmv - */ -public abstract class LockedTenant { - - final TenantName name; - final Instant createdAt; - final LastLoginInfo lastLoginInfo; - final Instant tenantRolesLastMaintained; - final List<CloudAccountInfo> cloudAccounts; - - private LockedTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained, List<CloudAccountInfo> cloudAccounts) { - this.name = requireNonNull(name); - this.createdAt = requireNonNull(createdAt); - this.lastLoginInfo = requireNonNull(lastLoginInfo); - this.tenantRolesLastMaintained = requireNonNull(tenantRolesLastMaintained); - this.cloudAccounts = requireNonNull(cloudAccounts); - } - - static LockedTenant of(Tenant tenant, Mutex lock) { - return switch (tenant.type()) { - case athenz -> new Athenz((AthenzTenant) tenant); - case cloud -> new Cloud((CloudTenant) tenant); - case deleted -> new Deleted((DeletedTenant) tenant); - }; - } - - /** Returns a read-only copy of this */ - public abstract Tenant get(); - - public abstract LockedTenant with(LastLoginInfo lastLoginInfo); - - public abstract LockedTenant with(Instant tenantRolesLastMaintained); - - public abstract LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts); - - public Deleted deleted(Instant deletedAt) { - return new Deleted(new DeletedTenant(name, createdAt, deletedAt)); - } - - @Override - public String toString() { - return "tenant '" + name + "'"; - } - - - /** A locked AthenzTenant. */ - public static class Athenz extends LockedTenant { - - private final AthenzDomain domain; - private final Property property; - private final Optional<PropertyId> propertyId; - private final Optional<Contact> contact; - - private Athenz(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, - Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained, List<CloudAccountInfo> cloudAccounts) { - super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); - this.domain = domain; - this.property = property; - this.propertyId = propertyId; - this.contact = contact; - } - - private Athenz(AthenzTenant tenant) { - this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.tenantRolesLastMaintained(), tenant.cloudAccounts()); - } - - @Override - public AthenzTenant get() { - return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); - } - - public Athenz with(AthenzDomain domain) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); - } - - public Athenz with(Property property) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); - } - - public Athenz with(PropertyId propertyId) { - return new Athenz(name, domain, property, Optional.of(propertyId), contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); - } - - public Athenz with(Contact contact) { - return new Athenz(name, domain, property, propertyId, Optional.of(contact), createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); - } - - @Override - public LockedTenant with(LastLoginInfo lastLoginInfo) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); - } - - @Override - public LockedTenant with(Instant tenantRolesLastMaintained) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); - } - - @Override - public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); - } - - } - - - /** A locked CloudTenant. */ - public static class Cloud extends LockedTenant { - - private final Optional<SimplePrincipal> creator; - private final BiMap<PublicKey, SimplePrincipal> developerKeys; - private final TenantInfo info; - private final List<TenantSecretStore> tenantSecretStores; - private final ArchiveAccess archiveAccess; - private final Optional<Instant> invalidateUserSessionsBefore; - private final Optional<BillingReference> billingReference; - private final PlanId planId; - - private Cloud(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<SimplePrincipal> creator, - BiMap<PublicKey, SimplePrincipal> developerKeys, TenantInfo info, - List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, - Optional<Instant> invalidateUserSessionsBefore, Instant tenantRolesLastMaintained, - List<CloudAccountInfo> cloudAccounts, Optional<BillingReference> billingReference, - PlanId planId) { - super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); - this.developerKeys = ImmutableBiMap.copyOf(developerKeys); - this.creator = creator; - this.info = info; - this.tenantSecretStores = tenantSecretStores; - this.archiveAccess = archiveAccess; - this.invalidateUserSessionsBefore = invalidateUserSessionsBefore; - this.billingReference = billingReference; - this.planId = planId; - } - - private Cloud(CloudTenant tenant) { - this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), - tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess(), tenant.invalidateUserSessionsBefore(), - tenant.tenantRolesLastMaintained(), tenant.cloudAccounts(), tenant.billingReference(), tenant.planId()); - } - - @Override - public CloudTenant get() { - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, - archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, - cloudAccounts, billingReference, planId); - } - - public Cloud withDeveloperKey(PublicKey key, Principal principal) { - BiMap<PublicKey, SimplePrincipal> keys = HashBiMap.create(developerKeys); - SimplePrincipal simplePrincipal = new SimplePrincipal(principal.getName()); - if (keys.containsKey(key)) - throw new IllegalArgumentException("Key " + KeyUtils.toPem(key) + " is already owned by " + keys.get(key)); - if (keys.inverse().containsKey(simplePrincipal)) - throw new IllegalArgumentException(principal + " is already associated with key " + KeyUtils.toPem(keys.inverse().get(simplePrincipal))); - keys.put(key, simplePrincipal); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, - invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, - billingReference, planId); - } - - public Cloud withoutDeveloperKey(PublicKey key) { - BiMap<PublicKey, SimplePrincipal> keys = HashBiMap.create(developerKeys); - keys.remove(key); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, - invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference, - planId); - } - - public Cloud withInfo(TenantInfo newInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, - archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, - billingReference, planId); - } - - @Override - public LockedTenant with(LastLoginInfo lastLoginInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, - archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, - billingReference, planId); - } - - public Cloud withSecretStore(TenantSecretStore tenantSecretStore) { - ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); - secretStores.add(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, - invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, - billingReference, planId); - } - - public Cloud withoutSecretStore(TenantSecretStore tenantSecretStore) { - ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); - secretStores.remove(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, - invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, - billingReference, planId); - } - - public Cloud withArchiveAccess(ArchiveAccess archiveAccess) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, - invalidateUserSessionsBefore,tenantRolesLastMaintained, cloudAccounts, - billingReference, planId); - } - - public Cloud withInvalidateUserSessionsBefore(Instant invalidateUserSessionsBefore) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, - Optional.of(invalidateUserSessionsBefore), tenantRolesLastMaintained, cloudAccounts, - billingReference, planId); - } - - @Override - public LockedTenant with(Instant tenantRolesLastMaintained) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, - invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, - billingReference, planId); - } - - @Override - public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, - invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, - billingReference, planId); - } - - public Cloud with(BillingReference billingReference) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, - invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, - Optional.of(billingReference), planId); - } - - public Cloud withPlanId(PlanId planId) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, - invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, - billingReference, planId); - } - } - - - /** A locked DeletedTenant. */ - public static class Deleted extends LockedTenant { - - private final Instant deletedAt; - - private Deleted(DeletedTenant tenant) { - super(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Instant.EPOCH, List.of()); - this.deletedAt = tenant.deletedAt(); - } - - @Override - public DeletedTenant get() { - return new DeletedTenant(name, createdAt, deletedAt); - } - - @Override - public LockedTenant with(LastLoginInfo lastLoginInfo) { - return this; - } - - @Override - public LockedTenant with(Instant tenantRolesLastMaintained) { - return this; - } - - @Override - public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { - return this; - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java deleted file mode 100644 index 064a2a39860..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller; - -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; - -/** - * An exception which indicates that a requested resource does not exist. - * - * @author Tony Vaagenes - */ -public class NotExistsException extends IllegalArgumentException { - - public NotExistsException(String message) { - super(message); - } - - /** - * Example message: Tenant 'myId' does not exist. - * - * @param capitalizedType e.g. Tenant, Application - * @param id The id of the entity that didn't exist. - * - */ - public NotExistsException(String capitalizedType, String id) { - super(Text.format("%s '%s' does not exist", capitalizedType, id)); - } - - public NotExistsException(Identifier id) { - this(id.capitalizedType(), id.id()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/OsController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/OsController.java deleted file mode 100644 index bec7c40d2a9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/OsController.java +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.CloudName; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.versions.CertifiedOsVersion; -import com.yahoo.vespa.hosted.controller.versions.OsVersion; -import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; -import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; - -import java.time.Instant; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.TreeSet; -import java.util.function.BinaryOperator; -import java.util.function.Function; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * A singleton owned by {@link Controller} which contains the methods and state for controlling OS upgrades. - * - * @author mpolden - */ -public record OsController(Controller controller) { - - private static final Logger LOG = Logger.getLogger(OsController.class.getName()); - - public OsController { - Objects.requireNonNull(controller); - } - - /** Returns the target OS version for infrastructure in this system. The controller will drive infrastructure OS - * upgrades to this version */ - public Optional<OsVersionTarget> target(CloudName cloud) { - return targets().stream().filter(target -> target.osVersion().cloud().equals(cloud)).findFirst(); - } - - /** Returns all target OS versions in this system */ - public Set<OsVersionTarget> targets() { - return curator().readOsVersionTargets(); - } - - /** - * Set the target OS version for given cloud in this system. - * - * @param version The target OS version - * @param cloud The cloud to upgrade - * @param force Allow downgrades, and override pinned target (if any) - * @param pin Pin this version. This prevents automatic scheduling of upgrades until version is unpinned - */ - public void upgradeTo(Version version, CloudName cloud, boolean force, boolean pin) { - requireNonEmpty(version); - requireCloud(cloud); - Instant scheduledAt = controller.clock().instant(); - try (Mutex lock = curator().lockOsVersions()) { - Map<CloudName, OsVersionTarget> targets = curator().readOsVersionTargets().stream() - .collect(Collectors.toMap(t -> t.osVersion().cloud(), - Function.identity())); - - OsVersionTarget currentTarget = targets.get(cloud); - boolean downgrade = false; - if (currentTarget != null) { - boolean versionChange = !currentTarget.osVersion().version().equals(version); - downgrade = version.isBefore(currentTarget.osVersion().version()); - if (versionChange && currentTarget.pinned() && !force) { - throw new IllegalArgumentException("Cannot " + (downgrade ? "downgrade" : "upgrade") + " cloud " + - cloud.value() + "' to version " + version.toFullString() + - ": Current target is pinned. Add 'force' parameter to override"); - } - if (downgrade && !force) { - throw new IllegalArgumentException("Cannot downgrade cloud '" + cloud.value() + "' to version " + - version.toFullString() + ": Missing 'force' parameter"); - } - if (!versionChange && currentTarget.pinned() == pin) return; // No change - } - - OsVersionTarget newTarget = new OsVersionTarget(new OsVersion(version, cloud), scheduledAt, pin, downgrade); - targets.put(cloud, newTarget); - curator().writeOsVersionTargets(new TreeSet<>(targets.values())); - LOG.info("Triggered OS " + (downgrade ? "downgrade" : "upgrade") + " to " + version.toFullString() + - " in cloud " + cloud.value()); - } - } - - /** Clear the target OS version for given cloud in this system */ - public void cancelUpgrade(CloudName cloudName) { - try (Mutex lock = curator().lockOsVersions()) { - Map<CloudName, OsVersionTarget> targets = curator().readOsVersionTargets().stream() - .collect(Collectors.toMap(t -> t.osVersion().cloud(), - Function.identity())); - if (targets.remove(cloudName) == null) { - throw new IllegalArgumentException("Cloud '" + cloudName.value() + " has no OS upgrade target"); - } - curator().writeOsVersionTargets(new TreeSet<>(targets.values())); - } - } - - /** Returns the current OS version status */ - public OsVersionStatus status() { - return curator().readOsVersionStatus(); - } - - /** Replace the current OS version status with a new one */ - public void updateStatus(OsVersionStatus newStatus) { - try (Mutex lock = curator().lockOsVersionStatus()) { - OsVersionStatus currentStatus = curator().readOsVersionStatus(); - for (CloudName cloud : controller.clouds()) { - Set<Version> newVersions = newStatus.versionsIn(cloud); - if (currentStatus.versionsIn(cloud).size() > 1 && newVersions.size() == 1) { - LOG.info("All nodes in " + cloud + " cloud upgraded to OS version " + - newVersions.iterator().next().toFullString()); - } - } - curator().writeOsVersionStatus(newStatus); - } - } - - /** Certify an OS version as compatible with given Vespa version */ - public CertifiedOsVersion certify(Version version, CloudName cloud, Version vespaVersion) { - requireNonEmpty(version); - requireNonEmpty(vespaVersion); - requireCloud(cloud); - try (Mutex lock = curator().lockCertifiedOsVersions()) { - OsVersion osVersion = new OsVersion(version, cloud); - Set<CertifiedOsVersion> certifiedVersions = readCertified(); - Optional<CertifiedOsVersion> matching = certifiedVersions.stream() - .filter(cv -> cv.osVersion().equals(osVersion)) - .findFirst(); - if (matching.isPresent()) { - return matching.get(); - } - certifiedVersions = new HashSet<>(certifiedVersions); - certifiedVersions.add(new CertifiedOsVersion(osVersion, vespaVersion)); - curator().writeCertifiedOsVersions(certifiedVersions); - return new CertifiedOsVersion(osVersion, vespaVersion); - } - } - - /** Revoke certification of an OS version */ - public void uncertify(Version version, CloudName cloud) { - try (Mutex lock = curator().lockCertifiedOsVersions()) { - OsVersion osVersion = new OsVersion(version, cloud); - Set<CertifiedOsVersion> certifiedVersions = readCertified(); - Optional<CertifiedOsVersion> existing = certifiedVersions.stream() - .filter(cv -> cv.osVersion().equals(osVersion)) - .findFirst(); - if (existing.isEmpty()) { - throw new IllegalArgumentException(osVersion + " is not certified"); - } - certifiedVersions = new HashSet<>(certifiedVersions); - certifiedVersions.remove(existing.get()); - curator().writeCertifiedOsVersions(certifiedVersions); - } - } - - /** Remove certifications for non-existent OS versions */ - public void removeStaleCertifications(OsVersionStatus currentStatus) { - try (Mutex lock = curator().lockCertifiedOsVersions()) { - Map<CloudName, Version> oldestVersionByCloud = currentStatus.versions().keySet().stream() - .filter(v -> !v.version().isEmpty()) - .collect(Collectors.toMap(OsVersion::cloud, - OsVersion::version, - BinaryOperator.minBy(Comparator.naturalOrder()))); - if (oldestVersionByCloud.isEmpty()) return; - - Set<CertifiedOsVersion> certifiedVersions = new HashSet<>(readCertified()); - boolean modified = certifiedVersions.removeIf(certifiedVersion -> { - Version oldestVersion = oldestVersionByCloud.get(certifiedVersion.osVersion().cloud()); - return oldestVersion == null || certifiedVersion.osVersion().version().isBefore(oldestVersion); - }); - if (modified) { - curator().writeCertifiedOsVersions(certifiedVersions); - } - } - } - - /** Returns whether given OS version is certified as compatible with the current system version */ - public boolean certified(OsVersion osVersion) { - if (controller.system().isCd()) return true; // Always certified (this is the system doing the certifying) - - Version systemVersion = controller.readSystemVersion(); - return readCertified().stream() - .anyMatch(certifiedOsVersion -> certifiedOsVersion.osVersion().equals(osVersion) && - // A later system version is fine, as we don't guarantee that - // an OS upgrade will always coincide with a Vespa release - !certifiedOsVersion.vespaVersion().isAfter(systemVersion)); - } - - /** Returns all certified versions */ - public Set<CertifiedOsVersion> readCertified() { - return controller.curator().readCertifiedOsVersions(); - } - - private void requireCloud(CloudName cloud) { - if (!controller.clouds().contains(cloud)) { - throw new IllegalArgumentException("Cloud '" + cloud + "' does not exist in this system"); - } - } - - private void requireNonEmpty(Version version) { - if (version.isEmpty()) { - throw new IllegalArgumentException("Invalid version '" + version.toFullString() + "'"); - } - } - - private CuratorDb curator() { - return controller.curator(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java deleted file mode 100644 index 5dec1449507..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ /dev/null @@ -1,624 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller; - -import com.google.common.hash.HashCode; -import com.google.common.hash.Hashing; -import com.google.common.io.BaseEncoding; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.AuthMethod; -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.StringFlag; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; -import com.yahoo.vespa.hosted.controller.application.Endpoint.Scope; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.EndpointList; -import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; -import com.yahoo.vespa.hosted.controller.routing.GeneratedEndpointList; -import com.yahoo.vespa.hosted.controller.routing.PreparedEndpoints; -import com.yahoo.vespa.hosted.controller.routing.RoutingId; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyList; -import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; -import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext.ExclusiveDeploymentRoutingContext; -import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext.SharedDeploymentRoutingContext; -import com.yahoo.vespa.hosted.controller.routing.context.ExclusiveZoneRoutingContext; -import com.yahoo.vespa.hosted.controller.routing.context.RoutingContext; -import com.yahoo.vespa.hosted.controller.routing.context.SharedZoneRoutingContext; -import com.yahoo.vespa.hosted.controller.routing.rotation.Rotation; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationRepository; -import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.TreeMap; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toMap; - -/** - * The routing controller is owned by {@link Controller} and encapsulates state and methods for inspecting and - * manipulating deployment endpoints in a hosted Vespa system. - * - * The one-stop shop for all your routing needs! - * - * @author mpolden - */ -public class RoutingController { - - private static final Logger LOG = Logger.getLogger(RoutingController.class.getName()); - - private final Controller controller; - private final RoutingPolicies routingPolicies; - private final RotationRepository rotationRepository; - private final StringFlag endpointConfig; - - public RoutingController(Controller controller, RotationsConfig rotationsConfig) { - this.controller = Objects.requireNonNull(controller, "controller must be non-null"); - this.routingPolicies = new RoutingPolicies(controller); - this.rotationRepository = new RotationRepository(Objects.requireNonNull(rotationsConfig, "rotationsConfig must be non-null"), - controller.applications(), - controller.curator()); - this.endpointConfig = Flags.ENDPOINT_CONFIG.bindTo(controller.flagSource()); - } - - /** Create a routing context for given deployment */ - public DeploymentRoutingContext of(DeploymentId deployment) { - if (usesSharedRouting(deployment.zoneId())) { - return new SharedDeploymentRoutingContext(deployment, - this, - controller.serviceRegistry().configServer(), - controller.clock()); - } - return new ExclusiveDeploymentRoutingContext(deployment, this); - } - - /** Create a routing context for given zone */ - public RoutingContext of(ZoneId zone) { - if (usesSharedRouting(zone)) { - return new SharedZoneRoutingContext(zone, controller.serviceRegistry().configServer()); - } - return new ExclusiveZoneRoutingContext(zone, routingPolicies); - } - - public RoutingPolicies policies() { - return routingPolicies; - } - - public RotationRepository rotations() { - return rotationRepository; - } - - /** Returns the endpoint config to use for given instance */ - public EndpointConfig endpointConfig(ApplicationId instance) { - String flagValue = endpointConfig.with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) - .with(FetchVector.Dimension.APPLICATION, TenantAndApplicationId.from(instance).serialized()) - .with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) - .value(); - return switch (flagValue) { - case "legacy" -> EndpointConfig.legacy; - case "combined" -> EndpointConfig.combined; - case "generated" -> EndpointConfig.generated; - default -> throw new IllegalArgumentException("Invalid endpoint-config flag value: '" + flagValue + "', must be " + - "'legacy', 'combined' or 'generated'"); - }; - } - - /** Prepares and returns the endpoints relevant for given deployment */ - public PreparedEndpoints prepare(DeploymentId deployment, BasicServicesXml services, EndpointCertificate certificate, LockedApplication application) { - EndpointList endpoints = EndpointList.EMPTY; - DeploymentSpec spec = application.get().deploymentSpec(); - - // Assign rotations to application - for (var instanceSpec : spec.instances()) { - if (instanceSpec.concerns(Environment.prod)) { - application = controller.routing().assignRotations(application, instanceSpec.name()); - } - } - - // Add zone-scoped endpoints - Map<EndpointId, List<GeneratedEndpoint>> generatedForDeclaredEndpoints = new HashMap<>(); - Set<ClusterSpec.Id> clustersWithToken = new HashSet<>(); - EndpointConfig config = endpointConfig(deployment.applicationId()); - RoutingPolicyList applicationPolicies = policies().read(TenantAndApplicationId.from(deployment.applicationId())); - RoutingPolicyList deploymentPolicies = applicationPolicies.deployment(deployment); - for (var container : services.containers()) { - ClusterSpec.Id clusterId = ClusterSpec.Id.from(container.id()); - boolean tokenSupported = container.authMethods().contains(BasicServicesXml.Container.AuthMethod.token); - if (tokenSupported) { - clustersWithToken.add(clusterId); - } - Optional<RoutingPolicy> clusterPolicy = deploymentPolicies.cluster(clusterId).first(); - List<GeneratedEndpoint> generatedForCluster = clusterPolicy.map(policy -> policy.generatedEndpoints().cluster().asList()) - .orElseGet(List::of); - // Generate endpoint for each auth method, if not present - generatedForCluster = generateEndpoints(AuthMethod.mtls, certificate, Optional.empty(), generatedForCluster); - if (tokenSupported) { - generatedForCluster = generateEndpoints(AuthMethod.token, certificate, Optional.empty(), generatedForCluster); - } - GeneratedEndpointList generatedEndpoints = config.supportsGenerated() ? GeneratedEndpointList.copyOf(generatedForCluster) : GeneratedEndpointList.EMPTY; - endpoints = endpoints.and(endpointsOf(deployment, clusterId, generatedEndpoints).scope(Scope.zone)); - } - - // Add global- and application-scoped endpoints - for (var container : services.containers()) { - ClusterSpec.Id clusterId = ClusterSpec.Id.from(container.id()); - applicationPolicies.cluster(clusterId).asList().stream() - .flatMap(policy -> policy.generatedEndpoints().declared().asList().stream()) - .forEach(ge -> { - List<GeneratedEndpoint> generated = generatedForDeclaredEndpoints.computeIfAbsent(ge.endpoint().get(), (k) -> new ArrayList<>()); - if (!generated.contains(ge)) { - generated.add(ge); - } - }); - } - // Generate endpoints if declared endpoint does not have any - Stream.concat(spec.endpoints().stream(), spec.instances().stream().flatMap(i -> i.endpoints().stream())) - .forEach(endpoint -> { - EndpointId endpointId = EndpointId.of(endpoint.endpointId()); - generatedForDeclaredEndpoints.compute(endpointId, (k, old) -> { - if (old == null) { - old = List.of(); - } - List<GeneratedEndpoint> generatedEndpoints = generateEndpoints(AuthMethod.mtls, certificate, Optional.of(endpointId), old); - boolean tokenSupported = clustersWithToken.contains(ClusterSpec.Id.from(endpoint.containerId())); - if (tokenSupported){ - generatedEndpoints = generateEndpoints(AuthMethod.token, certificate, Optional.of(endpointId), generatedEndpoints); - } - return generatedEndpoints; - }); - }); - Map<EndpointId, GeneratedEndpointList> generatedEndpoints = config.supportsGenerated() - ? generatedForDeclaredEndpoints.entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, kv -> GeneratedEndpointList.copyOf(kv.getValue()))) - : Map.of(); - endpoints = endpoints.and(declaredEndpointsOf(application.get().id(), spec, generatedEndpoints).targets(deployment)); - PreparedEndpoints prepared = new PreparedEndpoints(deployment, - endpoints, - application.get().require(deployment.applicationId().instance()).rotations(), - certificate); - - // Register rotation-backed endpoints in DNS - registerRotationEndpointsInDns(prepared); - - LOG.log(Level.FINE, () -> "Prepared endpoints: " + prepared); - - return prepared; - } - - // -------------- Implicit endpoints (scopes 'zone' and 'weighted') -------------- - - /** Returns the zone- and region-scoped endpoints of given deployment */ - public EndpointList endpointsOf(DeploymentId deployment, ClusterSpec.Id cluster, GeneratedEndpointList generatedEndpoints) { - requireGeneratedEndpoints(generatedEndpoints, false); - boolean generatedEndpointsAvailable = !generatedEndpoints.isEmpty(); - boolean tokenSupported = !generatedEndpoints.authMethod(AuthMethod.token).isEmpty(); - boolean isProduction = deployment.zoneId().environment().isProduction(); - RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(deployment.zoneId()); - List<Endpoint> endpoints = new ArrayList<>(); - Endpoint.EndpointBuilder zoneEndpoint = Endpoint.of(deployment.applicationId()) - .routingMethod(routingMethod) - .on(Port.fromRoutingMethod(routingMethod)) - .legacy(generatedEndpointsAvailable) - .target(cluster, deployment); - endpoints.add(zoneEndpoint.in(controller.system())); - ZoneApi zone = controller.zoneRegistry().zones().all().get(deployment.zoneId()).get(); - Endpoint.EndpointBuilder regionEndpoint = Endpoint.of(deployment.applicationId()) - .routingMethod(routingMethod) - .on(Port.fromRoutingMethod(routingMethod)) - .legacy(generatedEndpointsAvailable) - .targetRegion(cluster, - zone.getCloudNativeRegionName(), - zone.getCloudName()); - // Region endpoints are only used by global- and application-endpoints and are thus only needed in - // production environments - if (isProduction) { - endpoints.add(regionEndpoint.in(controller.system())); - } - for (var generatedEndpoint : generatedEndpoints) { - boolean include = switch (generatedEndpoint.authMethod()) { - case token -> tokenSupported; - case mtls -> true; - case none -> false; - }; - if (include) { - endpoints.add(zoneEndpoint.generatedFrom(generatedEndpoint) - .legacy(false) - .authMethod(generatedEndpoint.authMethod()) - .in(controller.system())); - // Only a single region endpoint is needed, not one per auth method - if (isProduction && generatedEndpoint.authMethod() == AuthMethod.mtls) { - GeneratedEndpoint weightedGeneratedEndpoint = generatedEndpoint.withClusterPart(weightedClusterPart(cluster, deployment)); - endpoints.add(regionEndpoint.generatedFrom(weightedGeneratedEndpoint) - .legacy(false) - .authMethod(AuthMethod.none) - .in(controller.system())); - } - } - } - return filterEndpoints(deployment.applicationId(), EndpointList.copyOf(endpoints)); - } - - /** Read routing policies and return zone- and region-scoped endpoints for given deployment */ - public EndpointList readEndpointsOf(DeploymentId deployment) { - Set<Endpoint> endpoints = new LinkedHashSet<>(); - for (var policy : routingPolicies.read(deployment)) { - endpoints.addAll(endpointsOf(deployment, policy.id().cluster(), policy.generatedEndpoints().cluster()).asList()); - } - return EndpointList.copyOf(endpoints); - } - - // -------------- Declared endpoints (scopes 'global' and 'application') -------------- - - /** Returns global endpoints pointing to given deployments */ - public EndpointList declaredEndpointsOf(RoutingId routingId, ClusterSpec.Id cluster, List<DeploymentId> deployments, GeneratedEndpointList generatedEndpoints) { - requireGeneratedEndpoints(generatedEndpoints, true); - var endpoints = new ArrayList<Endpoint>(); - var directMethods = 0; - var availableRoutingMethods = routingMethodsOfAll(deployments); - boolean generatedEndpointsAvailable = !generatedEndpoints.isEmpty(); - for (var method : availableRoutingMethods) { - if (method.isDirect() && ++directMethods > 1) { - throw new IllegalArgumentException("Invalid routing methods for " + routingId + ": Exceeded maximum " + - "direct methods"); - } - Endpoint.EndpointBuilder builder = Endpoint.of(routingId.instance()) - .target(routingId.endpointId(), cluster, deployments) - .on(Port.fromRoutingMethod(method)) - .legacy(generatedEndpointsAvailable) - .routingMethod(method); - endpoints.add(builder.in(controller.system())); - for (var ge : generatedEndpoints) { - endpoints.add(builder.generatedFrom(ge).legacy(false).authMethod(ge.authMethod()).in(controller.system())); - } - } - return filterEndpoints(routingId.instance(), EndpointList.copyOf(endpoints)); - } - - /** Returns application endpoints pointing to given deployments */ - public EndpointList declaredEndpointsOf(TenantAndApplicationId application, EndpointId endpoint, ClusterSpec.Id cluster, - Map<DeploymentId, Integer> deployments, GeneratedEndpointList generatedEndpoints) { - requireGeneratedEndpoints(generatedEndpoints, true); - ZoneId zone = deployments.keySet().iterator().next().zoneId(); // Where multiple zones are possible, they all have the same routing method. - RoutingMethod routingMethod = usesSharedRouting(zone) ? RoutingMethod.sharedLayer4 : RoutingMethod.exclusive; - boolean generatedEndpointsAvailable = !generatedEndpoints.isEmpty(); - Endpoint.EndpointBuilder builder = Endpoint.of(application) - .targetApplication(endpoint, - cluster, - deployments) - .routingMethod(routingMethod) - .legacy(generatedEndpointsAvailable) - .on(Port.fromRoutingMethod(routingMethod)); - List<Endpoint> endpoints = new ArrayList<>(); - endpoints.add(builder.in(controller.system())); - for (var ge : generatedEndpoints) { - endpoints.add(builder.generatedFrom(ge).legacy(false).authMethod(ge.authMethod()).in(controller.system())); - } - return EndpointList.copyOf(endpoints); - } - - /** Read application and return endpoints for all instances in application */ - public EndpointList readDeclaredEndpointsOf(Application application) { - return declaredEndpointsOf(application.id(), application.deploymentSpec(), readDeclaredGeneratedEndpoints(application.id())); - } - - /** Read application and return declared endpoints for given instance */ - public EndpointList readDeclaredEndpointsOf(ApplicationId instance) { - if (SystemApplication.matching(instance).isPresent()) return EndpointList.EMPTY; - Application application = controller.applications().requireApplication(TenantAndApplicationId.from(instance)); - return readDeclaredEndpointsOf(application).instance(instance.instance()); - } - - private EndpointList declaredEndpointsOf(TenantAndApplicationId application, DeploymentSpec deploymentSpec, Map<EndpointId, GeneratedEndpointList> generatedEndpoints) { - Set<Endpoint> endpoints = new LinkedHashSet<>(); - // Global endpoints - for (var spec : deploymentSpec.instances()) { - ApplicationId instance = application.instance(spec.name()); - for (var declaredEndpoint : spec.endpoints()) { - RoutingId routingId = RoutingId.of(instance, EndpointId.of(declaredEndpoint.endpointId())); - List<DeploymentId> deployments = declaredEndpoint.regions().stream() - .map(region -> new DeploymentId(instance, - ZoneId.from(Environment.prod, region))) - .toList(); - ClusterSpec.Id cluster = ClusterSpec.Id.from(declaredEndpoint.containerId()); - GeneratedEndpointList generatedForId = generatedEndpoints.getOrDefault(routingId.endpointId(), GeneratedEndpointList.EMPTY); - endpoints.addAll(declaredEndpointsOf(routingId, cluster, deployments, generatedForId).asList()); - } - } - // Application endpoints - for (var declaredEndpoint : deploymentSpec.endpoints()) { - Map<DeploymentId, Integer> deployments = declaredEndpoint.targets().stream() - .collect(toMap(t -> new DeploymentId(application.instance(t.instance()), - ZoneId.from(Environment.prod, t.region())), - t -> t.weight())); - ClusterSpec.Id cluster = ClusterSpec.Id.from(declaredEndpoint.containerId()); - EndpointId endpointId = EndpointId.of(declaredEndpoint.endpointId()); - GeneratedEndpointList generatedForId = generatedEndpoints.getOrDefault(endpointId, GeneratedEndpointList.EMPTY); - endpoints.addAll(declaredEndpointsOf(application, endpointId, cluster, deployments, generatedForId).asList()); - } - return EndpointList.copyOf(endpoints); - } - - // -------------- Other gunk related to endpoints and routing -------------- - - /** Read endpoints for use in deployment steps, for given deployments, grouped by their zone */ - public Map<ZoneId, List<Endpoint>> readStepRunnerEndpointsOf(Collection<DeploymentId> deployments) { - TreeMap<ZoneId, List<Endpoint>> endpoints = new TreeMap<>(Comparator.comparing(ZoneId::value)); - for (var deployment : deployments) { - EndpointList zoneEndpoints = readEndpointsOf(deployment).scope(Endpoint.Scope.zone) - .authMethod(AuthMethod.mtls) - .not().legacy(); - EndpointList directEndpoints = zoneEndpoints.direct(); - if (!directEndpoints.isEmpty()) { - zoneEndpoints = directEndpoints; // Use only direct endpoints if we have any - } - EndpointList generatedEndpoints = zoneEndpoints.generated(); - if (!generatedEndpoints.isEmpty()) { - zoneEndpoints = generatedEndpoints; // Use generated endpoints if we have any - } - if ( ! zoneEndpoints.isEmpty()) { - endpoints.put(deployment.zoneId(), zoneEndpoints.asList()); - } - } - return Collections.unmodifiableSortedMap(endpoints); - } - - /** Returns certificate DNS names (CN and SAN values) for given deployment */ - public List<String> certificateDnsNames(DeploymentId deployment, DeploymentSpec deploymentSpec, String generatedId, boolean legacy) { - List<String> endpointDnsNames = new ArrayList<>(); - if (legacy) { - endpointDnsNames.addAll(legacyCertificateDnsNames(deployment, deploymentSpec)); - } - for (Scope scope : List.of(Scope.zone, Scope.global, Scope.application)) { - endpointDnsNames.add(Endpoint.of(deployment.applicationId()) - .wildcardGenerated(generatedId, scope) - .routingMethod(RoutingMethod.exclusive) - .on(Port.tls()) - .certificateName() - .in(controller.system()) - .dnsName()); - } - return Collections.unmodifiableList(endpointDnsNames); - } - - private List<String> legacyCertificateDnsNames(DeploymentId deployment, DeploymentSpec deploymentSpec) { - List<String> endpointDnsNames = new ArrayList<>(); - - // We add first an endpoint name based on a hash of the application ID, - // as the certificate provider requires the first CN to be < 64 characters long. - endpointDnsNames.add(commonNameHashOf(deployment.applicationId(), controller.system())); - - List<Endpoint.EndpointBuilder> builders = new ArrayList<>(); - if (deployment.zoneId().environment().isProduction()) { - // Add default and wildcard names for global endpoints - builders.add(Endpoint.of(deployment.applicationId()).target(EndpointId.defaultId())); - builders.add(Endpoint.of(deployment.applicationId()).wildcard()); - - // Add default and wildcard names for each region targeted by application endpoints - List<DeploymentId> deploymentTargets = deploymentSpec.endpoints().stream() - .map(com.yahoo.config.application.api.Endpoint::targets) - .flatMap(Collection::stream) - .map(com.yahoo.config.application.api.Endpoint.Target::region) - .distinct() - .map(region -> new DeploymentId(deployment.applicationId(), ZoneId.from(Environment.prod, region))) - .toList(); - TenantAndApplicationId application = TenantAndApplicationId.from(deployment.applicationId()); - for (var targetDeployment : deploymentTargets) { - builders.add(Endpoint.of(application).targetApplication(EndpointId.defaultId(), targetDeployment)); - builders.add(Endpoint.of(application).wildcardApplication(targetDeployment)); - } - } - - // Add default and wildcard names for zone endpoints - builders.add(Endpoint.of(deployment.applicationId()).target(ClusterSpec.Id.from("default"), deployment)); - builders.add(Endpoint.of(deployment.applicationId()).wildcard(deployment)); - - // Build all certificate names - for (var builder : builders) { - Endpoint endpoint = builder.certificateName() - .routingMethod(RoutingMethod.exclusive) - .on(Port.tls()) - .in(controller.system()); - endpointDnsNames.add(endpoint.dnsName()); - } - return Collections.unmodifiableList(endpointDnsNames); - } - - /** Remove endpoints in DNS for all rotations assigned to given instance */ - public void removeRotationEndpointsFromDns(Application application, InstanceName instanceName) { - Set<Endpoint> endpointsToRemove = new LinkedHashSet<>(); - Instance instance = application.require(instanceName); - // Compute endpoints from rotations. When removing DNS records for rotation-based endpoints we cannot use the - // deployment spec, because submitting an empty deployment spec is the first step of removing an application - for (var rotation : instance.rotations()) { - var deployments = rotation.regions().stream() - .map(region -> new DeploymentId(instance.id(), ZoneId.from(Environment.prod, region))) - .toList(); - GeneratedEndpointList generatedForId = readDeclaredGeneratedEndpoints(application.id()).getOrDefault(rotation.endpointId(), GeneratedEndpointList.EMPTY); - endpointsToRemove.addAll(declaredEndpointsOf(RoutingId.of(instance.id(), rotation.endpointId()), - rotation.clusterId(), deployments, - generatedForId) - .asList()); - } - endpointsToRemove.forEach(endpoint -> controller.nameServiceForwarder() - .removeRecords(Record.Type.CNAME, - RecordName.from(endpoint.dnsName()), - Priority.normal, - Optional.of(application.id()))); - } - - private EndpointList filterEndpoints(ApplicationId instance, EndpointList endpoints) { - return endpointConfig(instance) == EndpointConfig.generated ? endpoints.generated() : endpoints; - } - - private void registerRotationEndpointsInDns(PreparedEndpoints prepared) { - TenantAndApplicationId owner = TenantAndApplicationId.from(prepared.deployment().applicationId()); - EndpointList globalEndpoints = prepared.endpoints().scope(Scope.global); - for (var assignedRotation : prepared.rotations()) { - EndpointList rotationEndpoints = globalEndpoints.named(assignedRotation.endpointId(), Scope.global) - .requiresRotation(); - // Skip rotations which do not apply to this zone - if (!assignedRotation.regions().contains(prepared.deployment().zoneId().region())) { - continue; - } - // Register names in DNS - Rotation rotation = rotationRepository.requireRotation(assignedRotation.rotationId()); - for (var endpoint : rotationEndpoints) { - controller.nameServiceForwarder().createRecord( - new Record(Record.Type.CNAME, RecordName.from(endpoint.dnsName()), RecordData.fqdn(rotation.name())), - Priority.normal, - Optional.of(owner) - ); - } - } - for (var endpoint : prepared.endpoints().scope(Scope.application).shared()) { // DNS for non-shared application endpoints is handled by RoutingPolicies - Set<ZoneId> targetZones = endpoint.targets().stream() - .map(t -> t.deployment().zoneId()) - .collect(Collectors.toUnmodifiableSet()); - if (targetZones.size() != 1) throw new IllegalArgumentException("Endpoint '" + endpoint.name() + - "' must target a single zone, got " + - targetZones); - ZoneId targetZone = targetZones.iterator().next(); - String vipHostname = controller.zoneRegistry().getVipHostname(targetZone) - .orElseThrow(() -> new IllegalArgumentException("No VIP configured for zone " + targetZone)); - controller.nameServiceForwarder().createRecord( - new Record(Record.Type.CNAME, RecordName.from(endpoint.dnsName()), RecordData.fqdn(vipHostname)), - Priority.normal, - Optional.of(owner)); - } - } - - /** Returns generated endpoints. A new endpoint is generated if no matching endpoint already exists */ - private List<GeneratedEndpoint> generateEndpoints(AuthMethod authMethod, EndpointCertificate certificate, - Optional<EndpointId> declaredEndpoint, - List<GeneratedEndpoint> current) { - if (current.stream().anyMatch(e -> e.authMethod() == authMethod && e.endpoint().equals(declaredEndpoint))) { - return current; - } - Optional<String> applicationPart = certificate.generatedId(); - if (applicationPart.isPresent()) { - current = new ArrayList<>(current); - current.add(new GeneratedEndpoint(GeneratedEndpoint.createPart(controller.random(true)), - applicationPart.get(), - authMethod, - declaredEndpoint)); - } - return current; - } - - /** Generate the cluster part of a {@link GeneratedEndpoint} for use in a {@link Endpoint.Scope#weighted} endpoint */ - private String weightedClusterPart(ClusterSpec.Id cluster, DeploymentId deployment) { - // This ID must be common for a given cluster in all deployments within the same cloud-native region - String cloudNativeRegion = controller.zoneRegistry().zones().all().get(deployment.zoneId()).get().getCloudNativeRegionName(); - HashCode hash = Hashing.sha256().newHasher() - .putString(cluster.value(), StandardCharsets.UTF_8) - .putString(":", StandardCharsets.UTF_8) - .putString(cloudNativeRegion, StandardCharsets.UTF_8) - .putString(":", StandardCharsets.UTF_8) - .putString(deployment.applicationId().serializedForm(), StandardCharsets.UTF_8) - .hash(); - String alphabet = "abcdef"; - char letter = alphabet.charAt(Math.abs(hash.asInt()) % alphabet.length()); - return letter + hash.toString().substring(0, 7); - } - - /** Returns existing generated endpoints, grouped by their {@link Scope#multiDeployment()} endpoint */ - private Map<EndpointId, GeneratedEndpointList> readDeclaredGeneratedEndpoints(TenantAndApplicationId application) { - Map<EndpointId, GeneratedEndpointList> endpoints = new HashMap<>(); - for (var policy : policies().read(application)) { - Map<EndpointId, GeneratedEndpointList> generatedForDeclared = policy.generatedEndpoints() - .not().cluster() - .groupingBy(ge -> ge.endpoint().get()); - generatedForDeclared.forEach(endpoints::putIfAbsent); - } - return endpoints; - } - - /** - * Assigns one or more global rotations to given application, if eligible. The given application is implicitly - * stored, ensuring that the assigned rotation(s) are persisted when this returns. - */ - private LockedApplication assignRotations(LockedApplication application, InstanceName instanceName) { - try (RotationLock rotationLock = rotationRepository.lock()) { - var rotations = rotationRepository.getOrAssignRotations(application.get().deploymentSpec(), - application.get().require(instanceName), - rotationLock); - application = application.with(instanceName, instance -> instance.with(rotations)); - controller.applications().store(application); // store assigned rotation even if deployment fails - } - return application; - } - - private boolean usesSharedRouting(ZoneId zone) { - return controller.zoneRegistry().routingMethod(zone).isShared(); - } - - /** Returns the routing methods that are available across all given deployments */ - private List<RoutingMethod> routingMethodsOfAll(Collection<DeploymentId> deployments) { - Map<RoutingMethod, Set<DeploymentId>> deploymentsByMethod = new HashMap<>(); - for (var deployment : deployments) { - RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(deployment.zoneId()); - deploymentsByMethod.computeIfAbsent(routingMethod, k -> new LinkedHashSet<>()) - .add(deployment); - } - List<RoutingMethod> routingMethods = new ArrayList<>(); - deploymentsByMethod.forEach((method, supportedDeployments) -> { - if (supportedDeployments.containsAll(deployments)) { - routingMethods.add(method); - } - }); - return Collections.unmodifiableList(routingMethods); - } - - private static void requireGeneratedEndpoints(GeneratedEndpointList generatedEndpoints, boolean declared) { - if (generatedEndpoints.asList().stream().anyMatch(ge -> ge.declared() != declared)) { - throw new IllegalStateException("All generated endpoints require declared=" + declared + - ", got " + generatedEndpoints); - } - } - - /** Create a common name based on a hash of given application. This must be less than 64 characters long. */ - private static String commonNameHashOf(ApplicationId application, SystemName system) { - @SuppressWarnings("deprecation") // for Hashing.sha1() - HashCode sha1 = Hashing.sha1().hashString(application.serializedForm(), StandardCharsets.UTF_8); - String base32 = BaseEncoding.base32().omitPadding().lowerCase().encode(sha1.asBytes()); - return 'v' + base32 + Endpoint.internalDnsSuffix(system); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java deleted file mode 100644 index 55269e2612f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.text.Text; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.concurrent.Once; -import com.yahoo.vespa.hosted.controller.notification.NotificationSource; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.security.AccessControl; -import com.yahoo.vespa.hosted.controller.security.Credentials; -import com.yahoo.vespa.hosted.controller.security.TenantSpec; -import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; -import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; -import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.time.Duration; -import java.time.Instant; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Consumer; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * A singleton owned by the {@link Controller} which contains the methods and state for controlling tenants. - * - * @author bratseth - * @author mpolden - */ -public class TenantController { - - private static final Logger log = Logger.getLogger(TenantController.class.getName()); - - private final Controller controller; - private final CuratorDb curator; - private final AccessControl accessControl; - - public TenantController(Controller controller, CuratorDb curator, AccessControl accessControl) { - this.controller = Objects.requireNonNull(controller, "controller must be non-null"); - this.curator = Objects.requireNonNull(curator, "curator must be non-null"); - this.accessControl = Objects.requireNonNull(accessControl, "accessControl must be non-null"); - - // Update serialization format of all tenants - Once.after(Duration.ofMinutes(1), () -> { - Instant start = controller.clock().instant(); - int count = 0; - for (TenantName name : curator.readTenantNames()) { - lockIfPresent(name, LockedTenant.class, this::store); - count++; - } - log.log(Level.INFO, Text.format("Wrote %d tenants in %s", count, - Duration.between(start, controller.clock().instant()))); - }); - } - - /** Returns a list of all known, non-deleted tenants sorted by name */ - public List<Tenant> asList() { - return asList(false); - } - - /** Returns a list of all known tenants sorted by name */ - public List<Tenant> asList(boolean includeDeleted) { - return curator.readTenants().stream() - .filter(tenant -> tenant.type() != Tenant.Type.deleted || includeDeleted) - .sorted(Comparator.comparing(Tenant::name)) - .toList(); - } - - /** Locks a tenant for modification and applies the given action. */ - public <T extends LockedTenant> void lockIfPresent(TenantName name, Class<T> token, Consumer<T> action) { - try (Mutex lock = lock(name)) { - get(name).map(tenant -> LockedTenant.of(tenant, lock)) - .map(token::cast) - .ifPresent(action); - } - } - - /** Lock a tenant for modification and apply action. Throws if the tenant does not exist */ - public <T extends LockedTenant> void lockOrThrow(TenantName name, Class<T> token, Consumer<T> action) { - try (Mutex lock = lock(name)) { - action.accept(token.cast(LockedTenant.of(require(name), lock))); - } - } - - /** Returns the tenant with the given name, or throws. */ - public Tenant require(TenantName name) { - return get(name).orElseThrow(() -> new IllegalArgumentException("No such tenant '" + name + "'.")); - } - - /** Returns the tenant with the given name, and ensures the type */ - public <T extends Tenant> T require(TenantName name, Class<T> tenantType) { - return get(name) - .map(t -> { - try { return tenantType.cast(t); } catch (ClassCastException e) { - throw new IllegalArgumentException("Tenant '" + name + "' was of type '" + t.getClass().getSimpleName() + "' and not '" + tenantType.getSimpleName() + "'"); - } - }) - .orElseThrow(() -> new IllegalArgumentException("No such tenant '" + name + "'.")); - } - - /** Replace and store any previous version of given tenant */ - public void store(LockedTenant tenant) { - curator.writeTenant(tenant.get()); - } - - /** Create a tenant, provided the given credentials are valid. */ - public void create(TenantSpec tenantSpec, Credentials credentials) { - try (Mutex lock = lock(tenantSpec.tenant())) { - TenantId.validate(tenantSpec.tenant().value()); - requireNonExistent(tenantSpec.tenant()); - curator.writeTenant(accessControl.createTenant(tenantSpec, controller.clock().instant(), credentials, asList())); - - // We should create tenant roles here but it takes too long - assuming the TenantRoleMaintainer will do it Soon™ - } - } - - /** Find tenant by name */ - public Optional<Tenant> get(TenantName name) { - return get(name, false); - } - - public Optional<Tenant> get(TenantName name, boolean includeDeleted) { - return curator.readTenant(name) - .filter(tenant -> tenant.type() != Tenant.Type.deleted || includeDeleted); - } - - /** Find tenant by name */ - public Optional<Tenant> get(String name) { - return get(TenantName.from(name)); - } - - /** Updates the tenant contained in the given tenant spec with new data. */ - public void update(TenantSpec tenantSpec, Credentials credentials) { - try (Mutex lock = lock(tenantSpec.tenant())) { - curator.writeTenant(accessControl.updateTenant(tenantSpec, credentials, asList(), - controller.applications().asList(tenantSpec.tenant()))); - } - } - - /** - * Update last login times for the given tenant at the given user levers with the given instant, but only if the - * new instant is later - */ - public void updateLastLogin(TenantName tenantName, List<LastLoginInfo.UserLevel> userLevels, Instant loggedInAt) { - try (Mutex lock = lock(tenantName)) { - Tenant tenant = require(tenantName); - LastLoginInfo loginInfo = tenant.lastLoginInfo(); - for (LastLoginInfo.UserLevel userLevel : userLevels) - loginInfo = loginInfo.withLastLoginIfLater(userLevel, loggedInAt); - - if (tenant.lastLoginInfo().equals(loginInfo)) return; // no change - curator.writeTenant(LockedTenant.of(tenant, lock).with(loginInfo).get()); - } - } - - public void updateLastTenantRolesMaintained(TenantName tenantName, Instant lastMaintained) { - try (Mutex lock = lock(tenantName)) { - var tenant = require(tenantName); - curator.writeTenant(LockedTenant.of(tenant, lock).with(lastMaintained).get()); - } - } - - public void updateCloudAccounts(TenantName tenantName, List<CloudAccountInfo> cloudAccounts) { - try (Mutex lock = lock(tenantName)) { - var tenant = require(tenantName); - if (tenant.cloudAccounts().equals(cloudAccounts)) return; // no change - curator.writeTenant(LockedTenant.of(tenant, lock).withCloudAccounts(cloudAccounts).get()); - } - } - - /** Deletes the given tenant. */ - public void delete(TenantName tenant, Optional<Credentials> credentials, boolean forget) { - try (Mutex lock = lock(tenant)) { - Tenant oldTenant = get(tenant, true) - .orElseThrow(() -> new NotExistsException("Could not delete tenant '" + tenant + "': Tenant not found")); - - if (oldTenant.type() != Tenant.Type.deleted) { - if (!controller.applications().asList(tenant).isEmpty()) - throw new IllegalArgumentException("Could not delete tenant '" + tenant.value() - + "': This tenant has active applications"); - - if (oldTenant.type() == Tenant.Type.athenz) { - credentials.ifPresent(creds -> accessControl.deleteTenant(tenant, creds)); - } else if (oldTenant.type() == Tenant.Type.cloud) { - accessControl.deleteTenant(tenant, null); - } else { - throw new IllegalArgumentException("Could not delete tenant '" + tenant.value() - + ": This tenant is of unhandled type " + oldTenant.type()); - } - - controller.notificationsDb().removeNotifications(NotificationSource.from(tenant)); - } - - if (forget) curator.removeTenant(tenant); - else curator.writeTenant(new DeletedTenant(tenant, oldTenant.createdAt(), controller.clock().instant())); - } - } - - private void requireNonExistent(TenantName name) { - var tenant = get(name, true); - if (tenant.isPresent() && tenant.get().type().equals(Tenant.Type.deleted)) { - throw new IllegalArgumentException("Tenant '" + name + "' cannot be created, try a different name"); - } - if (SystemApplication.TENANT.equals(name) - || tenant.isPresent() - // Underscores are allowed in existing tenant names, but tenants with - and _ cannot co-exist. E.g. - // my-tenant cannot be created if my_tenant exists. - || get(name.value().replace('-', '_')).isPresent()) { - throw new IllegalArgumentException("Tenant '" + name + "' already exists"); - } - } - - /** - * Returns a lock which provides exclusive rights to changing this tenant. - * Any operation which stores a tenant need to first acquire this lock, then read, modify - * and store the tenant, and finally release (close) the lock. - */ - private Mutex lock(TenantName tenant) { - return curator.lock(tenant); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java deleted file mode 100644 index d89f786714d..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import java.time.Instant; -import java.util.Collection; -import java.util.Comparator; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalDouble; -import java.util.function.Function; - -/** - * Recent activity in an application. - * - * @author mpolden - */ -public class ApplicationActivity { - - public static final ApplicationActivity none = new ApplicationActivity(Optional.empty(), Optional.empty(), - OptionalDouble.empty(), - OptionalDouble.empty()); - - private final Optional<Instant> lastQueried; - private final Optional<Instant> lastWritten; - private final OptionalDouble lastQueriesPerSecond; - private final OptionalDouble lastWritesPerSecond; - - private ApplicationActivity(Optional<Instant> lastQueried, Optional<Instant> lastWritten, - OptionalDouble lastQueriesPerSecond, OptionalDouble lastWritesPerSecond) { - this.lastQueried = Objects.requireNonNull(lastQueried, "lastQueried must be non-null"); - this.lastWritten = Objects.requireNonNull(lastWritten, "lastWritten must be non-null"); - this.lastQueriesPerSecond = Objects.requireNonNull(lastQueriesPerSecond, "lastQueriesPerSecond must be non-null"); - this.lastWritesPerSecond = Objects.requireNonNull(lastWritesPerSecond, "lastWritesPerSecond must be non-null"); - } - - /** The last time any deployment in this was queried */ - public Optional<Instant> lastQueried() { - return lastQueried; - } - - /** The last time any deployment in this was written */ - public Optional<Instant> lastWritten() { - return lastWritten; - } - - /** Query rate the last time this was queried */ - public OptionalDouble lastQueriesPerSecond() { - return lastQueriesPerSecond; - } - - /** Write rate the last time this was written */ - public OptionalDouble lastWritesPerSecond() { - return lastWritesPerSecond; - } - - public static ApplicationActivity from(Collection<Deployment> deployments) { - Optional<DeploymentActivity> lastActivityByQuery = lastActivityBy(DeploymentActivity::lastQueried, deployments); - Optional<DeploymentActivity> lastActivityByWrite = lastActivityBy(DeploymentActivity::lastWritten, deployments); - if (lastActivityByQuery.isEmpty() && lastActivityByWrite.isEmpty()) { - return none; - } - return new ApplicationActivity(lastActivityByQuery.flatMap(DeploymentActivity::lastQueried), - lastActivityByWrite.flatMap(DeploymentActivity::lastWritten), - lastActivityByQuery.map(DeploymentActivity::lastQueriesPerSecond) - .orElseGet(OptionalDouble::empty), - lastActivityByWrite.map(DeploymentActivity::lastWritesPerSecond) - .orElseGet(OptionalDouble::empty)); - } - - private static Optional<DeploymentActivity> lastActivityBy(Function<DeploymentActivity, Optional<Instant>> field, - Collection<Deployment> deployments) { - return deployments.stream() - .map(Deployment::activity) - .filter(activity -> field.apply(activity).isPresent()) - .max(Comparator.comparing(activity -> field.apply(activity).get())); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java deleted file mode 100644 index 32aae5c041c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.collections.AbstractFilteringList; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.ApplicationController; - -import java.util.Collection; - -/** - * A list of applications which can be filtered in various ways. - * - * @author jonmv - */ -public class ApplicationList extends AbstractFilteringList<Application, ApplicationList> { - - private ApplicationList(Collection<? extends Application> applications, boolean negate) { - super(applications, negate, ApplicationList::new); - } - - // ----------------------------------- Factories - - public static ApplicationList from(Collection<? extends Application> applications) { - return new ApplicationList(applications, false); - } - - public static ApplicationList from(Collection<ApplicationId> ids, ApplicationController applications) { - return from(ids.stream() - .map(TenantAndApplicationId::from) - .distinct() - .map(applications::requireApplication) - .toList()); - } - - // ----------------------------------- Filters - - /** Returns the subset of applications which have at least one production deployment */ - public ApplicationList withProductionDeployment() { - return matching(application -> application.instances().values().stream() - .anyMatch(instance -> instance.productionDeployments().size() > 0)); - } - - /** Returns the subset of applications with at least one declared job in deployment spec. */ - public ApplicationList withJobs() { - return matching(application -> application.deploymentSpec().steps().stream() - .anyMatch(step -> ! step.zones().isEmpty())); - } - - /** Returns the subset of applications which have a project ID */ - public ApplicationList withProjectId() { - return matching(application -> application.projectId().isPresent()); - } - - /** Returns the subset of application which have submitted a non-empty deployment spec. */ - public ApplicationList withDeploymentSpec() { - return matching(application -> ! DeploymentSpec.empty.equals(application.deploymentSpec())); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java deleted file mode 100644 index c2949e395e9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; - -import java.util.Objects; -import java.util.Set; - -/** - * Contains the tuple of [clusterId, endpointId, rotationId, regions[]], to keep track - * of which services have assigned which rotations under which name. - * - * @author ogronnesby - */ -public record AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, RotationId rotationId, Set<RegionName> regions) { - - public AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, RotationId rotationId, Set<RegionName> regions) { - this.clusterId = requireNonEmpty(clusterId, clusterId.value(), "clusterId"); - this.endpointId = Objects.requireNonNull(endpointId); - this.rotationId = Objects.requireNonNull(rotationId); - this.regions = Set.copyOf(Objects.requireNonNull(regions)); - } - - private static <T> T requireNonEmpty(T object, String value, String field) { - Objects.requireNonNull(object); - Objects.requireNonNull(value); - if (value.isEmpty()) { - throw new IllegalArgumentException("Field '" + field + "' was empty"); - } - return object; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java deleted file mode 100644 index b41b02011b4..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; - -import java.util.Objects; -import java.util.Optional; -import java.util.StringJoiner; - -import static java.util.Objects.requireNonNull; - -/** - * The changes to an application we currently wish to complete deploying. - * A goal of the system is to deploy platform and application versions separately. - * However, this goal must some times be traded against others, so a change can - * consist of both an application and platform version change. - * - * This is immutable. - * - * @author bratseth - */ -public final class Change { - - private static final Change empty = new Change(Optional.empty(), Optional.empty(), false, false); - - /** The platform version we are upgrading to, or empty if none */ - private final Optional<Version> platform; - - /** The application version we are changing to, or empty if none */ - private final Optional<RevisionId> revision; - - /** Whether this change is a pin to its contained Vespa version, or to the application's current. */ - private final boolean platformPinned; - - /** Whether this change is a pin to its contained application revision, or to the application's current. */ - private final boolean revisionPinned; - - private Change(Optional<Version> platform, Optional<RevisionId> revision, boolean platformPinned, boolean revisionPinned) { - this.platform = requireNonNull(platform, "platform cannot be null"); - this.revision = requireNonNull(revision, "revision cannot be null"); - if (revision.isPresent() && ( ! revision.get().isProduction())) { - throw new IllegalArgumentException("Application version to deploy must be a known version"); - } - this.platformPinned = platformPinned; - this.revisionPinned = revisionPinned; - } - - public Change withoutPlatform() { - return new Change(Optional.empty(), revision, platformPinned, revisionPinned); - } - - public Change withoutApplication() { - return new Change(platform, Optional.empty(), platformPinned, revisionPinned); - } - - /** Returns whether a change should currently be deployed */ - public boolean hasTargets() { - return platform.isPresent() || revision.isPresent(); - } - - /** Returns whether this is the empty change. */ - public boolean isEmpty() { - return ! hasTargets() && ! platformPinned && ! revisionPinned; - } - - /** Returns the platform version carried by this. */ - public Optional<Version> platform() { return platform; } - - /** Returns the application version carried by this. */ - public Optional<RevisionId> revision() { return revision; } - - public boolean isPlatformPinned() { return platformPinned; } - - public boolean isRevisionPinned() { return revisionPinned; } - - /** Returns an instance representing no change */ - public static Change empty() { return empty; } - - /** Returns a version of this change which replaces or adds this platform change */ - public Change with(Version platformVersion) { - if (platformPinned) - throw new IllegalArgumentException("Not allowed to set a platform version when pinned."); - - return new Change(Optional.of(platformVersion), revision, platformPinned, revisionPinned); - } - - /** Returns a version of this change which replaces or adds this revision change */ - public Change with(RevisionId revision) { - if (revisionPinned) - throw new IllegalArgumentException("Not allowed to set a revision when pinned."); - - return new Change(platform, Optional.of(revision), platformPinned, revisionPinned); - } - - /** Returns a change with the versions of this, and with the platform version pinned. */ - public Change withPlatformPin() { - return new Change(platform, revision, true, revisionPinned); - } - - /** Returns a change with the versions of this, and with the platform version unpinned. */ - public Change withoutPlatformPin() { - return new Change(platform, revision, false, revisionPinned); - } - - /** Returns a change with the versions of this, and with the platform version pinned. */ - public Change withRevisionPin() { - return new Change(platform, revision, platformPinned, true); - } - - /** Returns a change with the versions of this, and with the platform version unpinned. */ - public Change withoutRevisionPin() { - return new Change(platform, revision, platformPinned, false); - } - - /** Returns the change obtained when overwriting elements of the given change with any present in this */ - public Change onTopOf(Change other) { - if (platform.isPresent()) other = other.with(platform.get()); - if (revision.isPresent()) other = other.with(revision.get()); - if (platformPinned) other = other.withPlatformPin(); - if (revisionPinned) other = other.withRevisionPin(); - return other; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof Change)) return false; - Change change = (Change) o; - return platformPinned == change.platformPinned && - revisionPinned == change.revisionPinned && - Objects.equals(platform, change.platform) && - Objects.equals(revision, change.revision); - } - - @Override - public int hashCode() { - return Objects.hash(platform, revision, platformPinned, revisionPinned); - } - - @Override - public String toString() { - StringJoiner changes = new StringJoiner(" and "); - if (platformPinned) - changes.add("pin to " + platform.map(Version::toString).orElse("current platform")); - else - platform.ifPresent(version -> changes.add("upgrade to " + version)); - if (revisionPinned) - changes.add("pin to " + revision.map(RevisionId::toString).orElse("current revision")); - else - revision.ifPresent(revision -> changes.add("revision change to " + revision)); - changes.setEmptyValue("no change"); - return changes.toString(); - } - - public static Change of(RevisionId revision) { - return new Change(Optional.empty(), Optional.of(revision), false, false); - } - - public static Change of(Version platformChange) { - return new Change(Optional.of(platformChange), Optional.empty(), false, false); - } - - /** Returns whether this change carries a revision downgrade relative to the given revision. */ - public boolean downgrades(RevisionId revision) { - return this.revision.map(revision::compareTo).orElse(0) > 0; - } - - /** Returns whether this change carries a platform downgrade relative to the given version. */ - public boolean downgrades(Version version) { - return platform.map(version::compareTo).orElse(0) > 0; - } - - /** Returns whether this change carries a revision upgrade relative to the given revision. */ - public boolean upgrades(RevisionId revision) { - return this.revision.map(revision::compareTo).orElse(0) < 0; - } - - /** Returns whether this change carries a platform upgrade relative to the given version. */ - public boolean upgrades(Version version) { - return platform.map(version::compareTo).orElse(0) < 0; - } - -} - diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java deleted file mode 100644 index de26ca73cd8..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; - -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalDouble; - -/** - * A deployment of an application in a particular zone. - * - * @author bratseth - * @author smorgrav - */ -public class Deployment { - - private final ZoneId zone; - private final CloudAccount cloudAccount; - private final RevisionId revision; - private final Version version; - private final Instant deployTime; - private final DeploymentMetrics metrics; - private final DeploymentActivity activity; - private final QuotaUsage quota; - private final OptionalDouble cost; - private final Map<TokenId, Instant> dataPlaneTokens; - - public Deployment(ZoneId zone, CloudAccount cloudAccount, RevisionId revision, Version version, Instant deployTime, - DeploymentMetrics metrics, DeploymentActivity activity, QuotaUsage quota, OptionalDouble cost, - Map<TokenId, Instant> dataPlaneTokens) { - this.zone = Objects.requireNonNull(zone, "zone cannot be null"); - this.cloudAccount = Objects.requireNonNull(cloudAccount, "cloudAccount cannot be null"); - this.revision = Objects.requireNonNull(revision, "revision cannot be null"); - this.version = Objects.requireNonNull(version, "version cannot be null"); - this.deployTime = Objects.requireNonNull(deployTime, "deployTime cannot be null"); - this.metrics = Objects.requireNonNull(metrics, "deploymentMetrics cannot be null"); - this.activity = Objects.requireNonNull(activity, "activity cannot be null"); - this.quota = Objects.requireNonNull(quota, "usage cannot be null"); - this.cost = Objects.requireNonNull(cost, "cost cannot be null"); - this.dataPlaneTokens = Map.copyOf(dataPlaneTokens); - } - - /** Returns the zone this was deployed to */ - public ZoneId zone() { return zone; } - - /** Returns the cloud account this was deployed to */ - public CloudAccount cloudAccount() { return cloudAccount; } - - /** Returns the deployed application revision */ - public RevisionId revision() { return revision; } - - /** Returns the deployed Vespa version */ - public Version version() { return version; } - - /** Returns the time this was deployed */ - public Instant at() { return deployTime; } - - /** Returns metrics for this */ - public DeploymentMetrics metrics() { - return metrics; - } - - /** Returns activity for this */ - public DeploymentActivity activity() { return activity; } - - /** Returns quota usage for this */ - public QuotaUsage quota() { return quota; } - - /** Returns cost, in dollars per hour, for this */ - public OptionalDouble cost() { return cost; } - - /** Returns the data plane token IDs referenced by this deployment, and the last update time of this token at the time of deployment. */ - public Map<TokenId, Instant> dataPlaneTokens() { return dataPlaneTokens; } - - public Deployment recordActivityAt(Instant instant) { - return new Deployment(zone, cloudAccount, revision, version, deployTime, metrics, - activity.recordAt(instant, metrics), quota, cost, dataPlaneTokens); - } - - public Deployment withMetrics(DeploymentMetrics metrics) { - return new Deployment(zone, cloudAccount, revision, version, deployTime, metrics, activity, quota, cost, dataPlaneTokens); - } - - public Deployment withCost(double cost) { - if (this.cost.isPresent() && Double.compare(this.cost.getAsDouble(), cost) == 0) return this; - return new Deployment(zone, cloudAccount, revision, version, deployTime, metrics, activity, quota, OptionalDouble.of(cost), dataPlaneTokens); - } - - public Deployment withoutCost() { - if (cost.isEmpty()) return this; - return new Deployment(zone, cloudAccount, revision, version, deployTime, metrics, activity, quota, OptionalDouble.empty(), dataPlaneTokens); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Deployment that = (Deployment) o; - return Objects.equals(zone, that.zone) - && Objects.equals(cloudAccount, that.cloudAccount) - && Objects.equals(revision, that.revision) - && Objects.equals(version, that.version) - && Objects.equals(deployTime, that.deployTime) - && Objects.equals(metrics, that.metrics) - && Objects.equals(activity, that.activity) - && Objects.equals(quota, that.quota) - && Objects.equals(cost, that.cost) - && Objects.equals(dataPlaneTokens, that.dataPlaneTokens); - } - - @Override - public int hashCode() { - return Objects.hash(zone, cloudAccount, revision, version, deployTime, metrics, activity, quota, cost, dataPlaneTokens); - } - - @Override - public String toString() { - return "deployment to " + zone + " of " + revision + " on version " + version + " at " + deployTime; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java deleted file mode 100644 index d671f57f90f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import java.time.Instant; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalDouble; - -/** - * Recent activity in a deployment. - * - * @author mpolden - */ -public class DeploymentActivity { - - /** Query rates at or below this threshold indicate inactivity */ - private static final double inactivityThreshold = 0; - - public static final DeploymentActivity none = new DeploymentActivity(Optional.empty(), Optional.empty(), - OptionalDouble.empty(), - OptionalDouble.empty()); - - private final Optional<Instant> lastQueried; - private final Optional<Instant> lastWritten; - private final OptionalDouble lastQueriesPerSecond; - private final OptionalDouble lastWritesPerSecond; - - private DeploymentActivity(Optional<Instant> lastQueried, Optional<Instant> lastWritten, - OptionalDouble lastQueriesPerSecond, OptionalDouble lastWritesPerSecond) { - this.lastQueried = Objects.requireNonNull(lastQueried, "lastQueried must be non-null"); - this.lastWritten = Objects.requireNonNull(lastWritten, "lastWritten must be non-null"); - this.lastQueriesPerSecond = Objects.requireNonNull(lastQueriesPerSecond, "lastQueriesPerSecond must be non-null"); - this.lastWritesPerSecond = Objects.requireNonNull(lastWritesPerSecond, "lastWritesPerSecond must be non-null"); - } - - /** The last time this deployment received queries (search) */ - public Optional<Instant> lastQueried() { - return lastQueried; - } - - /** The last time this deployment received writes (feed) */ - public Optional<Instant> lastWritten() { - return lastWritten; - } - - /** Query rate the last time this deployment received queries (search) */ - public OptionalDouble lastQueriesPerSecond() { - return lastQueriesPerSecond; - } - - /** Write rate the last time this deployment received writes (feed) */ - public OptionalDouble lastWritesPerSecond() { - return lastWritesPerSecond; - } - - /** Record activity using given metrics */ - public DeploymentActivity recordAt(Instant instant, DeploymentMetrics metrics) { - return new DeploymentActivity(activityAt(instant, lastQueried, metrics.queriesPerSecond()), - activityAt(instant, lastWritten, metrics.writesPerSecond()), - activeRate(metrics.queriesPerSecond(), lastQueriesPerSecond), - activeRate(metrics.writesPerSecond(), lastWritesPerSecond)); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - DeploymentActivity that = (DeploymentActivity) o; - return lastQueried.equals(that.lastQueried) && lastWritten.equals(that.lastWritten) && lastQueriesPerSecond.equals(that.lastQueriesPerSecond) && lastWritesPerSecond.equals(that.lastWritesPerSecond); - } - - @Override - public int hashCode() { - return Objects.hash(lastQueried, lastWritten, lastQueriesPerSecond, lastWritesPerSecond); - } - - public static DeploymentActivity create(Optional<Instant> queriedAt, Optional<Instant> writtenAt, - OptionalDouble lastQueriesPerSecond, OptionalDouble lastWritesPerSecond) { - if (queriedAt.isEmpty() && writtenAt.isEmpty()) { - return none; - } - return new DeploymentActivity(queriedAt, writtenAt, lastQueriesPerSecond, lastWritesPerSecond); - } - - public static DeploymentActivity create(Optional<Instant> queriedAt, Optional<Instant> writtenAt) { - return create(queriedAt, writtenAt, OptionalDouble.empty(), OptionalDouble.empty()); - } - - private static OptionalDouble activeRate(double newRate, OptionalDouble oldRate) { - return newRate > inactivityThreshold ? OptionalDouble.of(newRate) : oldRate; - } - - private static Optional<Instant> activityAt(Instant newInstant, Optional<Instant> oldInstant, double rate) { - return rate > inactivityThreshold ? Optional.of(newInstant) : oldInstant; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java deleted file mode 100644 index ce652521a9f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import java.time.Instant; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -/** - * Metrics for a deployment of an application. This contains a snapshot of metrics gathered at a point in time, it does - * not contain any historical data. - * - * @author smorgrav - * @author mpolden - */ -public class DeploymentMetrics { - - public static final DeploymentMetrics none = new DeploymentMetrics(0, 0, 0, 0, 0, Optional.empty(), Map.of()); - - private final double queriesPerSecond; - private final double writesPerSecond; - private final double documentCount; - private final double queryLatencyMillis; - private final double writeLatencyMills; - private final Optional<Instant> instant; - private final Map<Warning, Integer> warnings; - - /* DO NOT USE. Public for serialization purposes */ - public DeploymentMetrics(double queriesPerSecond, double writesPerSecond, double documentCount, - double queryLatencyMillis, double writeLatencyMills, Optional<Instant> instant, - Map<Warning, Integer> warnings) { - this.queriesPerSecond = queriesPerSecond; - this.writesPerSecond = writesPerSecond; - this.documentCount = documentCount; - this.queryLatencyMillis = queryLatencyMillis; - this.writeLatencyMills = writeLatencyMills; - this.instant = Objects.requireNonNull(instant, "instant must be non-null"); - this.warnings = Map.copyOf(Objects.requireNonNull(warnings, "warnings must be non-null")); - if (warnings.entrySet().stream().anyMatch(kv -> kv.getValue() < 0)) { - throw new IllegalArgumentException("Warning count must be non-negative. Got " + warnings); - } - } - - /** Returns the number of queries per second */ - public double queriesPerSecond() { - return queriesPerSecond; - } - - /** Returns the number of writes per second */ - public double writesPerSecond() { - return writesPerSecond; - } - - /** Returns the number of documents */ - public double documentCount() { - return documentCount; - } - - /** Returns the average query latency in milliseconds */ - public double queryLatencyMillis() { - return queryLatencyMillis; - } - - /** Returns the average write latency in milliseconds */ - public double writeLatencyMillis() { - return writeLatencyMills; - } - - /** Returns the approximate time this was measured */ - public Optional<Instant> instant() { - return instant; - } - - /** Returns the number of warnings of the most recent deployment */ - public Map<Warning, Integer> warnings() { - return warnings; - } - - public DeploymentMetrics withQueriesPerSecond(double queriesPerSecond) { - return new DeploymentMetrics(queriesPerSecond, writesPerSecond, documentCount, queryLatencyMillis, - writeLatencyMills, instant, warnings); - } - - public DeploymentMetrics withWritesPerSecond(double writesPerSecond) { - return new DeploymentMetrics(queriesPerSecond, writesPerSecond, documentCount, queryLatencyMillis, - writeLatencyMills, instant, warnings); - } - - public DeploymentMetrics withDocumentCount(double documentCount) { - return new DeploymentMetrics(queriesPerSecond, writesPerSecond, documentCount, queryLatencyMillis, - writeLatencyMills, instant, warnings); - } - - public DeploymentMetrics withQueryLatencyMillis(double queryLatencyMillis) { - return new DeploymentMetrics(queriesPerSecond, writesPerSecond, documentCount, queryLatencyMillis, - writeLatencyMills, instant, warnings); - } - - public DeploymentMetrics withWriteLatencyMillis(double writeLatencyMills) { - return new DeploymentMetrics(queriesPerSecond, writesPerSecond, documentCount, queryLatencyMillis, - writeLatencyMills, instant, warnings); - } - - public DeploymentMetrics at(Instant instant) { - return new DeploymentMetrics(queriesPerSecond, writesPerSecond, documentCount, queryLatencyMillis, - writeLatencyMills, Optional.of(instant), warnings); - } - - public DeploymentMetrics with(Map<Warning, Integer> warnings) { - return new DeploymentMetrics(queriesPerSecond, writesPerSecond, documentCount, queryLatencyMillis, - writeLatencyMills, instant, warnings); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - DeploymentMetrics that = (DeploymentMetrics) o; - return Double.compare(that.queriesPerSecond, queriesPerSecond) == 0 && - Double.compare(that.writesPerSecond, writesPerSecond) == 0 && - Double.compare(that.documentCount, documentCount) == 0 && - Double.compare(that.queryLatencyMillis, queryLatencyMillis) == 0 && - Double.compare(that.writeLatencyMills, writeLatencyMills) == 0 && - instant.equals(that.instant) && - warnings.equals(that.warnings); - } - - @Override - public int hashCode() { - return Objects.hash(queriesPerSecond, writesPerSecond, documentCount, queryLatencyMillis, writeLatencyMills, instant, warnings); - } - - /** Types of deployment warnings. We currently have only one */ - public enum Warning { - all - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java deleted file mode 100644 index 4132b560fae..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; - -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.List; - -/** - * Calculates the quota to allocate to a deployment. - * - * @author ogronnesby - * @author andreer - */ -public class DeploymentQuotaCalculator { - - public static Quota calculate(Quota tenantQuota, - List<Application> tenantApps, - ApplicationId deployingApp, ZoneId deployingZone, - DeploymentSpec deploymentSpec) - { - if (tenantQuota.budget().isEmpty()) return tenantQuota; // Shortcut if there is no budget limit to care about. - if (deployingZone.environment().isTest()) return tenantQuota; - if (deployingZone.environment().isProduction()) return probablyEnoughForAll(tenantQuota, tenantApps, deployingApp, deploymentSpec); - return getMaximumAllowedQuota(tenantQuota, tenantApps, deployingApp, deployingZone); - } - - public static QuotaUsage calculateQuotaUsage(com.yahoo.vespa.hosted.controller.api.integration.configserver.Application application) { - var quotaUsageRate = application.clusters().values().stream() - .filter(cluster -> ! cluster.type().equals(ClusterSpec.Type.admin)) - .map(cluster -> largestQuotaUsage(cluster.current(), cluster.max())) - .mapToDouble(resources -> resources.nodes() * resources.nodeResources().cost()) - .sum(); - return QuotaUsage.create(quotaUsageRate); - } - - private static ClusterResources largestQuotaUsage(ClusterResources a, ClusterResources b) { - return a.cost() > b.cost() ? a : b; - } - - /** Just get the maximum quota we are allowed to use. */ - private static Quota getMaximumAllowedQuota(Quota tenantQuota, List<Application> applications, - ApplicationId application, ZoneId zone) { - var usageOutsideDeployment = applications.stream() - .map(app -> app.quotaUsage(application, zone)) - .reduce(QuotaUsage::add).orElse(QuotaUsage.none); - return tenantQuota.subtractUsage(usageOutsideDeployment.rate()); - } - - /** - * We want to avoid applying a resource change to an instance in production when it seems likely - * that there will not be enough quota to apply this change to _all_ production instances. - * <p> - * To achieve this, we must make the assumption that all production instances will use - * the same amount of resources, and so equally divide the quota among them. - */ - private static Quota probablyEnoughForAll(Quota tenantQuota, List<Application> tenantApps, - ApplicationId application, DeploymentSpec deploymentSpec) { - - TenantAndApplicationId deployingAppId = TenantAndApplicationId.from(application); - - var usageOutsideApplication = tenantApps.stream() - .filter(app -> !app.id().equals(deployingAppId)) - .map(Application::quotaUsage).reduce(QuotaUsage::add).orElse(QuotaUsage.none); - - QuotaUsage manualQuotaUsage = tenantApps.stream() - .filter(app -> app.id().equals(deployingAppId)).findFirst() - .map(Application::manualQuotaUsage).orElse(QuotaUsage.none); - - long productionDeployments = Math.max(1, deploymentSpec.instances().stream() - .flatMap(instance -> instance.zones().stream()) - .filter(zone -> zone.environment().isProduction()) - .count()); - - return tenantQuota.withBudget( - tenantQuota.subtractUsage(usageOutsideApplication.rate() + manualQuotaUsage.rate()) - .budget().get().divide(BigDecimal.valueOf(productionDeployments), - 5, RoundingMode.HALF_UP)); // 1/1000th of a cent should be accurate enough - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java deleted file mode 100644 index 39e1c89c202..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ /dev/null @@ -1,658 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.AuthMethod; -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; - -import java.net.URI; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Comparator.comparing; - -/** - * Represents an application or instance endpoint in hosted Vespa. - * <p> - * This encapsulates the logic for building URLs and DNS names for applications in all hosted Vespa systems. - * - * @author mpolden - */ -public class Endpoint { - - private static final String MAIN_OATH_DNS_SUFFIX = ".vespa.oath.cloud"; - private static final String CD_OATH_DNS_SUFFIX = ".cd.vespa.oath.cloud"; - private static final String PUBLIC_DNS_SUFFIX = ".vespa-app.cloud"; - private static final String PUBLIC_CD_DNS_SUFFIX = ".cd.vespa-app.cloud"; - - private final EndpointId id; - private final ClusterSpec.Id cluster; - private final Optional<InstanceName> instance; - private final URI url; - private final List<Target> targets; - private final Scope scope; - private final boolean legacy; - private final RoutingMethod routingMethod; - private final AuthMethod authMethod; - private final Optional<GeneratedEndpoint> generated; - - private Endpoint(TenantAndApplicationId application, Optional<InstanceName> instanceName, EndpointId id, - ClusterSpec.Id cluster, URI url, List<Target> targets, Scope scope, Port port, boolean legacy, - RoutingMethod routingMethod, boolean certificateName, AuthMethod authMethod, Optional<GeneratedEndpoint> generated) { - Objects.requireNonNull(application, "application must be non-null"); - Objects.requireNonNull(instanceName, "instanceName must be non-null"); - Objects.requireNonNull(cluster, "cluster must be non-null"); - Objects.requireNonNull(url, "url must be non-null"); - Objects.requireNonNull(targets, "deployment must be non-null"); - Objects.requireNonNull(scope, "scope must be non-null"); - Objects.requireNonNull(port, "port must be non-null"); - Objects.requireNonNull(routingMethod, "routingMethod must be non-null"); - Objects.requireNonNull(authMethod, "authMethod must be non-null"); - Objects.requireNonNull(generated, "generated must be non-null"); - this.id = requireEndpointId(id, scope, certificateName); - this.cluster = requireCluster(cluster, certificateName); - this.instance = requireInstance(instanceName, scope, certificateName, generated.isPresent()); - this.url = url; - this.targets = List.copyOf(requireTargets(targets, application, instanceName, scope, certificateName)); - this.scope = requireScope(scope, routingMethod); - this.legacy = legacy; - this.routingMethod = routingMethod; - this.authMethod = authMethod; - this.generated = generated; - } - - /** - * Returns the name of this endpoint (the first component of the DNS name). This can be one of the following: - * - * - The wildcard character '*' (for wildcard endpoints, with any scope) - * - The cluster ID ({@link Scope#zone} and {@link Scope#weighted} - * - The endpoint ID ({@link Scope#global} and {@link Scope#application}) - */ - public String name() { - return endpointOrClusterAsString(id, cluster); - } - - /** Returns the cluster ID to which this routes traffic */ - public ClusterSpec.Id cluster() { - return cluster; - } - - /** The specific instance this endpoint points to, if any */ - public Optional<InstanceName> instance() { - return instance; - } - - /** Returns the URL used to access this */ - public URI url() { - return url; - } - - /** Returns the DNS name of this */ - public String dnsName() { - // because getHost returns "null" for wildcard endpoints - return url.getAuthority().replaceAll(":.*", ""); - } - - /** Returns the target(s) to which this routes traffic */ - public List<Target> targets() { - return targets; - } - - /** Returns the deployments(s) to which this routes traffic */ - public List<DeploymentId> deployments() { - return targets.stream().map(Target::deployment).toList(); - } - - /** Returns the scope of this */ - public Scope scope() { - return scope; - } - - /** Returns whether this is considered a legacy DNS name intended to be removed at some point */ - public boolean legacy() { - return legacy; - } - - /** Returns the routing method used for this */ - public RoutingMethod routingMethod() { - return routingMethod; - } - - /** Returns whether this endpoint supports TLS connections */ - public boolean tls() { - return true; - } - - /** Returns whether this requires a rotation to be reachable */ - public boolean requiresRotation() { - return routingMethod.isShared() && scope == Scope.global; - } - - /** Returns whether this endpoint is generated by the system */ - public Optional<GeneratedEndpoint> generated() { - return generated; - } - - /** Returns the upstream name of given deployment. This *must* match what the routing layer generates */ - public String upstreamName(DeploymentId deployment) { - if (!routingMethod.isShared()) throw new IllegalArgumentException("Routing method " + routingMethod + " does not have upstream name"); - return upstreamName(cluster.value(), deployment.applicationId(), deployment.zoneId()); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Endpoint endpoint = (Endpoint) o; - return url.equals(endpoint.url); - } - - @Override - public int hashCode() { - return Objects.hash(url); - } - - @Override - public String toString() { - return Text.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s, authMethod=%s, name=%s]", url, scope, legacy, routingMethod, authMethod, name()); - } - - private static String endpointOrClusterAsString(EndpointId id, ClusterSpec.Id cluster) { - return id == null ? cluster.value() : id.id(); - } - - private static URI createUrl(String name, TenantAndApplicationId application, Optional<InstanceName> instance, - List<Target> targets, Scope scope, SystemName system, Port port, - Optional<GeneratedEndpoint> generated) { - - String separator = "."; - String portPart = port.isDefault() ? "" : ":" + port.port; - final String subdomain; - if (generated.isPresent()) { - subdomain = generatedPart(generated.get(), separator); - } else { - subdomain = sanitize(namePart(name, separator)) + - systemPart(system, separator) + - sanitize(instancePart(instance, separator)) + - sanitize(application.application().value()) + - separator + - sanitize(application.tenant().value()); - } - return URI.create("https://" + - subdomain + - "." + - scopePart(scope, targets, system, generated) + - dnsSuffix(system) + - portPart + - "/"); - } - - private static String generatedPart(GeneratedEndpoint generated, String separator) { - return generated.clusterPart() + separator + generated.applicationPart(); - } - - private static String sanitize(String part) { // TODO: Reject reserved words - return part.replace('_', '-'); - } - - private static String namePart(String name, String separator) { - if ("default".equals(name)) return ""; - return name + separator; - } - - private static String scopePart(Scope scope, List<Target> targets, SystemName system, Optional<GeneratedEndpoint> generated) { - String scopeSymbol = scopeSymbol(scope, system, generated); - if (scope == Scope.global) return scopeSymbol; - if (scope == Scope.application) return scopeSymbol; - if (scope == Scope.zone && generated.isPresent()) return scopeSymbol; - - ZoneId zone = targets.stream().map(target -> target.deployment.zoneId()).min(comparing(ZoneId::value)).get(); - String region = zone.region().value(); - String environment = zone.environment().isProduction() ? "" : "." + zone.environment().value(); - if (system.isPublic()) { - return region + environment + "." + scopeSymbol; - } - return region + (scopeSymbol.isEmpty() ? "" : "-" + scopeSymbol) + environment; - } - - private static String scopeSymbol(Scope scope, SystemName system, Optional<GeneratedEndpoint> generated) { - if (system.isPublic() || generated.isPresent()) { - return switch (scope) { - case zone -> "z"; - case weighted -> "w"; - case global -> "g"; - case application -> "a"; - }; - } - return switch (scope) { - case zone -> ""; - case weighted -> "w"; - case global -> "global"; - case application -> "a"; - }; - } - - private static String instancePart(Optional<InstanceName> instance, String separator) { - if (instance.isEmpty()) return ""; - if (instance.get().isDefault()) return ""; // Skip "default" - return instance.get().value() + separator; - } - - private static String systemPart(SystemName system, String separator) { - if (!system.isCd()) return ""; - if (system.isPublic()) return ""; - return system.value() + separator; - } - - /** Returns the DNS suffix used for endpoints in given system */ - private static String dnsSuffix(SystemName system) { - return switch (system) { - case cd -> CD_OATH_DNS_SUFFIX; - case main -> MAIN_OATH_DNS_SUFFIX; - case Public -> PUBLIC_DNS_SUFFIX; - case PublicCd -> PUBLIC_CD_DNS_SUFFIX; - default -> throw new IllegalArgumentException("No DNS suffix declared for system " + system); - }; - } - - /** Returns the DNS suffix used for internal names (i.e. names not exposed to tenants) in given system */ - public static String internalDnsSuffix(SystemName system) { - String suffix = dnsSuffix(system); - if (system.isPublic()) { - // Certificate provider requires special approval for three-level DNS names, e.g. foo.vespa-app.cloud. - // To avoid this in public we always add an extra level. - return ".internal" + suffix; - } - return suffix; - } - - private static String upstreamName(String name, ApplicationId application, ZoneId zone) { - return Stream.of(namePart(name, ""), - instancePart(Optional.of(application.instance()), ""), - application.application().value(), - application.tenant().value(), - zone.region().value(), - zone.environment().value()) - .filter(Predicate.not(String::isEmpty)) - .map(Endpoint::sanitizeUpstream) - .collect(Collectors.joining(".")); - } - - /** Remove any invalid characters from a upstream part */ - private static String sanitizeUpstream(String part) { - return truncate(part.toLowerCase() - .replace('_', '-') - .replaceAll("[^a-z0-9-]*", "")); - } - - /** Truncate the given part at the front so its length does not exceed 63 characters */ - private static String truncate(String part) { - return part.substring(Math.max(0, part.length() - 63)); - } - - private static ClusterSpec.Id requireCluster(ClusterSpec.Id cluster, boolean certificateName) { - if (!certificateName && cluster.value().equals("*")) throw new IllegalArgumentException("Wildcard found in cluster ID which is not a certificate name"); - return cluster; - } - - private static EndpointId requireEndpointId(EndpointId endpointId, Scope scope, boolean certificateName) { - if (scope.multiDeployment() && endpointId == null) throw new IllegalArgumentException("Endpoint ID must be set for multi-deployment endpoints"); - if (scope == Scope.zone && endpointId != null) throw new IllegalArgumentException("Endpoint ID cannot be set for " + scope + " endpoints"); - if (!certificateName && endpointId != null && endpointId.id().equals("*")) throw new IllegalArgumentException("Wildcard found in endpoint ID which is not a certificate name"); - return endpointId; - } - - private static Optional<InstanceName> requireInstance(Optional<InstanceName> instanceName, Scope scope, boolean certificateName, boolean generated) { - if (generated && certificateName) { - return instanceName; - } - if (scope == Scope.application) { - if (instanceName.isPresent()) throw new IllegalArgumentException("Instance cannot be set for scope " + scope); - } else { - if (instanceName.isEmpty()) throw new IllegalArgumentException("Instance must be set for scope " + scope); - } - return instanceName; - } - - private static Scope requireScope(Scope scope, RoutingMethod routingMethod) { - if (scope == Scope.application && !routingMethod.isDirect()) throw new IllegalArgumentException("Routing method " + routingMethod + " does not support " + scope + "-scoped endpoints"); - return scope; - } - - private static List<Target> requireTargets(List<Target> targets, TenantAndApplicationId application, Optional<InstanceName> instanceName, Scope scope, boolean certificateName) { - if (certificateName && targets.isEmpty()) return List.of(); - if (targets.isEmpty()) throw new IllegalArgumentException("At least one target must be given for " + scope + " endpoints"); - if (scope == Scope.zone && targets.size() != 1) throw new IllegalArgumentException("Exactly one target must be given for " + scope + " endpoints"); - for (var target : targets) { - if (scope == Scope.application) { - TenantAndApplicationId owner = TenantAndApplicationId.from(target.deployment().applicationId()); - if (!owner.equals(application)) { - throw new IllegalArgumentException("Endpoint has target owned by " + owner + - ", which does not match application of this endpoint: " + - application); - } - } else { - ApplicationId owner = target.deployment.applicationId(); - ApplicationId instance = application.instance(instanceName.get()); - if (!owner.equals(instance)) { - throw new IllegalArgumentException("Endpoint has target owned by " + owner + - ", which does not match instance of this endpoint: " + instance); - } - } - } - return targets; - } - - /** Returns the authentication method of this endpoint */ - public AuthMethod authMethod() { - return authMethod; - } - - /** An endpoint's scope */ - public enum Scope { - - /** - * Endpoint points to a multiple instances of an application, in the same region. - * - * Traffic is routed across instances according to weights specified in deployment.xml - */ - application, - - /** Endpoint points to one or more zones. Traffic is routed to the zone closest to the client */ - global, - - /** - * Endpoint points to one more zones in the same geographical region. Traffic is routed evenly across zones. - * - * This is for internal use only. Endpoints with this scope are not exposed directly to tenants. - */ - weighted, - - /** Endpoint points to a single zone */ - zone; - - /** Returns whether this scope may span multiple deployments */ - public boolean multiDeployment() { - return this == application || this == global; - } - - } - - /** Represents an endpoint's HTTP port */ - public record Port(int port) { - - private static final Port TLS_DEFAULT = new Port(443); - - public Port { - if (port < 1 || port > 65535) { - throw new IllegalArgumentException("Port must be between 1 and 65535, got " + port); - } - } - - private boolean isDefault() { - return port == TLS_DEFAULT.port; - } - - /** Returns the default HTTPS port */ - public static Port tls() { - return TLS_DEFAULT; - } - - /** Returns default port for the given routing method */ - public static Port fromRoutingMethod(RoutingMethod method) { - if (method.isDirect()) return Port.tls(); - return new Port(4443); - } - - } - - /** Build an endpoint for given instance */ - public static EndpointBuilder of(ApplicationId instance) { - return new EndpointBuilder(TenantAndApplicationId.from(instance), Optional.of(instance.instance())); - } - - /** Build an endpoint for given application */ - public static EndpointBuilder of(TenantAndApplicationId application) { - return new EndpointBuilder(application, Optional.empty()); - } - - /** A target of an endpoint */ - public static class Target { - - private final DeploymentId deployment; - private final int weight; - - private Target(DeploymentId deployment, int weight) { - this.deployment = Objects.requireNonNull(deployment); - this.weight = weight; - if (weight < 0 || weight > 100) { - throw new IllegalArgumentException("Endpoint target weight must be in range [0, 100], got " + weight); - } - } - - private Target(DeploymentId deployment) { - this(deployment, 1); - } - - /** Returns the deployment of this */ - public DeploymentId deployment() { - return deployment; - } - - /** Returns the assigned weight of this */ - public int weight() { - return weight; - } - - /** Returns whether this routes to given deployment */ - public boolean routesTo(DeploymentId deployment) { - return this.deployment.equals(deployment); - } - - } - - public static class EndpointBuilder { - - private final TenantAndApplicationId application; - private final Optional<InstanceName> instance; - - private Scope scope; - private List<Target> targets; - private ClusterSpec.Id cluster; - private EndpointId endpointId; - private Port port; - private RoutingMethod routingMethod = RoutingMethod.sharedLayer4; - private boolean legacy = false; - private boolean certificateName = false; - private AuthMethod authMethod = AuthMethod.mtls; - private Optional<GeneratedEndpoint> generated = Optional.empty(); - - private EndpointBuilder(TenantAndApplicationId application, Optional<InstanceName> instance) { - this.application = Objects.requireNonNull(application); - this.instance = Objects.requireNonNull(instance); - } - - /** Sets the zone target for this */ - public EndpointBuilder target(ClusterSpec.Id cluster, DeploymentId deployment) { - this.cluster = cluster; - this.scope = requireUnset(Scope.zone); - this.targets = List.of(new Target(deployment)); - return this; - } - - /** Sets the global target with given ID, deployments and cluster (as defined in deployments.xml) */ - public EndpointBuilder target(EndpointId endpointId, ClusterSpec.Id cluster, List<DeploymentId> deployments) { - this.endpointId = endpointId; - this.cluster = cluster; - this.targets = deployments.stream().map(Target::new).toList(); - this.scope = requireUnset(Scope.global); - return this; - } - - /** Sets the global target with given ID and pointing to the default cluster */ - public EndpointBuilder target(EndpointId endpointId) { - return target(endpointId, ClusterSpec.Id.from("default"), List.of()); - } - - /** Sets the application target with given ID and pointing to the default cluster */ - public EndpointBuilder targetApplication(EndpointId endpointId, DeploymentId deployment) { - return targetApplication(endpointId, ClusterSpec.Id.from("default"), Map.of(deployment, 1)); - } - - /** Sets the global wildcard target for this */ - public EndpointBuilder wildcard() { - return target(EndpointId.of("*"), ClusterSpec.Id.from("*"), List.of()); - } - - /** Sets the application wildcard target for this */ - public EndpointBuilder wildcardApplication(DeploymentId deployment) { - return targetApplication(EndpointId.of("*"), ClusterSpec.Id.from("*"), Map.of(deployment, 1)); - } - - /** Sets the zone wildcard target for this */ - public EndpointBuilder wildcard(DeploymentId deployment) { - return target(ClusterSpec.Id.from("*"), deployment); - } - - /** Sets the generated wildcard target for this */ - public EndpointBuilder wildcardGenerated(String applicationPart, Scope scope) { - this.cluster = ClusterSpec.Id.from("*"); - if (scope.multiDeployment()) { - this.endpointId = EndpointId.of("*"); - } - this.targets = List.of(); - this.scope = requireUnset(scope); - this.generated = Optional.of(new GeneratedEndpoint("*", applicationPart, AuthMethod.mtls, Optional.ofNullable(endpointId))); - return this; - } - - /** Sets the application target with given ID, cluster, deployments and their weights */ - public EndpointBuilder targetApplication(EndpointId endpointId, ClusterSpec.Id cluster, Map<DeploymentId, Integer> deployments) { - this.endpointId = endpointId; - this.cluster = cluster; - this.targets = deployments.entrySet().stream() - .map(kv -> new Target(kv.getKey(), kv.getValue())) - .toList(); - this.scope = Scope.application; - return this; - } - - /** Sets the region target for this, deduced from given zone */ - public EndpointBuilder targetRegion(ClusterSpec.Id cluster, String cloudNativeRegion, CloudName cloudName) { - this.cluster = cluster; - this.scope = requireUnset(Scope.weighted); - RegionName region = RegionName.from(cloudName.value() + "-" + cloudNativeRegion); - this.targets = List.of(new Target(new DeploymentId(application.instance(instance.get()), ZoneId.from(Environment.prod, region)))); - this.authMethod = AuthMethod.none; - return this; - } - - /** Sets the valid authentication method supported by this */ - public EndpointBuilder authMethod(AuthMethod authMethod) { - this.authMethod = authMethod; - return this; - } - - /** Sets the port of this */ - public EndpointBuilder on(Port port) { - this.port = port; - return this; - } - - /** Set whether this is a legacy endpoint */ - public EndpointBuilder legacy(boolean legacy) { - this.legacy = legacy; - return this; - } - - /** Sets the routing method for this */ - public EndpointBuilder routingMethod(RoutingMethod method) { - this.routingMethod = method; - return this; - } - - /** Sets whether we're building a name for inclusion in a certificate */ - public EndpointBuilder certificateName() { - this.certificateName = true; - return this; - } - - /** Sets the generated ID to use when building this */ - public EndpointBuilder generatedFrom(GeneratedEndpoint generated) { - this.generated = Optional.of(generated); - return this; - } - - /** Sets the system that owns this */ - public Endpoint in(SystemName system) { - String name = endpointOrClusterAsString(endpointId, Objects.requireNonNull(cluster, "cluster must be non-null")); - URI url = createUrl(name, - Objects.requireNonNull(application, "application must be non-null"), - Objects.requireNonNull(instance, "instance must be non-null"), - Objects.requireNonNull(targets, "targets must be non-null"), - Objects.requireNonNull(scope, "scope must be non-null"), - Objects.requireNonNull(system, "system must be non-null"), - Objects.requireNonNull(port, "port must be non-null"), - Objects.requireNonNull(generated) - ); - if (system.isPublic() && routingMethod != RoutingMethod.exclusive) { - throw illegal(url, "Public system only supports routing method " + RoutingMethod.exclusive + ", got " + routingMethod); - } - if (routingMethod.isDirect() && !port.isDefault()) { - throw illegal(url, "Routing method " + routingMethod + " can only use default port, got " + port); - } - if (authMethod == AuthMethod.token && generated.isEmpty()) { - throw illegal(url, authMethod + " is only supported for generated endpoints"); - } - if (scope != Scope.weighted && generated.isPresent() && generated.get().authMethod() != authMethod) { - throw illegal(url, "Authentication method of " + scope + " endpoint does not match authentication method of generated endpoint: " + generated.get().authMethod()); - } - if ((scope == Scope.weighted) != (authMethod == AuthMethod.none)) { - throw illegal(url, "Attempted to set unsupported authentication method " + authMethod + " on " + scope + " endpoint"); - } - if (scope.multiDeployment() && generated.isPresent() && (generated.get().endpoint().isEmpty() || !generated.get().endpoint().get().equals(endpointId))) { - throw illegal(url, "Generated endpoint must contain a matching endpoint ID, but got " + generated.get().endpoint()); - } - return new Endpoint(application, - instance, - endpointId, - cluster, - url, - targets, - scope, - port, - legacy, - routingMethod, - certificateName, - authMethod, - generated); - } - - private static IllegalArgumentException illegal(URI url, String reason) { - return new IllegalArgumentException("Invalid endpoint: " + url + ": " + reason); - } - - private Scope requireUnset(Scope scope) { - if (this.scope != null) { - throw new IllegalArgumentException("Cannot change endpoint scope. Already set to " + scope); - } - return scope; - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java deleted file mode 100644 index ef1f43eee69..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import java.util.Objects; - -/** - * A user-specified endpoint ID. This is typically the first part of an endpoint name. - * - * @author ogronnesby - */ -public class EndpointId implements Comparable<EndpointId> { - - private static final EndpointId DEFAULT = new EndpointId("default"); - - private final String id; - - private EndpointId(String id) { - this.id = requireNotEmpty(id); - } - - public String id() { return id; } - - @Override - public String toString() { - return "endpoint id '" + id + "'"; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - EndpointId that = (EndpointId) o; - return Objects.equals(id, that.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - private static String requireNotEmpty(String input) { - Objects.requireNonNull(input); - if (input.isEmpty()) { - throw new IllegalArgumentException("The value EndpointId was empty"); - } - return input; - } - - public static EndpointId defaultId() { return DEFAULT; } - - public static EndpointId of(String id) { return new EndpointId(id); } - - @Override - public int compareTo(EndpointId o) { - return id.compareTo(o.id); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java deleted file mode 100644 index 07fd6d9825d..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.collections.AbstractFilteringList; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.zone.AuthMethod; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; - -import java.util.Collection; -import java.util.List; -import java.util.Optional; - - -/** - * A list of endpoints for an application. - * - * @author mpolden - */ -public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> { - - public static final EndpointList EMPTY = EndpointList.copyOf(List.of()); - - private EndpointList(Collection<? extends Endpoint> endpoints, boolean negate) { - super(endpoints, negate, EndpointList::new); - if (endpoints.stream().distinct().count() != endpoints.size()) { - throw new IllegalArgumentException("Expected all endpoints to be distinct, got " + endpoints); - } - } - - /** Returns the subset of endpoints named according to given ID and scope */ - public EndpointList named(EndpointId id, Endpoint.Scope scope) { - return matching(endpoint -> endpoint.scope() == scope && // ID is only unique within a scope - endpoint.name().equals(id.id())); - } - - /** Returns the endpoint which has given DNS name, if any */ - public Optional<Endpoint> dnsName(String dnsName) { - return matching(endpoint -> endpoint.dnsName().equals(dnsName)).first(); - } - - /** Returns the subset of endpoints pointing to given cluster */ - public EndpointList cluster(ClusterSpec.Id cluster) { - return matching(endpoint -> endpoint.cluster().equals(cluster)); - } - - /** Returns the subset of endpoints pointing to given instance */ - public EndpointList instance(InstanceName instance) { - return matching(endpoint -> endpoint.instance().isPresent() && - endpoint.instance().get().equals(instance)); - } - - /** Returns the subset of endpoints which target all the given deployments */ - public EndpointList targets(List<DeploymentId> deployments) { - return matching(endpoint -> endpoint.deployments().containsAll(deployments)); - } - - /** Returns the subset of endpoints which target the given deployment */ - public EndpointList targets(DeploymentId deployment) { - return targets(List.of(deployment)); - } - - /** Returns the subset of endpoints that are considered legacy */ - public EndpointList legacy() { - return matching(Endpoint::legacy); - } - - /** Returns the subset of endpoints generated by the system */ - public EndpointList generated() { - return matching(endpoint -> endpoint.generated().isPresent()); - } - - /** Returns the subset of endpoints that require a rotation */ - public EndpointList requiresRotation() { - return matching(Endpoint::requiresRotation); - } - - /** Returns the subset of endpoints with given scope */ - public EndpointList scope(Endpoint.Scope scope) { - return matching(endpoint -> endpoint.scope() == scope); - } - - /** Returns the subset of endpoints that use direct routing */ - public EndpointList direct() { - return matching(endpoint -> endpoint.routingMethod().isDirect()); - } - - /** Returns the subset of endpoints that use shared routing */ - public EndpointList shared() { - return matching(endpoint -> endpoint.routingMethod().isShared()); - } - - /** Returns the subset of endpoints supporting given authentication method */ - public EndpointList authMethod(AuthMethod authMethod) { - return matching(endpoint -> endpoint.authMethod() == authMethod); - } - - public static EndpointList copyOf(Collection<Endpoint> endpoints) { - return new EndpointList(endpoints, false); - } - - public static EndpointList of(Endpoint ...endpoint) { - return copyOf(List.of(endpoint)); - } - - @Override - public String toString() { - return asList().toString(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java deleted file mode 100644 index 5f75d6105b5..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import ai.vespa.validation.Validation; -import com.yahoo.config.provision.zone.AuthMethod; - -import java.util.Objects; -import java.util.Optional; -import java.util.random.RandomGenerator; -import java.util.regex.Pattern; - -/** - * A system-generated endpoint, where the cluster and application parts are randomly generated. These become the - * first and second part of an endpoint name. See {@link Endpoint}. - * - * @author mpolden - */ -public record GeneratedEndpoint(String clusterPart, String applicationPart, AuthMethod authMethod, Optional<EndpointId> endpoint) { - - private static final Pattern CLUSTER_PART_PATTERN = Pattern.compile("^([a-f][a-f0-9]{7}|\\*)$"); - private static final Pattern APPLICATION_PART_PATTERN = Pattern.compile("^[a-f][a-f0-9]{7}$"); - - public GeneratedEndpoint { - Objects.requireNonNull(clusterPart); - Objects.requireNonNull(applicationPart); - Objects.requireNonNull(authMethod); - Objects.requireNonNull(endpoint); - - Validation.requireMatch(clusterPart, "Cluster part", CLUSTER_PART_PATTERN); - Validation.requireMatch(applicationPart, "Application part", APPLICATION_PART_PATTERN); - } - - /** Returns whether this was generated for an endpoint declared in {@link com.yahoo.config.application.api.DeploymentSpec} */ - public boolean declared() { - return endpoint.isPresent(); - } - - /** Returns whether this was generated for a cluster declared in {@link com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml} */ - public boolean cluster() { - return !declared(); - } - - /** Returns a copy of this with cluster part set to given value */ - public GeneratedEndpoint withClusterPart(String clusterPart) { - return new GeneratedEndpoint(clusterPart, applicationPart, authMethod, endpoint); - } - - /** Create a new endpoint part, using random as a source of randomness */ - public static String createPart(RandomGenerator random) { - String alphabet = "abcdef0123456789"; - StringBuilder sb = new StringBuilder(); - sb.append(alphabet.charAt(random.nextInt(6))); // Start with letter - for (int i = 0; i < 7; i++) { - sb.append(alphabet.charAt(random.nextInt(alphabet.length()))); - } - return sb.toString(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java deleted file mode 100644 index 939b3df9502..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.collections.AbstractFilteringList; -import com.yahoo.component.Version; -import com.yahoo.component.VersionCompatibility; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; - -import java.time.Instant; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; - -import static java.util.Comparator.comparing; -import static java.util.Comparator.naturalOrder; - -/** - * @author jonmv - */ -public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceList> { - - private final Map<ApplicationId, DeploymentStatus> instances; - - private InstanceList(Collection<? extends ApplicationId> items, boolean negate, Map<ApplicationId, DeploymentStatus> instances) { - super(items, negate, (i, n) -> new InstanceList(i, n, instances)); - this.instances = Map.copyOf(instances); - } - - /** - * Returns the subset of instances where all production deployments are compatible with the given version, - * and at least one known build is compatible with the given version. - * - * @param platform the version which applications returned are compatible with - */ - public InstanceList compatibleWithPlatform(Version platform, Function<ApplicationId, VersionCompatibility> compatibility) { - return matching(id -> instance(id).productionDeployments().values().stream() - .flatMap(deployment -> application(id).revisions().get(deployment.revision()).compileVersion().stream()) - .noneMatch(version -> compatibility.apply(id).refuse(platform, version)) - && application(id).revisions().production().stream() - .anyMatch(revision -> revision.compileVersion() - .map(compiled -> compatibility.apply(id).accept(platform, compiled)) - .orElse(true))); - } - - /** - * Returns the subset of instances whose application have a deployment on the given major, - * or specify it in deployment spec, - * or which are on a {@link VespaVersion.Confidence#legacy} platform, and do not specify that in deployment spec. - * - * @param targetMajorVersion the target major version which applications returned allows upgrading to - */ - public InstanceList allowingMajorVersion(int targetMajorVersion, VersionStatus versions) { - return matching(id -> { - Application application = application(id); - Optional<Integer> majorVersion = application.deploymentSpec().majorVersion(); - if (majorVersion.isPresent()) - return majorVersion.get() >= targetMajorVersion; - - for (List<Deployment> deployments : application.productionDeployments().values()) - for (Deployment deployment : deployments) { - if (deployment.version().getMajor() >= targetMajorVersion) return true; - if (versions.version(deployment.version()).confidence() == Confidence.legacy) return true; - } - return false; - }); - } - - /** Returns the subset of instances that are allowed to upgrade to the given version at the given time */ - public InstanceList canUpgradeAt(Version version, Instant instant) { - return matching(id -> instances.get(id).instanceSteps().get(id.instance()) - .readiness(Change.of(version)).okAt(instant)); - } - - /** Returns the subset of instances which have at least one production deployment */ - public InstanceList withProductionDeployment() { - return matching(id -> instance(id).productionDeployments().size() > 0); - } - - /** Returns the subset of instances which contain declared jobs */ - public InstanceList withDeclaredJobs() { - return matching(id -> instances.get(id).application().revisions().last().isPresent() - && instances.get(id).jobSteps().values().stream() - .anyMatch(job -> job.isDeclared() && job.job().get().application().equals(id))); - } - - /** Returns the subset of instances which have at least one deployment on a lower version than the given one, or which have no production deployments */ - public InstanceList onLowerVersionThan(Version version) { - return matching(id -> instance(id).productionDeployments().isEmpty() - || instance(id).productionDeployments().values().stream() - .anyMatch(deployment -> deployment.version().isBefore(version))); - } - - /** Returns the subset of instances that has completed deployment of given change */ - public InstanceList hasCompleted(Change change) { - return matching(id -> instances.get(id).hasCompleted(id.instance(), change)); - } - - /** Returns the subset of instances which are currently deploying a change */ - public InstanceList deploying() { - return matching(id -> instance(id).change().hasTargets()); - } - - /** Returns the subset of instances which are currently deploying a new revision */ - public InstanceList changingRevision() { - return matching(id -> instance(id).change().revision().isPresent()); - } - - /** Returns the subset of instances which currently have failing jobs on the given version */ - public InstanceList failingOn(Version version) { - return matching(id -> ! instances.get(id).instanceJobs().get(id).failingHard() - .lastCompleted().on(version).isEmpty()); - } - - /** Returns the subset of instances which are not pinned to a certain Vespa version. */ - public InstanceList unpinned() { - return matching(id -> ! instance(id).change().isPlatformPinned()); - } - - /** Returns the subset of instances which are currently failing a job. */ - public InstanceList failing() { - return matching(id -> ! instances.get(id).instanceJobs().get(id).failingHard().isEmpty()); - } - - /** Returns the subset of instances which are currently failing an upgrade. */ - public InstanceList failingUpgrade() { - return matching(id -> ! instances.get(id).instanceJobs().get(id).failingHard().not().failingApplicationChange().isEmpty()); - } - - /** Returns the subset of instances which are upgrading (to any version), not considering block windows. */ - public InstanceList upgrading() { - return matching(id -> instance(id).change().platform().isPresent()); - } - - /** Returns the subset of instances which are currently upgrading to the given version */ - public InstanceList upgradingTo(Version version) { - return upgradingTo(List.of(version)); - } - - - /** Returns the subset of instances which are currently upgrading to the given version */ - public InstanceList upgradingTo(Collection<Version> versions) { - return matching(id -> versions.stream().anyMatch(version -> instance(id).change().platform().equals(Optional.of(version)))); - } - - public InstanceList with(DeploymentSpec.UpgradePolicy policy) { - return matching(id -> application(id).deploymentSpec().requireInstance(id.instance()).upgradePolicy() == policy); - } - - /** Returns the subset of instances which started failing on the given version */ - public InstanceList startedFailingOn(Version version) { - return matching(id -> ! instances.get(id).instanceJobs().get(id).firstFailing().on(version).isEmpty()); - } - - /** Returns this list sorted by increasing oldest production deployment version. Applications without any deployments are ordered first. */ - public InstanceList byIncreasingDeployedVersion() { - return sortedBy(comparing(id -> instance(id).productionDeployments().values().stream() - .map(Deployment::version) - .min(naturalOrder()) - .orElse(Version.emptyVersion))); - } - - private Application application(ApplicationId id) { - return instances.get(id).application(); - } - - private Instance instance(ApplicationId id) { - return application(id).require(id.instance()); - } - - public static InstanceList from(DeploymentStatusList statuses) { - Map<ApplicationId, DeploymentStatus> instances = new HashMap<>(); - for (DeploymentStatus status : statuses.asList()) - for (InstanceName instance : status.application().deploymentSpec().instanceNames()) - instances.put(status.application().id().instance(instance), status); - return new InstanceList(instances.keySet(), false, instances); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/MailVerifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/MailVerifier.java deleted file mode 100644 index 9ff3206ee06..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/MailVerifier.java +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.LockedTenant; -import com.yahoo.vespa.hosted.controller.TenantController; -import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; -import com.yahoo.vespa.hosted.controller.notification.MailTemplating; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.PendingMailVerification; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; -import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; - -import java.time.Clock; -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.UUID; - - -/** - * @author olaa - */ -public class MailVerifier { - - private static final Duration VERIFICATION_DEADLINE = Duration.ofDays(7); - - private final TenantController tenantController; - private final Mailer mailer; - private final CuratorDb curatorDb; - private final Clock clock; - private final MailTemplating mailTemplating; - - public MailVerifier(ConsoleUrls consoleUrls, TenantController tenantController, Mailer mailer, CuratorDb curatorDb, Clock clock) { - this.tenantController = tenantController; - this.mailer = mailer; - this.curatorDb = curatorDb; - this.clock = clock; - this.mailTemplating = new MailTemplating(consoleUrls); - } - - public PendingMailVerification sendMailVerification(TenantName tenantName, String email, PendingMailVerification.MailType mailType) { - if (!email.contains("@")) { - throw new IllegalArgumentException("Invalid email address"); - } - - var verificationCode = UUID.randomUUID().toString(); - var verificationDeadline = clock.instant().plus(VERIFICATION_DEADLINE); - var pendingMailVerification = new PendingMailVerification(tenantName, email, verificationCode, verificationDeadline, mailType); - writePendingVerification(pendingMailVerification); - mailer.send(mailOf(pendingMailVerification)); - return pendingMailVerification; - } - - public Optional<PendingMailVerification> resendMailVerification(TenantName tenantName, String email, PendingMailVerification.MailType mailType) { - var oldPendingVerification = curatorDb.listPendingMailVerifications() - .stream() - .filter(pendingMailVerification -> - pendingMailVerification.getMailAddress().equals(email) && - pendingMailVerification.getMailType().equals(mailType) && - pendingMailVerification.getTenantName().equals(tenantName) - ).findFirst(); - - if (oldPendingVerification.isEmpty()) - return Optional.empty(); - - try (var lock = curatorDb.lockPendingMailVerification(oldPendingVerification.get().getVerificationCode())) { - curatorDb.deletePendingMailVerification(oldPendingVerification.get()); - } - - return Optional.of(sendMailVerification(tenantName, email, mailType)); - } - - public boolean verifyMail(String verificationCode) { - return curatorDb.getPendingMailVerification(verificationCode) - .filter(pendingMailVerification -> pendingMailVerification.getVerificationDeadline().isAfter(clock.instant())) - .map(pendingMailVerification -> { - var tenant = requireCloudTenant(pendingMailVerification.getTenantName()); - var oldTenantInfo = tenant.info(); - var updatedTenantInfo = switch (pendingMailVerification.getMailType()) { - case NOTIFICATIONS -> withTenantContacts(oldTenantInfo, pendingMailVerification); - case TENANT_CONTACT -> oldTenantInfo.withContact(oldTenantInfo.contact() - .withEmail(oldTenantInfo.contact().email().withVerification(true))); - case BILLING -> withVerifiedBillingMail(oldTenantInfo); - }; - - tenantController.lockOrThrow(tenant.name(), LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withInfo(updatedTenantInfo); - tenantController.store(lockedTenant); - }); - - try (var lock = curatorDb.lockPendingMailVerification(pendingMailVerification.getVerificationCode())) { - curatorDb.deletePendingMailVerification(pendingMailVerification); - } - return true; - }).orElse(false); - } - - private TenantInfo withTenantContacts(TenantInfo oldInfo, PendingMailVerification pendingMailVerification) { - var newContacts = oldInfo.contacts().ofType(TenantContacts.EmailContact.class) - .stream() - .map(contact -> { - if (pendingMailVerification.getMailAddress().equals(contact.email().getEmailAddress())) - return contact.withEmail(contact.email().withVerification(true)); - return contact; - }).toList(); - return oldInfo.withContacts(new TenantContacts(newContacts)); - } - - private TenantInfo withVerifiedBillingMail(TenantInfo oldInfo) { - var verifiedMail = oldInfo.billingContact().contact().email().withVerification(true); - var billingContact = oldInfo.billingContact() - .withContact(oldInfo.billingContact().contact().withEmail(verifiedMail)); - return oldInfo.withBilling(billingContact); - } - - private void writePendingVerification(PendingMailVerification pendingMailVerification) { - try (var lock = curatorDb.lockPendingMailVerification(pendingMailVerification.getVerificationCode())) { - curatorDb.writePendingMailVerification(pendingMailVerification); - } - } - - private CloudTenant requireCloudTenant(TenantName tenantName) { - return tenantController.get(tenantName) - .filter(tenant -> tenant.type() == Tenant.Type.cloud) - .map(CloudTenant.class::cast) - .orElseThrow(() -> new IllegalStateException("Mail verification is only applicable for cloud tenants")); - } - - private Mail mailOf(PendingMailVerification pendingMailVerification) { - var message = mailTemplating.generateMailVerificationHtml(pendingMailVerification); - return new Mail(List.of(pendingMailVerification.getMailAddress()), "Please verify your email", "", message); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/QuotaUsage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/QuotaUsage.java deleted file mode 100644 index f5642f44485..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/QuotaUsage.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import java.util.Objects; -import java.util.OptionalDouble; - -/** - * @author ogronnesby - */ -public class QuotaUsage { - - public static final QuotaUsage none = new QuotaUsage(0.0); - - private final double rate; - - private QuotaUsage(double rate) { - this.rate = rate; - } - - public double rate() { - return rate; - } - - public QuotaUsage add(QuotaUsage addend) { - return create(rate + addend.rate); - } - - public QuotaUsage sub(QuotaUsage subtrahend) { - return create(rate - subtrahend.rate); - } - - public static QuotaUsage create(OptionalDouble rate) { - if (rate.isEmpty()) { - return QuotaUsage.none; - } - return new QuotaUsage(rate.getAsDouble()); - } - - public static QuotaUsage create(double rate) { - return new QuotaUsage(rate); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - QuotaUsage that = (QuotaUsage) o; - return Double.compare(that.rate, rate) == 0; - } - - @Override - public int hashCode() { - return Objects.hash(rate); - } - - @Override - public String toString() { - return "QuotaUsage{" + - "rate=" + rate + - '}'; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java deleted file mode 100644 index d3ab2216539..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.text.Text; -import com.yahoo.vespa.applicationmodel.InfrastructureApplication; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; - -import java.util.Arrays; -import java.util.EnumSet; -import java.util.List; -import java.util.Optional; - -/** - * This represents a system-level application in hosted Vespa. All infrastructure nodes in a hosted Vespa zones are - * allocated to a system application. - * - * @author mpolden - */ -public enum SystemApplication { - - controllerHost(InfrastructureApplication.CONTROLLER_HOST), - configServerHost(InfrastructureApplication.CONFIG_SERVER_HOST), - configServer(InfrastructureApplication.CONFIG_SERVER), - proxyHost(InfrastructureApplication.PROXY_HOST), - proxy(InfrastructureApplication.PROXY, configServer), - tenantHost(InfrastructureApplication.TENANT_HOST); - - /** The tenant owning all system applications */ - public static final TenantName TENANT = TenantName.from("hosted-vespa"); - - private final InfrastructureApplication application; - private final List<SystemApplication> dependencies; - - SystemApplication(InfrastructureApplication application, SystemApplication... dependencies) { - this.application = application; - this.dependencies = List.of(dependencies); - } - - public ApplicationId id() { - return application.id(); - } - - /** The node type that is implicitly allocated to this */ - public NodeType nodeType() { - return application.nodeType(); - } - - /** Returns the system applications that should upgrade before this */ - public List<SystemApplication> dependencies() { return dependencies; } - - /** Returns whether this system application has an application package */ - public boolean hasApplicationPackage() { - return this == proxy; - } - - /** Returns whether config for this application has converged in given zone */ - public boolean configConvergedIn(ZoneId zone, Controller controller, Optional<Version> version) { - if (!hasApplicationPackage()) { - return true; - } - return controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id(), zone), version) - .map(ServiceConvergence::converged) - .orElse(false); - } - - /** Returns whether this should receive OS upgrades */ - public boolean shouldUpgradeOs() { - return nodeType().isHost(); - } - - /** All system applications that are not the controller */ - public static List<SystemApplication> notController() { - return List.copyOf(EnumSet.complementOf(EnumSet.of(SystemApplication.controllerHost))); - } - - /** All system applications */ - public static List<SystemApplication> all() { - return List.of(values()); - } - - /** Returns the system application matching given id, if any */ - public static Optional<SystemApplication> matching(ApplicationId id) { - return Arrays.stream(values()).filter(app -> app.id().equals(id)).findFirst(); - } - - @Override - public String toString() { - return Text.format("system application %s of type %s", id(), nodeType()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java deleted file mode 100644 index 9c9ec35fa80..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.TenantName; - -import java.util.Objects; - -/** - * Tenant and application name pair. - * - * @author jonmv - */ -public class TenantAndApplicationId implements Comparable<TenantAndApplicationId> { - - private final TenantName tenant; - private final ApplicationName application; - - private TenantAndApplicationId(TenantName tenant, ApplicationName application) { - requireNonBlank(tenant.value(), "Tenant name"); - requireNonBlank(application.value(), "Application name"); - this.tenant = tenant; - this.application = application; - } - - public static TenantAndApplicationId from(TenantName tenant, ApplicationName application) { - return new TenantAndApplicationId(tenant, application); - } - - public static TenantAndApplicationId from(String tenant, String application) { - return from(TenantName.from(tenant), ApplicationName.from(application)); - } - - public static TenantAndApplicationId fromSerialized(String value) { - String[] parts = value.split(":"); - if (parts.length != 2) - throw new IllegalArgumentException("Serialized value should be '<tenant>:<application>', but was '" + value + "'"); - - return from(parts[0], parts[1]); - } - - public static TenantAndApplicationId from(ApplicationId id) { - return from(id.tenant(), id.application()); - } - - public ApplicationId defaultInstance() { - return instance(InstanceName.defaultName()); - } - - public ApplicationId instance(String instance) { - return instance(InstanceName.from(instance)); - } - - public ApplicationId instance(InstanceName instance) { - return ApplicationId.from(tenant, application, instance); - } - - public String serialized() { - return tenant.value() + ":" + application.value(); - } - - public TenantName tenant() { - return tenant; - } - - public ApplicationName application() { - return application; - } - - @Override - public boolean equals(Object other) { - if (this == other) return true; - if (other == null || getClass() != other.getClass()) return false; - TenantAndApplicationId that = (TenantAndApplicationId) other; - return tenant.equals(that.tenant) && - application.equals(that.application); - } - - @Override - public int hashCode() { - return Objects.hash(tenant, application); - } - - @Override - public int compareTo(TenantAndApplicationId other) { - int tenantComparison = tenant.compareTo(other.tenant); - return tenantComparison != 0 ? tenantComparison : application.compareTo(other.application); - } - - @Override - public String toString() { - return tenant.value() + "." + application.value(); - } - - private static void requireNonBlank(String value, String name) { - Objects.requireNonNull(value, name + " cannot be null"); - if (name.isBlank()) - throw new IllegalArgumentException(name + " cannot be blank"); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java deleted file mode 100644 index 6a685281dbb..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Core application model - * - * @author bratseth - */ -@ExportPackage -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java deleted file mode 100644 index 27e45aa1e7d..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import com.google.common.hash.Hasher; -import com.google.common.hash.Hashing; -import com.google.common.hash.HashingOutputStream; -import com.yahoo.component.Version; -import com.yahoo.config.application.FileSystemWrapper; -import com.yahoo.config.application.FileSystemWrapper.FileWrapper; -import com.yahoo.config.application.XmlPreProcessor; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Tags; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.archive.ArchiveStreamReader; -import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile; -import com.yahoo.vespa.archive.ArchiveStreamReader.Options; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.deployment.ZipBuilder; -import com.yahoo.yolean.Exceptions; -import org.w3c.dom.Document; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.UncheckedIOException; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.function.Function; -import java.util.function.Predicate; - -import static com.yahoo.slime.Type.NIX; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.stream.Collectors.toMap; - -/** - * A representation of the content of an application package. - * Only meta-data content can be accessed as anything other than compressed data. - * A package is identified by a hash of the content. - * - * @author bratseth - * @author jonmv - */ -public class ApplicationPackage { - - public static final String deploymentFile = "deployment.xml"; - static final String trustedCertificatesDir = "security/"; - static final String trustedCertificatesFile = trustedCertificatesDir + "clients.pem"; - static final String buildMetaFile = "build-meta.json"; - static final String validationOverridesFile = "validation-overrides.xml"; - static final String servicesFile = "services.xml"; - static final Set<String> prePopulated = Set.of(deploymentFile, validationOverridesFile, servicesFile, buildMetaFile, trustedCertificatesFile); - - private static Hasher hasher() { return Hashing.murmur3_128().newHasher(); } - - private final String bundleHash; - private final byte[] zippedContent; - private final DeploymentSpec deploymentSpec; - private final ValidationOverrides validationOverrides; - private final ZipArchiveCache files; - private final Optional<Version> compileVersion; - private final Optional<Instant> buildTime; - private final Optional<Version> parentVersion; - - /** - * Creates an application package from its zipped content. - * This <b>assigns ownership</b> of the given byte array to this class; - * it must not be further changed by the caller. - */ - public ApplicationPackage(byte[] zippedContent) { - this(zippedContent, false, false); - } - - /** - * Creates an application package from its zipped content. - * This <b>assigns ownership</b> of the given byte array to this class; - * it must not be further changed by the caller. - * If 'requireFiles' is true, files needed by deployment orchestration must be present. - */ - public ApplicationPackage(byte[] zippedContent, boolean requireFiles, boolean checkCertificateFile) { - this.zippedContent = Objects.requireNonNull(zippedContent, "The application package content cannot be null"); - this.files = new ZipArchiveCache(zippedContent, prePopulated, checkCertificateFile); - - Optional<DeploymentSpec> deploymentSpec = files.get(deploymentFile).map(bytes -> new String(bytes, UTF_8)).map(DeploymentSpec::fromXml); - if (requireFiles && deploymentSpec.isEmpty()) - throw new IllegalArgumentException("Missing required file '" + deploymentFile + "'"); - this.deploymentSpec = deploymentSpec.orElse(DeploymentSpec.empty); - - this.validationOverrides = files.get(validationOverridesFile).map(bytes -> new String(bytes, UTF_8)).map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); - - Optional<Inspector> buildMetaObject = files.get(buildMetaFile).map(SlimeUtils::jsonToSlime).map(Slime::get); - this.compileVersion = buildMetaObject.flatMap(object -> parse(object, "compileVersion", field -> Version.fromString(field.asString()))); - this.buildTime = buildMetaObject.flatMap(object -> parse(object, "buildTime", field -> Instant.ofEpochMilli(field.asLong()))); - this.parentVersion = buildMetaObject.flatMap(object -> parse(object, "parentVersion", field -> Version.fromString(field.asString()))); - - this.bundleHash = calculateBundleHash(zippedContent); - - preProcessAndPopulateCache(); - } - - /** Hash of all files and settings that influence what is deployed to config servers. */ - public String bundleHash() { - return bundleHash; - } - - /** Returns the content of this package. The content <b>must not</b> be modified. */ - public byte[] zippedContent() { return zippedContent; } - - /** - * Returns the deployment spec from the deployment.xml file of the package content.<br> - * This is the DeploymentSpec.empty instance if this package does not contain a deployment.xml file.<br> - * <em>NB: <strong>Always</strong> read deployment spec from the {@link Application}, for deployment orchestration.</em> - */ - public DeploymentSpec deploymentSpec() { return deploymentSpec; } - - /** - * Returns the validation overrides from the validation-overrides.xml file of the package content. - * This is the ValidationOverrides.empty instance if this package does not contain a validation-overrides.xml file. - */ - public ValidationOverrides validationOverrides() { return validationOverrides; } - - /** Returns a basic variant of services.xml contained in this package, pre-processed according to given deployment and tags */ - public BasicServicesXml services(DeploymentId deployment, Tags tags) { - FileWrapper servicesXml = files.wrapper().wrap(Paths.get(servicesFile)); - if (!servicesXml.exists()) return BasicServicesXml.empty; - try { - Document document = new XmlPreProcessor(files.wrapper().wrap(Paths.get("./")), - new InputStreamReader(new ByteArrayInputStream(servicesXml.content()), UTF_8), - deployment.applicationId().instance(), - deployment.zoneId().environment(), - deployment.zoneId().region(), - tags).run(); - return BasicServicesXml.parse(document); - } catch (IllegalArgumentException e) { - throw e; - } catch (Exception e) { - throw new IllegalArgumentException(e); - } - } - - /** Returns the platform version which package was compiled against, if known. */ - public Optional<Version> compileVersion() { return compileVersion; } - - /** Returns the time this package was built, if known. */ - public Optional<Instant> buildTime() { return buildTime; } - - /** Returns the parent version used to compile the package, if known. */ - public Optional<Version> parentVersion() { return parentVersion; } - - private static <Type> Optional<Type> parse(Inspector buildMetaObject, String fieldName, Function<Inspector, Type> mapper) { - Inspector field = buildMetaObject.field(fieldName); - if ( ! field.valid() || field.type() == NIX) - return Optional.empty(); - try { - return Optional.of(mapper.apply(buildMetaObject.field(fieldName))); - } - catch (RuntimeException e) { - throw new IllegalArgumentException("Failed parsing \"" + fieldName + "\" in '" + buildMetaFile + "': " + Exceptions.toMessageString(e)); - } - } - - /** Creates a valid application package that will remove all application's deployments */ - public static ApplicationPackage deploymentRemoval() { - return new ApplicationPackage(filesZip(Map.of(validationOverridesFile, allValidationOverrides().xmlForm().getBytes(UTF_8), - deploymentFile, DeploymentSpec.empty.xmlForm().getBytes(UTF_8)))); - } - - /** Returns a zip containing metadata about deployments of this package by the given job. */ - public byte[] metaDataZip() { - return cacheZip(); - } - - private void preProcessAndPopulateCache() { - FileWrapper servicesXml = files.wrapper().wrap(Paths.get(servicesFile)); - if (servicesXml.exists()) - try { - new XmlPreProcessor(files.wrapper().wrap(Paths.get("./")), - new InputStreamReader(new ByteArrayInputStream(servicesXml.content()), UTF_8), - InstanceName.defaultName(), - Environment.prod, - RegionName.defaultName(), - Tags.empty()) - .run(); // Populates the zip archive cache with files that would be included. - } - catch (IllegalArgumentException e) { - throw e; - } - catch (Exception e) { - throw new IllegalArgumentException(e); - } - } - - private byte[] cacheZip() { - return filesZip(files.cache.entrySet().stream() - .filter(entry -> entry.getValue().isPresent()) - .collect(toMap(entry -> entry.getKey().toString(), - entry -> entry.getValue().get()))); - } - - public static byte[] filesZip(Map<String, byte[]> files) { - try (ZipBuilder zipBuilder = new ZipBuilder(files.values().stream().mapToInt(bytes -> bytes.length).sum() + 512)) { - files.forEach(zipBuilder::add); - zipBuilder.close(); - return zipBuilder.toByteArray(); - } - } - - private static ValidationOverrides allValidationOverrides() { - String until = DateTimeFormatter.ISO_LOCAL_DATE.format(Instant.now().plus(Duration.ofDays(25)).atZone(ZoneOffset.UTC)); - StringBuilder validationOverridesContents = new StringBuilder(1000); - validationOverridesContents.append("<validation-overrides version=\"1.0\">\n"); - for (ValidationId validationId: ValidationId.values()) - validationOverridesContents.append("\t<allow until=\"").append(until).append("\">").append(validationId.value()).append("</allow>\n"); - validationOverridesContents.append("</validation-overrides>\n"); - - return ValidationOverrides.fromXml(validationOverridesContents.toString()); - } - - // Hashes all files and settings that require a deployment to be forwarded to configservers - private String calculateBundleHash(byte[] zippedContent) { - Predicate<String> entryMatcher = name -> ! name.endsWith(deploymentFile) && ! name.endsWith(buildMetaFile); - Options options = Options.standard().pathPredicate(entryMatcher); - HashingOutputStream hashOut = new HashingOutputStream(Hashing.murmur3_128(-1), OutputStream.nullOutputStream()); - ArchiveFile file; - try (ArchiveStreamReader reader = ArchiveStreamReader.ofZip(new ByteArrayInputStream(zippedContent), options)) { - while ((file = reader.readNextTo(hashOut)) != null) { - hashOut.write(file.path().toString().getBytes(UTF_8)); - } - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - return hasher().putLong(hashOut.hash().asLong()) - .putInt(deploymentSpec.deployableHashCode()) - .hash().toString(); - } - - public static String calculateHash(byte[] bytes) { - return hasher().putBytes(bytes) - .hash().toString(); - } - - - /** Maps normalized paths to cached content read from a zip archive. */ - private static class ZipArchiveCache { - - /** Max size of each extracted file */ - private static final int maxSize = 10 << 20; // 10 Mb - - private final byte[] zip; - private final Map<Path, Optional<byte[]>> cache; - - public ZipArchiveCache(byte[] zip, Collection<String> prePopulated, boolean checkCertificateFile) { - this.zip = zip; - this.cache = new ConcurrentSkipListMap<>(); - this.cache.putAll(read(prePopulated)); - if (checkCertificateFile) - verifyThatTrustedCertificateExists(); - } - - public Optional<byte[]> get(String path) { - return get(Paths.get(path)); - } - - public Optional<byte[]> get(Path path) { - return cache.computeIfAbsent(path.normalize(), read(List.of(path.normalize().toString()))::get); - } - - public FileSystemWrapper wrapper() { - return FileSystemWrapper.ofFiles(Path.of("./"), // zip archive root - path -> get(path).isPresent(), // Assume content asked for will also be read ... - path -> get(path).orElseThrow(() -> new NoSuchFileException(path.toString()))); - } - - private Map<Path, Optional<byte[]>> read(Collection<String> names) { - var entries = findZipFileEntries(names::contains); - names.stream().map(Paths::get).forEach(path -> entries.putIfAbsent(path.normalize(), Optional.empty())); - return entries; - } - - - private void verifyThatTrustedCertificateExists() { - // Any name is valid for certificate files - var entries = findZipFileEntries((entry) -> entry.contains(trustedCertificatesDir) && entry.endsWith(".pem")); - if (entries.size() == 0) - throw new IllegalArgumentException("No client certificate found in " + trustedCertificatesDir + " in application package" + - ", see https://cloud.vespa.ai/en/security/guide"); - } - - private Map<Path, Optional<byte[]>> findZipFileEntries(Predicate<String> names) { - return ZipEntries.from(zip, names, maxSize, true) - .asList().stream() - .collect(toMap(entry -> Paths.get(entry.name()).normalize(), - ZipEntries.ZipEntryWithContent::content)); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java deleted file mode 100644 index bd08def6cec..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.vespa.hosted.controller.application.pkg.ZipEntries.ZipEntryWithContent; - -/** - * @author freva - */ -public class ApplicationPackageDiff { - - public static byte[] diffAgainstEmpty(ApplicationPackage right) { - byte[] emptyZip = new byte[]{80, 75, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - return diff(new ApplicationPackage(emptyZip), right); - } - - public static byte[] diff(ApplicationPackage left, ApplicationPackage right) { - return diff(left, right, 10 << 20, 1 << 20, 10 << 20); - } - - static byte[] diff(ApplicationPackage left, ApplicationPackage right, int maxFileSizeToDiff, int maxDiffSizePerFile, int maxTotalDiffSize) { - if (Arrays.equals(left.zippedContent(), right.zippedContent())) return "No diff\n".getBytes(StandardCharsets.UTF_8); - - Map<String, ZipEntryWithContent> leftContents = readContents(left, maxFileSizeToDiff); - Map<String, ZipEntryWithContent> rightContents = readContents(right, maxFileSizeToDiff); - - StringBuilder sb = new StringBuilder(); - List<String> files = Stream.of(leftContents, rightContents) - .flatMap(contents -> contents.keySet().stream()) - .sorted() - .distinct() - .toList(); - for (String file : files) { - if (sb.length() > maxTotalDiffSize) - sb.append("--- ").append(file).append('\n').append("Diff skipped: Total diff size >").append(maxTotalDiffSize).append("B)\n\n"); - else - diff(Optional.ofNullable(leftContents.get(file)), Optional.ofNullable(rightContents.get(file)), maxDiffSizePerFile) - .ifPresent(diff -> sb.append("--- ").append(file).append('\n').append(diff).append('\n')); - } - - return (sb.length() == 0 ? "No diff\n" : sb.toString()).getBytes(StandardCharsets.UTF_8); - } - - private static Optional<String> diff(Optional<ZipEntryWithContent> left, Optional<ZipEntryWithContent> right, int maxDiffSizePerFile) { - Optional<byte[]> leftContent = left.flatMap(ZipEntryWithContent::content); - Optional<byte[]> rightContent = right.flatMap(ZipEntryWithContent::content); - if (leftContent.isPresent() && rightContent.isPresent() && Arrays.equals(leftContent.get(), rightContent.get())) - return Optional.empty(); - - if (Stream.of(left, right).flatMap(Optional::stream).anyMatch(entry -> entry.content().isEmpty())) - return Optional.of(String.format("Diff skipped: File too large (%s -> %s)\n", - left.map(e -> e.size() + "B").orElse("new file"), right.map(e -> e.size() + "B").orElse("file deleted"))); - - if (Stream.of(leftContent, rightContent).flatMap(Optional::stream).anyMatch(c -> isBinary(c))) - return Optional.of(String.format("Diff skipped: File is binary (%s -> %s)\n", - left.map(e -> e.size() + "B").orElse("new file"), right.map(e -> e.size() + "B").orElse("file deleted"))); - - return LinesComparator.diff( - leftContent.map(c -> lines(c)).orElseGet(List::of), - rightContent.map(c -> lines(c)).orElseGet(List::of)) - .map(diff -> diff.length() > maxDiffSizePerFile ? "Diff skipped: Diff too large (" + diff.length() + "B)\n" : diff); - } - - private static Map<String, ZipEntryWithContent> readContents(ApplicationPackage app, int maxFileSizeToDiff) { - return ZipEntries.from(app.zippedContent(), entry -> true, maxFileSizeToDiff, false).asList().stream() - .collect(Collectors.toMap(ZipEntryWithContent::name, e -> e)); - } - - private static List<String> lines(byte[] data) { - List<String> lines = new ArrayList<>(Math.min(16, data.length / 100)); - try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(data), StandardCharsets.UTF_8))) { - String line; - while ((line = bufferedReader.readLine()) != null) { - lines.add(line); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return lines; - } - - private static boolean isBinary(byte[] data) { - if (data.length == 0) return false; - - int lengthToCheck = Math.min(data.length, 10000); - int ascii = 0; - - for (int i = 0; i < lengthToCheck; i++) { - byte b = data[i]; - if (b < 0x9) return true; - - // TAB, newline/line feed, carriage return - if (b == 0x9 || b == 0xA || b == 0xD) ascii++; - else if (b >= 0x20 && b <= 0x7E) ascii++; - } - - return (double) ascii / lengthToCheck < 0.95; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java deleted file mode 100644 index e13dd2acbdb..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import java.io.ByteArrayOutputStream; -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.nio.file.attribute.FileTime; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.function.UnaryOperator; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; - -import static java.io.OutputStream.nullOutputStream; -import static java.lang.Math.min; - -/** - * Wraps a zipped application package stream. - * This allows replacing content as the input stream is read. - * This also retains a truncated {@link ApplicationPackage}, containing only the specified set of files, - * which can be accessed when this stream is fully exhausted. - * - * @author jonmv - */ -public class ApplicationPackageStream { - - private final Supplier<Replacer> replacer; - private final Supplier<Predicate<String>> filter; - private final Supplier<InputStream> in; - private final AtomicReference<ApplicationPackage> truncatedPackage = new AtomicReference<>(); - private final FileTime createdAt = FileTime.fromMillis(System.currentTimeMillis()); - - /** Stream that copies application meta and other XML files from the input stream to its {@link #truncatedPackage()} when exhausted. */ - public ApplicationPackageStream(Supplier<InputStream> in) { - this(in, () -> name -> ApplicationPackage.prePopulated.contains(name) || name.endsWith(".xml"), Map.of()); - } - - /** Stream that copies the indicated entries from the input stream to its {@link #truncatedPackage()} when exhausted. */ - public ApplicationPackageStream(Supplier<InputStream> in, Supplier<Predicate<String>> truncation) { - this(in, truncation, Map.of()); - } - - /** Stream that replaces the indicated entries, and copies the filtered entries to its {@link #truncatedPackage()} when exhausted. */ - public ApplicationPackageStream(Supplier<InputStream> in, Supplier<Predicate<String>> truncation, Map<String, UnaryOperator<InputStream>> replacements) { - this(in, truncation, Replacer.of(replacements)); - } - - /** Stream that uses the given replacer to modify content, and copies the filtered entries to its {@link #truncatedPackage()} when exhausted. */ - public ApplicationPackageStream(Supplier<InputStream> in, Supplier<Predicate<String>> truncation, Supplier<Replacer> replacer) { - this.in = in; - this.filter = truncation; - this.replacer = replacer; - } - - /** - * Returns a new stream containing the zipped application package this wraps. Separate streams may exist concurrently, - * and the first to be exhausted will populate the truncated application package. - */ - public InputStream zipStream() { - return new Stream(in.get(), replacer.get(), filter.get(), createdAt, truncatedPackage); - } - - /** - * Returns the application package backed by only the files indicated by the truncation filter. - * Throws if no instances of {@link #zipStream()} have been exhausted yet. - */ - public ApplicationPackage truncatedPackage() { - ApplicationPackage truncated = truncatedPackage.get(); - if (truncated == null) throw new IllegalStateException("must completely exhaust input before reading package"); - return truncated; - } - - private static class Stream extends InputStream { - - private final byte[] inBuffer = new byte[1 << 16]; - private final ByteArrayOutputStream teeOut = new ByteArrayOutputStream(1 << 16); - private final ZipOutputStream teeZip = new ZipOutputStream(teeOut); - private final ByteArrayOutputStream out = new ByteArrayOutputStream(1 << 16); - private final ZipOutputStream outZip = new ZipOutputStream(out); - private final AtomicReference<ApplicationPackage> truncatedPackage; - private final InputStream in; - private final ZipInputStream inZip; - private final Replacer replacer; - private final Predicate<String> filter; - private final FileTime createdAt; - private byte[] currentOut = new byte[0]; - private InputStream currentIn = InputStream.nullInputStream(); - private boolean includeCurrent = false; - private int pos = 0; - private boolean closed = false; - private boolean done = false; - - private Stream(InputStream in, Replacer replacer, Predicate<String> filter, FileTime createdAt, AtomicReference<ApplicationPackage> truncatedPackage) { - this.in = in; - this.inZip = new ZipInputStream(in); - this.replacer = replacer; - this.filter = filter; - this.createdAt = createdAt; - this.truncatedPackage = truncatedPackage; - } - - private void fill() throws IOException { - if (done) return; - while (out.size() == 0) { - // Exhaust current entry first. - int i, n = out.size(); - while (out.size() == 0 && (i = currentIn.read(inBuffer)) != -1) { - if (includeCurrent) teeZip.write(inBuffer, 0, i); - outZip.write(inBuffer, 0, i); - n += i; - } - - // Current entry exhausted, look for next. - if (n == 0) { - next(); - if (done) break; - } - } - - currentOut = out.toByteArray(); - out.reset(); - pos = 0; - } - - private void next() throws IOException { - if (includeCurrent) teeZip.closeEntry(); - outZip.closeEntry(); - - ZipEntry next = inZip.getNextEntry(); - String name; - FileTime modifiedAt; - InputStream content = null; - if (next == null) { - // We may still have replacements to fill in, but if we don't, we're done filling, forever! - name = replacer.next(); - modifiedAt = createdAt; - if (name == null) { - outZip.close(); // This typically makes new output available, so must check for that after this. - teeZip.close(); - currentIn = nullInputStream(); - truncatedPackage.compareAndSet(null, new ApplicationPackage(teeOut.toByteArray())); - done = true; - return; - } - } - else { - name = next.getName(); - modifiedAt = next.getLastModifiedTime(); - content = new FilterInputStream(inZip) { @Override public void close() { } }; // Protect inZip from replacements closing it. - } - - includeCurrent = truncatedPackage.get() == null && filter.test(name); - currentIn = replacer.modify(name, content); - if (currentIn == null) { - currentIn = InputStream.nullInputStream(); - } - else { - if (includeCurrent) teeZip.putNextEntry(new ZipEntry(name) {{ setLastModifiedTime(modifiedAt); }}); - outZip.putNextEntry(new ZipEntry(name) {{ setLastModifiedTime(modifiedAt); }}); - } - } - - @Override - public int read() throws IOException { - if (closed) throw new IOException("stream closed"); - if (pos == currentOut.length) { - fill(); - if (pos == currentOut.length) return -1; - } - return 0xff & currentOut[pos++]; - } - - @Override - public int read(byte[] out, int off, int len) throws IOException { - if (closed) throw new IOException("stream closed"); - if ((off | len | (off + len) | (out.length - (off + len))) < 0) throw new IndexOutOfBoundsException(); - if (pos == currentOut.length) { - fill(); - if (pos == currentOut.length) return -1; - } - int n = min(currentOut.length - pos, len); - System.arraycopy(currentOut, pos, out, off, n); - pos += n; - return n; - } - - @Override - public int available() throws IOException { - return pos == currentOut.length && done ? 0 : 1; - } - - @Override - public void close() { - if ( ! closed) try { - transferTo(nullOutputStream()); // Finish reading the zip, to populate the truncated package in case of errors. - in.transferTo(nullOutputStream()); // For some inane reason, ZipInputStream doesn't exhaust its wrapped input. - inZip.close(); - closed = true; - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - } - - /** Replaces entries in a zip stream as they are encountered, then appends remaining entries at the end. */ - public interface Replacer { - - /** Called when the entries of the original zip stream are exhausted. Return remaining names, or {@code null} when none left. */ - String next(); - - /** Modify content for a given name; return {@code null} for removal; in is {@code null} for entries not present in the input. */ - InputStream modify(String name, InputStream in); - - /** - * Wraps a map of fixed replacements, and: - * <ul> - * <li>Removes entries whose value is {@code null}.</li> - * <li>Modifies entries present in both input and the map.</li> - * <li>Appends entries present exclusively in the map.</li> - * <li>Writes all other entries as they are.</li> - * </ul> - */ - static Supplier<Replacer> of(Map<String, UnaryOperator<InputStream>> replacements) { - return () -> new Replacer() { - final Map<String, UnaryOperator<InputStream>> remaining = new HashMap<>(replacements); - @Override public String next() { - return remaining.isEmpty() ? null : remaining.keySet().iterator().next(); - } - @Override public InputStream modify(String name, InputStream in) { - UnaryOperator<InputStream> mapper = remaining.remove(name); - return mapper == null ? in : mapper.apply(in); - } - }; - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java deleted file mode 100644 index 5412fdf03a3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.DeploymentSpec.DeclaredZone; -import com.yahoo.config.application.api.Endpoint; -import com.yahoo.config.application.api.Endpoint.Level; -import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneEndpoint; -import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.joining; - -/** - * This contains validators for a {@link ApplicationPackage} that depend on a {@link Controller} to perform validation. - * - * @author mpolden - */ -public class ApplicationPackageValidator { - - private final Controller controller; - - public ApplicationPackageValidator(Controller controller) { - this.controller = Objects.requireNonNull(controller, "controller must be non-null"); - } - - /** - * Validate the given application package - * - * @throws IllegalArgumentException if any validations fail - */ - public void validate(Application application, ApplicationPackage applicationPackage, Instant instant) { - validateSteps(applicationPackage.deploymentSpec()); - validateEndpointRegions(applicationPackage.deploymentSpec()); - validateEndpointChange(application, applicationPackage, instant); - validateCompactedEndpoint(applicationPackage); - validateDeprecatedElements(applicationPackage); - validateCloudAccounts(application, applicationPackage); - } - - private void validateCloudAccounts(Application application, ApplicationPackage applicationPackage) { - Set<CloudAccount> tenantAccounts = new TreeSet<>(controller.applications().accountsOf(application.id().tenant())); - Set<CloudAccount> declaredAccounts = new TreeSet<>(applicationPackage.deploymentSpec().cloudAccounts().values()); - for (DeploymentInstanceSpec instance : applicationPackage.deploymentSpec().instances()) - for (ZoneId zone : controller.zoneRegistry().zones().controllerUpgraded().ids()) - declaredAccounts.addAll(instance.cloudAccounts(zone.environment(), zone.region()).values()); - - declaredAccounts.removeIf(tenantAccounts::contains); - declaredAccounts.removeIf(CloudAccount::isUnspecified); - if ( ! declaredAccounts.isEmpty()) - throw new IllegalArgumentException("cloud accounts " + - declaredAccounts.stream().map(CloudAccount::value).collect(joining(", ", "[", "]")) + - " are not valid for tenant " + - application.id().tenant()); - } - - /** Verify that deployment spec does not use elements deprecated on a major version older than wanted major version */ - private void validateDeprecatedElements(ApplicationPackage applicationPackage) { - int wantedMajor = applicationPackage.compileVersion().map(Version::getMajor) - .or(() -> applicationPackage.deploymentSpec().majorVersion()) - .orElseGet(() -> controller.readSystemVersion().getMajor()); - for (var deprecatedElement : applicationPackage.deploymentSpec().deprecatedElements()) { - if (deprecatedElement.majorVersion() >= wantedMajor) continue; - throw new IllegalArgumentException(deprecatedElement.humanReadableString()); - } - } - - /** Verify that each of the production zones listed in the deployment spec exist in this system */ - private void validateSteps(DeploymentSpec deploymentSpec) { - for (var spec : deploymentSpec.instances()) { - for (var zone : spec.zones()) { - Environment environment = zone.environment(); - if (zone.region().isEmpty()) continue; - ZoneId zoneId = ZoneId.from(environment, zone.region().get()); - if (!controller.zoneRegistry().hasZone(zoneId)) { - throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!"); - } - } - } - } - - /** Verify that: - * <ul> - * <li>no single endpoint contains regions in different clouds</li> - * <li>application endpoints with different regions must be contained in CGP and AWS</li> - * </ul> - */ - private void validateEndpointRegions(DeploymentSpec deploymentSpec) { - for (var instance : deploymentSpec.instances()) { - validateEndpointRegions(instance.endpoints(), instance); - } - validateEndpointRegions(deploymentSpec.endpoints(), null); - } - - private void validateEndpointRegions(List<Endpoint> endpoints, DeploymentInstanceSpec instance) { - for (var endpoint : endpoints) { - RegionName[] regions = new HashSet<>(endpoint.regions()).toArray(RegionName[]::new); - Set<CloudName> clouds = controller.zoneRegistry().zones().all().in(Environment.prod) - .in(regions) - .zones().stream() - .map(ZoneApi::getCloudName) - .collect(Collectors.toSet()); - String endpointString = instance == null ? "Application endpoint '" + endpoint.endpointId() + "'" - : "Endpoint '" + endpoint.endpointId() + "' in " + instance; - if (Set.of(CloudName.GCP, CloudName.AWS).containsAll(clouds)) { } // Everything is fine! - else if (Set.of(CloudName.YAHOO).containsAll(clouds) || Set.of(CloudName.DEFAULT).containsAll(clouds)) { - if (endpoint.level() == Level.application && regions.length != 1) { - throw new IllegalArgumentException(endpointString + " cannot contain different regions: " + - endpoint.regions().stream().sorted().toList()); - } - } - else if (clouds.size() == 1) { - throw new IllegalArgumentException("unknown cloud '" + clouds.iterator().next() + "'"); - } - else { - throw new IllegalArgumentException(endpointString + " cannot contain regions in different clouds: " + - endpoint.regions().stream().sorted().toList()); - } - } - } - - /** Verify endpoint configuration of given application package */ - private void validateEndpointChange(Application application, ApplicationPackage applicationPackage, Instant instant) { - for (DeploymentInstanceSpec instance : applicationPackage.deploymentSpec().instances()) { - validateGlobalEndpointChanges(application, instance.name(), applicationPackage, instant); - validateZoneEndpointChanges(application, instance.name(), applicationPackage, instant); - } - } - - /** Verify that compactable endpoint parts (instance name and endpoint ID) do not clash */ - private void validateCompactedEndpoint(ApplicationPackage applicationPackage) { - Map<List<String>, InstanceEndpoint> instanceEndpoints = new HashMap<>(); - for (var instanceSpec : applicationPackage.deploymentSpec().instances()) { - for (var endpoint : instanceSpec.endpoints()) { - List<String> nonCompactableIds = nonCompactableIds(instanceSpec.name(), endpoint); - InstanceEndpoint instanceEndpoint = new InstanceEndpoint(instanceSpec.name(), endpoint.endpointId()); - InstanceEndpoint existingEndpoint = instanceEndpoints.get(nonCompactableIds); - if (existingEndpoint != null) { - throw new IllegalArgumentException("Endpoint with ID '" + endpoint.endpointId() + "' in instance '" - + instanceSpec.name().value() + - "' clashes with endpoint '" + existingEndpoint.endpointId + - "' in instance '" + existingEndpoint.instance + "'"); - } - instanceEndpoints.put(nonCompactableIds, instanceEndpoint); - } - } - } - - /** Verify changes to endpoint configuration by comparing given application package to the existing one, if any */ - private void validateGlobalEndpointChanges(Application application, InstanceName instanceName, ApplicationPackage applicationPackage, Instant instant) { - var validationId = ValidationId.globalEndpointChange; - if (applicationPackage.validationOverrides().allows(validationId, instant)) return; - - var endpoints = application.deploymentSpec().instance(instanceName) - .map(deploymentInstanceSpec1 -> deploymentInstanceSpec1.endpoints()) - .orElseGet(List::of); - DeploymentInstanceSpec deploymentInstanceSpec = applicationPackage.deploymentSpec().requireInstance(instanceName); - var newEndpoints = new ArrayList<>(deploymentInstanceSpec.endpoints()); - - if (newEndpoints.containsAll(endpoints)) return; // Adding new endpoints is fine - if (containsAllDestinationsOf(endpoints, newEndpoints)) return; // Adding destinations is fine - - var removedEndpoints = new ArrayList<>(endpoints); - removedEndpoints.removeAll(newEndpoints); - newEndpoints.removeAll(endpoints); - throw new IllegalArgumentException(validationId.value() + ": application '" + application.id() + - (instanceName.isDefault() ? "" : "." + instanceName.value()) + - "' has endpoints " + endpoints + - ", but does not include all of these in deployment.xml. Deploying given " + - "deployment.xml will remove " + removedEndpoints + - (newEndpoints.isEmpty() ? "" : " and add " + newEndpoints) + - ". " + ValidationOverrides.toAllowMessage(validationId)); - } - - /** Verify changes to endpoint configuration by comparing given application package to the existing one, if any */ - private void validateZoneEndpointChanges(Application application, InstanceName instance, ApplicationPackage applicationPackage, Instant now) { - ValidationId validationId = ValidationId.zoneEndpointChange; - if (applicationPackage.validationOverrides().allows(validationId, now)) return;; - - String prefix = validationId + ": application '" + application.id() + - (instance.isDefault() ? "" : "." + instance.value()) + "' "; - DeploymentInstanceSpec spec = applicationPackage.deploymentSpec().requireInstance(instance); - for (DeclaredZone zone : spec.zones()) { - if (zone.environment() == Environment.prod) { - Map<ClusterSpec.Id, ZoneEndpoint> newEndpoints = spec.zoneEndpoints(ZoneId.from(zone.environment(), zone.region().get())); - application.deploymentSpec().instance(instance) // If old spec has this instance ... - .filter(oldSpec -> oldSpec.concerns(zone.environment(), zone.region())) // ... and deploys to this zone ... - .map(oldSpec -> oldSpec.zoneEndpoints(ZoneId.from(zone.environment(), zone.region().get()))) - .ifPresent(oldEndpoints -> { // ... then we compare the endpoints present in both. - oldEndpoints.forEach((cluster, oldEndpoint) -> { - ZoneEndpoint newEndpoint = newEndpoints.getOrDefault(cluster, ZoneEndpoint.defaultEndpoint); - if ( ! newEndpoint.allowedUrns().containsAll(oldEndpoint.allowedUrns())) - throw new IllegalArgumentException(prefix + "allows access to cluster '" + cluster.value() + - "' in '" + zone.region().get().value() + "' to " + - oldEndpoint.allowedUrns().stream().map(AllowedUrn::toString).collect(joining(", ", "[", "]")) + - ", but does not include all these in the new deployment spec. " + - "Deploying with the new settings will allow access to " + - (newEndpoint.allowedUrns().isEmpty() ? "no one" : newEndpoint.allowedUrns().stream().map(AllowedUrn::toString).collect(joining(", ", "[", "]")) + - ". " + ValidationOverrides.toAllowMessage(validationId))); - }); - newEndpoints.forEach((cluster, newEndpoint) -> { - ZoneEndpoint oldEndpoint = oldEndpoints.getOrDefault(cluster, ZoneEndpoint.defaultEndpoint); - if (oldEndpoint.isPublicEndpoint() && ! newEndpoint.isPublicEndpoint()) - throw new IllegalArgumentException(prefix + "has a public endpoint for cluster '" + cluster.value() + - "' in '" + zone.region().get().value() + "', but the new deployment spec " + - "disables this. " + ValidationOverrides.toAllowMessage(validationId)); - }); - }); - } - } - } - - /** Returns whether newEndpoints contains all destinations in endpoints */ - private static boolean containsAllDestinationsOf(List<Endpoint> endpoints, List<Endpoint> newEndpoints) { - var containsAllRegions = true; - var hasSameCluster = true; - for (var endpoint : endpoints) { - var endpointContainsAllRegions = false; - var endpointHasSameCluster = false; - for (var newEndpoint : newEndpoints) { - if (endpoint.endpointId().equals(newEndpoint.endpointId())) { - endpointContainsAllRegions = newEndpoint.regions().containsAll(endpoint.regions()); - endpointHasSameCluster = newEndpoint.containerId().equals(endpoint.containerId()); - } - } - containsAllRegions &= endpointContainsAllRegions; - hasSameCluster &= endpointHasSameCluster; - } - return containsAllRegions && hasSameCluster; - } - - /** Returns a list of the non-compactable IDs of given instance and endpoint */ - private static List<String> nonCompactableIds(InstanceName instance, Endpoint endpoint) { - List<String> ids = new ArrayList<>(2); - if (!instance.isDefault()) { - ids.add(instance.value()); - } - if (!"default".equals(endpoint.endpointId())) { - ids.add(endpoint.endpointId()); - } - return ids; - } - - private record InstanceEndpoint(InstanceName instance, String endpointId) {} - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXml.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXml.java deleted file mode 100644 index da08ce108e3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXml.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import com.yahoo.text.XML; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId; -import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml.Container.AuthMethod; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -/** - * A partially parsed variant of services.xml, for use by the {@link com.yahoo.vespa.hosted.controller.Controller}. - * - * @author mpolden - */ -public record BasicServicesXml(List<Container> containers) { - - public static final BasicServicesXml empty = new BasicServicesXml(List.of()); - - private static final String SERVICES_TAG = "services"; - private static final String CONTAINER_TAG = "container"; - private static final String CLIENTS_TAG = "clients"; - private static final String CLIENT_TAG = "client"; - private static final String TOKEN_TAG = "token"; - - public BasicServicesXml(List<Container> containers) { - this.containers = List.copyOf(Objects.requireNonNull(containers)); - } - - /** Parse a services.xml from given document */ - public static BasicServicesXml parse(Document document) { - Element root = document.getDocumentElement(); - if (!root.getTagName().equals("services")) { - throw new IllegalArgumentException("Root tag must be <" + SERVICES_TAG + ">"); - } - List<BasicServicesXml.Container> containers = new ArrayList<>(); - for (var childNode : XML.getChildren(root)) { - if (childNode.getTagName().equals(CONTAINER_TAG)) { - String id = childNode.getAttribute("id"); - if (id.isEmpty()) { - id = CONTAINER_TAG; // ID defaults to tag name when unset. See ConfigModelBuilder::getIdString - } - List<Container.AuthMethod> methods = new ArrayList<>(); - List<TokenId> tokens = new ArrayList<>(); - parseAuthMethods(childNode, methods, tokens); - containers.add(new Container(id, methods, tokens)); - } - } - return new BasicServicesXml(containers); - } - - private static void parseAuthMethods(Element containerNode, List<AuthMethod> methods, List<TokenId> tokens) { - for (var node : XML.getChildren(containerNode)) { - if (node.getTagName().equals(CLIENTS_TAG)) { - for (var clientNode : XML.getChildren(node)) { - if (clientNode.getTagName().equals(CLIENT_TAG)) { - boolean tokenEnabled = false; - for (var child : XML.getChildren(clientNode)) { - if (TOKEN_TAG.equals(child.getTagName())) { - tokenEnabled = true; - tokens.add(TokenId.of(child.getAttribute("id"))); - } - } - methods.add(tokenEnabled ? Container.AuthMethod.token : Container.AuthMethod.mtls); - } - } - } - } - if (methods.isEmpty()) { - methods.add(Container.AuthMethod.mtls); - } - } - - /** - * A Vespa container service. - * - * @param id ID of container - * @param authMethods Authentication methods supported by this container - */ - public record Container(String id, List<AuthMethod> authMethods, List<TokenId> dataPlaneTokens) { - - public Container(String id, List<AuthMethod> authMethods, List<TokenId> dataPlaneTokens) { - this.id = Objects.requireNonNull(id); - this.authMethods = Objects.requireNonNull(authMethods).stream() - .distinct() - .sorted() - .toList(); - if (authMethods.isEmpty()) throw new IllegalArgumentException("Container must have at least one auth method"); - this.dataPlaneTokens = dataPlaneTokens.stream().sorted().distinct().toList(); - } - - public enum AuthMethod { - mtls, - token, - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/LinesComparator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/LinesComparator.java deleted file mode 100644 index 8b4791c6b1b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/LinesComparator.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Line based variant of Apache commons-text StringComparator - * https://github.com/apache/commons-text/blob/3b1a0a5a47ee9fa2b36f99ca28e2e1d367a10a11/src/main/java/org/apache/commons/text/diff/StringsComparator.java - */ - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.yahoo.vespa.hosted.controller.application.pkg; - -import com.yahoo.collections.Pair; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -/** - * <p> - * It is guaranteed that the comparisons will always be done as - * {@code o1.equals(o2)} where {@code o1} belongs to the first - * sequence and {@code o2} belongs to the second sequence. This can - * be important if subclassing is used for some elements in the first - * sequence and the {@code equals} method is specialized. - * </p> - * <p> - * Comparison can be seen from two points of view: either as giving the smallest - * modification allowing to transform the first sequence into the second one, or - * as giving the longest sequence which is a subsequence of both initial - * sequences. The {@code equals} method is used to compare objects, so any - * object can be put into sequences. Modifications include deleting, inserting - * or keeping one object, starting from the beginning of the first sequence. - * </p> - * <p> - * This class implements the comparison algorithm, which is the very efficient - * algorithm from Eugene W. Myers - * <a href="http://www.cis.upenn.edu/~bcpierce/courses/dd/papers/diff.ps"> - * An O(ND) Difference Algorithm and Its Variations</a>. - */ -public class LinesComparator { - - private final List<String> left; - private final List<String> right; - private final int[] vDown; - private final int[] vUp; - - private LinesComparator(List<String> left, List<String> right) { - this.left = left; - this.right = right; - - int size = left.size() + right.size() + 2; - vDown = new int[size]; - vUp = new int[size]; - } - - private void buildScript(int start1, int end1, int start2, int end2, List<Pair<LineOperation, String>> result) { - Snake middle = getMiddleSnake(start1, end1, start2, end2); - - if (middle == null - || middle.start == end1 && middle.diag == end1 - end2 - || middle.end == start1 && middle.diag == start1 - start2) { - - int i = start1; - int j = start2; - while (i < end1 || j < end2) { - if (i < end1 && j < end2 && left.get(i).equals(right.get(j))) { - result.add(new Pair<>(LineOperation.keep, left.get(i))); - ++i; - ++j; - } else { - if (end1 - start1 > end2 - start2) { - result.add(new Pair<>(LineOperation.delete, left.get(i))); - ++i; - } else { - result.add(new Pair<>(LineOperation.insert, right.get(j))); - ++j; - } - } - } - - } else { - buildScript(start1, middle.start, start2, middle.start - middle.diag, result); - for (int i = middle.start; i < middle.end; ++i) { - result.add(new Pair<>(LineOperation.keep, left.get(i))); - } - buildScript(middle.end, end1, middle.end - middle.diag, end2, result); - } - } - - private Snake buildSnake(final int start, final int diag, final int end1, final int end2) { - int end = start; - while (end - diag < end2 && end < end1 && left.get(end).equals(right.get(end - diag))) { - ++end; - } - return new Snake(start, end, diag); - } - - private Snake getMiddleSnake(final int start1, final int end1, final int start2, final int end2) { - final int m = end1 - start1; - final int n = end2 - start2; - if (m == 0 || n == 0) { - return null; - } - - final int delta = m - n; - final int sum = n + m; - final int offset = (sum % 2 == 0 ? sum : sum + 1) / 2; - vDown[1 + offset] = start1; - vUp[1 + offset] = end1 + 1; - - for (int d = 0; d <= offset; ++d) { - // Down - for (int k = -d; k <= d; k += 2) { - // First step - - final int i = k + offset; - if (k == -d || k != d && vDown[i - 1] < vDown[i + 1]) { - vDown[i] = vDown[i + 1]; - } else { - vDown[i] = vDown[i - 1] + 1; - } - - int x = vDown[i]; - int y = x - start1 + start2 - k; - - while (x < end1 && y < end2 && left.get(x).equals(right.get(y))) { - vDown[i] = ++x; - ++y; - } - // Second step - if (delta % 2 != 0 && delta - d <= k && k <= delta + d) { - if (vUp[i - delta] <= vDown[i]) { // NOPMD - return buildSnake(vUp[i - delta], k + start1 - start2, end1, end2); - } - } - } - - // Up - for (int k = delta - d; k <= delta + d; k += 2) { - // First step - final int i = k + offset - delta; - if (k == delta - d || k != delta + d && vUp[i + 1] <= vUp[i - 1]) { - vUp[i] = vUp[i + 1] - 1; - } else { - vUp[i] = vUp[i - 1]; - } - - int x = vUp[i] - 1; - int y = x - start1 + start2 - k; - while (x >= start1 && y >= start2 && left.get(x).equals(right.get(y))) { - vUp[i] = x--; - y--; - } - // Second step - if (delta % 2 == 0 && -d <= k && k <= d) { - if (vUp[i] <= vDown[i + delta]) { // NOPMD - return buildSnake(vUp[i], k + start1 - start2, end1, end2); - } - } - } - } - - // this should not happen - throw new RuntimeException("Internal Error"); - } - - private static class Snake { - private final int start; - private final int end; - private final int diag; - - private Snake(int start, int end, int diag) { - this.start = start; - this.end = end; - this.diag = diag; - } - } - - private enum LineOperation { - keep(" "), delete("- "), insert("+ "); - private final String prefix; - LineOperation(String prefix) { - this.prefix = prefix; - } - } - - /** @return line-based diff in unified format. Empty contents are identical. */ - public static Optional<String> diff(List<String> left, List<String> right) { - List<Pair<LineOperation, String>> changes = new ArrayList<>(Math.max(left.size(), right.size())); - new LinesComparator(left, right).buildScript(0, left.size(), 0, right.size(), changes); - - // After we have a list of keep, delete, insert for each line from left and right input, generate a unified - // diff by printing all delete and insert operations with contextLines of keep lines before and after. - // Make sure the change windows are non-overlapping by continuously growing the window - int contextLines = 3; - List<int[]> changeWindows = new ArrayList<>(); - int[] last = null; - for (int i = 0, leftIndex = 0, rightIndex = 0; i < changes.size(); i++) { - if (changes.get(i).getFirst() == LineOperation.keep) { - leftIndex++; - rightIndex++; - continue; - } - - // We found a new change and it is too far away from the previous change to be combined into the same window - if (last == null || i - last[1] > contextLines) { - last = new int[]{Math.max(i - contextLines, 0), Math.min(i + contextLines + 1, changes.size()), Math.max(leftIndex - contextLines, 0), Math.max(rightIndex - contextLines, 0)}; - changeWindows.add(last); - } else // otherwise, extend the previous change window - last[1] = Math.min(i + contextLines + 1, changes.size()); - - if (changes.get(i).getFirst() == LineOperation.delete) leftIndex++; - else rightIndex++; - } - if (changeWindows.isEmpty()) return Optional.empty(); - - StringBuilder sb = new StringBuilder(); - for (int[] changeWindow: changeWindows) { - int start = changeWindow[0], end = changeWindow[1], leftIndex = changeWindow[2], rightIndex = changeWindow[3]; - Map<LineOperation, Long> counts = IntStream.range(start, end) - .mapToObj(i -> changes.get(i).getFirst()) - .collect(Collectors.groupingBy(i -> i, Collectors.counting())); - sb.append("@@ -").append(leftIndex + 1).append(',').append(end - start - counts.getOrDefault(LineOperation.insert, 0L)) - .append(" +").append(rightIndex + 1).append(',').append(end - start - counts.getOrDefault(LineOperation.delete, 0L)).append(" @@\n"); - for (int i = start; i < end; i++) - sb.append(changes.get(i).getFirst().prefix).append(changes.get(i).getSecond()).append('\n'); - } - return Optional.of(sb.toString()); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java deleted file mode 100644 index dc55472bcc2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.DeploymentSpec.Step; -import com.yahoo.config.provision.AthenzDomain; -import com.yahoo.config.provision.AthenzService; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.path.Path; -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.X509CertificateBuilder; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream.Replacer; -import com.yahoo.vespa.hosted.controller.config.ControllerConfig; -import com.yahoo.vespa.hosted.controller.config.ControllerConfig.Steprunner.Testerapp; -import com.yahoo.yolean.Exceptions; - -import javax.security.auth.x500.X500Principal; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.math.BigInteger; -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.function.Supplier; -import java.util.function.UnaryOperator; -import java.util.jar.JarInputStream; -import java.util.jar.Manifest; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Pattern; - -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.production; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging_setup; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.system; -import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.deploymentFile; -import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.servicesFile; -import static java.io.InputStream.nullInputStream; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNullElse; -import static java.util.function.UnaryOperator.identity; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.mapping; -import static java.util.stream.Collectors.toList; - -/** - * Validation and manipulation of test package. - * - * @author jonmv - */ -public class TestPackage { - - private static final Logger log = Logger.getLogger(TestPackage.class.getName()); - - // Must match exactly the advertised resources of an AWS instance type. Also consider that the container - // will have ~1.8 GB less memory than equivalent resources in AWS (VESPA-16259). - static final NodeResources DEFAULT_TESTER_RESOURCES_CLOUD = new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); - static final NodeResources DEFAULT_TESTER_RESOURCES = new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any); - - private final ApplicationPackageStream applicationPackageStream; - private final X509Certificate certificate; - - public TestPackage(Supplier<InputStream> inZip, boolean isPublicSystem, CloudName cloud, RunId id, Testerapp testerApp, - DeploymentSpec fallbackSpec, Instant certificateValidFrom, Duration certificateValidDuration) { - KeyPair keyPair; - if (certificateValidFrom != null) { - keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - X500Principal subject = new X500Principal("CN=" + id.tester().id().toFullString() + "." + id.type() + "." + id.number()); - this.certificate = X509CertificateBuilder.fromKeypair(keyPair, - subject, - certificateValidFrom, - certificateValidFrom.plus(certificateValidDuration), - SignatureAlgorithm.SHA512_WITH_RSA, - BigInteger.valueOf(1)) - .build(); - } - else { - keyPair = null; - this.certificate = null; - } - this.applicationPackageStream = new ApplicationPackageStream(inZip, () -> name -> name.endsWith(".xml"), () -> new Replacer() { - - // Initially skips all declared entries, ensuring they're generated and appended after all input entries. - final Map<String, UnaryOperator<InputStream>> entries = new LinkedHashMap<>(); - final Map<String, UnaryOperator<InputStream>> replacements = new LinkedHashMap<>(); - boolean hasLegacyTests = false; - DeploymentSpec containedSpec; - - @Override - public String next() { - if (entries.isEmpty()) return null; - String next = entries.keySet().iterator().next(); - replacements.put(next, entries.remove(next)); - return next; - } - - @Override - public InputStream modify(String name, InputStream in) { - hasLegacyTests |= name.startsWith("artifacts/") && name.endsWith("-tests.jar"); - - // Pick out the deployment.xml stored in the package, if any. - if (entries.containsKey(deploymentFile) && name.equals(deploymentFile)) - containedSpec = DeploymentSpec.fromXml(new InputStreamReader(in)); - - return entries.containsKey(name) ? null // Skip entry for now, as it will be appended later when we get here again after {@link #next()}. - : replacements.getOrDefault(name, identity()).apply(in); // Modify entry, if needed. - } - - { - // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip. - entries.put("artifacts/.ignore-" + UUID.randomUUID(), __ -> nullInputStream()); - entries.put("tests/.ignore-" + UUID.randomUUID(), __ -> nullInputStream()); - - entries.put(servicesFile, - __ -> { - DeploymentSpec spec = requireNonNullElse(containedSpec, fallbackSpec); - boolean isEnclave = isPublicSystem && ! spec.cloudAccount(cloud, id.application().instance(), id.type().zone()).isUnspecified(); - return new ByteArrayInputStream(servicesXml( ! isPublicSystem, - certificateValidFrom != null, - hasLegacyTests, - testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance()), isEnclave), - testerApp)); - }); - - entries.put(deploymentFile, - __ -> new ByteArrayInputStream(deploymentXml(id.tester(), - id.application().instance(), - cloud, - id.type().zone(), - requireNonNullElse(containedSpec, fallbackSpec)))); - - if (certificate != null) { - entries.put("artifacts/key", __ -> new ByteArrayInputStream(KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8))); - entries.put("artifacts/cert", __ -> new ByteArrayInputStream(X509CertificateUtils.toPem(certificate).getBytes(UTF_8))); - } - } - }); - } - - public ApplicationPackageStream asApplicationPackage() { - return applicationPackageStream; - } - - public X509Certificate certificate() { - return Objects.requireNonNull(certificate); - } - - public static TestSummary validateTests(DeploymentSpec spec, byte[] testPackage) { - return validateTests(expectedSuites(spec.steps()), testPackage); - } - - static TestSummary validateTests(Collection<Suite> expectedSuites, byte[] testPackage) { - List<String> problems = new ArrayList<>(); - Set<Suite> suites = new LinkedHashSet<>(); - ZipEntries.from(testPackage, __ -> true, 0, false).asList().stream() - .map(entry -> Path.fromString(entry.name())) - .collect(groupingBy(path -> path.elements().size() > 1 ? path.elements().get(0) : "", - mapping(path -> (path.elements().size() > 1 ? path.getChildPath() : path).getRelative(), toList()))) - .forEach((directory, paths) -> { - switch (directory) { - case "components": { - for (String path : paths) { - if (path.endsWith("-tests.jar")) { - try { - byte[] testsJar = ZipEntries.readFile(testPackage, "components/" + path, 1 << 30); - Manifest manifest = new JarInputStream(new ByteArrayInputStream(testsJar)).getManifest(); - String bundleCategoriesHeader = manifest.getMainAttributes().getValue("X-JDisc-Test-Bundle-Categories"); - if (bundleCategoriesHeader == null) continue; - for (String suite : bundleCategoriesHeader.split(",")) - if ( ! suite.isBlank()) switch (suite.trim()) { - case "SystemTest" -> suites.add(system); - case "StagingSetup" -> suites.add(staging_setup); - case "StagingTest" -> suites.add(staging); - case "ProductionTest" -> suites.add(production); - default -> problems.add("unexpected test suite name '" + suite + "' in bundle manifest"); - } - } - catch (Exception e) { - problems.add("failed reading test bundle manifest: " + Exceptions.toMessageString(e)); - } - } - } - } - break; - case "tests": { - if (paths.stream().anyMatch(Pattern.compile("system-test/.+\\.json").asMatchPredicate())) suites.add(system); - if (paths.stream().anyMatch(Pattern.compile("staging-setup/.+\\.json").asMatchPredicate())) suites.add(staging_setup); - if (paths.stream().anyMatch(Pattern.compile("staging-test/.+\\.json").asMatchPredicate())) suites.add(staging); - if (paths.stream().anyMatch(Pattern.compile("production-test/.+\\.json").asMatchPredicate())) suites.add(production); - } - break; - case "artifacts": { - if (paths.stream().anyMatch(Pattern.compile(".+-tests.jar").asMatchPredicate())) - suites.addAll(expectedSuites); // ಠ_ಠ - - for (String forbidden : List.of("key", "cert")) - if (paths.contains(forbidden)) - problems.add("test package contains 'artifacts/" + forbidden + - "'; this conflicts with credentials used to run tests in Vespa Cloud"); - } - break; - } - }); - - if (expectedSuites.contains(system) && ! suites.contains(system)) - problems.add("test package has no system tests, but <test /> is declared in deployment.xml"); - - if (suites.contains(staging) != suites.contains(staging_setup)) - problems.add("test package has " + (suites.contains(staging) ? "staging tests" : "staging setup") + - ", so it should also include " + (suites.contains(staging) ? "staging setup" : "staging tests")); - else if (expectedSuites.contains(staging) && ! suites.contains(staging)) - problems.add("test package has no staging setup and tests, but <staging /> is declared in deployment.xml"); - - if (suites.contains(production) != expectedSuites.contains(production)) - problems.add("test package has " + (suites.contains(production) ? "" : "no ") + "production tests, " + - "but " + (suites.contains(production) ? "no " : "") + "production tests are declared in deployment.xml"); - - if ( ! problems.isEmpty()) - problems.add("see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"); - - return new TestSummary(problems, suites); - } - - static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec, boolean isEnclave) { - NodeResources nodeResources = spec.steps().stream() - .filter(step -> step.concerns(zone.environment())) - .findFirst() - .flatMap(step -> step.zones().get(0).testerFlavor()) - .map(NodeResources::fromLegacyName) - .orElse(zone.region().value().matches("^(aws|gcp)-.*") ? DEFAULT_TESTER_RESOURCES_CLOUD - : DEFAULT_TESTER_RESOURCES); - if (isEnclave) nodeResources = nodeResources.with(NodeResources.Architecture.x86_64); - return nodeResources.with(NodeResources.DiskSpeed.any); - } - - /** Returns the generated services.xml content for the tester application. */ - static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, boolean hasLegacyTests, - NodeResources resources, ControllerConfig.Steprunner.Testerapp config) { - int jdiscMemoryGb = 2; // 2Gb memory for tester application which uses Maven. - int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb()); - - // Of the remaining memory, split 50/50 between Surefire running the tests and the rest - int testMemoryMb = (int) (1024 * (resources.memoryGb() - jdiscMemoryGb) / 2); - - String resourceString = Text.format("<resources vcpu=\"%.2f\" memory=\"%.2fGb\" disk=\"%.2fGb\" disk-speed=\"%s\" storage-type=\"%s\" architecture=\"%s\"/>", - resources.vcpu(), resources.memoryGb(), resources.diskGb(), resources.diskSpeed().name(), resources.storageType().name(), resources.architecture().name()); - - String runtimeProviderClass = config.runtimeProviderClass(); - String tenantCdBundle = config.tenantCdBundle(); - - String servicesXml = - "<?xml version='1.0' encoding='UTF-8'?>\n" + - "<services xmlns:deploy='vespa' version='1.0'>\n" + - " <container version='1.0' id='tester'>\n" + - "\n" + - " <component id=\"com.yahoo.vespa.hosted.testrunner.TestRunner\" bundle=\"vespa-testrunner-components\">\n" + - " <config name=\"com.yahoo.vespa.hosted.testrunner.test-runner\">\n" + - " <artifactsPath>artifacts</artifactsPath>\n" + - " <surefireMemoryMb>" + testMemoryMb + "</surefireMemoryMb>\n" + - " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + - " <useTesterCertificate>" + useTesterCertificate + "</useTesterCertificate>\n" + - " </config>\n" + - " </component>\n" + - "\n" + - " <handler id=\"com.yahoo.vespa.testrunner.TestRunnerHandler\" bundle=\"vespa-osgi-testrunner\">\n" + - " <binding>http://*/tester/v1/*</binding>\n" + - " </handler>\n" + - "\n" + - " <component id=\"" + runtimeProviderClass + "\" bundle=\"" + tenantCdBundle + "\" />\n" + - "\n" + - " <component id=\"com.yahoo.vespa.testrunner.JunitRunner\" bundle=\"vespa-osgi-testrunner\">\n" + - " <config name=\"com.yahoo.vespa.testrunner.junit-test-runner\">\n" + - " <artifactsPath>artifacts</artifactsPath>\n" + - " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + - " </config>\n" + - " </component>\n" + - "\n" + - " <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" + - " <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" + - " <artifactsPath>artifacts</artifactsPath>\n" + - " <testsPath>tests</testsPath>\n" + - " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + - " </config>\n" + - " </component>\n" + - "\n" + - " <nodes count=\"1\">\n" + - (hasLegacyTests ? " <jvm allocated-memory=\"" + jdiscMemoryPct + "%\"/>\n" : "" ) + - " " + resourceString + "\n" + - " </nodes>\n" + - " </container>\n" + - "</services>\n"; - - return servicesXml.getBytes(UTF_8); - } - - /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */ - static byte[] deploymentXml(TesterId id, InstanceName instance, CloudName cloud, ZoneId zone, DeploymentSpec original) { - Optional<AthenzDomain> athenzDomain = original.athenzDomain(); - Optional<AthenzService> athenzService = original.requireInstance(instance) - .athenzService(zone.environment(), zone.region()); - Optional<CloudAccount> cloudAccount = Optional.of(original.cloudAccount(cloud, instance, zone)) - .filter(account -> ! account.isUnspecified()); - Optional<Duration> hostTTL = (zone.environment().isProduction() - ? original.requireInstance(instance) - .steps().stream().filter(step -> step.isTest() && step.concerns(zone.environment(), Optional.of(zone.region()))) - .findFirst().flatMap(Step::hostTTL) - : original.requireInstance(instance).hostTTL(zone.environment(), Optional.of(zone.region()))) - .filter(__ -> cloudAccount.isPresent()); - String deploymentSpec = - "<?xml version='1.0' encoding='UTF-8'?>\n" + - "<deployment version='1.0'" + - athenzDomain.map(domain -> " athenz-domain='" + domain.value() + "'").orElse("") + - athenzService.map(service -> " athenz-service='" + service.value() + "'").orElse("") + - cloudAccount.map(account -> " cloud-account='" + account.value() + "'").orElse("") + - hostTTL.map(ttl -> " empty-host-ttl='" + ttl.getSeconds() / 60 + "m'").orElse("") + - ">" + - " <instance id='" + id.id().instance().value() + "' />" + - "</deployment>"; - return deploymentSpec.getBytes(UTF_8); - } - - static Set<Suite> expectedSuites(List<Step> steps) { - Set<Suite> suites = new HashSet<>(); - if (steps.isEmpty()) return suites; - for (Step step : steps) { - if (step.isTest()) { - if (step.concerns(Environment.prod)) suites.add(production); - if (step.concerns(Environment.test)) suites.add(system); - if (step.concerns(Environment.staging)) { suites.add(staging); suites.add(staging_setup); } - } - else - suites.addAll(expectedSuites(step.steps())); - } - return suites; - } - - - public static class TestSummary { - - private final List<String> problems; - private final List<Suite> suites; - - public TestSummary(List<String> problems, Set<Suite> suites) { - this.problems = List.copyOf(problems); - this.suites = List.copyOf(suites); - } - - public List<String> problems() { - return problems; - } - - public List<Suite> suites() { - return suites; - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java deleted file mode 100644 index 90e7acf9e77..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import com.yahoo.vespa.archive.ArchiveStreamReader; -import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile; -import com.yahoo.vespa.archive.ArchiveStreamReader.Options; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Predicate; - -/** - * A list of entries read from a ZIP archive, and their contents. - * - * @author bratseth - */ -public class ZipEntries { - - private final List<ZipEntryWithContent> entries; - - private ZipEntries(List<ZipEntryWithContent> entries) { - this.entries = List.copyOf(Objects.requireNonNull(entries)); - } - - /** Read ZIP entries from inputStream */ - public static ZipEntries from(byte[] zip, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes, boolean throwIfEntryExceedsMaxSize) { - - Options options = Options.standard() - .pathPredicate(entryNameMatcher) - .maxSize(2L << 30) // 2 GB - .maxEntrySize(maxEntrySizeInBytes) - .maxEntries(1024) - .truncateEntry(!throwIfEntryExceedsMaxSize); - List<ZipEntryWithContent> entries = new ArrayList<>(); - try (ArchiveStreamReader reader = ArchiveStreamReader.ofZip(new ByteArrayInputStream(zip), options)) { - ArchiveFile file; - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - while ((file = reader.readNextTo(baos)) != null) { - entries.add(new ZipEntryWithContent(file.path().toString(), - Optional.of(baos.toByteArray()).filter(b -> b.length > 0), - file.size())); - baos.reset(); - } - } - return new ZipEntries(entries); - } - - public static byte[] readFile(byte[] zip, String name, int maxEntrySizeInBytes) { - return from(zip, name::equals, maxEntrySizeInBytes, true).asList().get(0).contentOrThrow(); - } - - public List<ZipEntryWithContent> asList() { return entries; } - - public static class ZipEntryWithContent { - - private final String name; - private final Optional<byte[]> content; - private final long size; - - public ZipEntryWithContent(String name, Optional<byte[]> content, long size) { - this.name = name; - this.content = content; - this.size = size; - } - - public String name() { return name; } - public byte[] contentOrThrow() { return content.orElseThrow(() -> new NoSuchElementException("'" + name + "' has no content")); } - public Optional<byte[]> content() { return content; } - public long size() { return size; } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java deleted file mode 100644 index d2be561a520..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.archive; - -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBuckets; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; -import com.yahoo.vespa.hosted.controller.api.integration.archive.TenantManagedArchiveBucket; -import com.yahoo.vespa.hosted.controller.api.integration.archive.VespaManagedArchiveBucket; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.net.URI; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -/** - * This class decides which tenant goes in what bucket, and creates new buckets when required. - * - * @author andreer - */ -public class CuratorArchiveBucketDb { - - private static final Duration ENCLAVE_BUCKET_CACHE_LIFETIME = Duration.ofMinutes(60); - - /** - * Archive URIs are often requested because they are returned in /application/v4 API. Since they - * never change, it's safe to cache them and only update on misses - */ - private final Map<ZoneId, Map<TenantName, String>> archiveUriCache = new ConcurrentHashMap<>(); - private final Map<ZoneId, Map<CloudAccount, TenantManagedArchiveBucket>> tenantArchiveCache = new ConcurrentHashMap<>(); - - private final ArchiveService archiveService; - private final CuratorDb curatorDb; - private final Clock clock; - - public CuratorArchiveBucketDb(Controller controller) { - this.archiveService = controller.serviceRegistry().archiveService(); - this.curatorDb = controller.curator(); - this.clock = controller.clock(); - } - - public Optional<URI> archiveUriFor(ZoneId zoneId, TenantName tenant, boolean createIfMissing) { - return getBucketNameFromCache(zoneId, tenant) - .or(() -> createIfMissing ? Optional.of(assignToBucket(zoneId, tenant)) : Optional.empty()) - .map(bucketName -> archiveService.bucketURI(zoneId, bucketName)); - } - - public Optional<URI> archiveUriFor(ZoneId zoneId, CloudAccount account, boolean searchIfMissing) { - Instant updatedAfter = searchIfMissing ? clock.instant().minus(ENCLAVE_BUCKET_CACHE_LIFETIME) : Instant.MIN; - return getBucketNameFromCache(zoneId, account, updatedAfter) - .or(() -> { - if (!searchIfMissing) return Optional.empty(); - try (var lock = curatorDb.lockArchiveBuckets(zoneId)) { - ArchiveBuckets archiveBuckets = buckets(zoneId); - updateArchiveUriCache(zoneId, archiveBuckets); - - return getBucketNameFromCache(zoneId, account, updatedAfter) - .or(() -> archiveService.findEnclaveArchiveBucket(zoneId, account) - .map(bucketName -> { - var bucket = new TenantManagedArchiveBucket(bucketName, account, clock.instant()); - ArchiveBuckets updated = archiveBuckets.with(bucket); - curatorDb.writeArchiveBuckets(zoneId, updated); - updateArchiveUriCache(zoneId, updated); - return bucket; - })); - } - }) - .map(TenantManagedArchiveBucket::bucketName) - .map(bucketName -> archiveService.bucketURI(zoneId, bucketName)); - } - - private String assignToBucket(ZoneId zoneId, TenantName tenant) { - try (var lock = curatorDb.lockArchiveBuckets(zoneId)) { - ArchiveBuckets archiveBuckets = buckets(zoneId); - updateArchiveUriCache(zoneId, archiveBuckets); - - return getBucketNameFromCache(zoneId, tenant) // Some other thread might have assigned it before we grabbed the lock - .orElseGet(() -> { - // If not, find an existing bucket with space - VespaManagedArchiveBucket bucketToAssignTo = archiveBuckets.vespaManaged().stream() - .filter(bucket -> archiveService.canAddTenantToBucket(zoneId, bucket)) - .findAny() - // Or create a new one - .orElseGet(() -> archiveService.createArchiveBucketFor(zoneId)); - - ArchiveBuckets updated = archiveBuckets.with(bucketToAssignTo.withTenant(tenant)); - curatorDb.writeArchiveBuckets(zoneId, updated); - updateArchiveUriCache(zoneId, updated); - - return bucketToAssignTo.bucketName(); - }); - } - } - - public ArchiveBuckets buckets(ZoneId zoneId) { - return curatorDb.readArchiveBuckets(zoneId); - } - - private Optional<String> getBucketNameFromCache(ZoneId zoneId, TenantName tenantName) { - return Optional.ofNullable(archiveUriCache.get(zoneId)).map(map -> map.get(tenantName)); - } - - private Optional<TenantManagedArchiveBucket> getBucketNameFromCache(ZoneId zoneId, CloudAccount cloudAccount, Instant updatedAfter) { - return Optional.ofNullable(tenantArchiveCache.get(zoneId)) - .map(map -> map.get(cloudAccount)) - .filter(bucket -> bucket.updatedAt().isAfter(updatedAfter)); - } - - private void updateArchiveUriCache(ZoneId zoneId, ArchiveBuckets archiveBuckets) { - Map<TenantName, String> bucketNameByTenant = archiveBuckets.vespaManaged().stream() - .flatMap(bucket -> bucket.tenants().stream().map(tenant -> Map.entry(tenant, bucket.bucketName()))) - .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); - archiveUriCache.put(zoneId, bucketNameByTenant); - - Map<CloudAccount, TenantManagedArchiveBucket> bucketByAccount = archiveBuckets.tenantManaged().stream() - .collect(Collectors.toUnmodifiableMap(TenantManagedArchiveBucket::cloudAccount, bucket -> bucket)); - tenantArchiveCache.put(zoneId, bucketByAccount); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/HostedAthenzIdentities.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/HostedAthenzIdentities.java deleted file mode 100644 index f68c13ec0d4..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/HostedAthenzIdentities.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; - -/** - * @author bjorncs - */ -public class HostedAthenzIdentities { - - public static final AthenzDomain SCREWDRIVER_DOMAIN = new AthenzDomain("cd.screwdriver.project"); - - private HostedAthenzIdentities() {} - - public static AthenzUser from(UserId userId) { - return AthenzUser.fromUserId(userId.id()); - } - - public static AthenzService from(ScrewdriverId screwdriverId) { - return new AthenzService(SCREWDRIVER_DOMAIN, "sd" + screwdriverId.id()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/config/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/config/package-info.java deleted file mode 100644 index aceee5f70f4..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/config/package-info.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Required for using {@link com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig} outside controller-server module. - * - * @author bjorncs - */ -@ExportPackage -package com.yahoo.vespa.hosted.controller.athenz.config; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java deleted file mode 100644 index e3f53b5606f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.impl; - -import ai.vespa.metrics.ControllerMetrics; -import com.yahoo.component.annotation.Inject; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.client.ErrorHandler; -import com.yahoo.vespa.athenz.client.zms.DefaultZmsClient; -import com.yahoo.vespa.athenz.client.zms.ZmsClient; -import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; -import com.yahoo.vespa.athenz.client.zts.ZtsClient; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; - -import java.net.URI; -import java.util.HashMap; -import java.util.Map; - -/** - * @author bjorncs - */ -public class AthenzClientFactoryImpl implements AthenzClientFactory { - - private static final String METRIC_NAME = ControllerMetrics.ATHENZ_REQUEST_ERROR.baseName(); - private static final String ATHENZ_SERVICE_DIMENSION = "athenz-service"; - private static final String EXCEPTION_DIMENSION = "exception"; - - private final AthenzConfig config; - private final ServiceIdentityProvider identityProvider; - private final Metric metrics; - private final Map<String, Metric.Context> metricContexts; - - @Inject - public AthenzClientFactoryImpl(ServiceIdentityProvider identityProvider, AthenzConfig config, Metric metrics) { - this.identityProvider = identityProvider; - this.config = config; - this.metrics = metrics; - this.metricContexts = new HashMap<>(); - } - - @Override - public AthenzIdentity getControllerIdentity() { - return identityProvider.identity(); - } - - /** - * @return A ZMS client instance with the service identity as principal. - */ - @Override - public ZmsClient createZmsClient() { - return new DefaultZmsClient(URI.create(config.zmsUrl()), identityProvider, this::reportMetricErrorHandler); - } - - /** - * @return A ZTS client instance with the service identity as principal. - */ - @Override - public ZtsClient createZtsClient() { - return new DefaultZtsClient.Builder(URI.create(config.ztsUrl())).withIdentityProvider(identityProvider).build(); - } - - @Override - public boolean cacheLookups() { - return true; - } - - private void reportMetricErrorHandler(ErrorHandler.RequestProperties request, Exception error) { - Metric.Context context = metricContexts.computeIfAbsent(request.hostname(), host -> metrics.createContext( - Map.of(ATHENZ_SERVICE_DIMENSION, host, - EXCEPTION_DIMENSION, error.getClass().getSimpleName()))); - metrics.add(METRIC_NAME, 1, context); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java deleted file mode 100644 index ec5fb9af902..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ /dev/null @@ -1,372 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.impl; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.restapi.RestApiException; -import com.yahoo.text.Text; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.AthenzResourceName; -import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.athenz.api.OAuthCredentials; -import com.yahoo.vespa.athenz.client.zms.RoleAction; -import com.yahoo.vespa.athenz.client.zms.ZmsClient; -import com.yahoo.vespa.athenz.client.zms.ZmsClientException; -import com.yahoo.vespa.athenz.client.zts.ZtsClient; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.security.AccessControl; -import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; -import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; -import com.yahoo.vespa.hosted.controller.security.Credentials; -import com.yahoo.vespa.hosted.controller.security.TenantSpec; -import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.time.Instant; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * @author bjorncs - * @author jonmv - */ -public class AthenzFacade implements AccessControl { - - private static final Logger log = Logger.getLogger(AthenzFacade.class.getName()); - private final ZmsClient zmsClient; - private final ZtsClient ztsClient; - private final AthenzIdentity service; - private final Function<AthenzIdentity, List<AthenzDomain>> userDomains; - private final Predicate<AccessTuple> accessRights; - - @Inject - public AthenzFacade(AthenzClientFactory factory) { - this.zmsClient = factory.createZmsClient(); - this.ztsClient = factory.createZtsClient(); - this.service = factory.getControllerIdentity(); - this.userDomains = factory.cacheLookups() - ? CacheBuilder.newBuilder() - .expireAfterWrite(10, TimeUnit.SECONDS) - .build(CacheLoader.from(this::getUserDomains))::getUnchecked - : this::getUserDomains; - this.accessRights = factory.cacheLookups() - ? CacheBuilder.newBuilder() - .expireAfterWrite(10, TimeUnit.SECONDS) - .build(CacheLoader.from(this::lookupAccess))::getUnchecked - : this::lookupAccess; - } - - private List<AthenzDomain> getUserDomains(AthenzIdentity userIdentity) { - return ztsClient.getTenantDomains(service, userIdentity, "admin"); - } - - @Override - public Tenant createTenant(TenantSpec tenantSpec, Instant createdAt, Credentials credentials, List<Tenant> existing) { - AthenzTenantSpec spec = (AthenzTenantSpec) tenantSpec; - AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; - AthenzDomain domain = spec.domain(); - - verifyIsDomainAdmin(athenzCredentials.user().getIdentity(), domain); - - Optional<Tenant> existingWithSameDomain = existing.stream() - .filter(tenant -> tenant.type() == Tenant.Type.athenz - && domain.equals(((AthenzTenant) tenant).domain())) - .findAny(); - - AthenzTenant tenant = AthenzTenant.create(spec.tenant(), - domain, - spec.property(), - spec.propertyId(), - createdAt); - - if (existingWithSameDomain.isPresent()) { // Throw if domain is already taken. - throw new IllegalArgumentException("Could not create tenant '" + spec.tenant().value() + - "': The Athens domain '" + - domain.getName() + "' is already connected to tenant '" + - existingWithSameDomain.get().name().value() + "'"); - } - else { // Create tenant resources in Athenz if domain is not already taken. - log("createTenancy(tenantDomain=%s, service=%s)", domain, service); - zmsClient.createTenancy(domain, service, athenzCredentials.oAuthCredentials()); - } - - return tenant; - } - - @Override - public Tenant updateTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing, List<Application> applications) { - AthenzTenantSpec spec = (AthenzTenantSpec) tenantSpec; - AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; - AthenzDomain newDomain = spec.domain(); - AthenzDomain oldDomain = athenzCredentials.domain(); - - verifyIsDomainAdmin(athenzCredentials.user().getIdentity(), newDomain); - - Optional<Tenant> existingWithSameDomain = existing.stream() - .filter(tenant -> tenant.type() == Tenant.Type.athenz - && newDomain.equals(((AthenzTenant) tenant).domain())) - .findAny(); - Instant createdAt = existing.stream() - .filter(tenant -> tenant.name().equals(spec.tenant())) - .findAny().orElseThrow() // Should not happen, we assert that the tenant exists before the method is called - .createdAt(); - - Tenant tenant = AthenzTenant.create(spec.tenant(), - newDomain, - spec.property(), - spec.propertyId(), - createdAt); - - if (existingWithSameDomain.isPresent()) { // Throw if domain taken by someone else, or do nothing if taken by this tenant. - if ( ! existingWithSameDomain.get().equals(tenant)) // Equality by name. - throw new IllegalArgumentException("Could not create tenant '" + spec.tenant().value() + - "': The Athens domain '" + - newDomain.getName() + "' is already connected to tenant '" + - existingWithSameDomain.get().name().value() + "'"); - - return tenant; // Short-circuit here if domain is still the same. - } - else { // Delete and recreate tenant, and optionally application, resources in Athenz otherwise. - log("createTenancy(tenantDomain=%s, service=%s)", newDomain, service); - zmsClient.createTenancy(newDomain, service, athenzCredentials.oAuthCredentials()); - for (Application application : applications) - createApplication(newDomain, application.id().application(), athenzCredentials.oAuthCredentials()); - - log("deleteTenancy(tenantDomain=%s, service=%s)", oldDomain, service); - for (Application application : applications) - deleteApplication(oldDomain, application.id().application(), athenzCredentials.oAuthCredentials()); - zmsClient.deleteTenancy(oldDomain, service, athenzCredentials.oAuthCredentials()); - } - - return tenant; - } - - @Override - public void deleteTenant(TenantName tenant, Credentials credentials) { - AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; - AthenzDomain tenantDomain = athenzCredentials.domain(); - log("deleteTenancy(tenantDomain=%s, service=%s)", tenantDomain, service); - try { - zmsClient.deleteTenancy(tenantDomain, service, athenzCredentials.oAuthCredentials()); - } catch (ZmsClientException e) { - if (e.getErrorCode() == 404) { - log.log(Level.WARNING, - "Failed to cleanup tenant " + tenant.value() + " with domain '" + tenantDomain.getName() - + "' in Athenz due to non-existing tenant domain", - e); - } else { - throw e; - } - } - } - - @Override - public void createApplication(TenantAndApplicationId id, Credentials credentials) { - AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; - createApplication(athenzCredentials.domain(), id.application(), athenzCredentials.oAuthCredentials()); - } - - private void createApplication(AthenzDomain domain, ApplicationName application, OAuthCredentials oAuthCredentials) { - Set<RoleAction> tenantRoleActions = createTenantRoleActions(); - log("createProviderResourceGroup(" + - "tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s, roleActions=%s)", - domain, service.getDomain().getName(), service.getName(), application, tenantRoleActions); - try { - zmsClient.createProviderResourceGroup(domain, service, application.value(), tenantRoleActions, oAuthCredentials); - } - catch (ZmsClientException e) { - if (e.getErrorCode() == com.yahoo.jdisc.Response.Status.FORBIDDEN) - throw new RestApiException.Forbidden("Not authorized to create application", e); - else - throw e; - } - } - - @Override - public void deleteApplication(TenantAndApplicationId id, Credentials credentials) { - AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; - log("deleteProviderResourceGroup(tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s)", - athenzCredentials.domain(), service.getDomain().getName(), service.getName(), id.application()); - try { - zmsClient.deleteProviderResourceGroup(athenzCredentials.domain(), service, id.application().value(), - athenzCredentials.oAuthCredentials()); - } catch (ZmsClientException e) { - if (e.getErrorCode() == 404) { - log.log(Level.WARNING, - "Failed to cleanup application '" + id.serialized() - + "' in Athenz due to non-existing tenant domain or resource group", - e); - } else { - throw e; - } - } - } - - /** - * Returns the list of tenants to which a user has access. - * @param tenants the list of all known tenants - * @param credentials the credentials of user whose tenants to list - * @return the list of tenants the given user has access to - */ - // TODO jonmv: Remove - public List<Tenant> accessibleTenants(List<Tenant> tenants, Credentials credentials) { - AthenzIdentity identity = ((AthenzPrincipal) credentials.user()).getIdentity(); - return tenants.stream() - .filter(tenant -> tenant.type() == Tenant.Type.athenz - && userDomains.apply(identity).contains(((AthenzTenant) tenant).domain())) - .toList(); - } - - public void addTenantAdmin(AthenzDomain tenantDomain, AthenzUser user) { - zmsClient.addRoleMember(new AthenzRole(tenantDomain, "tenancy." + service.getFullName() + ".admin"), user, Optional.empty()); - } - - private void deleteApplication(AthenzDomain domain, ApplicationName application, OAuthCredentials oAuthCredentials) { - log("deleteProviderResourceGroup(tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s)", - domain, service.getDomain().getName(), service.getName(), application); - zmsClient.deleteProviderResourceGroup(domain, service, application.value(), oAuthCredentials); - } - - public boolean hasApplicationAccess( - AthenzIdentity identity, ApplicationAction action, AthenzDomain tenantDomain, ApplicationName applicationName, Optional<ZoneId> zone) { - return hasAccess( - action.name(), applicationResourceString(tenantDomain, applicationName, zone), identity); - } - - public boolean hasTenantAdminAccess(AthenzIdentity identity, AthenzDomain tenantDomain) { - return hasAccess(TenantAction._modify_.name(), tenantResourceString(tenantDomain), identity); - } - - public boolean hasHostedOperatorAccess(AthenzIdentity identity) { - return hasAccess("modify", service.getDomain().getName() + ":hosted-vespa", identity); - } - - public boolean hasHostedSupporterAccess(AthenzIdentity identity) { - return hasAccess("read", service.getDomain().getName() + ":hosted-vespa", identity); - } - - public boolean canLaunch(AthenzIdentity principal, AthenzService service) { - return hasAccess("launch", service.getDomain().getName() + ":service."+service.getName(), principal); - } - - public boolean hasSystemFlagsAccess(AthenzIdentity identity, boolean dryRun) { - return hasAccess(dryRun ? "dryrun" : "deploy", new AthenzResourceName(service.getDomain(), "system-flags").toResourceNameString(), identity); - } - - public boolean hasPaymentCallbackAccess(AthenzIdentity identity) { - return hasAccess("callback", new AthenzResourceName(service.getDomain().getName(), "payment-notification-resource").toResourceNameString(), identity); - } - - public boolean hasAccountingAccess(AthenzIdentity identity) { - return hasAccess("modify", new AthenzResourceName(service.getDomain().getName(), "hosted-accounting-resource").toResourceNameString(), identity); - } - - /** - * Used when creating tenancies. As there are no tenancy policies at this point, - * we cannot use {@link #hasTenantAdminAccess(AthenzIdentity, AthenzDomain)} - */ - private void verifyIsDomainAdmin(AthenzIdentity identity, AthenzDomain domain) { - log("getMembership(domain=%s, role=%s, principal=%s)", domain, "admin", identity); - if ( ! zmsClient.getMembership(new AthenzRole(domain, "admin"), identity)) - throw new RestApiException.Forbidden( - Text.format("The user '%s' is not admin in Athenz domain '%s'", identity.getFullName(), domain.getName())); - } - - public List<AthenzDomain> getDomainList(String prefix) { - log.log(Level.FINE, "getDomainList(prefix=%s)", prefix); - return zmsClient.getDomainList(prefix); - } - - private static Set<RoleAction> createTenantRoleActions() { - return Arrays.stream(ApplicationAction.values()) - .map(action -> new RoleAction(action.roleName, action.name())) - .collect(Collectors.toSet()); - } - - private boolean hasAccess(String action, String resource, AthenzIdentity identity) { - return accessRights.test(new AccessTuple(resource, action, identity)); - } - - private boolean lookupAccess(AccessTuple t) { - boolean result = ztsClient.hasAccess(AthenzResourceName.fromString(t.resource), t.action, t.identity); - log("getAccess(action=%s, resource=%s, principal=%s) = %b", t.action, t.resource, t.identity, result); - return result; - } - - private static void log(String format, Object... args) { - log.log(Level.FINE, String.format(format, args)); - } - - private String resourceStringPrefix(AthenzDomain tenantDomain) { - return Text.format("%s:service.%s.tenant.%s", - service.getDomain().getName(), service.getName(), tenantDomain.getName()); - } - - private String tenantResourceString(AthenzDomain tenantDomain) { - return resourceStringPrefix(tenantDomain) + ".wildcard"; - } - - private String applicationResourceString(AthenzDomain tenantDomain, ApplicationName applicationName, Optional<ZoneId> zone) { - // If environment is not provided, add .wildcard to match .* in the policy resource (* is not allowed in the request) - String environment = zone.map(ZoneId::environment).map(Environment::value).orElse("wildcard"); - return resourceStringPrefix(tenantDomain) + "." + "res_group" + "." + applicationName.value() + "." + environment; - } - - private enum TenantAction { - // This is meant to match only the '*' action of the 'admin' role. - // If needed, we can replace it with 'create', 'delete' etc. later. - _modify_ - } - - - private static class AccessTuple { - - private final String resource; - private final String action; - private final AthenzIdentity identity; - - private AccessTuple(String resource, String action, AthenzIdentity identity) { - this.resource = resource; - this.action = action; - this.identity = identity; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AccessTuple that = (AccessTuple) o; - return resource.equals(that.resource) && - action.equals(that.action) && - identity.equals(that.identity); - } - - @Override - public int hashCode() { - return Objects.hash(resource, action, identity); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java deleted file mode 100644 index dfc660442b9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.auditlog; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * This represents the audit log of a hosted Vespa system. The audit log contains manual actions performed through - * operator APIs served by the controller. - * - * Entries of the audit log are sorted by their timestamp, in descending order. - * - * @author mpolden - */ -public record AuditLog(List<Entry> entries) { - - public static final AuditLog empty = new AuditLog(List.of()); - - /** DO NOT USE. Public for serialization purposes */ - public AuditLog(List<Entry> entries) { - this.entries = Objects.requireNonNull(entries).stream().sorted().toList(); - } - - /** Returns a new audit log without entries older than given instant */ - public AuditLog pruneBefore(Instant instant) { - List<Entry> entries = new ArrayList<>(this.entries); - entries.removeIf(entry -> entry.at().isBefore(instant)); - return new AuditLog(entries); - } - - /** Returns copy of this with given entry added */ - public AuditLog with(Entry entry) { - List<Entry> entries = new ArrayList<>(this.entries); - entries.add(entry); - return new AuditLog(entries); - } - - /** Returns the first n entries in this. Since entries are sorted descendingly, this will be the n newest entries */ - public AuditLog first(int n) { - if (entries.size() < n) return this; - return new AuditLog(entries.subList(0, n)); - } - - /** An entry in the audit log. This describes an HTTP request */ - public record Entry(Instant at, String principal, Method method, String resource, Optional<String> data, - Client client) implements Comparable<Entry> { - - final static int maxDataLength = 1024; - private final static Comparator<Entry> comparator = Comparator.comparing(Entry::at).reversed(); - - public Entry(Instant at, Client client, String principal, Method method, String resource, byte[] data) { - this(Objects.requireNonNull(at, "at must be non-null"), - Objects.requireNonNull(principal, "principal must be non-null"), - Objects.requireNonNull(method, "method must be non-null"), - Objects.requireNonNull(resource, "resource must be non-null"), - sanitize(data), - Objects.requireNonNull(client, "client must be non-null")); - } - - /** Time of the request */ - public Instant at() { - return at; - } - - /** - * The client that performed this request. This may be based on user-controlled input, e.g. User-Agent header - * and is thus not guaranteed to be accurate. - */ - public Client client() { - return client; - } - - /** The principal performing the request */ - public String principal() { - return principal; - } - - /** Request method */ - public Method method() { - return method; - } - - /** API resource (URL path) */ - public String resource() { - return resource; - } - - /** Request data. This may be truncated if request data logged in this entry was too large */ - public Optional<String> data() { - return data; - } - - @Override - public int compareTo(Entry that) { - return comparator.compare(this, that); - } - - /** HTTP methods that should be logged */ - public enum Method { - POST, - PATCH, - PUT, - DELETE - } - - /** Known clients of the audit log */ - public enum Client { - /** The Vespa Cloud Console */ - console, - /** Vespa CLI */ - cli, - /** Operator tools */ - hv, - /** Other clients, e.g. curl */ - other, - } - - private static Optional<String> sanitize(byte[] data) { - StringBuilder sb = new StringBuilder(); - for (byte b : data) { - char c = (char) b; - if (!printableAscii(c) && !tabOrLineBreak(c)) { - return Optional.empty(); - } - sb.append(c); - if (sb.length() == maxDataLength) { - break; - } - } - return Optional.of(sb.toString()).filter(s -> !s.isEmpty()); - } - - private static boolean printableAscii(char c) { - return c >= 32 && c <= 126; - } - - private static boolean tabOrLineBreak(char c) { - return c == 9 || c == 10 || c == 13; - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java deleted file mode 100644 index ad541599475..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.auditlog; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.jdisc.http.HttpHeaders; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLog.Entry; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.io.SequenceInputStream; -import java.net.URI; -import java.security.Principal; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.Objects; -import java.util.Optional; - -import static com.yahoo.yolean.Exceptions.uncheck; -import static java.util.Objects.requireNonNullElse; - -/** - * This provides read and write operations for the audit log. - * - * @author mpolden - */ -public class AuditLogger { - - /** The TTL of log entries. Entries older than this will be removed when the log is updated */ - private static final Duration entryTtl = Duration.ofDays(14); - private static final int maxEntries = 2000; - - private final CuratorDb db; - private final Clock clock; - - public AuditLogger(CuratorDb db, Clock clock) { - this.db = Objects.requireNonNull(db, "db must be non-null"); - this.clock = Objects.requireNonNull(clock, "clock must be non-null"); - } - - /** Read the current audit log */ - public AuditLog readLog() { - return db.readAuditLog(); - } - - /** - * Write a log entry for given request to the audit log. - * - * Note that data contained in the given request may be consumed. Callers should use the returned HttpRequest for - * further processing. - */ - public HttpRequest log(HttpRequest request) { - Optional<AuditLog.Entry.Method> method = auditableMethod(request); - if (method.isEmpty()) return request; // Nothing to audit, e.g. a GET request - - Principal principal = request.getJDiscRequest().getUserPrincipal(); - if (principal == null) { - throw new IllegalStateException("Cannot audit " + request.getMethod() + " " + request.getUri() + - " as no principal was found in the request. This is likely caused by a " + - "misconfiguration and should not happen"); - } - - InputStream requestData = requireNonNullElse(request.getData(), InputStream.nullInputStream()); - byte[] data = uncheck(() -> requestData.readNBytes(Entry.maxDataLength)); - - AuditLog.Entry.Client client = parseClient(request); - Instant now = clock.instant(); - AuditLog.Entry entry = new AuditLog.Entry(now, client, principal.getName(), method.get(), - pathAndQueryOf(request.getUri()), data); - try (Mutex lock = db.lockAuditLog()) { - AuditLog auditLog = db.readAuditLog() - .pruneBefore(now.minus(entryTtl)) - .with(entry) - .first(maxEntries); - db.writeAuditLog(auditLog); - } - - // Create a new input stream to allow callers to consume request body - return new HttpRequest(request.getJDiscRequest(), - new SequenceInputStream(new ByteArrayInputStream(data), requestData), - request.propertyMap()); - } - - private static AuditLog.Entry.Client parseClient(HttpRequest request) { - String userAgent = request.getHeader(HttpHeaders.Names.USER_AGENT); - if (userAgent != null) { - if (userAgent.startsWith("Vespa CLI/")) { - return AuditLog.Entry.Client.cli; - } else if (userAgent.startsWith("Vespa Hosted Client ")) { - return AuditLog.Entry.Client.hv; - } - } - if (request.getPort() == 443) { - return AuditLog.Entry.Client.console; - } - return AuditLog.Entry.Client.other; - } - - /** Returns the auditable method of given request, if any */ - private static Optional<AuditLog.Entry.Method> auditableMethod(HttpRequest request) { - try { - return Optional.of(AuditLog.Entry.Method.valueOf(request.getMethod().name())); - } catch (IllegalArgumentException e) { - return Optional.empty(); - } - } - - private static String pathAndQueryOf(URI url) { - String pathAndQuery = url.getPath(); - String query = url.getQuery(); - if (query != null) { - pathAndQuery += "?" + query; - } - return pathAndQuery; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggingRequestHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggingRequestHandler.java deleted file mode 100644 index d73b5ef1d15..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggingRequestHandler.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.auditlog; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.jdisc.handler.ContentChannel; - -/** - * A handler that logs requests to the audit log. Handlers that need audit logging should extend this and implement - * {@link AuditLoggingRequestHandler#auditAndHandle(HttpRequest)}. - * - * @author mpolden - */ -public abstract class AuditLoggingRequestHandler extends ThreadedHttpRequestHandler { - - private final AuditLogger auditLogger; - - public AuditLoggingRequestHandler(Context ctx, AuditLogger auditLogger) { - super(ctx); - this.auditLogger = auditLogger; - } - - @Override - public final HttpResponse handle(HttpRequest request) { - return auditAndHandle(auditLogger.log(request)); - } - - @Override - public final HttpResponse handle(HttpRequest request, ContentChannel channel) { - return super.handle(request, channel); - } - - public abstract HttpResponse auditAndHandle(HttpRequest request); - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/package-info.java deleted file mode 100644 index daafbf7c767..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author mpolden - */ -@ExportPackage -package com.yahoo.vespa.hosted.controller.auditlog; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java deleted file mode 100644 index 49e2dc5bb0d..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.certificate; - -import com.yahoo.config.provision.InstanceName; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; - -import java.util.Optional; - -/** - * Represents a certificate and its owner. A certificate is either assigned to all instances of an application, or a - * specific one. - * - * @author mpolden - */ -public record AssignedCertificate(TenantAndApplicationId application, - Optional<InstanceName> instance, - EndpointCertificate certificate, - boolean shouldValidate) { - - public AssignedCertificate with(EndpointCertificate certificate) { - return new AssignedCertificate(application, instance, certificate, shouldValidate); - } - - public AssignedCertificate withoutInstance() { - return new AssignedCertificate(application, Optional.empty(), certificate, shouldValidate); - } - - public AssignedCertificate withShouldValidate(boolean shouldValidate) { - return new AssignedCertificate(application, instance, certificate, shouldValidate); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java deleted file mode 100644 index 391c9806f0a..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.certificate; - -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.text.Text; -import com.yahoo.transaction.Mutex; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.StringFlag; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateValidator; -import com.yahoo.vespa.hosted.controller.api.integration.secrets.GcpSecretStore; -import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; - -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.Comparator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate.State; - -/** - * This provisions, assigns and updates the certificate for a given deployment. - * - * See also {@link com.yahoo.vespa.hosted.controller.maintenance.EndpointCertificateMaintainer}, which handles - * refreshes, deletions and triggers deployments. - * - * @author andreer - * @author mpolden - */ -public class EndpointCertificates { - - private static final Logger LOG = Logger.getLogger(EndpointCertificates.class.getName()); - private static final Duration GCP_CERTIFICATE_EXPIRY_TIME = Duration.ofDays(100); // 100 days, 10 more than notAfter time - - private final Controller controller; - private final CuratorDb curator; - private final Clock clock; - private final EndpointCertificateProvider certificateProvider; - private final EndpointCertificateValidator certificateValidator; - private final BooleanFlag useAlternateCertProvider; - private final StringFlag endpointCertificateAlgo; - - public EndpointCertificates(Controller controller, EndpointCertificateProvider certificateProvider, - EndpointCertificateValidator certificateValidator) { - this.controller = controller; - this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); - this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); - this.curator = controller.curator(); - this.clock = controller.clock(); - this.certificateProvider = certificateProvider; - this.certificateValidator = certificateValidator; - } - - /** Returns a suitable certificate for endpoints of given deployment */ - public EndpointCertificate get(DeploymentId deployment, DeploymentSpec deploymentSpec, Mutex applicationLock) { - Objects.requireNonNull(applicationLock); - Instant start = clock.instant(); - EndpointConfig config = controller.routing().endpointConfig(deployment.applicationId()); - EndpointCertificate certificate = assignTo(deployment, deploymentSpec, config); - Duration duration = Duration.between(start, clock.instant()); - if (duration.toSeconds() > 30) { - LOG.log(Level.INFO, Text.format("Getting endpoint certificate for %s took %d seconds!", deployment.applicationId().serializedForm(), duration.toSeconds())); - } - if (isGcp(deployment)) { - // This is needed until CKMS is available from GCP - return validateGcpCertificate(deployment, deploymentSpec, certificate, config); - } - return certificate; - } - - private boolean isGcp(DeploymentId deployment) { - return controller.zoneRegistry().zones().all().in(CloudName.GCP).ids().contains(deployment.zoneId()); - } - - private EndpointCertificate validateGcpCertificate(DeploymentId deployment, DeploymentSpec deploymentSpec, EndpointCertificate certificate, EndpointConfig config) { - // Validate before copying cert to GCP. This will ensure we don't bug out on the first deployment, but will take more time - List<String> dnsNames = controller.routing().certificateDnsNames(deployment, deploymentSpec, certificate.generatedId().get(), config.supportsLegacy()); - certificateValidator.validate(certificate, deployment.applicationId().serializedForm(), deployment.zoneId(), dnsNames); - GcpSecretStore gcpSecretStore = controller.serviceRegistry().gcpSecretStore(); - String mangledCertName = "endpointCert_" + certificate.certName().replace('.', '_') + "-v" + certificate.version(); // Google cloud does not accept dots in secrets, but they accept underscores - String mangledKeyName = "endpointCert_" + certificate.keyName().replace('.', '_') + "-v" + certificate.version(); // Google cloud does not accept dots in secrets, but they accept underscores - if (gcpSecretStore.getLatestSecretVersion(mangledCertName) == null) { - gcpSecretStore.setSecret(mangledCertName, - Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), - "endpoint-cert-accessor"); - gcpSecretStore.addSecretVersion(mangledCertName, - controller.secretStore().getSecret(certificate.certName(), certificate.version())); - } - if (gcpSecretStore.getLatestSecretVersion(mangledKeyName) == null) { - gcpSecretStore.setSecret(mangledKeyName, - Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), - "endpoint-cert-accessor"); - gcpSecretStore.addSecretVersion(mangledKeyName, - controller.secretStore().getSecret(certificate.keyName(), certificate.version())); - } - return certificate.withVersion(1).withKeyName(mangledKeyName).withCertName(mangledCertName); - } - - private AssignedCertificate assignFromPool(TenantAndApplicationId application, Optional<InstanceName> instanceName, ZoneId zone) { - try (Mutex lock = controller.curator().lockCertificatePool()) { - Optional<UnassignedCertificate> candidate = curator.readUnassignedCertificates().stream() - .filter(pc -> pc.state() == State.ready) - .min(Comparator.comparingLong(pc -> pc.certificate().lastRequested())); - if (candidate.isEmpty()) { - throw new IllegalArgumentException("No endpoint certificate available in pool, for deployment of " + - application + instanceName.map(i -> "." + i.value()).orElse("") - + " in " + zone); - } - try (NestedTransaction transaction = new NestedTransaction()) { - curator.removeUnassignedCertificate(candidate.get(), transaction); - AssignedCertificate assigned = new AssignedCertificate(application, instanceName, candidate.get().certificate(), false); - curator.writeAssignedCertificate(assigned, transaction); - transaction.commit(); - return assigned; - } - } - } - - private AssignedCertificate instanceLevelCertificate(DeploymentId deployment, DeploymentSpec deploymentSpec, boolean allowPool) { - TenantAndApplicationId application = TenantAndApplicationId.from(deployment.applicationId()); - Optional<InstanceName> instance = Optional.of(deployment.applicationId().instance()); - Optional<AssignedCertificate> currentCertificate = curator.readAssignedCertificate(application, instance); - final AssignedCertificate assignedCertificate; - if (currentCertificate.isEmpty()) { - Optional<String> generatedId = Optional.empty(); - // Re-use the generated ID contained in an existing certificate (matching this application, this instance, - // or any other instance present in deployment sec), if any. If this exists we provision a new certificate - // containing the same ID - if (!deployment.zoneId().environment().isManuallyDeployed()) { - generatedId = curator.readAssignedCertificates().stream() - .filter(ac -> { - boolean matchingInstance = ac.instance().isPresent() && - deploymentSpec.instance(ac.instance().get()).isPresent(); - return (matchingInstance || ac.instance().isEmpty()) && - ac.application().equals(application); - }) - .map(AssignedCertificate::certificate) - .flatMap(ac -> ac.generatedId().stream()) - .findFirst(); - } - if (allowPool && generatedId.isEmpty()) { - assignedCertificate = assignFromPool(application, instance, deployment.zoneId()); - } else { - if (generatedId.isEmpty()) { - generatedId = Optional.of(generateId()); - } - EndpointCertificate provisionedCertificate = provision(deployment, Optional.empty(), deploymentSpec, generatedId.get()); - // We do not validate the certificate if one has never existed before - because we do not want to - // wait for it to be available before we deploy. This allows the config server to start - // provisioning nodes ASAP, and the risk is small for a new deployment. - assignedCertificate = new AssignedCertificate(application, instance, provisionedCertificate, false); - } - } else { - assignedCertificate = currentCertificate.get().withShouldValidate(!allowPool); - } - return assignedCertificate; - } - - private AssignedCertificate applicationLevelCertificate(DeploymentId deployment) { - if (deployment.zoneId().environment().isManuallyDeployed()) { - throw new IllegalArgumentException(deployment + " is manually deployed and cannot assign an application-level certificate"); - } - TenantAndApplicationId application = TenantAndApplicationId.from(deployment.applicationId()); - Optional<AssignedCertificate> applicationLevelCertificate = curator.readAssignedCertificate(application, Optional.empty()); - if (applicationLevelCertificate.isEmpty()) { - Optional<AssignedCertificate> instanceLevelCertificate = curator.readAssignedCertificate(application, Optional.of(deployment.applicationId().instance())); - // Migrate from instance-level certificate - if (instanceLevelCertificate.isPresent()) { - try (var transaction = new NestedTransaction()) { - AssignedCertificate assignedCertificate = instanceLevelCertificate.get().withoutInstance(); - curator.removeAssignedCertificate(application, Optional.of(deployment.applicationId().instance()), transaction); - curator.writeAssignedCertificate(assignedCertificate, transaction); - transaction.commit(); - return assignedCertificate; - } - } else { - return assignFromPool(application, Optional.empty(), deployment.zoneId()); - } - } - return applicationLevelCertificate.get(); - } - - /** Assign a certificate to given deployment. A new certificate is provisioned (possibly from a pool) and reconfigured as necessary */ - private EndpointCertificate assignTo(DeploymentId deployment, DeploymentSpec deploymentSpec, EndpointConfig config) { - // Assign certificate based on endpoint config - AssignedCertificate assignedCertificate = switch (config) { - case legacy, combined -> instanceLevelCertificate(deployment, deploymentSpec, false); - case generated -> deployment.zoneId().environment().isManuallyDeployed() - ? instanceLevelCertificate(deployment, deploymentSpec, true) - : applicationLevelCertificate(deployment); - }; - - // Generate ID if not already present in certificate - Optional<String> generatedId = assignedCertificate.certificate().generatedId(); - if (generatedId.isEmpty()) { - generatedId = Optional.of(generateId()); - assignedCertificate = assignedCertificate.with(assignedCertificate.certificate().withGeneratedId(generatedId.get())); - } - - // Ensure all wanted names are present in certificate - List<String> wantedNames = controller.routing().certificateDnsNames(deployment, deploymentSpec, generatedId.get(), config.supportsLegacy()); - Set<String> currentNames = Set.copyOf(assignedCertificate.certificate().requestedDnsSans()); - // TODO(mpolden): Consider requiring exact match for generated as we likely want to remove any legacy names in this case - if (!currentNames.containsAll(wantedNames)) { - EndpointCertificate updatedCertificate = provision(deployment, Optional.of(assignedCertificate.certificate()), deploymentSpec, generatedId.get()); - // Validation is unlikely to succeed in this case, as certificate must be available first. Controller will retry - assignedCertificate = assignedCertificate.with(updatedCertificate) - .withShouldValidate(true); - } - - // Require that generated ID is always set, for any kind of certificate - if (assignedCertificate.certificate().generatedId().isEmpty()) { - throw new IllegalArgumentException("Certificate for " + deployment + " does not contain generated ID: " + - assignedCertificate.certificate()); - } - - // Update the time we last requested this certificate. This field is used by EndpointCertificateMaintainer to - // determine stale certificates - assignedCertificate = assignedCertificate.with(assignedCertificate.certificate().withLastRequested(clock.instant().getEpochSecond())); - curator.writeAssignedCertificate(assignedCertificate); - - // Validate if we're re-assigned an existing certificate, or if we updated the names of an existing certificate - if (assignedCertificate.shouldValidate()) { - certificateValidator.validate(assignedCertificate.certificate(), deployment.applicationId().serializedForm(), - deployment.zoneId(), wantedNames); - } - - return assignedCertificate.certificate(); - } - - private String generateId() { - List<String> unassignedIds = curator.readUnassignedCertificates().stream() - .map(UnassignedCertificate::id) - .toList(); - List<String> assignedIds = curator.readAssignedCertificates().stream() - .map(AssignedCertificate::certificate) - .map(EndpointCertificate::generatedId) - .flatMap(Optional::stream) - .toList(); - Set<String> allIds = Stream.concat(unassignedIds.stream(), assignedIds.stream()).collect(Collectors.toSet()); - String id; - do { - id = GeneratedEndpoint.createPart(controller.random(true)); - } while (allIds.contains(id)); - return id; - } - - private EndpointCertificate provision(DeploymentId deployment, - Optional<EndpointCertificate> current, - DeploymentSpec deploymentSpec, - String generatedId) { - List<ZoneId> zonesInSystem = controller.zoneRegistry().zones().controllerUpgraded().ids(); - Set<ZoneId> requiredZones = new LinkedHashSet<>(); - requiredZones.add(deployment.zoneId()); - if (!deployment.zoneId().environment().isManuallyDeployed()) { - // If not deploying to a dev or perf zone, require all prod zones in deployment spec + test and staging - Optional<DeploymentInstanceSpec> instanceSpec = deploymentSpec.instance(deployment.applicationId().instance()); - zonesInSystem.stream() - .filter(zone -> zone.environment().isTest() || - (instanceSpec.isPresent() && - instanceSpec.get().deploysTo(zone.environment(), zone.region()))) - .forEach(requiredZones::add); - } - Set<String> wantedNames = requiredZones.stream() - .flatMap(zone -> controller.routing().certificateDnsNames(new DeploymentId(deployment.applicationId(), zone), - deploymentSpec, generatedId, true) - .stream()) - .collect(Collectors.toCollection(LinkedHashSet::new)); - - // Preserve any currently present names that are still valid (i.e. the name points to a zone found in this system) - Set<String> currentNames = current.map(EndpointCertificate::requestedDnsSans) - .map(Set::copyOf) - .orElseGet(Set::of); - for (var zone : zonesInSystem) { - List<String> wantedNamesZone = controller.routing().certificateDnsNames(new DeploymentId(deployment.applicationId(), zone), - deploymentSpec, - generatedId, - true); - if (currentNames.containsAll(wantedNamesZone)) { - wantedNames.addAll(wantedNamesZone); - } - } - - // Request certificate - LOG.log(Level.INFO, String.format("Requesting new endpoint certificate for application %s", deployment.applicationId().serializedForm())); - String algo = endpointCertificateAlgo.with(FetchVector.Dimension.INSTANCE_ID, deployment.applicationId().serializedForm()).value(); - boolean useAlternativeProvider = useAlternateCertProvider.with(FetchVector.Dimension.INSTANCE_ID, deployment.applicationId().serializedForm()).value(); - String keyPrefix = deployment.applicationId().toFullString(); - Instant t0 = controller.clock().instant(); - EndpointCertificate endpointCertificate = certificateProvider.requestCaSignedCertificate(keyPrefix, List.copyOf(wantedNames), current, algo, useAlternativeProvider); - Instant t1 = controller.clock().instant(); - LOG.log(Level.INFO, String.format("Endpoint certificate request for application %s returned after %s", deployment.applicationId().serializedForm(), Duration.between(t0, t1))); - return endpointCertificate.withGeneratedId(generatedId); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/UnassignedCertificate.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/UnassignedCertificate.java deleted file mode 100644 index 1d1f4938758..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/UnassignedCertificate.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.certificate; - -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; - -/** - * An unassigned certificate, which exists in a pre-provisioned pool of certificates. Once assigned to an application, - * the certificate is removed from the pool. - * - * @param certificate Details of the certificate - * @param state Current state of this - * - * @author andreer - */ -public record UnassignedCertificate(EndpointCertificate certificate, UnassignedCertificate.State state) { - - public UnassignedCertificate { - if (certificate.generatedId().isEmpty()) { - throw new IllegalArgumentException("generatedId must be set for a pooled certificate"); - } - } - - public String id() { - return certificate.generatedId().get(); - } - - public UnassignedCertificate withState(State state) { - return new UnassignedCertificate(certificate, state); - } - - public enum State { - /** The certificate is ready for assignment */ - ready, - - /** The certificate is requested and is being provisioned */ - requested - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Once.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Once.java deleted file mode 100644 index 2e717f16d0e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Once.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.concurrent; - -import java.time.Duration; -import java.util.Objects; -import java.util.Timer; -import java.util.TimerTask; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Execute a runnable exactly once in a background thread. - * - * @author mpolden - */ -public class Once extends TimerTask { - - private static final Logger log = Logger.getLogger(Once.class.getName()); - - private final Runnable runnable; - private final Timer timer = new Timer(true); - - // private to avoid exposing run method - private Once(Runnable runnable, Duration delay) { - this.runnable = Objects.requireNonNull(runnable, "runnable must be non-null"); - Objects.requireNonNull(delay, "delay must be non-null"); - timer.schedule(this, delay.toMillis()); - } - - /** Execute runnable after given delay */ - public static void after(Duration delay, Runnable runnable) { - new Once(runnable, delay); - } - - @Override - public void run() { - try { - runnable.run(); - } catch (Throwable t) { - log.log(Level.WARNING, "Task '" + runnable + "' failed", t); - } finally { - timer.cancel(); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java deleted file mode 100644 index 2b1d00ada95..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import java.util.Objects; - -/** - * Summary of node and service status during a deployment job. - * - * @author jonmv - */ -public class ConvergenceSummary { - - private final long nodes; - private final long down; - private final long upgradingOs; - private final long upgradingFirmware; - private final long needPlatformUpgrade; - private final long upgradingPlatform; - private final long needReboot; - private final long rebooting; - private final long needRestart; - private final long restarting; - private final long services; - private final long needNewConfig; - private final long retiring; - - public ConvergenceSummary(long nodes, long down, long upgradingOs, long upgradingFirmware, long needPlatformUpgrade, long upgradingPlatform, - long needReboot, long rebooting, long needRestart, long restarting, long services, long needNewConfig, long retiring) { - this.nodes = nodes; - this.down = down; - this.upgradingOs = upgradingOs; - this.upgradingFirmware = upgradingFirmware; - this.needPlatformUpgrade = needPlatformUpgrade; - this.upgradingPlatform = upgradingPlatform; - this.needReboot = needReboot; - this.rebooting = rebooting; - this.needRestart = needRestart; - this.restarting = restarting; - this.services = services; - this.needNewConfig = needNewConfig; - this.retiring = retiring; - } - - /** Number of nodes in the application. */ - public long nodes() { - return nodes; - } - - /** Number of nodes allowed to be down. */ - public long down() { - return down; - } - - /** Number of nodes down for OS upgrade. */ - public long upgradingOs() { - return upgradingOs; - } - - /** Number of nodes down for firmware upgrade. */ - public long upgradingFirmware() { - return upgradingFirmware; - } - - /** Number of nodes in need of a platform upgrade. */ - public long needPlatformUpgrade() { - return needPlatformUpgrade; - } - - /** Number of nodes down for platform upgrade. */ - public long upgradingPlatform() { - return upgradingPlatform; - } - - /** Number of nodes in need of a reboot. */ - public long needReboot() { - return needReboot; - } - - /** Number of nodes down for reboot. */ - public long rebooting() { - return rebooting; - } - - /** Number of nodes in need of a restart. */ - public long needRestart() { - return needRestart; - } - - /** Number of nodes down for restart. */ - public long restarting() { - return restarting; - } - - /** Number of services in the application. */ - public long services() { - return services; - } - - /** Number of services with outdated config generation. */ - public long needNewConfig() { - return needNewConfig; - } - - /** Number of nodes that are retiring. */ - public long retiring() { - return retiring; - } - - /** Whether the convergence is done. */ - public boolean converged() { - return nodes > 0 - && needPlatformUpgrade == 0 - && needReboot == 0 - && needRestart == 0 - && services > 0 - && needNewConfig == 0; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ConvergenceSummary that = (ConvergenceSummary) o; - return nodes == that.nodes && - down == that.down && - upgradingOs == that.upgradingOs && - upgradingFirmware == that.upgradingFirmware && - needPlatformUpgrade == that.needPlatformUpgrade && - upgradingPlatform == that.upgradingPlatform && - needReboot == that.needReboot && - rebooting == that.rebooting && - needRestart == that.needRestart && - restarting == that.restarting && - services == that.services && - needNewConfig == that.needNewConfig && - retiring == that.retiring; - } - - @Override - public int hashCode() { - return Objects.hash(nodes, down, upgradingOs, upgradingFirmware, needPlatformUpgrade, upgradingPlatform, needReboot, rebooting, needRestart, restarting, services, needNewConfig, retiring); - } - -} - - diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java deleted file mode 100644 index 223ba546b3e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ /dev/null @@ -1,1285 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.google.common.collect.ImmutableMap; -import com.yahoo.component.Version; -import com.yahoo.component.VersionCompatibility; -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.DeploymentSpec.DeclaredTest; -import com.yahoo.config.application.api.DeploymentSpec.DeclaredZone; -import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; -import com.yahoo.config.application.api.DeploymentSpec.UpgradeRollout; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.stream.CustomCollectors; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.deployment.Run.Reason; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; - -import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.collections.Iterables.reversed; -import static com.yahoo.config.application.api.DeploymentSpec.RevisionTarget.next; -import static com.yahoo.config.provision.Environment.prod; -import static com.yahoo.config.provision.Environment.staging; -import static com.yahoo.config.provision.Environment.test; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.cancelled; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.invalidApplication; -import static java.util.Comparator.comparing; -import static java.util.Comparator.naturalOrder; -import static java.util.Comparator.reverseOrder; -import static java.util.Objects.requireNonNull; -import static java.util.function.BinaryOperator.maxBy; -import static java.util.stream.Collectors.collectingAndThen; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.mapping; -import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; - -/** - * Status of the deployment jobs of an {@link Application}. - * - * @author jonmv - */ -public class DeploymentStatus { - - private static <T> List<T> union(List<T> first, List<T> second) { - return Stream.concat(first.stream(), second.stream()).distinct().toList(); - } - - private final Application application; - private final JobList allJobs; - private final VersionStatus versionStatus; - private final Version systemVersion; - private final Function<InstanceName, VersionCompatibility> versionCompatibility; - private final ZoneRegistry zones; - private final Instant now; - private final Map<JobId, StepStatus> jobSteps; - private final List<StepStatus> allSteps; - - public DeploymentStatus(Application application, Function<JobId, JobStatus> allJobs, ZoneRegistry zones, VersionStatus versionStatus, - Version systemVersion, Function<InstanceName, VersionCompatibility> versionCompatibility, Instant now) { - this.application = requireNonNull(application); - this.zones = zones; - this.versionStatus = requireNonNull(versionStatus); - this.systemVersion = requireNonNull(systemVersion); - this.versionCompatibility = versionCompatibility; - this.now = requireNonNull(now); - List<StepStatus> allSteps = new ArrayList<>(); - Map<JobId, JobStatus> jobs = new HashMap<>(); - this.jobSteps = jobDependencies(application.deploymentSpec(), allSteps, job -> jobs.computeIfAbsent(job, allJobs)); - this.allSteps = Collections.unmodifiableList(allSteps); - this.allJobs = JobList.from(jobSteps.keySet().stream().map(allJobs).toList()); - } - - private JobType systemTest(JobType dependent) { - return JobType.systemTest(zones, dependent == null ? null : findCloud(dependent)); - } - - private JobType stagingTest(JobType dependent) { - return JobType.stagingTest(zones, dependent == null ? null : findCloud(dependent)); - } - - /** The application this deployment status concerns. */ - public Application application() { - return application; - } - - /** A filterable list of the status of all jobs for this application. */ - public JobList jobs() { - return allJobs; - } - - /** Whether any jobs both dependent on the dependency, and a dependency for the dependent, are failing. */ - private boolean hasFailures(StepStatus dependency, StepStatus dependent) { - Set<StepStatus> dependents = new HashSet<>(); - fillDependents(dependency, new HashSet<>(), dependents, dependent); - Set<JobId> criticalJobs = dependents.stream().flatMap(step -> step.job().stream()).collect(toSet()); - - return ! allJobs.matching(job -> criticalJobs.contains(job.id())) - .failingHard() - .isEmpty(); - } - - private boolean fillDependents(StepStatus dependency, Set<StepStatus> visited, Set<StepStatus> dependents, StepStatus current) { - if (visited.contains(current)) - return dependents.contains(current); - - if (dependency == current) - dependents.add(current); - else - for (StepStatus dep : current.dependencies) - if (fillDependents(dependency, visited, dependents, dep)) - dependents.add(current); - - visited.add(current); - return dependents.contains(current); - } - - /** Whether any job is failing on versions selected by the given filter, with errors other than lack of capacity in a test zone.. */ - public boolean hasFailures(Predicate<RevisionId> revisionFilter) { - return ! allJobs.failingHard() - .matching(job -> revisionFilter.test(job.lastTriggered().get().versions().targetRevision())) - .isEmpty(); - } - - /** Whether any jobs of this application are failing with other errors than lack of capacity in a test zone. */ - public boolean hasFailures() { - return ! allJobs.failingHard().isEmpty(); - } - - /** All job statuses, by job type, for the given instance. */ - public Map<JobType, JobStatus> instanceJobs(InstanceName instance) { - return allJobs.asList().stream() - .filter(job -> job.id().application().equals(application.id().instance(instance))) - .collect(CustomCollectors.toLinkedMap(job -> job.id().type(), Function.identity())); - } - - /** Filterable job status lists for each instance of this application. */ - public Map<ApplicationId, JobList> instanceJobs() { - return allJobs.groupingBy(job -> job.id().application()); - } - - /** Returns change potentially with a compatibility platform added, if required for the change to roll out to the given instance. */ - public Change withPermittedPlatform(Change change, InstanceName instance, boolean allowOutdatedPlatform) { - Change augmented = withCompatibilityPlatform(change, instance); - if (allowOutdatedPlatform) - return augmented; - - // If compatibility platform is present, require that jobs have previously been run on that platform's major. - // If platform is not present, app is already on the (old) platform iff. it has production deployments. - boolean alreadyDeployedOnPlatform = augmented.platform().map(platform -> allJobs.production().not().test().asList().stream() - .anyMatch(job -> job.runs().values().stream() - .anyMatch(run -> run.versions().targetPlatform().getMajor() == platform.getMajor()))) - .orElse( ! application.productionDeployments().values().stream().allMatch(List::isEmpty)); - - // Verify target platform is either current, or was previously deployed for this app. - if (augmented.platform().isPresent() && ! versionStatus.isOnCurrentMajor(augmented.platform().get()) && ! alreadyDeployedOnPlatform) - throw new IllegalArgumentException("platform version " + augmented.platform().get() + " is not on a current major version in this system"); - - Version latestHighConfidencePlatform = null; - for (VespaVersion platform : versionStatus.deployableVersions()) - if (platform.confidence().equalOrHigherThan(Confidence.high)) - latestHighConfidencePlatform = platform.versionNumber(); - - // Verify package is compatible with the current major, or newer, or that there already are deployments on a compatible, outdated platform. - if (latestHighConfidencePlatform != null) { - Version target = latestHighConfidencePlatform; - augmented.revision().flatMap(revision -> application.revisions().get(revision).compileVersion()) - .filter(target::isAfter) - .ifPresent(compiled -> { - if (versionCompatibility.apply(instance).refuse(target, compiled) && ! alreadyDeployedOnPlatform) - throw new IllegalArgumentException("compile version " + compiled + " is incompatible with the current major version of this system"); - }); - } - - return augmented; - } - - private Change withCompatibilityPlatform(Change change, InstanceName instance) { - if (change.revision().isEmpty()) - return change; - - Optional<Version> compileVersion = change.revision() - .map(application.revisions()::get) - .flatMap(ApplicationVersion::compileVersion); - - // If the revision requires a certain platform for compatibility, add that here, unless we're already deploying a compatible platform. - VersionCompatibility compatibility = versionCompatibility.apply(instance); - Predicate<Version> compatibleWithCompileVersion = version -> compileVersion.map(compiled -> compatibility.accept(version, compiled)).orElse(true); - if (change.platform().map(compatibleWithCompileVersion::test).orElse(false)) - return change; - - if ( application.productionDeployments().isEmpty() - || application.productionDeployments().getOrDefault(instance, List.of()).stream() - .anyMatch(deployment -> ! compatibleWithCompileVersion.test(deployment.version()))) { - for (Version platform : targetsForPolicy(versionStatus, systemVersion, application.deploymentSpec().requireInstance(instance).upgradePolicy())) - if (compatibleWithCompileVersion.test(platform)) - return change.withoutPlatformPin().with(platform); - } - return change; - } - - /** Returns target versions for given confidence, by descending version number. */ - public static List<Version> targetsForPolicy(VersionStatus versions, Version systemVersion, DeploymentSpec.UpgradePolicy policy) { - if (policy == DeploymentSpec.UpgradePolicy.canary) - return List.of(systemVersion); - - VespaVersion.Confidence target = policy == DeploymentSpec.UpgradePolicy.defaultPolicy ? VespaVersion.Confidence.normal : VespaVersion.Confidence.high; - return versions.deployableVersions().stream() - .filter(version -> version.confidence().equalOrHigherThan(target)) - .map(VespaVersion::versionNumber) - .sorted(reverseOrder()) - .toList(); - } - - - /** - * The set of jobs that need to run for the changes of each instance of the application to be considered complete, - * and any test jobs for any outstanding change, which will likely be needed to later deploy this change. - */ - public Map<JobId, List<Job>> jobsToRun() { - if (application.revisions().last().isEmpty()) return Map.of(); - - Map<InstanceName, Change> changes = new LinkedHashMap<>(); - for (InstanceName instance : application.deploymentSpec().instanceNames()) - changes.put(instance, application.require(instance).change()); - Map<JobId, List<Job>> jobs = jobsToRun(changes); - - // Add test jobs for any outstanding change. - Map<InstanceName, Change> outstandingChanges = new LinkedHashMap<>(); - for (InstanceName instance : application.deploymentSpec().instanceNames()) { - Change outstanding = outstandingChange(instance); - if (outstanding.hasTargets()) - outstandingChanges.put(instance, outstanding.onTopOf(application.require(instance).change().withoutRevisionPin())); - } - var testJobs = jobsToRun(outstandingChanges, true).entrySet().stream() - .filter(entry -> ! entry.getKey().type().isProduction()); - - return Stream.concat(jobs.entrySet().stream(), testJobs) - .collect(collectingAndThen(toMap(Map.Entry::getKey, - Map.Entry::getValue, - DeploymentStatus::union, - LinkedHashMap::new), - Collections::unmodifiableMap)); - } - - private Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes, boolean eagerTests) { - if (application.revisions().last().isEmpty()) return Map.of(); - - Map<JobId, List<Job>> productionJobs = new LinkedHashMap<>(); - changes.forEach((instance, change) -> productionJobs.putAll(productionJobs(instance, change, eagerTests))); - Map<JobId, List<Job>> testJobs = testJobs(productionJobs); - Map<JobId, List<Job>> jobs = new LinkedHashMap<>(testJobs); - jobs.putAll(productionJobs); - // Add runs for idle, declared test jobs if they have no successes on their instance's change's versions. - jobSteps.forEach((job, step) -> { - if ( ! step.isDeclared() || job.type().isProduction() || jobs.containsKey(job)) - return; - - Change change = changes.get(job.application().instance()); - if (change == null || ! change.hasTargets()) - return; - - Map<CloudName, Optional<JobId>> firstProductionJobsWithDeployment = firstDependentProductionJobsWithDeployment(job.application().instance()); - firstProductionJobsWithDeployment.forEach((cloud, firstProductionJobWithDeploymentInCloud) -> { - Versions versions = Versions.from(change, - application, - firstProductionJobWithDeploymentInCloud.flatMap(this::deploymentFor), - fallbackPlatform(change, job)); - if (step.completedAt(change, firstProductionJobWithDeploymentInCloud).isEmpty()) { - JobType typeWithZone = job.type().isSystemTest() ? JobType.systemTest(zones, cloud) : JobType.stagingTest(zones, cloud); - Readiness readiness = step.readiness(change, firstProductionJobWithDeploymentInCloud); - jobs.merge(job, List.of(new Job(typeWithZone, - versions, - readiness.okAt(now) && jobs().get(job).get().isRunning() ? readiness.running() : readiness, - change, - null)), DeploymentStatus::union); - } - }); - }); - return Collections.unmodifiableMap(jobs); - } - - /** - * Returns the clouds, and their first production deployments, that depend on this instance; or, - * if no such deployments exist, all clouds the application deploy to, and their first production deployments; or - * if no clouds are deployed to at all, the system default cloud. - */ - public Map<CloudName, Optional<JobId>> firstDependentProductionJobsWithDeployment(InstanceName testInstance) { - // Find instances' dependencies on each other: these are topologically ordered, so a simple traversal does it. - Map<InstanceName, Set<InstanceName>> dependencies = new HashMap<>(); - instanceSteps().forEach((name, step) -> { - dependencies.put(name, new HashSet<>()); - dependencies.get(name).add(name); - for (StepStatus dependency : step.dependencies()) { - dependencies.get(name).add(dependency.instance()); - dependencies.get(name).addAll(dependencies.get(dependency.instance)); - } - }); - - Map<CloudName, Optional<JobId>> independentJobsPerCloud = new HashMap<>(); - Map<CloudName, Optional<JobId>> jobsPerCloud = new HashMap<>(); - jobSteps.forEach((job, step) -> { - if ( ! job.type().isProduction() || ! job.type().isDeployment()) - return; - - (dependencies.get(step.instance()).contains(testInstance) ? jobsPerCloud - : independentJobsPerCloud) - .merge(findCloud(job.type()), - Optional.of(job), - (o, n) -> o.filter(v -> deploymentFor(v).isPresent()) // Keep first if its deployment is present. - .or(() -> n.filter(v -> deploymentFor(v).isPresent())) // Use next if only its deployment is present. - .or(() -> o)); // Keep first if none have deployments. - }); - - if (jobsPerCloud.isEmpty()) - jobsPerCloud.putAll(independentJobsPerCloud); - - if (jobsPerCloud.isEmpty()) - jobsPerCloud.put(zones.systemZone().getCloudName(), Optional.empty()); - - return jobsPerCloud; - } - - - /** Fall back to the newest, deployable platform, which is compatible with what we want to deploy. */ - public Supplier<Version> fallbackPlatform(Change change, JobId job) { - return () -> { - InstanceName instance = job.application().instance(); - Optional<Version> compileVersion = change.revision().map(application.revisions()::get).flatMap(ApplicationVersion::compileVersion); - List<Version> targets = targetsForPolicy(versionStatus, - systemVersion, - application.deploymentSpec().instance(instance) - .map(DeploymentInstanceSpec::upgradePolicy) - .orElse(UpgradePolicy.defaultPolicy)); - - // Prefer fallback with proper confidence. - for (Version target : targets) - if (compileVersion.isEmpty() || versionCompatibility.apply(instance).accept(target, compileVersion.get())) - return target; - - // Try fallback with any confidence. - for (VespaVersion target : reversed(versionStatus.deployableVersions())) - if (compileVersion.isEmpty() || versionCompatibility.apply(instance).accept(target.versionNumber(), compileVersion.get())) - return target.versionNumber(); - - return compileVersion.orElseThrow(() -> new IllegalArgumentException("no legal platform version exists in this system for compile version " + compileVersion.get())); - }; - } - - - /** The set of jobs that need to run for the given changes to be considered complete. */ - public boolean hasCompleted(InstanceName instance, Change change) { - DeploymentInstanceSpec spec = application.deploymentSpec().requireInstance(instance); - if ((spec.concerns(test) || spec.concerns(staging)) && ! spec.concerns(prod)) { - if (newestTested(instance, run -> run.versions().targetRevision()).map(change::downgrades).orElse(false)) return true; - if (newestTested(instance, run -> run.versions().targetPlatform()).map(change::downgrades).orElse(false)) return true; - } - - return jobsToRun(Map.of(instance, change), false).isEmpty(); - } - - /** The set of jobs that need to run for the given changes to be considered complete. */ - public Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes) { - return jobsToRun(changes, false); - } - - /** The step status for all steps in the deployment spec of this, which are jobs, in the same order as in the deployment spec. */ - public Map<JobId, StepStatus> jobSteps() { return jobSteps; } - - public Map<InstanceName, StepStatus> instanceSteps() { - ImmutableMap.Builder<InstanceName, StepStatus> instances = ImmutableMap.builder(); - for (StepStatus status : allSteps) - if (status instanceof InstanceStatus) - instances.put(status.instance(), status); - return instances.build(); - } - - /** The step status for all relevant steps in the deployment spec of this, in the same order as in the deployment spec. */ - public List<StepStatus> allSteps() { - return allSteps; - } - - public Optional<Deployment> deploymentFor(JobId job) { - return Optional.ofNullable(application.require(job.application().instance()) - .deployments().get(job.type().zone())); - } - - private <T extends Comparable<T>> Optional<T> newestTested(InstanceName instance, Function<Run, T> runMapper) { - Set<CloudName> clouds = Stream.concat(Stream.of(zones.systemZone().getCloudName()), - jobSteps.keySet().stream() - .filter(job -> job.type().isProduction()) - .map(job -> findCloud(job.type()))) - .collect(toSet()); - List<ZoneId> testZones = new ArrayList<>(); - if (application.deploymentSpec().requireInstance(instance).concerns(test)) - for (CloudName cloud: clouds) testZones.add(JobType.systemTest(zones, cloud).zone()); - if (application.deploymentSpec().requireInstance(instance).concerns(staging)) - for (CloudName cloud: clouds) testZones.add(JobType.stagingTest(zones, cloud).zone()); - - Map<ZoneId, Optional<T>> newestPerZone = instanceJobs().get(application.id().instance(instance)) - .type(systemTest(null), stagingTest(null)) - .asList().stream().flatMap(jobs -> jobs.runs().values().stream()) - .filter(Run::hasSucceeded) - .collect(groupingBy(run -> run.id().type().zone(), - mapping(runMapper, Collectors.maxBy(naturalOrder())))); - return newestPerZone.keySet().containsAll(testZones) - ? testZones.stream().map(newestPerZone::get) - .reduce((o, n) -> o.isEmpty() || n.isEmpty() ? Optional.empty() : n.get().compareTo(o.get()) < 0 ? n : o) - .orElse(Optional.empty()) - : Optional.empty(); - } - - /** - * The change to a revision which all dependencies of the given instance has completed, - * which does not downgrade any deployments in the instance, - * which is not already rolling out to the instance, and - * which causes at least one job to run if deployed to the instance. - * For the "next" revision target policy it is the oldest such revision; otherwise, it is the latest. - */ - public Change outstandingChange(InstanceName instance) { - StepStatus status = instanceSteps().get(instance); - if (status == null) return Change.empty(); - DeploymentInstanceSpec spec = application.deploymentSpec().requireInstance(instance); - boolean ascending = next == spec.revisionTarget(); - int cumulativeRisk = 0; - int nextRisk = 0; - int skippedCumulativeRisk = 0; - Instant readySince = now; - - Optional<RevisionId> newestRevision = application.productionDeployments() - .getOrDefault(instance, List.of()).stream() - .map(Deployment::revision).max(naturalOrder()); - Change candidate = Change.empty(); - for (ApplicationVersion version : application.revisions().deployable(ascending)) { - // A revision is only a candidate if it upgrades, and does not downgrade, this instance. - Change change = Change.of(version.id()); - if ( newestRevision.isPresent() && change.downgrades(newestRevision.get()) - || ! application.require(instance).change().revision().map(change::upgrades).orElse(true) - || hasCompleted(instance, change)) { - if (ascending) continue; // Keep looking for the next revision which is an upgrade, or ... - else return Change.empty(); // ... if the latest is already complete, there's nothing outstanding. - } - - // This revision contains something new, so start aggregating the risk score. - skippedCumulativeRisk += version.risk(); - nextRisk = nextRisk > 0 ? nextRisk : version.risk(); - // If it's not yet ready to roll out, we keep looking. - Optional<Instant> readyAt = status.dependenciesCompletedAt(Change.of(version.id()), Optional.empty()); - if (readyAt.map(now::isBefore).orElse(true)) continue; - - // It's ready. If looking for the latest, max risk is 0, and we'll return now; otherwise, we _may_ keep on looking for more. - cumulativeRisk += skippedCumulativeRisk; - skippedCumulativeRisk = 0; - nextRisk = 0; - if (cumulativeRisk >= spec.maxRisk()) - return candidate.equals(Change.empty()) ? change : candidate; // If the first candidate exceeds max risk, we have to accept that. - - // Otherwise, we may note this as a candidate, and keep looking for a newer revision, unless that makes us exceed max risk. - if (readyAt.get().isBefore(readySince)) readySince = readyAt.get(); - candidate = change; - } - // If min risk is ready, or max idle time has passed, we return the candidate. Otherwise, no outstanding change is ready. - return instanceJobs(instance).values().stream().allMatch(jobs -> jobs.lastTriggered().isEmpty()) - || cumulativeRisk >= spec.minRisk() - || cumulativeRisk + nextRisk > spec.maxRisk() - || ! now.isBefore(readySince.plus(Duration.ofHours(spec.maxIdleHours()))) - ? candidate : Change.empty(); - } - - /** Earliest instant when job was triggered with given versions, or both system and staging tests were successful. */ - public Readiness verifiedAt(JobId job, Versions versions) { - Readiness triggered = allJobs.get(job) - .flatMap(status -> status.runs().values().stream() - .filter(run -> run.versions().equals(versions)) - .findFirst()) - .map(Run::start) - .map(Readiness::new) - .orElse(Readiness.unverified); - Readiness systemTested = testedAt(job, systemTest(job.type()), versions); - Readiness stagingTested = testedAt(job, stagingTest(job.type()), versions); - if (! systemTested.ok() || ! stagingTested.ok()) return triggered; - Readiness tested = min(systemTested, stagingTested); - return triggered.ok() && triggered.at().isBefore(tested.at) ? triggered : tested; - } - - /** Earliest instant when versions were tested for the given instance. */ - private Readiness testedAt(JobId job, JobType type, Versions versions) { - return prerequisiteTests(job, type).stream() - .map(test -> allJobs.get(test).stream() - .flatMap(status -> RunList.from(status) - .on(versions) - .matching(run -> run.id().type().zone().equals(type.zone())) - .matching(Run::hasSucceeded) - .asList().stream() - .map(run -> run.end().get())) - .min(naturalOrder())) - .map(testedAt -> testedAt.map(Readiness::new).orElse(Readiness.unverified)) - .reduce(Readiness.empty, DeploymentStatus::max); - } - - private Map<JobId, List<Job>> productionJobs(InstanceName instance, Change change, boolean assumeUpgradesSucceed) { - Map<JobId, List<Job>> jobs = new LinkedHashMap<>(); - for (Entry<JobId, StepStatus> entry : reversed(List.copyOf(jobSteps.entrySet()))) { - JobId job = entry.getKey(); - StepStatus step = entry.getValue(); - if ( ! job.application().instance().equals(instance) || ! job.type().isProduction()) - continue; - - // Signal strict completion criterion by depending on job itself. - if (step.completedAt(change, Optional.of(job)).isPresent()) - continue; - - // When computing eager test jobs for outstanding changes, assume current change completes successfully. - Optional<Deployment> deployment = deploymentFor(job); - Optional<Version> existingPlatform = deployment.map(Deployment::version); - Optional<RevisionId> existingRevision = deployment.map(Deployment::revision); - boolean deployingCompatibilityChange = areIncompatible(existingPlatform, change.revision(), job) - || areIncompatible(change.platform(), existingRevision, job); - if (assumeUpgradesSucceed) { - if (deployingCompatibilityChange) // No eager tests for this. - continue; - - Change currentChange = application.require(instance).change(); - Versions target = Versions.from(currentChange, application, deployment, fallbackPlatform(currentChange, job)); - existingPlatform = Optional.of(target.targetPlatform()); - existingRevision = Optional.of(target.targetRevision()); - } - List<Job> toRun = new ArrayList<>(); - List<Change> changes = deployingCompatibilityChange - || allJobs.get(job).flatMap(status -> status.lastCompleted()).isEmpty() - ? List.of(change) - : changes(job, step, change); - for (Change partial : changes) { - Versions versions = Versions.from(partial, application, existingPlatform, existingRevision, fallbackPlatform(partial, job)); - Readiness readiness = step.readiness(partial, Optional.of(job)); - // This job is blocked if it is already running ... - readiness = jobs().get(job).get().isRunning() && readiness.okAt(now) ? readiness.running() : readiness; - // ... or if it is a deployment, and a test job for the current state is not yet complete, - // which is the case when the next versions to run that test with is not the same as we want to deploy here. - List<Job> tests = job.type().isTest() ? null : jobs.get(new JobId(job.application(), JobType.productionTestOf(job.type().zone()))); - readiness = tests != null && ! versions.targetsMatch(tests.get(0).versions) && readiness.okAt(now) ? readiness.blocked() : readiness; - toRun.add(new Job(job.type(), versions, readiness, partial, null)); - // Assume first partial change is applied before the second. - existingPlatform = Optional.of(versions.targetPlatform()); - existingRevision = Optional.of(versions.targetRevision()); - } - jobs.put(job, toRun); - } - Map<JobId, List<Job>> jobsInOrder = new LinkedHashMap<>(); - for (Entry<JobId, List<Job>> entry : reversed(List.copyOf(jobs.entrySet()))) - jobsInOrder.put(entry.getKey(), entry.getValue()); - return jobsInOrder; - } - - private boolean areIncompatible(Optional<Version> platform, Optional<RevisionId> revision, JobId job) { - Optional<Version> compileVersion = revision.map(application.revisions()::get) - .flatMap(ApplicationVersion::compileVersion); - return platform.isPresent() - && compileVersion.isPresent() - && versionCompatibility.apply(job.application().instance()).refuse(platform.get(), compileVersion.get()); - } - - /** Changes to deploy with the given job, possibly split in two steps. */ - private List<Change> changes(JobId job, StepStatus step, Change change) { - if ( change.platform().isEmpty() || change.revision().isEmpty() - || change.isPlatformPinned() || change.isRevisionPinned()) - return List.of(change); - - if ( step.completedAt(change.withoutApplication(), Optional.of(job)).isPresent() - || step.completedAt(change.withoutPlatform(), Optional.of(job)).isPresent()) - return List.of(change); - - // For a dual change, where both targets remain, we determine what to run by looking at when the two parts became ready: - // for deployments, we look at dependencies; for production tests, this may be overridden by what is already deployed. - JobId deployment = new JobId(job.application(), JobType.deploymentTo(job.type().zone())); - UpgradeRollout rollout = application.deploymentSpec().requireInstance(job.application().instance()).upgradeRollout(); - if (job.type().isTest()) { - Optional<Instant> platformDeployedAt = jobSteps.get(deployment).completedAt(change.withoutApplication(), Optional.of(deployment)); - Optional<Instant> revisionDeployedAt = jobSteps.get(deployment).completedAt(change.withoutPlatform(), Optional.of(deployment)); - - // If only the revision has deployed, then we expect to test that first. - if (platformDeployedAt.isEmpty() && revisionDeployedAt.isPresent()) return List.of(change.withoutPlatform(), change); - - // If only the upgrade has deployed, then we expect to test that first, with one exception: - // The revision has caught up to the upgrade at the deployment job; and either - // the upgrade is failing between deployment and here, or - // the specified rollout is leading or simultaneous; and - // the revision is now blocked by waiting for the production test to verify the upgrade. - // In this case we must abandon the production test on the pure upgrade, so the revision can be deployed. - if (platformDeployedAt.isPresent() && revisionDeployedAt.isEmpty()) { - if (jobSteps.get(deployment).readiness(change, Optional.of(deployment)).okAt(now)) { - return switch (rollout) { - // If separate rollout, this test should keep blocking the revision, unless there are failures. - case separate -> hasFailures(jobSteps.get(deployment), jobSteps.get(job)) ? List.of(change) : List.of(change.withoutApplication(), change); - // If leading rollout, this test should now expect the two changes to fuse and roll together. - case leading -> List.of(change); - // If simultaneous rollout, this test should now expect the revision to run ahead. - case simultaneous -> List.of(change.withoutPlatform(), change); - }; - } - return List.of(change.withoutApplication(), change); - } - // If neither is deployed, then neither is ready, and we assume the same order of changes as for the deployment job. - if (platformDeployedAt.isEmpty()) - return changes(deployment, jobSteps.get(deployment), change); - - // If both are deployed, then we need to follow normal logic for what is ready. - } - - Optional<Instant> platformReadyAt = step.dependenciesCompletedAt(change.withoutApplication(), Optional.of(job)); - Optional<Instant> revisionReadyAt = step.dependenciesCompletedAt(change.withoutPlatform(), Optional.of(job)); - - boolean failingUpgradeOnlyTests = ! jobs().type(systemTest(job.type()), stagingTest(job.type())) - .failingHardOn(Versions.from(change.withoutApplication(), application, deploymentFor(job), () -> systemVersion)) - .isEmpty(); - - // If neither change is ready, we guess based on the specified rollout. - if (platformReadyAt.isEmpty() && revisionReadyAt.isEmpty()) { - return switch (rollout) { - case separate -> ! failingUpgradeOnlyTests - ? List.of(change.withoutApplication(), change) // Platform should stay ahead ... - : List.of(change); // ... unless upgrade-only is failing tests. - case leading -> List.of(change); // They should eventually join. - case simultaneous -> List.of(change.withoutPlatform(), change); // Revision should get ahead. - }; - } - - // If only the revision is ready, we run that first. - if (platformReadyAt.isEmpty()) return List.of(change.withoutPlatform(), change); - - // If only the platform is ready, we run that first. - if (revisionReadyAt.isEmpty()) return List.of(change.withoutApplication(), change); - - // Both changes are ready for this step, and we look to the specified rollout to decide. - boolean platformReadyFirst = platformReadyAt.get().isBefore(revisionReadyAt.get()); - boolean revisionReadyFirst = revisionReadyAt.get().isBefore(platformReadyAt.get()); - return switch (rollout) { - case separate -> // Let whichever change rolled out first, keep rolling first, unless upgrade alone is failing. - (platformReadyFirst || platformReadyAt.get().equals(Instant.EPOCH)) // Assume platform was first if no jobs have run yet. - ? step.job().flatMap(jobs()::get).flatMap(JobStatus::firstFailing).isPresent() || failingUpgradeOnlyTests - ? List.of(change) // Platform was first, but is failing. - : List.of(change.withoutApplication(), change) // Platform was first, and is OK. - : revisionReadyFirst - ? List.of(change.withoutPlatform(), change) // Revision was first. - : List.of(change); // Both ready at the same time, probably due to earlier failure. - case leading -> // When one change catches up, they fuse and continue together. - List.of(change); - case simultaneous -> // Revisions are allowed to run ahead, but the job where it caught up should have both changes. - platformReadyFirst ? List.of(change) : List.of(change.withoutPlatform(), change); - }; - } - - /** The test jobs that need to run prior to the given production deployment jobs. */ - public Map<JobId, List<Job>> testJobs(Map<JobId, List<Job>> jobs) { - Map<JobId, List<Job>> testJobs = new LinkedHashMap<>(); - jobs.forEach((job, versionsList) -> { - if (job.type().isProduction() && job.type().isDeployment()) { - for (JobType testType : List.of(systemTest(job.type()), stagingTest(job.type()))) { - prerequisiteTests(job, testType).forEach(testJob -> { - for (Job productionJob : versionsList) - if (allJobs.successOn(testType, productionJob.versions()) - .instance(testJob.application().instance()) - .asList().isEmpty()) { - Readiness readiness = jobSteps().get(testJob).readiness(productionJob.change, Optional.of(job)); - testJobs.merge(testJob, List.of(new Job(testJob.type(), - productionJob.versions(), - readiness.okAt(now) && jobs().get(testJob).get().isRunning() ? readiness.running() : readiness, - productionJob.change, - job)), - DeploymentStatus::union); - - } - }); - } - } - }); - return Collections.unmodifiableMap(testJobs); - } - - private CloudName findCloud(JobType job) { - return zones.zones().all().get(job.zone()).map(ZoneApi::getCloudName).orElse(zones.systemZone().getCloudName()); - } - - private JobId firstDeclaredOrElseImplicitTest(JobType testJob) { - return application.deploymentSpec().instanceNames().stream() - .map(name -> new JobId(application.id().instance(name), testJob)) - .filter(jobSteps::containsKey) - .min(comparing(id -> ! jobSteps.get(id).isDeclared())).orElseThrow(); - } - - /** JobId of any declared test of the given type, for the given instance. */ - private Optional<JobId> declaredTest(ApplicationId instanceId, JobType testJob) { - JobId jobId = new JobId(instanceId, testJob); - return jobSteps.containsKey(jobId) && jobSteps.get(jobId).isDeclared() ? Optional.of(jobId) : Optional.empty(); - } - - /** A DAG of the dependencies between the primitive steps in the spec, with iteration order equal to declaration order. */ - private Map<JobId, StepStatus> jobDependencies(DeploymentSpec spec, List<StepStatus> allSteps, Function<JobId, JobStatus> jobs) { - if (DeploymentSpec.empty.equals(spec)) - return Map.of(); - - Map<JobId, StepStatus> dependencies = new LinkedHashMap<>(); - List<StepStatus> previous = List.of(); - for (DeploymentSpec.Step step : spec.steps()) - previous = fillStep(dependencies, allSteps, step, previous, null, jobs, - instanceWithImplicitTest(test, spec), - instanceWithImplicitTest(staging, spec)); - - return Collections.unmodifiableMap(dependencies); - } - - private static InstanceName instanceWithImplicitTest(Environment environment, DeploymentSpec spec) { - InstanceName first = null; - for (DeploymentInstanceSpec step : spec.instances()) { - if (step.concerns(environment)) return null; - first = first != null ? first : step.name(); - } - return first; - } - - /** - * Returns set of declared tests directly reachable from the given production job, or the first declared (or implicit) test. - * A test in instance {@code I} is directly reachable from a job in instance {@code K} if a chain of instances {@code I, J, ..., K} - * exists, such that only {@code I} has a declared test of the particular type. - * These are the declared tests that should be OK before we proceed with the corresponding production deployment. - * If no such tests exist, the first declared test, or a test in the first declared instance, is used instead. - */ - private List<JobId> prerequisiteTests(JobId prodJob, JobType testType) { - List<JobId> tests = new ArrayList<>(); - Set<InstanceName> seen = new LinkedHashSet<>(); - Deque<InstanceName> pending = new ArrayDeque<>(); - pending.add(prodJob.application().instance()); - while ( ! pending.isEmpty()) { - InstanceName instance = pending.poll(); - Optional<JobId> test = declaredTest(application().id().instance(instance), testType); - if (test.isPresent()) tests.add(test.get()); - else instanceSteps().get(instance).dependencies().stream().map(StepStatus::instance).forEach(dependency -> { - if (seen.add(dependency)) pending.add(dependency); - }); - } - if (tests.isEmpty()) tests.add(firstDeclaredOrElseImplicitTest(testType)); - return tests; - } - - /** Adds the primitive steps contained in the given step, which depend on the given previous primitives, to the dependency graph. */ - private List<StepStatus> fillStep(Map<JobId, StepStatus> dependencies, List<StepStatus> allSteps, DeploymentSpec.Step step, - List<StepStatus> previous, InstanceName instance, Function<JobId, JobStatus> jobs, - InstanceName implicitSystemTest, InstanceName implicitStagingTest) { - if (step.steps().isEmpty() && ! (step instanceof DeploymentInstanceSpec)) { - if (instance == null) - return previous; // Ignore test and staging outside all instances. - - if ( ! step.delay().isZero()) { - StepStatus stepStatus = new DelayStatus((DeploymentSpec.Delay) step, previous, instance); - allSteps.add(stepStatus); - return List.of(stepStatus); - } - - JobType jobType; - JobId jobId; - StepStatus stepStatus; - if (step.concerns(test) || step.concerns(staging)) { - jobType = step.concerns(test) ? systemTest(null) : stagingTest(null); - jobId = new JobId(application.id().instance(instance), jobType); - stepStatus = JobStepStatus.ofTestDeployment((DeclaredZone) step, List.of(), this, jobs.apply(jobId), true); - previous = new ArrayList<>(previous); - previous.add(stepStatus); - } - else if (step.isTest()) { - jobType = JobType.test(((DeclaredTest) step).region()); - jobId = new JobId(application.id().instance(instance), jobType); - stepStatus = JobStepStatus.ofProductionTest((DeclaredTest) step, previous, this, jobs.apply(jobId)); - previous = List.of(stepStatus); - } - else if (step.concerns(prod)) { - jobType = JobType.prod(((DeclaredZone) step).region().get()); - jobId = new JobId(application.id().instance(instance), jobType); - stepStatus = JobStepStatus.ofProductionDeployment((DeclaredZone) step, previous, this, jobs.apply(jobId)); - previous = List.of(stepStatus); - } - else return previous; // Empty container steps end up here, and are simply ignored. - allSteps.add(stepStatus); - dependencies.put(jobId, stepStatus); - return previous; - } - - if (step instanceof DeploymentInstanceSpec) { - DeploymentInstanceSpec spec = ((DeploymentInstanceSpec) step); - StepStatus instanceStatus = new InstanceStatus(spec, previous, now, application.require(spec.name()), this); - instance = spec.name(); - allSteps.add(instanceStatus); - previous = List.of(instanceStatus); - if (instance.equals(implicitSystemTest)) { - JobId job = new JobId(application.id().instance(instance), systemTest(null)); - JobStepStatus testStatus = JobStepStatus.ofTestDeployment(new DeclaredZone(test), List.of(), - this, jobs.apply(job), false); - dependencies.put(job, testStatus); - allSteps.add(testStatus); - } - if (instance.equals(implicitStagingTest)) { - JobId job = new JobId(application.id().instance(instance), stagingTest(null)); - JobStepStatus testStatus = JobStepStatus.ofTestDeployment(new DeclaredZone(staging), List.of(), - this, jobs.apply(job), false); - dependencies.put(job, testStatus); - allSteps.add(testStatus); - } - } - - if (step.isOrdered()) { - for (DeploymentSpec.Step nested : step.steps()) - previous = fillStep(dependencies, allSteps, nested, previous, instance, jobs, implicitSystemTest, implicitStagingTest); - - return previous; - } - - List<StepStatus> parallel = new ArrayList<>(); - for (DeploymentSpec.Step nested : step.steps()) - parallel.addAll(fillStep(dependencies, allSteps, nested, previous, instance, jobs, implicitSystemTest, implicitStagingTest)); - - return List.copyOf(parallel); - } - - - public enum StepType { - - /** An instance — completion marks a change as ready for the jobs contained in it. */ - instance, - - /** A timed delay. */ - delay, - - /** A system, staging or production test. */ - test, - - /** A production deployment. */ - deployment, - } - - /** - * Used to represent all steps — explicit and implicit — that may run in order to complete deployment of a change. - * - * Each node contains a step describing the node, - * a list of steps which need to be complete before the step may start, - * a list of jobs from which completion of the step is computed, and - * optionally, an instance name used to identify a job type for the step, - * - * The completion criterion for each type of step is implemented in subclasses of this. - */ - public static abstract class StepStatus { - - private final StepType type; - private final DeploymentSpec.Step step; - private final List<StepStatus> dependencies; // All direct dependencies of this step. - private final InstanceName instance; - - private StepStatus(StepType type, DeploymentSpec.Step step, List<StepStatus> dependencies, InstanceName instance) { - this.type = requireNonNull(type); - this.step = requireNonNull(step); - this.dependencies = List.copyOf(dependencies); - this.instance = instance; - } - - /** The type of step this is. */ - public final StepType type() { return type; } - - /** The step defining this. */ - public final DeploymentSpec.Step step() { return step; } - - /** The list of steps that need to be complete before this may start. */ - public final List<StepStatus> dependencies() { return dependencies; } - - /** The instance of this. */ - public final InstanceName instance() { return instance; } - - /** The id of the job this corresponds to, if any. */ - public Optional<JobId> job() { return Optional.empty(); } - - /** The time at which this is, or was, complete on the given change and / or versions. */ - public Optional<Instant> completedAt(Change change) { return completedAt(change, Optional.empty()); } - - /** The time at which this is, or was, complete on the given change and / or versions. */ - abstract Optional<Instant> completedAt(Change change, Optional<JobId> dependent); - - /** The time at which this step is ready to run the specified change and / or versions. */ - public Readiness readiness(Change change) { return readiness(change, Optional.empty()); } - - /** The time at which this step is ready to run the specified change and / or versions. */ - Readiness readiness(Change change, Optional<JobId> dependent) { - return dependenciesCompletedAt(change, dependent) - .map(Readiness::new) - .map(ready -> Stream.of(blockedUntil(change), - pausedUntil(), - coolingDownUntil(change, dependent)) - .reduce(ready, maxBy(naturalOrder()))) - .orElse(Readiness.notReady); - } - - /** The time at which all dependencies completed on the given change and / or versions. */ - Optional<Instant> dependenciesCompletedAt(Change change, Optional<JobId> dependent) { - Instant latest = Instant.EPOCH; - for (StepStatus step : dependencies) { - Optional<Instant> completedAt = step.completedAt(change, dependent); - if (completedAt.isEmpty()) return Optional.empty(); - latest = latest.isBefore(completedAt.get()) ? completedAt.get() : latest; - } - return Optional.of(latest); - } - - /** The time until which this step is blocked by a change blocker. */ - public Readiness blockedUntil(Change change) { return Readiness.empty; } - - /** The time until which this step is paused by user intervention. */ - public Readiness pausedUntil() { return Readiness.empty; } - - /** The time until which this step is cooling down, due to consecutive failures. */ - public Readiness coolingDownUntil(Change change, Optional<JobId> dependent) { return Readiness.empty; } - - /** Whether this step is declared in the deployment spec, or is an implicit step. */ - public boolean isDeclared() { return true; } - - } - - - private static class DelayStatus extends StepStatus { - - private DelayStatus(DeploymentSpec.Delay step, List<StepStatus> dependencies, InstanceName instance) { - super(StepType.delay, step, dependencies, instance); - } - - @Override - Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { - return Optional.ofNullable(readiness(change, dependent).at()) - .map(completion -> completion.plus(step().delay())); - } - - } - - - private static class InstanceStatus extends StepStatus { - - private final DeploymentInstanceSpec spec; - private final Instant now; - private final Instance instance; - private final DeploymentStatus status; - - private InstanceStatus(DeploymentInstanceSpec spec, List<StepStatus> dependencies, Instant now, - Instance instance, DeploymentStatus status) { - super(StepType.instance, spec, dependencies, spec.name()); - this.spec = spec; - this.now = now; - this.instance = instance; - this.status = status; - } - - /** The time at which this step is ready to run the specified change and / or versions. */ - @Override - public Readiness readiness(Change change) { - return status.jobSteps.keySet().stream() - .filter(job -> job.type().isProduction() && job.application().instance().equals(instance.name())) - .map(job -> super.readiness(change, Optional.of(job))) - .reduce((a, b) -> ! a.ok() ? a : ! b.ok() ? b : min(a, b)) - .orElseGet(() -> super.readiness(change, Optional.empty())); - } - - /** - * Time of completion of its dependencies, if all parts of the given change are contained in the change - * for this instance, or if no more jobs should run for this instance for the given change. - */ - @Override - Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { - return ( (change.platform().isEmpty() || change.platform().equals(instance.change().platform())) - && (change.revision().isEmpty() || change.revision().equals(instance.change().revision())) - || step().steps().stream().noneMatch(step -> step.concerns(prod))) - ? dependenciesCompletedAt(change, dependent).or(() -> Optional.of(Instant.EPOCH).filter(__ -> change.hasTargets())) - : Optional.empty(); - } - - @Override - public Readiness blockedUntil(Change change) { - for (Instant current = now; now.plus(Duration.ofDays(7)).isAfter(current); ) { - boolean blocked = false; - for (DeploymentSpec.ChangeBlocker blocker : spec.changeBlocker()) { - while ( blocker.window().includes(current) - && now.plus(Duration.ofDays(7)).isAfter(current) - && ( change.platform().isPresent() && blocker.blocksVersions() - || change.revision().isPresent() && blocker.blocksRevisions())) { - blocked = true; - current = current.plus(Duration.ofHours(1)).truncatedTo(ChronoUnit.HOURS); - } - } - if ( ! blocked) - return current == now ? Readiness.empty : new Readiness(current, DelayCause.changeBlocked); - } - return new Readiness(now.plusSeconds(1 << 30), DelayCause.changeBlocked); // Some time in the future that doesn't look like anything you'd expect. - } - - } - - - private static abstract class JobStepStatus extends StepStatus { - - private final JobStatus job; - private final DeploymentStatus status; - - private JobStepStatus(StepType type, DeploymentSpec.Step step, List<StepStatus> dependencies, JobStatus job, - DeploymentStatus status) { - super(type, step, dependencies, job.id().application().instance()); - this.job = requireNonNull(job); - this.status = requireNonNull(status); - } - - @Override - public Optional<JobId> job() { return Optional.of(job.id()); } - - @Override - public Readiness pausedUntil() { - return status.application().require(job.id().application().instance()).jobPause(job.id().type()) - .map(pause -> new Readiness(pause, DelayCause.paused)) - .orElse(Readiness.empty); - } - - @Override - public Readiness coolingDownUntil(Change change, Optional<JobId> dependent) { - if (job.lastTriggered().isEmpty()) return Readiness.empty; - if (job.lastCompleted().isEmpty()) return Readiness.empty; - if (job.firstFailing().isEmpty() || ! job.firstFailing().get().hasEnded()) return Readiness.empty; - Versions lastVersions = job.lastCompleted().get().versions(); - Versions toRun = Versions.from(change, status.application, dependent.flatMap(status::deploymentFor), status.fallbackPlatform(change, job.id())); - if ( ! toRun.targetsMatch(lastVersions)) return Readiness.empty; - if ( job.id().type().environment().isTest() - && ! dependent.map(JobId::type).map(status::findCloud).map(List.of(CloudName.AWS, CloudName.GCP)::contains).orElse(true) - && job.isNodeAllocationFailure()) return Readiness.empty; - - if (job.lastStatus().get() == invalidApplication) return new Readiness(status.now.plus(Duration.ofSeconds(1 << 30)), DelayCause.invalidPackage); - if (job.lastStatus().get() == cancelled) return new Readiness(status.now.plus(Duration.ofSeconds(1 << 30)), DelayCause.coolingDown); - Instant firstFailing = job.firstFailing().get().end().get(); - Instant lastCompleted = job.lastCompleted().get().end().get(); - - Duration penalty = firstFailing.equals(lastCompleted) ? Duration.ZERO - : Duration.ofMinutes(10) - .plus(Duration.between(firstFailing, lastCompleted) - .dividedBy(2)); - return lastCompleted.plus(penalty).isAfter(status.now) ? new Readiness(lastCompleted.plus(penalty), DelayCause.coolingDown) - : Readiness.empty; - } - - private static JobStepStatus ofProductionDeployment(DeclaredZone step, List<StepStatus> dependencies, - DeploymentStatus status, JobStatus job) { - ZoneId zone = ZoneId.from(step.environment(), step.region().get()); - Optional<Deployment> existingDeployment = Optional.ofNullable(status.application().require(job.id().application().instance()) - .deployments().get(zone)); - - return new JobStepStatus(StepType.deployment, step, dependencies, job, status) { - - @Override - public Readiness readiness(Change change, Optional<JobId> dependent) { - Readiness readyAt = super.readiness(change, dependent); - Readiness testedAt = status.verifiedAt(job.id(), Versions.from(change, status.application, existingDeployment, status.fallbackPlatform(change, job.id()))); - return max(readyAt, testedAt); - } - - /** Complete if deployment is on pinned version, and last successful deployment, or if given versions is strictly a downgrade, and this isn't forced by a pin. */ - @Override - Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { - if ( change.isPlatformPinned() - && change.platform().isPresent() - && ! existingDeployment.map(Deployment::version).equals(change.platform())) - return Optional.empty(); - - if ( change.revision().isPresent() - && change.isRevisionPinned() - && ! existingDeployment.map(Deployment::revision).equals(change.revision())) - return Optional.empty(); - - Change fullChange = status.application().require(job.id().application().instance()).change(); - if (existingDeployment.map(deployment -> ! (change.upgrades(deployment.version()) || change.upgrades(deployment.revision())) - && (fullChange.downgrades(deployment.version()) || fullChange.downgrades(deployment.revision()))) - .orElse(false)) - return job.lastCompleted().flatMap(Run::end); - - Optional<Instant> end = Optional.empty(); - for (Run run : job.runs().descendingMap().values()) { - if (run.versions().targetsMatch(change)) { - if (run.hasSucceeded()) end = run.end(); - } - else if (dependent.equals(job())) // If strict completion, consider only last time this change was deployed. - break; - } - return end; - } - }; - } - - private static JobStepStatus ofProductionTest(DeclaredTest step, List<StepStatus> dependencies, - DeploymentStatus status, JobStatus job) { - JobId prodId = new JobId(job.id().application(), JobType.deploymentTo(job.id().type().zone())); - return new JobStepStatus(StepType.test, step, dependencies, job, status) { - @Override - Readiness readiness(Change change, Optional<JobId> dependent) { - Readiness readyAt = super.readiness(change, dependent); - Readiness deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId)) - .map(Readiness::new).orElse(Readiness.notReady); - return max(readyAt, deployedAt); - } - - @Override - Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { - Optional<Instant> deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId)); - Versions target = Versions.from(change, status.application(), status.deploymentFor(job.id()), status.fallbackPlatform(change, job.id())); - Change applied = Change.empty(); - if (change.platform().isPresent()) - applied = applied.with(target.targetPlatform()); - if (change.revision().isPresent()) - applied = applied.with(target.targetRevision()); - Change relevant = applied; - - return (dependent.equals(job()) ? job.lastTriggered().filter(run -> deployedAt.map(at -> ! run.start().isBefore(at)).orElse(false)).stream() - : job.runs().values().stream()) - .filter(Run::hasSucceeded) - .filter(run -> run.versions().targetsMatch(relevant)) - .flatMap(run -> run.end().stream()).findFirst(); - } - }; - } - - private static JobStepStatus ofTestDeployment(DeclaredZone step, List<StepStatus> dependencies, - DeploymentStatus status, JobStatus job, boolean declared) { - return new JobStepStatus(StepType.test, step, dependencies, job, status) { - @Override - Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { - Optional<ZoneId> requiredTestZone = dependent.map(dep -> job.id().type().isSystemTest() ? status.systemTest(dep.type()).zone() - : status.stagingTest(dep.type()).zone()); - return RunList.from(job) - .matching(run -> dependent.flatMap(status::deploymentFor) - .map(deployment -> run.versions().targetsMatch(Versions.from(change, - status.application, - Optional.of(deployment), - status.fallbackPlatform(change, dependent.get())))) - .orElseGet(() -> (change.platform().isEmpty() || change.platform().get().equals(run.versions().targetPlatform())) - && (change.revision().isEmpty() || change.revision().get().equals(run.versions().targetRevision())))) - .matching(Run::hasSucceeded) - .matching(run -> requiredTestZone.isEmpty() || requiredTestZone.get().equals(run.id().type().zone())) - .asList().stream() - .map(run -> run.end().get()) - .max(naturalOrder()); - } - - @Override - public boolean isDeclared() { return declared; } - }; - } - - } - - public static class Job { - - private final JobType type; - private final Versions versions; - private final Readiness readiness; - private final Change change; - private final JobId dependent; - - public Job(JobType type, Versions versions, Readiness readiness, Change change, JobId dependent) { - this.type = type; - this.versions = type.isSystemTest() ? versions.withoutSources() : versions; - this.readiness = readiness; - this.change = change; - this.dependent = dependent; - } - - public JobType type() { - return type; - } - - public Versions versions() { - return versions; - } - - public Readiness readiness() { - return readiness; - } - - public Reason reason() { - return new Reason(Optional.empty(), Optional.ofNullable(dependent), Optional.ofNullable(change)); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Job job = (Job) o; - return type.zone().equals(job.type.zone()) && versions.equals(job.versions) && readiness.equals(job.readiness) && change.equals(job.change); - } - - @Override - public int hashCode() { - return Objects.hash(type.zone(), versions, readiness, change); - } - - @Override - public String toString() { - return change + " with versions " + versions + ", " + readiness; - } - - } - - public enum DelayCause { none, unverified, notReady, blocked, running, coolingDown, invalidPackage, changeBlocked, paused } - public record Readiness(Instant at, DelayCause cause) implements Comparable<Readiness> { - public static final Readiness unverified = new Readiness(null, DelayCause.unverified); - public static final Readiness notReady = new Readiness(null, DelayCause.notReady); - public static final Readiness empty = new Readiness(Instant.EPOCH, DelayCause.none); - public Readiness(Instant at) { this(at, DelayCause.none); } - public Readiness blocked() { return new Readiness(at, DelayCause.blocked); } - public Readiness running() { return new Readiness(at, DelayCause.running); } - public boolean ok() { return at != null; } - public boolean okAt(Instant at) { return ok() && cause != DelayCause.running && cause != DelayCause.blocked && ! at.isBefore(this.at); } - @Override public int compareTo(Readiness o) { - return at == null ? o.at == null ? 0 : 1 - : o.at == null ? -1 : at.compareTo(o.at); - } - @Override public String toString() { - return ok() ? "ready at " + at + switch (cause) { - case none -> ""; - case coolingDown -> ": cooling down after repeated failures"; - case blocked -> ": waiting for verification test to complete"; - case running -> ": waiting for current run to complete"; - case invalidPackage -> ": invalid application package, must resubmit"; - case changeBlocked -> ": deployment configuration blocks changes"; - case paused -> ": manually paused"; - default -> throw new IllegalStateException(cause + " should not have an instant at which it is ready"); - } - : "not ready" + switch (cause) { - case unverified -> ": waiting for verification test to complete"; - case notReady -> ": waiting for dependencies to complete"; - default -> throw new IllegalStateException(cause + " should have an instant at which it is ready"); - }; - } - } - - static <T extends Comparable<T>> T min(T a, T b) { - return a.compareTo(b) > 0 ? b : a; - } - - static <T extends Comparable<T>> T max(T a, T b) { - return a.compareTo(b) < 0 ? b : a; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java deleted file mode 100644 index 16bd5bd9bb2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.collections.AbstractFilteringList; -import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.application.Change; - -import java.time.Instant; -import java.util.Collection; - -/** - * List for filtering deployment status of applications, for inspection and decision making. - * - * @author jonmv - */ -public class DeploymentStatusList extends AbstractFilteringList<DeploymentStatus, DeploymentStatusList> { - - private DeploymentStatusList(Collection<? extends DeploymentStatus> items, boolean negate) { - super(items, negate, DeploymentStatusList::new); - } - - public static DeploymentStatusList from(Collection<? extends DeploymentStatus> status) { - return new DeploymentStatusList(status, false); - } - - /** Returns the subset of applications which have changes left to deploy; blocked, or deploying */ - public DeploymentStatusList withChanges() { - return matching(status -> status.application().productionInstances().values().stream() - .anyMatch(instance -> instance.change().hasTargets() || status.outstandingChange(instance.name()).hasTargets())); - } - - /** Returns the subset of applications which have been failing an upgrade to the given version since the given instant */ - public DeploymentStatusList failingUpgradeToVersionSince(Version version, Instant threshold) { - return matching(status -> status.instanceJobs().values().stream() - .anyMatch(jobs -> failingUpgradeToVersionSince(jobs, version, threshold))); - } - - /** Returns the subset of applications which have been failing an application change since the given instant */ - public DeploymentStatusList failingApplicationChangeSince(Instant threshold) { - return matching(status -> status.instanceJobs().entrySet().stream() - .anyMatch(jobs -> failingApplicationChangeSince(jobs.getValue(), - status.application().require(jobs.getKey().instance()).change(), - threshold))); - } - - private static boolean failingUpgradeToVersionSince(JobList jobs, Version version, Instant threshold) { - return ! jobs.not().failingApplicationChange() - .firstFailing().endedNoLaterThan(threshold) - .lastCompleted().on(version) - .isEmpty(); - } - - private static boolean failingApplicationChangeSince(JobList jobs, Change change, Instant threshold) { - return change.revision().map(revision -> ! jobs.failingWithBrokenRevisionSince(revision, threshold).isEmpty()).orElse(false); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java deleted file mode 100644 index 834efa81d26..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ /dev/null @@ -1,505 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.ApplicationController; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness; -import com.yahoo.vespa.hosted.controller.deployment.Run.Reason; - -import java.math.BigDecimal; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.function.Predicate; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static java.util.Comparator.comparing; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.toMap; - -/** - * Responsible for scheduling deployment jobs in a build system and keeping - * {@link Instance#change()} in sync with what is scheduled. - * - * This class is multi-thread safe. - * - * @author bratseth - * @author mpolden - * @author jonmv - */ -public class DeploymentTrigger { - - public static final Duration maxPause = Duration.ofDays(3); - public static final Duration maxFailingRevisionTime = Duration.ofDays(5); - private final static Logger log = Logger.getLogger(DeploymentTrigger.class.getName()); - - private final Controller controller; - private final Clock clock; - private final JobController jobs; - - public DeploymentTrigger(Controller controller, Clock clock) { - this.controller = Objects.requireNonNull(controller, "controller cannot be null"); - this.clock = Objects.requireNonNull(clock, "clock cannot be null"); - this.jobs = controller.jobController(); - } - - /** - * Propagates the latest revision to ready instances. - * Ready instances are those whose dependencies are complete, and which aren't blocked, and, additionally, - * which aren't upgrading, or are already deploying an application change, or failing upgrade. - */ - public void triggerNewRevision(TenantAndApplicationId id) { - applications().lockApplicationIfPresent(id, application -> { - DeploymentStatus status = jobs.deploymentStatus(application.get()); - for (InstanceName instanceName : application.get().deploymentSpec().instanceNames()) { - Change outstanding = status.outstandingChange(instanceName); - boolean deployOutstanding = outstanding.hasTargets() - && status.instanceSteps().get(instanceName) - .readiness(outstanding).okAt(clock.instant()) - && acceptNewRevision(status, instanceName, outstanding.revision().get()); - application = application.with(instanceName, - instance -> withRemainingChange(instance, - deployOutstanding ? outstanding.onTopOf(instance.change()) - : instance.change(), - status, - false)); - } - - // If app has been broken since it was first submitted, and not fixed for a long time, we stop managing it until a new submission comes in. - if (applicationWasAlwaysBroken(status)) - application = application.withProjectId(OptionalLong.empty()); - - applications().store(application); - }); - } - - private boolean applicationWasAlwaysBroken(DeploymentStatus status) { - // If application has a production deployment, we cannot forget it. - if (status.application().instances().values().stream().anyMatch(instance -> ! instance.productionDeployments().isEmpty())) - return false; - - // Then, we need a job that always failed, and failed on the last revision for at least 30 days. - RevisionId last = status.application().revisions().last().get().id(); - Instant threshold = clock.instant().minus(Duration.ofDays(30)); - for (JobStatus job : status.jobs().asList()) - for (Run run : job.runs().descendingMap().values()) - if (run.hasEnded() && ! run.hasFailed() || ! run.versions().targetRevision().equals(last)) break; - else if (run.start().isBefore(threshold)) return true; - - return false; - } - - /** - * Records information when a job completes (successfully or not). This information is used when deciding what to - * trigger next. - */ - public void notifyOfCompletion(ApplicationId id) { - if (applications().getInstance(id).isEmpty()) { - log.log(Level.WARNING, "Ignoring completion of job of unknown application '" + id + "'"); - return; - } - - applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - if (application.get().deploymentSpec().instance(id.instance()).isPresent()) - applications().store(application.with(id.instance(), - instance -> withRemainingChange(instance, - instance.change(), - jobs.deploymentStatus(application.get()), - true))); - }); - } - - /** - * Finds and triggers jobs that can and should run but are currently not, and returns the number of triggered jobs. - * Only one job per type is triggered each run for test jobs, since their environments have limited capacity. - */ - public TriggerResult triggerReadyJobs() { - List<Job> readyJobs = computeReadyJobs(); - - var prodJobs = new ArrayList<Job>(); - var testJobs = new ArrayList<Job>(); - for (Job job : readyJobs) - (job.jobType().isProduction() ? prodJobs : testJobs).add(job); - - // Flat list of prod jobs, grouped by application id, retaining the step order - List<Job> sortedProdJobs = prodJobs.stream() - .collect(groupingBy(Job::applicationId)) - .values().stream() - .flatMap(List::stream) - .toList(); - - // Map of test jobs, a list for each job type. Jobs in each list are sorted by priority. - Map<JobType, List<Job>> sortedTestJobsByType = testJobs.stream() - .sorted(comparing(Job::isRetry) - .thenComparing(Job::applicationUpgrade) - .reversed() - .thenComparing(Job::availableSince)) - .collect(groupingBy(Job::jobType)); - - // Trigger all prod jobs - long triggeredJobs = 0; - long failedJobs = 0; - for (Job job : sortedProdJobs) { - if (trigger(job)) ++triggeredJobs; - else ++failedJobs; - } - - // Trigger max one test job per type - for (Collection<Job> jobs: sortedTestJobsByType.values()) - for (Job job : jobs) - if (trigger(job)) { ++triggeredJobs; break; } - else ++failedJobs; - - return new TriggerResult(triggeredJobs, failedJobs); - } - - public record TriggerResult(long triggered, long failed) { } - - /** Attempts to trigger the given job. */ - private boolean trigger(Job job) { - try { - log.log(Level.FINE, () -> "Triggering " + job); - applications().lockApplicationOrThrow(TenantAndApplicationId.from(job.applicationId()), application -> { - jobs.start(job.applicationId(), job.jobType, job.versions, false, job.reason); - applications().store(application.with(job.applicationId().instance(), instance -> - instance.withJobPause(job.jobType, OptionalLong.empty()))); - }); - return true; - } - catch (Exception e) { - log.log(Level.WARNING, "Failed triggering " + job.jobType() + " for " + job.instanceId, e); - return false; - } - } - - /** Force triggering of a job for given instance, with same versions as last run. */ - public JobId reTrigger(ApplicationId applicationId, JobType jobType, String reason) { - Application application = applications().requireApplication(TenantAndApplicationId.from(applicationId)); - Instance instance = application.require(applicationId.instance()); - JobId job = new JobId(instance.id(), jobType); - JobStatus jobStatus = jobs.jobStatus(new JobId(applicationId, jobType)); - Run last = jobStatus.lastTriggered() - .orElseThrow(() -> new IllegalArgumentException(job + " has never been triggered")); - trigger(deploymentJob(instance, last.versions(), last.id().type(), jobStatus.isNodeAllocationFailure(), clock.instant(), - new Reason(Optional.ofNullable(reason), last.reason().dependent(), last.reason().change()))); - return job; - } - - /** Force triggering of a job for given instance. */ - public List<JobId> forceTrigger(ApplicationId applicationId, JobType jobType, String reason, boolean requireTests, - boolean upgradeRevision, boolean upgradePlatform) { - Application application = applications().requireApplication(TenantAndApplicationId.from(applicationId)); - Instance instance = application.require(applicationId.instance()); - DeploymentStatus status = jobs.deploymentStatus(application); - if (jobType.environment().isTest()) { - CloudName cloud = status.firstDependentProductionJobsWithDeployment(applicationId.instance()).keySet().stream().findFirst() - .orElse(controller.zoneRegistry().systemZone().getCloudName()); - jobType = jobType.isSystemTest() ? JobType.systemTest(controller.zoneRegistry(), cloud) - : JobType.stagingTest(controller.zoneRegistry(), cloud); - } - JobId job = new JobId(instance.id(), jobType); - if (job.type().environment().isManuallyDeployed()) - return forceTriggerManualJob(job, reason); - - Change change = instance.change(); - if ( ! upgradeRevision && change.revision().isPresent()) change = change.withoutApplication(); - if ( ! upgradePlatform && change.platform().isPresent()) change = change.withoutPlatform(); - Versions versions = Versions.from(change, application, status.deploymentFor(job), status.fallbackPlatform(change, job)); - DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(job.type(), versions, new Readiness(controller.clock().instant()), instance.change(), null); - Map<JobId, List<DeploymentStatus.Job>> testJobs = status.testJobs(Map.of(job, List.of(toTrigger))); - - Map<JobId, List<DeploymentStatus.Job>> jobs = testJobs.isEmpty() || ! requireTests - ? Map.of(job, List.of(toTrigger)) - : testJobs.entrySet().stream() - .filter(entry -> controller.jobController().last(entry.getKey()).map(Run::hasEnded).orElse(true)) - .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); - - jobs.forEach((jobId, jobList) -> { - trigger(deploymentJob(application.require(jobId.application().instance()), - jobList.get(0).versions(), - jobId.type(), - status.jobs().get(jobId).get().isNodeAllocationFailure(), - clock.instant(), - new Reason(Optional.of(reason), jobList.get(0).reason().dependent(), jobList.get(0).reason().change()))); - }); - return List.copyOf(jobs.keySet()); - } - - private List<JobId> forceTriggerManualJob(JobId job, String reason) { - Run last = jobs.last(job).orElseThrow(() -> new IllegalArgumentException(job + " has never been run")); - Versions target = new Versions(controller.readSystemVersion(), - last.versions().targetRevision(), - Optional.of(last.versions().targetPlatform()), - Optional.of(last.versions().targetRevision())); - jobs.start(job.application(), job.type(), target, true, Reason.because(reason)); - return List.of(job); - } - - /** Retrigger job. If the job is already running, it will be canceled, and retrigger enqueued. */ - public Optional<JobId> reTriggerOrAddToQueue(DeploymentId deployment, String reason) { - JobType jobType = JobType.deploymentTo(deployment.zoneId()); - Optional<Run> existingRun = controller.jobController().active(deployment.applicationId()).stream() - .filter(run -> run.id().type().equals(jobType)) - .findFirst(); - - if (existingRun.isPresent()) { - Run run = existingRun.get(); - try (Mutex lock = controller.curator().lockDeploymentRetriggerQueue()) { - List<RetriggerEntry> retriggerEntries = controller.curator().readRetriggerEntries(); - List<RetriggerEntry> newList = new ArrayList<>(retriggerEntries); - RetriggerEntry requiredEntry = new RetriggerEntry(new JobId(deployment.applicationId(), jobType), run.id().number() + 1); - if (newList.stream().noneMatch(entry -> entry.jobId().equals(requiredEntry.jobId()) && entry.requiredRun() >= requiredEntry.requiredRun())) { - newList.add(requiredEntry); - } - newList = newList.stream() - .filter(entry -> !(entry.jobId().equals(requiredEntry.jobId()) && entry.requiredRun() < requiredEntry.requiredRun())) - .toList(); - controller.curator().writeRetriggerEntries(newList); - } - controller.jobController().abort(run.id(), "force re-triggered", false); - return Optional.empty(); - } else { - return Optional.of(reTrigger(deployment.applicationId(), jobType, reason)); - } - } - - /** Prevents jobs of the given type from starting, until the given time. */ - public void pauseJob(ApplicationId id, JobType jobType, Instant until) { - if (until.isAfter(clock.instant().plus(maxPause))) - throw new IllegalArgumentException("Pause only allowed for up to " + maxPause); - - applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> - applications().store(application.with(id.instance(), - instance -> instance.withJobPause(jobType, OptionalLong.of(until.toEpochMilli()))))); - } - - /** Resumes a previously paused job, letting it be triggered normally. */ - public void resumeJob(ApplicationId id, JobType jobType) { - applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> - applications().store(application.with(id.instance(), - instance -> instance.withJobPause(jobType, OptionalLong.empty())))); - } - - /** Overrides the given instance's platform and application changes with any contained in the given change. */ - public void forceChange(ApplicationId instanceId, Change change) { - forceChange(instanceId, change, true); - } - - /** Overrides the given instance's platform and application changes with any contained in the given change. */ - public void forceChange(ApplicationId instanceId, Change change, boolean allowOutdatedPlatform) { - applications().lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> { - applications().store(application.with(instanceId.instance(), - instance -> withRemainingChange(instance, - change.onTopOf(instance.change()), - jobs.deploymentStatus(application.get()), - allowOutdatedPlatform))); - }); - } - - /** Cancels the indicated part of the given application's change. */ - public void cancelChange(ApplicationId instanceId, ChangesToCancel cancellation) { - applications().lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> { - Change change = switch (cancellation) { - case ALL -> Change.empty(); - case PLATFORM -> application.get().require(instanceId.instance()).change().withoutPlatform(); - case APPLICATION -> application.get().require(instanceId.instance()).change().withoutApplication(); - case PIN -> application.get().require(instanceId.instance()).change().withoutPlatformPin(); - case PLATFORM_PIN -> application.get().require(instanceId.instance()).change().withoutPlatformPin(); - case APPLICATION_PIN -> application.get().require(instanceId.instance()).change().withoutRevisionPin(); - }; - applications().store(application.with(instanceId.instance(), - instance -> withRemainingChange(instance, - change, - jobs.deploymentStatus(application.get()), - true))); - }); - } - - public enum ChangesToCancel { ALL, PLATFORM, APPLICATION, PIN, PLATFORM_PIN, APPLICATION_PIN } - - // ---------- Conveniences ---------- - - private ApplicationController applications() { - return controller.applications(); - } - - // ---------- Ready job computation ---------- - - /** Returns the set of all jobs which have changes to propagate from the upstream steps. */ - private List<Job> computeReadyJobs() { - return jobs.deploymentStatuses(ApplicationList.from(applications().readable()) - .withProjectId() // Need to keep this, as we have applications with deployment spec that shouldn't be orchestrated. - .withJobs()) - .withChanges() - .asList().stream() - .filter(status -> ! hasExceededQuota(status.application().id().tenant())) - .map(this::computeReadyJobs) - .flatMap(Collection::stream) - .toList(); - } - - /** Finds the next step to trigger for the given application, if any, and returns these as a list. */ - private List<Job> computeReadyJobs(DeploymentStatus status) { - List<Job> jobs = new ArrayList<>(); - Map<JobId, List<DeploymentStatus.Job>> jobsToRun = status.jobsToRun(); - jobsToRun.forEach((jobId, jobsList) -> { - abortIfOutdated(status, jobsToRun, jobId); - DeploymentStatus.Job job = jobsList.get(0); - if ( job.readiness().okAt(clock.instant()) - && ! controller.jobController().isDisabled(new JobId(jobId.application(), job.type())) - && ! (jobId.type().isProduction() && isUnhealthyInAnotherZone(status.application(), jobId))) { - jobs.add(deploymentJob(status.application().require(jobId.application().instance()), - job.versions(), - job.type(), - status.instanceJobs(jobId.application().instance()).get(jobId.type()).isNodeAllocationFailure(), - job.readiness().at(), - job.reason())); - } - }); - return Collections.unmodifiableList(jobs); - } - - private boolean hasExceededQuota(TenantName tenant) { - return controller.serviceRegistry().billingController().getQuota(tenant).budget().equals(Optional.of(BigDecimal.ZERO)); - } - - /** Returns whether the application is healthy in all other production zones. */ - private boolean isUnhealthyInAnotherZone(Application application, JobId job) { - for (Deployment deployment : application.require(job.application().instance()).productionDeployments().values()) { - if ( ! deployment.zone().equals(job.type().zone()) - && ! controller.applications().isHealthy(new DeploymentId(job.application(), deployment.zone()))) - return true; - } - return false; - } - - private void abortIfOutdated(JobStatus job, List<DeploymentStatus.Job> jobs) { - job.lastTriggered() - .filter(last -> ! last.hasEnded() && last.reason().reason().isEmpty()) - .ifPresent(last -> { - if (jobs.stream().noneMatch(versions -> versions.versions().targetsMatch(last.versions()) - && versions.versions().sourcesMatchIfPresent(last.versions()))) { - String blocked = jobs.stream() - .map(scheduled -> scheduled.versions().toString()) - .collect(Collectors.joining(", ")); - log.log(Level.INFO, "Aborting outdated run " + last + ", which is blocking runs: " + blocked); - controller.jobController().abort(last.id(), "run no longer scheduled, and is blocking scheduled runs: " + blocked, false); - } - }); - } - - /** Returns whether the job is free to start, and also aborts it if it's running with outdated versions. */ - private void abortIfOutdated(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) { - Readiness readiness = jobs.get(job).get(0).readiness(); - if (readiness.cause() == DelayCause.running) - abortIfOutdated(status.jobs().get(job).get(), jobs.get(job)); - if (readiness.cause() == DelayCause.blocked && ! job.type().isTest()) - status.jobs().get(new JobId(job.application(), JobType.productionTestOf(job.type().zone()))) - .ifPresent(jobStatus -> abortIfOutdated(jobStatus, jobs.get(jobStatus.id()))); - } - - // ---------- Change management o_O ---------- - - private boolean acceptNewRevision(DeploymentStatus status, InstanceName instance, RevisionId revision) { - if (status.application().deploymentSpec().instance(instance).isEmpty()) return false; // Unknown instance. - if (status.application().get(instance).map(Instance::change).map(Change::isRevisionPinned).orElse(false)) return false; - if ( ! status.jobs().failingWithBrokenRevisionSince(revision, clock.instant().minus(maxFailingRevisionTime)) - .isEmpty()) return false; // Don't deploy a broken revision. - boolean isChangingRevision = status.application().require(instance).change().revision().isPresent(); - DeploymentInstanceSpec spec = status.application().deploymentSpec().requireInstance(instance); - Predicate<RevisionId> revisionFilter = spec.revisionTarget() == DeploymentSpec.RevisionTarget.next - ? failing -> status.application().require(instance).change().revision().get().compareTo(failing) == 0 - : failing -> revision.compareTo(failing) > 0; - return switch (spec.revisionChange()) { - case whenClear -> ! isChangingRevision; - case whenFailing -> ! isChangingRevision || status.hasFailures(revisionFilter); - case always -> true; - }; - } - - private Instance withRemainingChange(Instance instance, Change change, DeploymentStatus status, boolean allowOutdatedPlatform) { - Change remaining = change; - if (status.hasCompleted(instance.name(), change.withoutApplication())) - remaining = remaining.withoutPlatform(); - if (status.hasCompleted(instance.name(), change.withoutPlatform())) - remaining = remaining.withoutApplication(); - - return instance.withChange(status.withPermittedPlatform(remaining, instance.name(), allowOutdatedPlatform)); - } - - // ---------- Version and job helpers ---------- - - private Job deploymentJob(Instance instance, Versions versions, JobType jobType, boolean isNodeAllocationFailure, Instant availableSince, Reason reason) { - return new Job(instance, versions, jobType, availableSince, isNodeAllocationFailure, instance.change().revision().isPresent(), reason); - } - - // ---------- Data containers ---------- - - - private static class Job { - - private final ApplicationId instanceId; - private final JobType jobType; - private final Versions versions; - private final Instant availableSince; - private final boolean isRetry; - private final boolean isApplicationUpgrade; - private final Run.Reason reason; - - private Job(Instance instance, Versions versions, JobType jobType, Instant availableSince, - boolean isRetry, boolean isApplicationUpgrade, Run.Reason reason) { - this.instanceId = instance.id(); - this.jobType = jobType; - this.versions = versions; - this.availableSince = availableSince; - this.isRetry = isRetry; - this.isApplicationUpgrade = isApplicationUpgrade; - this.reason = reason; - } - - ApplicationId applicationId() { return instanceId; } - JobType jobType() { return jobType; } - Instant availableSince() { return availableSince; } // TODO jvenstad: This is 95% broken now. Change.at() can restore it. - boolean isRetry() { return isRetry; } - boolean applicationUpgrade() { return isApplicationUpgrade; } - Reason reason() { return reason; } - - @Override - public String toString() { - return jobType + " for " + instanceId + - " on (" + versions.targetPlatform() + versions.sourcePlatform().map(version -> " <-- " + version).orElse("") + - ", " + versions.targetRevision() + versions.sourceRevision().map(version -> " <-- " + version).orElse("") + - "), ready since " + availableSince; - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java deleted file mode 100644 index 9bfa2674754..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ /dev/null @@ -1,1063 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import ai.vespa.http.HttpURL; -import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.Notifications; -import com.yahoo.config.application.api.Notifications.When; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.EndpointsChecker; -import com.yahoo.config.provision.EndpointsChecker.Availability; -import com.yahoo.config.provision.EndpointsChecker.Status; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateException; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; -import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream; -import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage; -import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; -import com.yahoo.vespa.hosted.controller.notification.Notification; -import com.yahoo.vespa.hosted.controller.notification.NotificationSource; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; -import com.yahoo.yolean.Exceptions; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.io.UncheckedIOException; -import java.net.InetAddress; -import java.security.cert.CertificateExpiredException; -import java.security.cert.CertificateNotYetValidException; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.config.application.api.Notifications.Role.author; -import static com.yahoo.config.application.api.Notifications.When.failing; -import static com.yahoo.config.application.api.Notifications.When.failingCommit; -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.invalidApplication; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.quotaExceeded; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.report; -import static com.yahoo.yolean.Exceptions.uncheck; -import static com.yahoo.yolean.Exceptions.uncheckInterruptedAndRestoreFlag; -import static java.lang.Math.min; -import static java.util.Objects.requireNonNull; -import static java.util.function.Predicate.not; -import static java.util.logging.Level.FINE; -import static java.util.logging.Level.INFO; -import static java.util.logging.Level.WARNING; -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toSet; - -/** - * Runs steps of a deployment job against its provided controller. - * - * A dual-purpose logger is set up for each step run here: - * 1. all messages are logged to a buffer which is stored in an external log storage at the end of execution, and - * 2. all messages are also logged through the usual logging framework; by default, any messages of level - * {@code Level.INFO} or higher end up in the Vespa log, and all messages may be sent there by means of log-control. - * - * @author jonmv - */ -public class InternalStepRunner implements StepRunner { - - private static final Logger logger = Logger.getLogger(InternalStepRunner.class.getName()); - - private final Controller controller; - private final TestConfigSerializer testConfigSerializer; - private final DeploymentFailureMails mails; - private final Timeouts timeouts; - - public InternalStepRunner(Controller controller) { - this.controller = controller; - this.testConfigSerializer = new TestConfigSerializer(controller.system()); - this.mails = new DeploymentFailureMails(controller.serviceRegistry().consoleUrls()); - this.timeouts = Timeouts.of(controller.system()); - } - - @Override - public Optional<RunStatus> run(LockedStep step, RunId id) { - DualLogger logger = new DualLogger(id, step.get()); - try { - return switch (step.get()) { - case deployTester -> deployTester(id, logger); - case installTester -> installTester(id, logger); - case deployInitialReal -> deployInitialReal(id, logger); - case installInitialReal -> installInitialReal(id, logger); - case deployReal -> deployReal(id, logger); - case installReal -> installReal(id, logger); - case startStagingSetup -> startTests(id, true, logger); - case endStagingSetup -> endTests(id, true, logger); - case startTests -> startTests(id, false, logger); - case endTests -> endTests(id, false, logger); - case copyVespaLogs -> copyVespaLogs(id, logger); - case deactivateReal -> deactivateReal(id, logger); - case deactivateTester -> deactivateTester(id, logger); - case report -> report(id, logger); - }; - } catch (UncheckedIOException e) { - logger.logWithInternalException(INFO, "IO exception running " + id + ": " + Exceptions.toMessageString(e), e); - return Optional.empty(); - } catch (RuntimeException | LinkageError e) { - logger.log(WARNING, "Unexpected exception running " + id, e); - if (step.get().alwaysRun() && ! (e instanceof LinkageError)) { - logger.log("Will keep trying, as this is a cleanup step."); - return Optional.empty(); - } - return Optional.of(error); - } - } - - private Optional<RunStatus> deployInitialReal(RunId id, DualLogger logger) { - Versions versions = controller.jobController().run(id).versions(); - logger.log("Deploying platform version " + - versions.sourcePlatform().orElse(versions.targetPlatform()) + - " and application " + - versions.sourceRevision().orElse(versions.targetRevision()) + " ..."); - return deployReal(id, true, logger); - } - - private Optional<RunStatus> deployReal(RunId id, DualLogger logger) { - Versions versions = controller.jobController().run(id).versions(); - logger.log("Deploying platform version " + versions.targetPlatform() + - " and application " + versions.targetRevision() + " ..."); - return deployReal(id, false, logger); - } - - private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, DualLogger logger) { - Optional<X509Certificate> testerCertificate = controller.jobController().run(id).testerCertificate(); - return deploy(() -> controller.applications().deploy(id.job(), - setTheStage, - logger::log, - account -> getAndSetCloudAccountWithOverrideForStaging(id, account)), - controller.jobController().run(id) - .stepInfo(setTheStage ? deployInitialReal : deployReal).get() - .startTime().get(), - id, - logger) - .filter(result -> { - // If no tester cert, or deployment failed, propagate original result. - if ( ! useTesterCertificate(id) || result != running) - return true; - // If tester cert, ensure real is deployed with the tester cert whose key was successfully deployed. - return controller.jobController().run(id).stepStatus(deployTester).get() == succeeded - && testerCertificate.equals(controller.jobController().run(id).testerCertificate()); - }); - } - - private Optional<RunStatus> deployTester(RunId id, DualLogger logger) { - Version platform = testerPlatformVersion(id); - logger.log("Deploying the tester container on platform " + platform + " ..."); - return deploy(() -> controller.applications().deployTester(id.tester(), - testerPackage(id), - id.type().zone(), - platform, - cloudAccount -> setCloudAccountForStaging(id, cloudAccount)), - controller.jobController().run(id) - .stepInfo(deployTester).get() - .startTime().get(), - id, - logger); - } - - private Optional<CloudAccount> setCloudAccountForStaging(RunId id, Optional<CloudAccount> account) { - if (id.type().environment() == Environment.staging) { - controller.jobController().locked(id, run -> run.with(account.orElse(CloudAccount.empty))); - } - return account; - } - - private Optional<CloudAccount> getAndSetCloudAccountWithOverrideForStaging(RunId id, Optional<CloudAccount> account) { - if (id.type().environment() == Environment.staging) { - Instant doom = controller.clock().instant().plusSeconds(60); // Sleeping is bad, but we're already in a sleepy code path: deployment. - while (true) { - Run run = controller.jobController().run(id); - Optional<CloudAccount> stored = run.cloudAccount(); - if (stored.isPresent()) - return stored.filter(not(CloudAccount.empty::equals)); - - long millisToDoom = Duration.between(controller.clock().instant(), doom).toMillis(); - if (millisToDoom > 0) - uncheckInterruptedAndRestoreFlag(() -> Thread.sleep(min(millisToDoom, 5000))); - else - throw new CloudAccountNotSetException("Cloud account not yet set; must deploy tests first"); - } - } - account.ifPresent(cloudAccount -> controller.jobController().locked(id, run -> run.with(cloudAccount))); - return account; - } - - private Optional<RunStatus> deploy(Supplier<DeploymentResult> deployment, Instant startTime, RunId id, DualLogger logger) { - try { - DeploymentResult result = deployment.get(); - logger.logAll(result.log().stream() - .map(entry -> new LogEntry(0, // Sequenced by BufferedLogStore. - Instant.ofEpochMilli(entry.epochMillis()), - LogEntry.typeOf(entry.level()), - entry.message())) - .toList()); - - logger.log("Deployment successful."); - logger.log(result.message()); - - return Optional.of(running); - } - catch (ConfigServerException e) { - // Retry certain failures for up to one hour. - Optional<RunStatus> result = startTime.isBefore(controller.clock().instant().minus(Duration.ofHours(1))) - ? Optional.of(deploymentFailed) : Optional.empty(); - if (result.isPresent()) - logger.log(WARNING, "Deployment failed for one hour; giving up now!"); - - switch (e.code()) { - case CERTIFICATE_NOT_READY -> { - logger.log("No valid CA signed certificate for app available to config server"); - if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) { - logger.log(WARNING, "CA signed certificate for app not available to config server within " + - timeouts.endpointCertificate().toMinutes() + " minutes"); - return Optional.of(RunStatus.endpointCertificateTimeout); - } - return result; - } - case ACTIVATION_CONFLICT, APPLICATION_LOCK_FAILURE, CONFIG_NOT_CONVERGED -> { - logger.log("Deployment failed with possibly transient error " + e.code() + - ", will retry: " + e.getMessage()); - return result; - } - case INTERNAL_SERVER_ERROR -> { - // Log only error code, to avoid exposing internal data in error message - logger.log("Deployment failed with possibly transient error " + e.code() + ", will retry"); - return result; - } - case LOAD_BALANCER_NOT_READY, PARENT_HOST_NOT_READY -> { - logger.log(e.message()); // Consider splitting these messages in summary and details, on config server. - Instant someTimeAfterStart = startTime.plusSeconds(200); - if (someTimeAfterStart.isAfter(controller.clock().instant())) - controller.jobController().locked(id, run -> run.sleepingUntil(someTimeAfterStart)); - return result; - } - case NODE_ALLOCATION_FAILURE -> { - logger.log(e.message()); - return controller.system().isCd() && startTime.plus(timeouts.capacity()).isAfter(controller.clock().instant()) - ? result - : Optional.of(nodeAllocationFailure); - } - case INVALID_APPLICATION_PACKAGE -> { - logger.log(WARNING, e.getMessage()); - return Optional.of(invalidApplication); - } - case BAD_REQUEST -> { - logger.log(WARNING, e.getMessage()); - return Optional.of(deploymentFailed); - } - case QUOTA_EXCEEDED -> { - logger.log(WARNING, e.getMessage()); - return Optional.of(quotaExceeded); - } - } - - throw e; - } - catch (CloudAccountNotSetException e) { - logger.log(INFO, "Timed out waiting for cloud account to be set for " + id + ": " + e.getMessage()); - return Optional.empty(); - } - catch (IllegalArgumentException e) { - logger.log(WARNING, e.getMessage()); - return Optional.of(deploymentFailed); - } - catch (EndpointCertificateException e) { - switch (e.type()) { - case CERT_NOT_AVAILABLE: - // Same as CERTIFICATE_NOT_READY above, only from the controller - logger.log("Retrieving CA signed certificate for the application. " + - "This may take up to " + timeouts.endpointCertificate().toMinutes() + " minutes on first deployment."); - if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) { - logger.log(WARNING, "CA signed certificate for app not available within " + - timeouts.endpointCertificate().toMinutes() + " minutes: " + Exceptions.toMessageString(e)); - return Optional.of(RunStatus.endpointCertificateTimeout); - } - return Optional.empty(); - default: - throw e; // Should be surfaced / fail deployment - } - } - } - - private Optional<RunStatus> installInitialReal(RunId id, DualLogger logger) { - return installReal(id, true, logger); - } - - private Optional<RunStatus> installReal(RunId id, DualLogger logger) { - return installReal(id, false, logger); - } - - private Optional<RunStatus> installReal(RunId id, boolean setTheStage, DualLogger logger) { - Optional<Deployment> deployment = deployment(id.application(), id.type()); - if (deployment.isEmpty()) { - logger.log("Deployment expired before installation was successful."); - return Optional.of(installationFailed); - } - - Versions versions = controller.jobController().run(id).versions(); - Version platform = setTheStage ? versions.sourcePlatform().orElse(versions.targetPlatform()) : versions.targetPlatform(); - - Run run = controller.jobController().run(id); - // In manually deployed zones it is allowed for some model versions not being built (e.g due to incompatibility) - // but deployment still succeeding, so we cannot use version when checking for config convergence - Optional<Version> platformVersion = id.type().environment().isManuallyDeployed() ? Optional.empty() : Optional.of(platform); - Optional<ServiceConvergence> services = configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone()), - platformVersion); - if (services.isEmpty()) { - logger.log("Config status not currently available -- will retry."); - return Optional.empty(); - } - List<Node> nodes = configServer().nodeRepository().list(id.type().zone(), - NodeFilter.all() - .applications(id.application()) - .states(active)); - - Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); - List<Node> parents = configServer().nodeRepository().list(id.type().zone(), - NodeFilter.all() - .hostnames(parentHostnames)); - boolean firstTick = run.convergenceSummary().isEmpty(); - NodeList nodeList = NodeList.of(nodes, parents, services.get()); - ConvergenceSummary summary = nodeList.summary(); - if (firstTick) { // Run the first time (for each convergence step). - logger.log("######## Details for all nodes ########"); - logger.log(nodeList.asList().stream() - .flatMap(node -> nodeDetails(node, true)) - .toList()); - } - else if ( ! summary.converged()) { - logger.log("Waiting for convergence of " + summary.services() + " services across " + summary.nodes() + " nodes"); - if (summary.needPlatformUpgrade() > 0) - logger.log(summary.upgradingPlatform() + "/" + summary.needPlatformUpgrade() + " nodes upgrading platform"); - if (summary.needReboot() > 0) - logger.log(summary.rebooting() + "/" + summary.needReboot() + " nodes rebooting"); - if (summary.needRestart() > 0) - logger.log(summary.restarting() + "/" + summary.needRestart() + " nodes restarting"); - if (summary.retiring() > 0) - logger.log(summary.retiring() + " nodes retiring"); - if (summary.upgradingFirmware() > 0) - logger.log(summary.upgradingFirmware() + " nodes upgrading firmware"); - if (summary.upgradingOs() > 0) - logger.log(summary.upgradingOs() + " nodes upgrading OS"); - if (summary.needNewConfig() > 0) - logger.log(summary.needNewConfig() + " application services still deploying"); - } - if (summary.converged()) { - controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); - Availability availability = endpointsAvailable(id.application(), id.type().zone(), deployment.get(), run.versions().sourceRevision().isEmpty(), logger); - if (availability.status() == Status.available) { - if (controller.routing().policies().processDnsChallenges(new DeploymentId(id.application(), id.type().zone()))) { - logger.log("Installation succeeded!"); - return Optional.of(running); - } - logger.log("Waiting for DNS challenges for private endpoints to be processed"); - return Optional.empty(); - } - logger.log(availability.message()); - if (availability.status() == Status.endpointsUnavailable && timedOut(id, deployment.get(), timeouts.endpoint())) { - logger.log(WARNING, "Endpoints failed to show up within " + timeouts.endpoint().toMinutes() + " minutes!"); - return Optional.of(error); - } - } - - String failureReason = null; - - NodeList suspendedTooLong = nodeList.isStateful() - .suspendedSince(controller.clock().instant().minus(timeouts.statefulNodesDown())) - .and(nodeList.not().isStateful() - .suspendedSince(controller.clock().instant().minus(timeouts.statelessNodesDown())) - ); - if ( ! suspendedTooLong.isEmpty() && deployment.get().at().plus(timeouts.statelessNodesDown()).isBefore(controller.clock().instant())) { - failureReason = "Some nodes have been suspended for more than the allowed threshold:\n" + - suspendedTooLong.asList().stream().map(node -> node.node().hostname().value()).collect(joining("\n")); - } - - if (run.noNodesDownSince() - .map(since -> since.isBefore(controller.clock().instant().minus(timeouts.noNodesDown()))) - .orElse(false)) { - if (summary.needPlatformUpgrade() > 0 || summary.needReboot() > 0 || summary.needRestart() > 0) - failureReason = "Timed out after waiting " + timeouts.noNodesDown().toMinutes() + " minutes for " + - "nodes to suspend. This is normal if the cluster is excessively busy. " + - "Nodes will continue to attempt suspension to progress installation independently of " + - "this run."; - else - failureReason = "Nodes not able to start with new application package."; - } - - Duration timeout = JobRunner.jobTimeout.minusHours(1); // Time out before job dies. - if (timedOut(id, deployment.get(), timeout)) { - failureReason = "Installation failed to complete within " + timeout.toHours() + "hours!"; - } - - if (failureReason != null) { - logger.log("######## Details for all nodes ########"); - logger.log(nodeList.asList().stream() - .flatMap(node -> nodeDetails(node, true)) - .toList()); - logger.log("######## Details for nodes with pending changes ########"); - logger.log(nodeList.not().in(nodeList.not().needsNewConfig() - .not().needsPlatformUpgrade() - .not().needsReboot() - .not().needsRestart() - .not().needsFirmwareUpgrade() - .not().needsOsUpgrade()) - .asList().stream() - .flatMap(node -> nodeDetails(node, true)) - .toList()); - logger.log(INFO, failureReason); - return Optional.of(installationFailed); - } - - if ( ! firstTick) - logger.log(FINE, nodeList.expectedDown().and(nodeList.needsNewConfig()).asList().stream() - .distinct() - .flatMap(node -> nodeDetails(node, false)) - .toList()); - - controller.jobController().locked(id, lockedRun -> { - Instant noNodesDownSince = nodeList.allowedDown().isEmpty() ? lockedRun.noNodesDownSince().orElse(controller.clock().instant()) : null; - return lockedRun.noNodesDownSince(noNodesDownSince).withSummary(summary); - }); - - return Optional.empty(); - } - - private Version testerPlatformVersion(RunId id) { - Version targetPlatform = controller.jobController().run(id).versions().targetPlatform(); - Version systemVersion = controller.readSystemVersion(); - boolean incompatible = controller.applications().versionCompatibility(id.application()).refuse(targetPlatform, systemVersion); - return incompatible || application(id.application()).change().isPlatformPinned() ? targetPlatform : systemVersion; - } - - private Optional<RunStatus> installTester(RunId id, DualLogger logger) { - Run run = controller.jobController().run(id); - Version platform = testerPlatformVersion(id); - ZoneId zone = id.type().zone(); - ApplicationId testerId = id.tester().id(); - - Optional<ServiceConvergence> services = configServer().serviceConvergence(new DeploymentId(testerId, zone), - Optional.of(platform)); - if (services.isEmpty()) { - if (run.stepInfo(installTester).get().startTime().get().isBefore(controller.clock().instant().minus(Duration.ofMinutes(30)))) { - logger.log(WARNING, "Config status not available after 30 minutes; giving up!"); - return Optional.of(error); - } - else { - logger.log("Config status not currently available -- will retry."); - return Optional.empty(); - } - } - List<Node> nodes = configServer().nodeRepository().list(zone, - NodeFilter.all() - .applications(testerId) - .states(active, reserved)); - Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); - List<Node> parents = configServer().nodeRepository().list(zone, - NodeFilter.all() - .hostnames(parentHostnames)); - NodeList nodeList = NodeList.of(nodes, parents, services.get()); - logger.log(nodeList.asList().stream() - .flatMap(node -> nodeDetails(node, false)) - .toList()); - - if (nodeList.summary().converged() && testerContainersAreUp(testerId, zone, logger)) { - logger.log("Tester container successfully installed!"); - return Optional.of(running); - } - - if (run.stepInfo(installTester).get().startTime().get().plus(timeouts.tester()).isBefore(controller.clock().instant())) { - logger.log(WARNING, "Installation of tester failed to complete within " + timeouts.tester().toMinutes() + " minutes!"); - return Optional.of(error); - } - - return Optional.empty(); - } - - private ConfigServer configServer() { return controller.serviceRegistry().configServer(); } - - /** Returns true iff all containers in the tester deployment give 100 consecutive 200 OK responses on /status.html. */ - private boolean testerContainersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { - DeploymentId deploymentId = new DeploymentId(id, zoneId); - if (controller.jobController().cloud().testerReady(deploymentId)) { - return true; - } else { - logger.log("Failed to get 100 consecutive OKs from tester container for " + deploymentId); - return false; - } - } - - private Availability endpointsAvailable(ApplicationId id, ZoneId zone, Deployment deployment, boolean initialDeployment, DualLogger logger) { - DeploymentId deploymentId = new DeploymentId(id, zone); - Map<ZoneId, List<Endpoint>> endpoints = controller.routing().readStepRunnerEndpointsOf(Set.of(deploymentId)); - logEndpoints(endpoints, logger); - DeploymentRoutingContext context = controller.routing().of(deploymentId); - boolean resolveEndpoints = context.routingMethod() == RoutingMethod.exclusive; - return controller.serviceRegistry().testerCloud().verifyEndpoints( - deploymentId, - endpoints.getOrDefault(zone, List.of()) - .stream() - .map(endpoint -> { - ClusterSpec.Id cluster = ClusterSpec.Id.from(endpoint.name()); - RoutingPolicy policy = context.routingPolicy(cluster).get(); - return new EndpointsChecker.Endpoint(id, - cluster, - HttpURL.from(endpoint.url()), - policy.ipAddress().filter(__ -> resolveEndpoints).map(uncheck(InetAddress::getByName)), - policy.canonicalName().filter(__ -> resolveEndpoints), - policy.isPublic(), - deployment.cloudAccount()); - }).toList(), - initialDeployment); - } - - private void logEndpoints(Map<ZoneId, List<Endpoint>> zoneEndpoints, DualLogger logger) { - List<String> messages = new ArrayList<>(); - messages.add("Found endpoints:"); - zoneEndpoints.forEach((zone, endpoints) -> { - messages.add("- " + zone); - for (Endpoint endpoint : endpoints) - messages.add(" |-- " + endpoint.url() + " (cluster '" + endpoint.name() + "')"); - }); - logger.log(messages); - } - - private Stream<String> nodeDetails(NodeWithServices node, boolean printAllServices) { - return Stream.concat(Stream.of(node.node().hostname() + ": " + humanize(node.node().serviceState()) + (node.node().suspendedSince().map(since -> " since " + since).orElse("")), - "--- platform " + wantedPlatform(node.node()) + (node.needsPlatformUpgrade() - ? " <-- " + currentPlatform(node.node()) - : "") + - (node.needsOsUpgrade() && node.isAllowedDown() - ? ", upgrading OS (" + node.parent().wantedOsVersion() + " <-- " + node.parent().currentOsVersion() + ")" - : "") + - (node.needsFirmwareUpgrade() && node.isAllowedDown() - ? ", upgrading firmware" - : "") + - (node.needsRestart() - ? ", restart pending (" + node.node().wantedRestartGeneration() + " <-- " + node.node().restartGeneration() + ")" - : "") + - (node.needsReboot() - ? ", reboot pending (" + node.node().wantedRebootGeneration() + " <-- " + node.node().rebootGeneration() + ")" - : "")), - node.services().stream() - .filter(service -> printAllServices || node.needsNewConfig()) - .map(service -> "--- " + service.type() + " on port " + service.port() + (service.currentGeneration() == -1 - ? " has not started " - : " has config generation " + service.currentGeneration() + ", wanted is " + node.wantedConfigGeneration()))); - } - - - private String wantedPlatform(Node node) { - return node.wantedDockerImage().repository() + ":" + node.wantedVersion(); - } - - private String currentPlatform(Node node) { - String currentRepo = node.currentDockerImage().repository(); - String wantedRepo = node.wantedDockerImage().repository(); - return (currentRepo.equals(wantedRepo) ? "" : currentRepo + ":") + node.currentVersion(); - } - - private String humanize(Node.ServiceState state) { - switch (state) { - case allowedDown: return "allowed to be DOWN"; - case expectedUp: return "expected to be UP"; - case permanentlyDown: return "permanently DOWN"; - case unorchestrated: return "unorchestrated"; - default: return state.name(); - } - } - - private Optional<RunStatus> startTests(RunId id, boolean isSetup, DualLogger logger) { - Optional<Deployment> deployment = deployment(id.application(), id.type()); - if (deployment.isEmpty()) { - logger.log(INFO, "Deployment expired before tests could start."); - return Optional.of(error); - } - - var deployments = controller.applications().requireInstance(id.application()) - .productionDeployments().keySet().stream() - .map(zone -> new DeploymentId(id.application(), zone)) - .collect(Collectors.toSet()); - ZoneId zoneId = id.type().zone(); - deployments.add(new DeploymentId(id.application(), zoneId)); - - logger.log("Attempting to find endpoints ..."); - var endpoints = controller.routing().readStepRunnerEndpointsOf(deployments); - if ( ! endpoints.containsKey(zoneId)) { - logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!"); - return Optional.of(error); - } - logEndpoints(endpoints, logger); - - if (!controller.jobController().cloud().testerReady(getTesterDeploymentId(id))) { - logger.log(WARNING, "Tester container went bad!"); - return Optional.of(error); - } - - logger.log("Starting tests ..."); - TesterCloud.Suite suite = TesterCloud.Suite.of(id.type(), isSetup); - byte[] config = testConfigSerializer.configJson(id.application(), - id.type(), - true, - deployment.get().version(), - deployment.get().revision(), - deployment.get().at(), - endpoints, - controller.applications().reachableContentClustersByZone(deployments)); - controller.jobController().cloud().startTests(getTesterDeploymentId(id), suite, config); - return Optional.of(running); - } - - @SuppressWarnings("fallthrough") - private Optional<RunStatus> endTests(RunId id, boolean isSetup, DualLogger logger) { - Optional<Deployment> deployment = deployment(id.application(), id.type()); - if (deployment.isEmpty()) { - logger.log(INFO, "Deployment expired before tests could complete."); - return Optional.of(error); - } - - Optional<X509Certificate> testerCertificate = controller.jobController().run(id).testerCertificate(); - if (testerCertificate.isPresent()) { - try { - testerCertificate.get().checkValidity(Date.from(controller.clock().instant())); - } - catch (CertificateExpiredException | CertificateNotYetValidException e) { - logger.log(WARNING, "Tester certificate expired before tests could complete."); - return Optional.of(error); - } - } - - controller.jobController().updateTestLog(id); - - TesterCloud.Status testStatus = controller.jobController().cloud().getStatus(getTesterDeploymentId(id)); - switch (testStatus) { - case NOT_STARTED: - throw new IllegalStateException("Tester reports tests not started, even though they should have!"); - case RUNNING: - return Optional.empty(); - case FAILURE: - logger.log("Tests failed."); - controller.jobController().updateTestReport(id); - return Optional.of(testFailure); - case INCONCLUSIVE: - controller.jobController().updateTestReport(id); - controller.jobController().locked(id, run -> { - Instant nextAttemptAt = run.start(); - while ( ! nextAttemptAt.isAfter(controller.clock().instant())) nextAttemptAt = nextAttemptAt.plusSeconds(1800); - logger.log("Tests were inconclusive, and will run again at " + nextAttemptAt + "."); - return run.sleepingUntil(nextAttemptAt); - }); - return Optional.of(reset); - case ERROR: - logger.log(INFO, "Tester failed running its tests!"); - controller.jobController().updateTestReport(id); - return Optional.of(error); - case NO_TESTS: - if ( ! isSetup) { // TODO: consider changing this Later™ - TesterCloud.Suite suite = TesterCloud.Suite.of(id.type(), isSetup); - logger.log(INFO, "No tests were found in the test package, for test suite '" + suite + "'"); - logger.log(INFO, "The test package should either contain basic HTTP tests under 'tests/<suite-name>/', " + - "or a Java test bundle under 'components/' with at least one test with the annotation " + - "for this suite. See docs.vespa.ai/en/testing.html for details."); - controller.jobController().updateTestReport(id); - - DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); - boolean requireTests = spec.steps().stream().anyMatch(step -> step.concerns(id.type().environment())); - logger.log(WARNING, "No tests were actually run, but this test suite is explicitly declared in 'deployment.xml'. " + - "Either add tests, ensure they're correctly configured, or remove the test declaration."); - return Optional.of(requireTests ? testFailure : noTests); - } - case SUCCESS: - logger.log("Tests completed successfully."); - controller.jobController().updateTestReport(id); - return Optional.of(running); - default: - throw new IllegalStateException("Unknown status '" + testStatus + "'!"); - } - } - - private Optional<RunStatus> copyVespaLogs(RunId id, DualLogger logger) { - if (deployment(id.application(), id.type()).isPresent()) - try { - controller.jobController().updateVespaLog(id); - } - // Hitting a config server which doesn't have this particular app loaded causes a 404. - catch (ConfigServerException e) { - Instant doom = controller.jobController().run(id).stepInfo(copyVespaLogs).get().startTime().get() - .plus(Duration.ofMinutes(3)); - if (e.code() == ConfigServerException.ErrorCode.NOT_FOUND && controller.clock().instant().isBefore(doom)) { - logger.log(INFO, "Found no logs, but will retry"); - return Optional.empty(); - } - else { - logger.log(INFO, "Failure getting vespa logs for " + id, e); - return Optional.of(error); - } - } - catch (Exception e) { - logger.log(INFO, "Failure getting vespa logs for " + id, e); - return Optional.of(error); - } - return Optional.of(running); - } - - private Optional<RunStatus> deactivateReal(RunId id, DualLogger logger) { - try { - logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone() + " ..."); - controller.applications().deactivate(id.application(), id.type().zone()); - return Optional.of(running); - } - catch (RuntimeException e) { - logger.log(WARNING, "Failed deleting application " + id.application(), e); - Instant startTime = controller.jobController().run(id).stepInfo(deactivateReal).get().startTime().get(); - return startTime.isBefore(controller.clock().instant().minus(Duration.ofHours(1))) - ? Optional.of(error) - : Optional.empty(); - } - } - - private Optional<RunStatus> deactivateTester(RunId id, DualLogger logger) { - try { - logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone() + " ..."); - controller.jobController().deactivateTester(id.tester(), id.type()); - return Optional.of(running); - } - catch (RuntimeException e) { - logger.log(WARNING, "Failed deleting tester of " + id.application(), e); - Instant startTime = controller.jobController().run(id).stepInfo(deactivateTester).get().startTime().get(); - return startTime.isBefore(controller.clock().instant().minus(Duration.ofHours(1))) - ? Optional.of(error) - : Optional.empty(); - } - } - - private Optional<RunStatus> report(RunId id, DualLogger logger) { - try { - boolean isRemoved = ! id.type().environment().isManuallyDeployed() - && ! controller.jobController().deploymentStatus(controller.applications().requireApplication(TenantAndApplicationId.from(id.application()))) - .jobSteps().containsKey(id.job()); - - controller.jobController().active(id).ifPresent(run -> { - if (run.status() == reset) - return; - - if (run.hasFailed() && ! isRemoved) - sendEmailNotification(run, logger); - - updateConsoleNotification(run, isRemoved); - }); - } - catch (IllegalStateException e) { - logger.log(INFO, "Job '" + id.type() + "' no longer supposed to run?", e); - return Optional.of(error); - } - catch (RuntimeException e) { - Instant start = controller.jobController().run(id).stepInfo(report).get().startTime().get(); - return (controller.clock().instant().isAfter(start.plusSeconds(180))) - ? Optional.empty() - : Optional.of(error); - } - return Optional.of(running); - } - - /** Sends a mail with a notification of a failed run, if one should be sent. */ - private void sendEmailNotification(Run run, DualLogger logger) { - if ( ! isNewFailure(run)) - return; - - Application application = controller.applications().requireApplication(TenantAndApplicationId.from(run.id().application())); - Notifications notifications = application.deploymentSpec().requireInstance(run.id().application().instance()).notifications(); - boolean newCommit = application.require(run.id().application().instance()).change().revision() - .map(run.versions().targetRevision()::equals) - .orElse(false); - When when = newCommit ? failingCommit : failing; - - List<String> recipients = new ArrayList<>(notifications.emailAddressesFor(when)); - if (notifications.emailRolesFor(when).contains(author)) - application.revisions().get(run.versions().targetRevision()).authorEmail().ifPresent(recipients::add); - - if (recipients.isEmpty()) - return; - - try { - logger.log(INFO, "Sending failure notification to " + String.join(", ", recipients)); - mailOf(run, recipients).ifPresent(controller.serviceRegistry().mailer()::send); - } - catch (RuntimeException e) { - logger.log(WARNING, "Exception trying to send mail for " + run.id(), e); - } - } - - private boolean isNewFailure(Run run) { - return controller.jobController().lastCompleted(run.id().job()) - .map(previous -> ! previous.hasFailed() || ! previous.versions().targetsMatch(run.versions())) - .orElse(true); - } - - private void updateConsoleNotification(Run run, boolean isRemoved) { - NotificationSource source = NotificationSource.from(run.id()); - Consumer<String> updater = msg -> controller.notificationsDb().setDeploymentNotification(run.id(), msg); - switch (isRemoved ? success : run.status()) { - case aborted, cancelled: return; // wait and see how the next run goes. - case noTests: - case running: - case success: - controller.notificationsDb().removeNotification(source, Notification.Type.deployment); - return; - case nodeAllocationFailure: - if ( ! run.id().type().environment().isTest()) updater.accept("could not allocate the requested capacity to your tenant. Please contact Vespa Cloud support."); - return; - case invalidApplication: - updater.accept("invalid application configuration. Please review warnings and errors in the deployment job log."); - return; - case deploymentFailed: - updater.accept("failure processing application configuration. Please review warnings and errors in the deployment job log."); - return; - case installationFailed: - updater.accept("nodes were not able to deploy to the new configuration. Please check the Vespa log for errors, and contact Vespa Cloud support if unable to resolve these."); - return; - case testFailure: - updater.accept("one or more verification tests against the deployment failed. Please review test output in the deployment job log."); - return; - case error: - case endpointCertificateTimeout: - break; - case quotaExceeded: - updater.accept("quota exceeded. Contact support to upgrade your plan."); - return; - default: - logger.log(WARNING, "Don't know what to set console notification to for run status '" + run.status() + "'"); - } - updater.accept("something in the deployment framework went wrong. Such errors are " + - "usually transient. Please contact Vespa Cloud support if the problem persists."); - } - - private Optional<Mail> mailOf(Run run, List<String> recipients) { - switch (run.status()) { - case running: - case aborted: - case cancelled: - case noTests: - case success: - return Optional.empty(); - case nodeAllocationFailure: - return run.id().type().isProduction() ? Optional.of(mails.nodeAllocationFailure(run.id(), recipients)) : Optional.empty(); - case deploymentFailed: - case invalidApplication: - return Optional.of(mails.deploymentFailure(run.id(), recipients)); - case installationFailed: - return Optional.of(mails.installationFailure(run.id(), recipients)); - case testFailure: - return Optional.of(mails.testFailure(run.id(), recipients)); - case error: - case endpointCertificateTimeout: - break; - default: - logger.log(WARNING, "Don't know what mail to send for run status '" + run.status() + "'"); - } - return Optional.of(mails.systemError(run.id(), recipients)); - } - - /** Returns the deployment of the real application in the zone of the given job, if it exists. */ - private Optional<Deployment> deployment(ApplicationId id, JobType type) { - return Optional.ofNullable(application(id).deployments().get(type.zone())); - } - - /** Returns the real application with the given id. */ - private Instance application(ApplicationId id) { - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), __ -> { }); // Memory fence. - return controller.applications().requireInstance(id); - } - - /** - * Returns whether the time since deployment is more than the zone deployment expiry, or the given timeout. - * - * We time out the job before the deployment expires, for zones where deployments are not persistent, - * to be able to collect the Vespa log from the deployment. Thus, the lower of the zone's deployment expiry, - * and the given default installation timeout, minus one minute, is used as a timeout threshold. - */ - private boolean timedOut(RunId id, Deployment deployment, Duration defaultTimeout) { - // TODO jonmv: This is a workaround for new deployment writes not yet being visible in spite of Curator locking. - // TODO Investigate what's going on here, and remove this workaround. - Run run = controller.jobController().run(id); - if ( ! controller.system().isCd() && run.start().isAfter(deployment.at())) - return false; - - Duration timeout = controller.zoneRegistry().getDeploymentTimeToLive(deployment.zone()) - .filter(zoneTimeout -> zoneTimeout.compareTo(defaultTimeout) < 0) - .orElse(defaultTimeout); - return deployment.at().isBefore(controller.clock().instant().minus(timeout.minus(Duration.ofMinutes(1)))); - } - - private boolean useTesterCertificate(RunId id) { - return controller.system().isPublic() && id.type().environment().isTest(); - } - - /** Returns the application package for the tester application, assembled from a generated config, fat-jar and services.xml. */ - private ApplicationPackageStream testerPackage(RunId id) { - RevisionId revision = controller.jobController().run(id).versions().targetRevision(); - DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); - boolean useTesterCertificate = useTesterCertificate(id); - - TestPackage testPackage = new TestPackage(() -> controller.applications().applicationStore().streamTester(id.application().tenant(), - id.application().application(), revision), - controller.system().isPublic(), - controller.zoneRegistry().get(id.type().zone()).getCloudName(), - id, - controller.controllerConfig().steprunner().testerapp(), - spec, - useTesterCertificate ? controller.clock().instant() : null, - timeouts.testerCertificate()); - if (useTesterCertificate) controller.jobController().storeTesterCertificate(id, testPackage.certificate()); - - return testPackage.asApplicationPackage(); - } - - private DeploymentId getTesterDeploymentId(RunId runId) { - ZoneId zoneId = runId.type().zone(); - return new DeploymentId(runId.tester().id(), zoneId); - } - - /** Logger which logs to a {@link JobController}, as well as to the parent class' {@link Logger}. */ - private class DualLogger { - - private final RunId id; - private final Step step; - - private DualLogger(RunId id, Step step) { - this.id = id; - this.step = step; - } - - private void log(String... messages) { - log(INFO, List.of(messages)); - } - - private void log(Level level, String... messages) { - log(level, List.of(messages)); - } - - private void logAll(List<LogEntry> messages) { - controller.jobController().log(id, step, messages); - } - - private void log(List<String> messages) { - log(INFO, messages); - } - - private void log(Level level, List<String> messages) { - controller.jobController().log(id, step, level, messages); - } - - private void log(Level level, String message) { - log(level, message, null); - } - - // Print stack trace in our logs, but don't expose it to end users - private void logWithInternalException(Level level, String message, Throwable thrown) { - logger.log(level, id + " at " + step + ": " + message, thrown); - controller.jobController().log(id, step, level, message); - } - - private void log(Level level, String message, Throwable thrown) { - logger.log(level, id + " at " + step + ": " + message, thrown); - - if (thrown != null) { - ByteArrayOutputStream traceBuffer = new ByteArrayOutputStream(); - thrown.printStackTrace(new PrintStream(traceBuffer)); - message += "\n" + traceBuffer; - } - controller.jobController().log(id, step, level, message); - } - - } - - - static class Timeouts { - - private final SystemName system; - - private Timeouts(SystemName system) { - this.system = requireNonNull(system); - } - - public static Timeouts of(SystemName system) { - return new Timeouts(system); - } - - Duration capacity() { return Duration.ofMinutes(system.isCd() ? 15 : 0); } - Duration endpoint() { return Duration.ofMinutes(15); } - Duration endpointCertificate() { return Duration.ofMinutes(20); } - Duration tester() { return Duration.ofMinutes(30); } - Duration statelessNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 60); } - Duration statefulNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 720); } - Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 240); } - Duration testerCertificate() { return Duration.ofMinutes(300); } - - } - - private static class CloudAccountNotSetException extends RuntimeException { - CloudAccountNotSetException(String message) { super(message); } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java deleted file mode 100644 index ae6bcdea00c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ /dev/null @@ -1,932 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.google.common.collect.ImmutableSortedMap; -import com.yahoo.component.Version; -import com.yahoo.component.VersionCompatibility; -import com.yahoo.concurrent.UncheckedTimeoutException; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.flags.FetchVector.Dimension; -import com.yahoo.vespa.flags.ListFlag; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.ApplicationController; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.LockedApplication; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff; -import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage; -import com.yahoo.vespa.hosted.controller.deployment.Run.Reason; -import com.yahoo.vespa.hosted.controller.notification.Notification.Type; -import com.yahoo.vespa.hosted.controller.notification.NotificationSource; -import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; - -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Deque; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import java.util.function.UnaryOperator; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Stream; - -import static com.yahoo.collections.Iterables.reversed; -import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.deploymentFile; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.cancelled; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup; -import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.report; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.time.temporal.ChronoUnit.SECONDS; -import static java.util.Comparator.comparing; -import static java.util.Comparator.naturalOrder; -import static java.util.function.Predicate.not; -import static java.util.logging.Level.INFO; -import static java.util.logging.Level.WARNING; - -/** - * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs. - * - * Keys are the {@link ApplicationId} of the real application, for which the deployment job is run, the - * {@link JobType} to run, and the strictly increasing run number of this combination. - * The deployment jobs run tests using regular applications, but these tester application IDs are not to be used elsewhere. - * - * Jobs consist of sets of {@link Step}s, defined in {@link JobProfile}s. - * Each run is represented by a {@link Run}, which holds the status of each step of the run, as well as - * some other meta data. - * - * @author jonmv - */ -public class JobController { - - public static final Duration maxHistoryAge = Duration.ofDays(60); - public static final Duration obsoletePackageExpiry = Duration.ofDays(7); - - private static final Logger log = Logger.getLogger(JobController.class.getName()); - - private final int historyLength; - private final Controller controller; - private final CuratorDb curator; - private final BufferedLogStore logs; - private final TesterCloud cloud; - private final JobMetrics metric; - private final ListFlag<String> disabledZones; - - private final AtomicReference<Consumer<Run>> runner = new AtomicReference<>(__ -> { }); - - public JobController(Controller controller) { - this.historyLength = controller.system().isCd() ? 256 : 64; - this.controller = controller; - this.curator = controller.curator(); - this.logs = new BufferedLogStore(curator, controller.serviceRegistry().runDataStore()); - this.cloud = controller.serviceRegistry().testerCloud(); - this.metric = new JobMetrics(controller.metric()); - this.disabledZones = PermanentFlags.DISABLED_DEPLOYMENT_ZONES.bindTo(controller.flagSource()); - } - - public TesterCloud cloud() { return cloud; } - public int historyLength() { return historyLength; } - public void setRunner(Consumer<Run> runner) { this.runner.set(runner); } - - /** Rewrite all job data with the newest format. */ - public void updateStorage() { - for (ApplicationId id : instances()) - for (JobType type : jobs(id)) { - locked(id, type, runs -> { // Runs are not modified here, and are written as they were. - curator.readLastRun(id, type).ifPresent(curator::writeLastRun); - }); - } - } - - public boolean isDisabled(JobId id) { - return disabledZones.with(Dimension.INSTANCE_ID, id.application().serializedForm()).value().contains(id.type().zone().value()); - } - - /** Returns all entries currently logged for the given run. */ - public Optional<RunLog> details(RunId id) { - return details(id, -1); - } - - /** Returns the logged entries for the given run, which are after the given id threshold. */ - public Optional<RunLog> details(RunId id, long after) { - try (Mutex __ = curator.lock(id.application(), id.type())) { - Run run = runs(id.application(), id.type()).get(id); - if (run == null) - return Optional.empty(); - - return active(id).isPresent() - ? Optional.of(logs.readActive(id.application(), id.type(), after)) - : logs.readFinished(id, after); - } - } - - /** Stores the given log entries for the given run and step. */ - public void log(RunId id, Step step, List<LogEntry> entries) { - locked(id, __ -> { - logs.append(id.application(), id.type(), step, entries, true); - return __; - }); - } - - /** Stores the given log messages for the given run and step. */ - public void log(RunId id, Step step, Level level, List<String> messages) { - log(id, step, messages.stream() - .map(message -> new LogEntry(0, controller.clock().instant(), LogEntry.typeOf(level), message)) - .toList()); - } - - /** Stores the given log message for the given run and step. */ - public void log(RunId id, Step step, Level level, String message) { - log(id, step, level, Collections.singletonList(message)); - } - - /** Fetches any new Vespa log entries, and records the timestamp of the last of these, for continuation. */ - public void updateVespaLog(RunId id) { - locked(id, run -> { - if ( ! run.hasStep(copyVespaLogs)) - return run; - - storeVespaLogs(id); - - // TODO jonmv: remove all the below around start of 2023. - ZoneId zone = id.type().zone(); - Optional<Deployment> deployment = Optional.ofNullable(controller.applications().requireInstance(id.application()) - .deployments().get(zone)); - if (deployment.isEmpty() || deployment.get().at().isBefore(run.start())) - return run; - - List<LogEntry> log; - Optional<Instant> deployedAt; - Instant from; - if ( ! run.id().type().isProduction()) { - deployedAt = run.stepInfo(installInitialReal).or(() -> run.stepInfo(installReal)).flatMap(StepInfo::startTime); - if (deployedAt.isPresent()) { - from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.get().minusSeconds(10); - log = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() - .getLogs(new DeploymentId(id.application(), zone), - Map.of("from", Long.toString(from.toEpochMilli()))), - from); - } - else log = List.of(); - } - else log = List.of(); - - if (id.type().isTest()) { - deployedAt = run.stepInfo(installTester).flatMap(StepInfo::startTime); - if (deployedAt.isPresent()) { - from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.get().minusSeconds(10); - List<LogEntry> testerLog = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() - .getLogs(new DeploymentId(id.tester().id(), zone), - Map.of("from", Long.toString(from.toEpochMilli()))), - from); - - Instant justNow = controller.clock().instant().minusSeconds(2); - log = Stream.concat(log.stream(), testerLog.stream()) - .filter(entry -> entry.at().isBefore(justNow)) - .sorted(comparing(LogEntry::at)) - .toList(); - } - } - if (log.isEmpty()) - return run; - - logs.append(id.application(), id.type(), Step.copyVespaLogs, log, false); - return run.with(log.get(log.size() - 1).at()); - }); - } - - public InputStream getVespaLogs(RunId id, long fromMillis, boolean tester) { - Run run = run(id); - return run.stepStatus(copyVespaLogs).map(succeeded::equals).orElse(false) - ? controller.serviceRegistry().runDataStore().getLogs(id, tester) - : getVespaLogsFromLogserver(run, fromMillis, tester).orElse(InputStream.nullInputStream()); - } - - public static Optional<Instant> deploymentCompletedAt(Run run, boolean tester) { - return (tester ? run.stepInfo(installTester) - : run.stepInfo(installInitialReal).or(() -> run.stepInfo(installReal))) - .flatMap(StepInfo::startTime).map(start -> start.minusSeconds(10)); - } - - public void storeVespaLogs(RunId id) { - Run run = run(id); - if ( ! id.type().isProduction()) { - getVespaLogsFromLogserver(run, 0, false).ifPresent(logs -> { - try (logs) { - controller.serviceRegistry().runDataStore().putLogs(id, false, logs); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - } - if (id.type().isTest()) { - getVespaLogsFromLogserver(run, 0, true).ifPresent(logs -> { - try (logs) { - controller.serviceRegistry().runDataStore().putLogs(id, true, logs); - } - catch(IOException e){ - throw new UncheckedIOException(e); - } - }); - } - } - - private Optional<InputStream> getVespaLogsFromLogserver(Run run, long fromMillis, boolean tester) { - return deploymentCompletedAt(run, tester).map(at -> - controller.serviceRegistry().configServer().getLogs(new DeploymentId(tester ? run.id().tester().id() : run.id().application(), - run.id().type().zone()), - Map.of("from", Long.toString(Math.max(fromMillis, at.toEpochMilli())), - "to", Long.toString(run.end().orElse(controller.clock().instant()).toEpochMilli())))); -} - - /** Fetches any new test log entries, and records the id of the last of these, for continuation. */ - public void updateTestLog(RunId id) { - locked(id, run -> { - Optional<Step> step = Stream.of(endStagingSetup, endTests) - .filter(run.readySteps()::contains) - .findAny(); - if (step.isEmpty()) - return run; - - List<LogEntry> entries = cloud.getLog(new DeploymentId(id.tester().id(), id.type().zone()), - run.lastTestLogEntry()); - if (entries.isEmpty()) - return run; - - logs.append(id.application(), id.type(), step.get(), entries, false); - return run.with(entries.stream().mapToLong(LogEntry::id).max().getAsLong()); - }); - } - - public void updateTestReport(RunId id) { - locked(id, run -> { - Optional<TestReport> report = cloud.getTestReport(new DeploymentId(id.tester().id(), id.type().zone())); - if (report.isEmpty()) { - return run; - } - logs.writeTestReport(id, report.get()); - return run; - }); - } - - public Optional<String> getTestReports(RunId id) { - return logs.readTestReports(id); - } - - /** Stores the given certificate as the tester certificate for this run, or throws if it's already set. */ - public void storeTesterCertificate(RunId id, X509Certificate testerCertificate) { - locked(id, run -> run.with(testerCertificate)); - } - - /** Returns a list of all instances of applications which have registered. */ - public List<ApplicationId> instances() { - return controller.applications().readable().stream() - .flatMap(application -> application.instances().values().stream()) - .map(Instance::id).toList(); - } - - /** Returns all job types which have been run for the given application. */ - private List<JobType> jobs(ApplicationId id) { - return JobType.allIn(controller.zoneRegistry()).stream() - .filter(type -> last(id, type).isPresent()).toList(); - } - - /** Returns an immutable map of all known runs for the given application and job type. */ - public NavigableMap<RunId, Run> runs(JobId id) { - return runs(id.application(), id.type()); - } - - /** Lists the start time of non-redeployment runs of the given job, in order of increasing age. */ - public List<Instant> jobStarts(JobId id) { - return runs(id).descendingMap().values().stream() - .filter(run -> !run.isRedeployment()) - .map(Run::start).toList(); - } - - /** Returns when given deployment last started deploying, falling back to time of deployment if it cannot be determined from job runs */ - public Instant lastDeploymentStart(ApplicationId instanceId, Deployment deployment) { - return jobStarts(new JobId(instanceId, JobType.deploymentTo(deployment.zone()))).stream() - .findFirst() - .orElseGet(deployment::at); - } - - /** Returns an immutable map of all known runs for the given application and job type. */ - public NavigableMap<RunId, Run> runs(ApplicationId id, JobType type) { - ImmutableSortedMap.Builder<RunId, Run> runs = ImmutableSortedMap.orderedBy(Comparator.comparing(RunId::number)); - Optional<Run> last = last(id, type); - curator.readHistoricRuns(id, type).forEach((runId, run) -> { - if (last.isEmpty() || ! runId.equals(last.get().id())) - runs.put(runId, run); - }); - last.ifPresent(run -> runs.put(run.id(), run)); - return runs.build(); - } - - /** Returns the run with the given id, or throws if no such run exists. */ - public Run run(RunId id) { - return runs(id.application(), id.type()).values().stream() - .filter(run -> run.id().equals(id)) - .findAny() - .orElseThrow(() -> new NoSuchElementException("no run with id '" + id + "' exists")); - } - - /** Returns the last run of the given type, for the given application, if one has been run. */ - public Optional<Run> last(JobId job) { - return curator.readLastRun(job.application(), job.type()); - } - - /** Returns the last run of the given type, for the given application, if one has been run. */ - public Optional<Run> last(ApplicationId id, JobType type) { - return curator.readLastRun(id, type); - } - - /** Returns the last completed of the given job. */ - public Optional<Run> lastCompleted(JobId id) { - return JobStatus.lastCompleted(runs(id)); - } - - /** Returns the first failing of the given job. */ - public Optional<Run> firstFailing(JobId id) { - return JobStatus.firstFailing(runs(id)); - } - - /** Returns the last success of the given job. */ - public Optional<Run> lastSuccess(JobId id) { - return JobStatus.lastSuccess(runs(id)); - } - - /** Returns the run with the given id, provided it is still active. */ - public Optional<Run> active(RunId id) { - return last(id.application(), id.type()) - .filter(run -> ! run.hasEnded()) - .filter(run -> run.id().equals(id)); - } - - /** Returns a list of all active runs. */ - public List<Run> active() { - return controller.applications().idList().stream() - .flatMap(id -> active(id).stream()) - .toList(); - } - - /** Returns a list of all active runs for the given application. */ - public List<Run> active(TenantAndApplicationId id) { - return controller.applications().requireApplication(id).instances().keySet().stream() - .flatMap(name -> JobType.allIn(controller.zoneRegistry()).stream() - .map(type -> last(id.instance(name), type)) - .flatMap(Optional::stream) - .filter(run -> ! run.hasEnded())) - .toList(); - } - - /** Returns a list of all active runs for the given instance. */ - public List<Run> active(ApplicationId id) { - return JobType.allIn(controller.zoneRegistry()).stream() - .map(type -> last(id, type)) - .flatMap(Optional::stream) - .filter(run -> !run.hasEnded()) - .toList(); - } - - /** Returns the job status of the given job, possibly empty. */ - public JobStatus jobStatus(JobId id) { - return new JobStatus(id, runs(id)); - } - - /** Returns the deployment status of the given application. */ - public DeploymentStatus deploymentStatus(Application application) { - VersionStatus versionStatus = controller.readVersionStatus(); - return deploymentStatus(application, versionStatus, controller.systemVersion(versionStatus)); - } - - private DeploymentStatus deploymentStatus(Application application, VersionStatus versionStatus, Version systemVersion) { - return new DeploymentStatus(application, - this::jobStatus, - controller.zoneRegistry(), - versionStatus, - systemVersion, - instance -> controller.applications().versionCompatibility(application.id().instance(instance)), - controller.clock().instant()); - } - - /** Adds deployment status to each of the given applications. */ - public DeploymentStatusList deploymentStatuses(ApplicationList applications, VersionStatus versionStatus) { - Version systemVersion = controller.systemVersion(versionStatus); - return DeploymentStatusList.from(applications.asList().stream() - .map(application -> deploymentStatus(application, versionStatus, systemVersion)) - .toList()); - } - - /** Adds deployment status to each of the given applications. Calling this will do an implicit read of the controller's version status */ - public DeploymentStatusList deploymentStatuses(ApplicationList applications) { - VersionStatus versionStatus = controller.readVersionStatus(); - return deploymentStatuses(applications, versionStatus); - } - - /** Changes the status of the given step, for the given run, provided it is still active. */ - public void update(RunId id, RunStatus status, LockedStep step) { - locked(id, run -> run.with(status, step)); - } - - /** - * Changes the status of the given run to inactive, and stores it as a historic run. - * Throws TimeoutException if some step in this job is still being run. - */ - public void finish(RunId id) throws TimeoutException { - Deque<Mutex> locks = new ArrayDeque<>(); - try { - // Ensure no step is still running before we finish the run — report depends transitively on all the other steps. - Run unlockedRun = run(id); - locks.push(curator.lock(id.application(), id.type(), report)); - for (Step step : report.allPrerequisites(unlockedRun.steps().keySet())) - locks.push(curator.lock(id.application(), id.type(), step)); - - locked(id, run -> { - // If run should be reset, just return here. - if (run.status() == reset) { - for (Step step : run.steps().keySet()) - log(id, step, INFO, List.of("### Run will reset, and start over at " + run.sleepUntil().orElse(controller.clock().instant()).truncatedTo(SECONDS), "")); - return run.reset(); - } - if (run.status() == running && run.stepStatuses().values().stream().anyMatch(not(succeeded::equals))) return run; - - // Store the modified run after it has been written to history, in case the latter fails. - Run finishedRun = run.finished(controller.clock().instant()); - locked(id.application(), id.type(), runs -> { - runs.put(run.id(), finishedRun); - long last = id.number(); - long successes = runs.values().stream().filter(Run::hasSucceeded).count(); - var oldEntries = runs.entrySet().iterator(); - for (var old = oldEntries.next(); - old.getKey().number() <= last - historyLength - || old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge)); - old = oldEntries.next()) { - - // Make sure we keep the last success and the first failing - if ( successes == 1 - && old.getValue().hasSucceeded() - && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { - oldEntries.next(); - continue; - } - - logs.delete(old.getKey()); - oldEntries.remove(); - } - }); - logs.flush(id); - metric.jobFinished(run.id().job(), finishedRun.status()); - pruneRevisions(unlockedRun); - - return finishedRun; - }); - } - finally { - for (Mutex lock : locks) { - try { - lock.close(); - } catch (Throwable t) { - log.log(WARNING, "Failed to close the lock " + lock + ": the lock may or may not " + - "have been released in ZooKeeper, and if not this controller " + - "must be restarted to release the lock", t); - } - } - } - } - - /** Marks the given run as aborted; no further normal steps will run, but run-always steps will try to succeed. */ - public void abort(RunId id, String reason, boolean cancelledByHumans) { - locked(id, run -> { - if (run.status() == aborted || run.status() == cancelled) - return run; - - run.stepStatuses().entrySet().stream() - .filter(entry -> entry.getValue() == unfinished) - .forEach(entry -> log(id, entry.getKey(), INFO, "Aborting run: " + reason)); - return run.aborted(cancelledByHumans); - }); - } - - /** Accepts and stores a new application package and test jar pair under a generated application version key. */ - public ApplicationVersion submit(TenantAndApplicationId id, Submission submission, long projectId) { - ApplicationController applications = controller.applications(); - AtomicReference<ApplicationVersion> version = new AtomicReference<>(); - applications.lockApplicationOrThrow(id, application -> { - Optional<ApplicationVersion> previousVersion = application.get().revisions().last(); - Optional<ApplicationPackage> previousPackage = previousVersion.flatMap(previous -> applications.applicationStore().find(id.tenant(), id.application(), previous.buildNumber())) - .map(ApplicationPackage::new); - long previousBuild = previousVersion.map(latestVersion -> latestVersion.buildNumber()).orElse(0L); - version.set(submission.toApplicationVersion(1 + previousBuild)); - - byte[] diff = previousPackage.map(previous -> ApplicationPackageDiff.diff(previous, submission.applicationPackage())) - .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(submission.applicationPackage())); - applications.applicationStore().put(id.tenant(), - id.application(), - version.get().id(), - submission.applicationPackage().zippedContent(), - withDeploymentSpec(submission.testPackage(), - submission.applicationPackage().deploymentSpec()), - diff); - applications.applicationStore().putMeta(id.tenant(), - id.application(), - controller.clock().instant(), - submission.applicationPackage().metaDataZip()); - - application = application.withProjectId(projectId == -1 ? OptionalLong.empty() : OptionalLong.of(projectId)); - application = application.withRevisions(revisions -> revisions.with(version.get())); - application = withPrunedPackages(application, version.get().id()); - version.set(application.get().revisions().get(version.get().id())); - - validate(id, submission); - - List<InstanceName> newInstances = applications.storeWithUpdatedConfig(application, submission.applicationPackage()); - if (application.get().projectId().isPresent()) - applications.deploymentTrigger().triggerNewRevision(id); - for (InstanceName instance : newInstances) - controller.applications().deploymentTrigger().forceChange(id.instance(instance), Change.of(version.get().id())); - }); - return version.get(); - } - - static byte[] withDeploymentSpec(byte[] testZip, DeploymentSpec spec) { - ZipBuilder zip = new ZipBuilder(testZip.length + (1 << 12)); - try (zip) { - zip.add(testZip, name -> !name.equals(deploymentFile)); - zip.add(deploymentFile, spec.xmlForm().getBytes(UTF_8)); - } - return zip.toByteArray(); - } - - private void validate(TenantAndApplicationId id, Submission submission) { - controller.notificationsDb().removeNotification(NotificationSource.from(id), Type.testPackage); - controller.notificationsDb().removeNotification(NotificationSource.from(id), Type.submission); - - validateTests(id, submission); - validateMajorVersion(id, submission); - } - - private void validateTests(TenantAndApplicationId id, Submission submission) { - var testSummary = TestPackage.validateTests(submission.applicationPackage().deploymentSpec(), submission.testPackage()); - if ( ! testSummary.problems().isEmpty()) - controller.notificationsDb().setTestPackageNotification(id, testSummary.problems()); - } - - private void validateMajorVersion(TenantAndApplicationId id, Submission submission) { - submission.applicationPackage().deploymentSpec().majorVersion().ifPresent(explicitMajor -> { - if ( ! controller.readVersionStatus().isOnCurrentMajor(new Version(explicitMajor))) - controller.notificationsDb().setSubmissionNotification(id, - "Vespa " + explicitMajor + " will soon reach end of life, upgrade to [Vespa " + (explicitMajor + 1) + " now](" + - "https://cloud.vespa.ai/en/vespa" + (explicitMajor + 1) + "-release-notes.html)"); // ∠( ᐛ 」∠)_ - }); - } - - private LockedApplication withPrunedPackages(LockedApplication application, RevisionId latest) { - TenantAndApplicationId id = application.get().id(); - Application wrapped = application.get(); - RevisionId oldestDeployed = application.get().oldestDeployedRevision() - .or(() -> wrapped.instances().values().stream() - .flatMap(instance -> instance.change().revision().stream()) - .min(naturalOrder())) - .orElse(latest); - RevisionId oldestToKeep = null; - Instant now = controller.clock().instant(); - for (ApplicationVersion version : application.get().revisions().withPackage()) { - if (version.id().compareTo(oldestDeployed) < 0) { - if (version.obsoleteAt().isEmpty()) { - application = application.withRevisions(revisions -> revisions.with(version.obsoleteAt(now))); - if (oldestToKeep == null) - oldestToKeep = version.id(); - } - else { - if (oldestToKeep == null && !version.obsoleteAt().get().isBefore(now.minus(obsoletePackageExpiry))) - oldestToKeep = version.id(); - } - } - } - - if (oldestToKeep != null) { - controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestToKeep); - for (ApplicationVersion version : application.get().revisions().withPackage()) - if (version.id().compareTo(oldestToKeep) < 0) - application = application.withRevisions(revisions -> revisions.with(version.withoutPackage())); - } - return application; - } - - /** Forget revisions no longer present in any relevant job history. */ - private void pruneRevisions(Run run) { - TenantAndApplicationId applicationId = TenantAndApplicationId.from(run.id().application()); - boolean isProduction = run.versions().targetRevision().isProduction(); - (isProduction ? deploymentStatus(controller.applications().requireApplication(applicationId)).jobs().asList().stream() - : Stream.of(jobStatus(run.id().job()))) - .flatMap(jobs -> jobs.runs().values().stream()) - .map(r -> r.versions().targetRevision()) - .filter(id -> id.isProduction() == isProduction) - .min(naturalOrder()) - .ifPresent(oldestRevision -> { - controller.applications().lockApplicationOrThrow(applicationId, application -> { - if (isProduction) { - controller.applications().applicationStore().pruneDiffs(run.id().application().tenant(), run.id().application().application(), oldestRevision.number()); - controller.applications().store(application.withRevisions(revisions -> revisions.withoutOlderThan(oldestRevision))); - } - else { - controller.applications().applicationStore().pruneDevDiffs(new DeploymentId(run.id().application(), run.id().job().type().zone()), oldestRevision.number()); - controller.applications().store(application.withRevisions(revisions -> revisions.withoutOlderThan(oldestRevision, run.id().job()))); - } - }); - }); - } - - /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ - public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, Reason reason) { - start(id, type, versions, isRedeployment, JobProfile.of(type), reason); - } - - /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ - public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, JobProfile profile, Reason reason) { - ApplicationVersion revision = controller.applications().requireApplication(TenantAndApplicationId.from(id)).revisions().get(versions.targetRevision()); - if (revision.compileVersion() - .map(version -> controller.applications().versionCompatibility(id).refuse(versions.targetPlatform(), version)) - .orElse(false)) - throw new IllegalArgumentException("Will not start " + type + " for " + id + " with incompatible platform version (" + - versions.targetPlatform() + ") " + "and compile versions (" + revision.compileVersion().get() + ")"); - - locked(id, type, __ -> { - Optional<Run> last = last(id, type); - if (last.flatMap(run -> active(run.id())).isPresent()) - throw new IllegalArgumentException("Cannot start " + type + " for " + id + "; it is already running!"); - - RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); - curator.writeLastRun(Run.initial(newId, versions, isRedeployment, controller.clock().instant(), profile, reason)); - metric.jobStarted(newId.job()); - }); - } - - - /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment. */ - public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage) { - deploy(id, type, platform, applicationPackage, false, false); - } - - /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment.*/ - public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage, - boolean dryRun, boolean allowOutdatedPlatform) { - if ( ! controller.zoneRegistry().hasZone(type.zone())) - throw new IllegalArgumentException(type.zone() + " is not present in this system"); - - VersionStatus versionStatus = controller.readVersionStatus(); - if ( ! controller.system().isCd() - && platform.isPresent() - && versionStatus.deployableVersions().stream().map(VespaVersion::versionNumber).noneMatch(platform.get()::equals)) - throw new IllegalArgumentException("platform version " + platform.get() + " is not present in this system"); - - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - if ( ! application.get().instances().containsKey(id.instance())) - application = controller.applications().withNewInstance(application, id); - // TODO(mpolden): Enable for public CD once all tests have been updated - if (controller.system() != SystemName.PublicCd) { - controller.applications().validatePackage(applicationPackage, application.get()); - controller.applications().decideCloudAccountOf(new DeploymentId(id, type.zone()), applicationPackage.deploymentSpec()); - } - controller.applications().store(application); - }); - - DeploymentId deploymentId = new DeploymentId(id, type.zone()); - Optional<Run> lastRun = last(id, type); - lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id(), Duration.ofMinutes(2))); - - long build = 1 + lastRun.map(run -> run.versions().targetRevision().number()).orElse(0L); - RevisionId revisionId = RevisionId.forDevelopment(build, new JobId(id, type)); - ApplicationVersion version = ApplicationVersion.forDevelopment(revisionId, applicationPackage.compileVersion(), applicationPackage.deploymentSpec().majorVersion()); - - byte[] diff = getDiff(applicationPackage, deploymentId, lastRun); - - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, deploymentId, application.get().get(id.instance()), versionStatus)); - if ( ! allowOutdatedPlatform - && ! controller.readVersionStatus().isOnCurrentMajor(targetPlatform) - && runs(id, type).values().stream().noneMatch(run -> run.versions().targetPlatform().getMajor() == targetPlatform.getMajor())) - throw new IllegalArgumentException("platform version " + targetPlatform + " is not on a current major version in this system"); - - controller.applications().applicationStore().putDev(deploymentId, version.id(), applicationPackage.zippedContent(), diff); - controller.applications().store(application.withRevisions(revisions -> revisions.with(version))); - Optional<Deployment> existing = application.get().get(id.instance()).map(instance -> instance.deployments().get(type.zone())); - start(id, - type, - new Versions(targetPlatform, version.id(), existing.map(Deployment::version), existing.map(Deployment::revision)), - false, - dryRun ? JobProfile.developmentDryRun : JobProfile.development, - Reason.empty()); - }); - - locked(id, type, __ -> { - runner.get().accept(last(id, type).get()); - }); - } - - /* Application package diff against previous version, or against empty version if previous does not exist or is invalid */ - private byte[] getDiff(ApplicationPackage applicationPackage, DeploymentId deploymentId, Optional<Run> lastRun) { - return lastRun.map(run -> run.versions().targetRevision()) - .map(prevVersion -> { - ApplicationPackage previous; - try { - previous = new ApplicationPackage(controller.applications().applicationStore().get(deploymentId, prevVersion)); - } catch (RuntimeException e) { - return ApplicationPackageDiff.diffAgainstEmpty(applicationPackage); - } - return ApplicationPackageDiff.diff(previous, applicationPackage); - }) - .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage)); - } - - private Version findTargetPlatform(ApplicationPackage applicationPackage, DeploymentId id, Optional<Instance> instance, VersionStatus versionStatus) { - // Prefer previous platform if possible. Candidates are all deployable, ascending, with existing version appended; then reversed. - Version systemVersion = controller.systemVersion(versionStatus); - - List<Version> versions = new ArrayList<>(List.of(systemVersion)); - for (VespaVersion version : versionStatus.deployableVersions()) - if (version.confidence().equalOrHigherThan(Confidence.normal)) - versions.add(version.versionNumber()); - - instance.map(Instance::deployments) - .map(deployments -> deployments.get(id.zoneId())) - .map(Deployment::version) - .filter(versions::contains) // Don't deploy versions that are no longer known. - .ifPresent(versions::add); - - // Remove all versions that are older than the compile version. - versions.removeIf(version -> applicationPackage.compileVersion().map(version::isBefore).orElse(false)); - if (versions.isEmpty()) { - // Fall back to the newest deployable version, if all the ones with normal confidence were too old. - Iterator<VespaVersion> descending = reversed(versionStatus.deployableVersions()).iterator(); - if ( ! descending.hasNext()) - throw new IllegalStateException("no deployable platform version found in the system"); - else - versions.add(descending.next().versionNumber()); - } - - VersionCompatibility compatibility = controller.applications().versionCompatibility(id.applicationId()); - List<Version> compatibleVersions = new ArrayList<>(); - for (Version target : reversed(versions)) - if (applicationPackage.compileVersion().isEmpty() || compatibility.accept(target, applicationPackage.compileVersion().get())) - compatibleVersions.add(target); - - if (compatibleVersions.isEmpty()) - throw new IllegalArgumentException("no platforms are compatible with compile version " + applicationPackage.compileVersion().get()); - - Optional<Integer> major = applicationPackage.deploymentSpec().majorVersion(); - List<Version> versionOnRightMajor = new ArrayList<>(); - for (Version target : reversed(versions)) - if (major.isEmpty() || major.get() == target.getMajor()) - versionOnRightMajor.add(target); - - if (versionOnRightMajor.isEmpty()) - throw new IllegalArgumentException("no platforms were found for major version " + major.get() + " specified in deployment.xml"); - - for (Version target : compatibleVersions) - if (versionOnRightMajor.contains(target)) - return target; - - throw new IllegalArgumentException("no platforms on major version " + major.get() + " specified in deployment.xml " + - "are compatible with compile version " + applicationPackage.compileVersion().get()); - } - - /** Aborts a run and waits for it complete. */ - private void abortAndWait(RunId id, Duration timeout) { - abort(id, "replaced by new deployment", true); - runner.get().accept(last(id.application(), id.type()).get()); - - Instant doom = controller.clock().instant().plus(timeout); - Duration sleep = Duration.ofMillis(100); - while ( ! last(id.application(), id.type()).get().hasEnded()) { - if (controller.clock().instant().plus(sleep).isAfter(doom)) - throw new UncheckedTimeoutException("timeout waiting for " + id + " to abort and finish"); - try { - Thread.sleep(sleep.toMillis()); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - } - } - - /** Deletes run data and tester deployments for applications which are unknown, or no longer built internally. */ - public void collectGarbage() { - Set<ApplicationId> applicationsToBuild = new HashSet<>(instances()); - curator.applicationsWithJobs().stream() - .filter(id -> ! applicationsToBuild.contains(id)) - .forEach(id -> { - try { - TesterId tester = TesterId.of(id); - for (JobType type : jobs(id)) - locked(id, type, deactivateTester, __ -> { - try (Mutex ___ = curator.lock(id, type)) { - try { - deactivateTester(tester, type); - } - catch (Exception e) { - // It's probably already deleted, so if we fail, that's OK. - } - curator.deleteRunData(id, type); - } - }); - logs.delete(id); - curator.deleteRunData(id); - } - catch (Exception e) { - log.log(WARNING, "failed cleaning up after deleted application", e); - } - }); - } - - public void deactivateTester(TesterId id, JobType type) { - controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone())); - } - - /** Locks all runs and modifies the list of historic runs for the given application and job type. */ - private void locked(ApplicationId id, JobType type, Consumer<SortedMap<RunId, Run>> modifications) { - try (Mutex __ = curator.lock(id, type)) { - SortedMap<RunId, Run> runs = new TreeMap<>(curator.readHistoricRuns(id, type)); - modifications.accept(runs); - curator.writeHistoricRuns(id, type, runs.values()); - } - } - - /** Locks and modifies the run with the given id, provided it is still active. */ - public void locked(RunId id, UnaryOperator<Run> modifications) { - try (Mutex __ = curator.lock(id.application(), id.type())) { - active(id).ifPresent(run -> { - Run modified = modifications.apply(run); - if (modified != null) curator.writeLastRun(modified); - }); - } - } - - /** Locks the given step and checks none of its prerequisites are running, then performs the given actions. */ - public void locked(ApplicationId id, JobType type, Step step, Consumer<LockedStep> action) throws TimeoutException { - try (Mutex lock = curator.lock(id, type, step)) { - for (Step prerequisite : step.allPrerequisites(last(id, type).get().steps().keySet())) // Check that no prerequisite is still running. - try (Mutex __ = curator.lock(id, type, prerequisite)) { ; } - - action.accept(new LockedStep(lock, step)); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java deleted file mode 100644 index 95ea3ff1ffb..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.collections.AbstractFilteringList; -import com.yahoo.component.Version; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; - -import java.time.Instant; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.function.Predicate; - -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.cancelled; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; - -/** - * A list of deployment jobs that can be filtered in various ways. - * - * @author jonmv - */ -public class JobList extends AbstractFilteringList<JobStatus, JobList> { - - private JobList(Collection<? extends JobStatus> jobs, boolean negate) { - super(jobs, negate, JobList::new); - } - - // ----------------------------------- Factories - - public static JobList from(Collection<? extends JobStatus> jobs) { - return new JobList(jobs, false); - } - - // ----------------------------------- Basic filters - - /** Returns the status of the job of the given type, if it is contained in this. */ - public Optional<JobStatus> get(JobId id) { - return asList().stream().filter(job -> job.id().equals(id)).findAny(); - } - - /** Returns the subset of jobs which are currently upgrading */ - public JobList upgrading() { - return matching(job -> job.isRunning() - && job.lastSuccess().isPresent() - && job.lastSuccess().get().versions().targetPlatform().isBefore(job.lastTriggered().get().versions().targetPlatform())); - } - - /** Returns the subset of jobs which are currently failing */ - public JobList failing() { - return matching(job -> job.lastCompleted().isPresent() && ! job.isSuccess()); - } - - /** Returns the subset of jobs which are currently failing, not out of test capacity, and not aborted. */ - public JobList failingHard() { - return failing().not().outOfTestCapacity().not().withStatus(aborted).not().withStatus(cancelled); - } - - public JobList outOfTestCapacity() { - return matching(job -> job.isNodeAllocationFailure() && job.id().type().environment().isTest()); - } - - public JobList running() { - return matching(job -> job.isRunning()); - } - - /** Returns the subset of jobs which must be failing due to an application change */ - public JobList failingApplicationChange() { - return matching(JobList::failingApplicationChange); - } - - /** Returns the subset of jobs which are failing because of an application change, and have been since the threshold, on the given revision. */ - public JobList failingWithBrokenRevisionSince(RevisionId broken, Instant threshold) { - return failingApplicationChange().matching(job -> job.runs().values().stream() - .anyMatch(run -> run.versions().targetRevision().equals(broken) - && run.hasFailed() - && run.start().isBefore(threshold))); - } - - /** Returns the subset of jobs which are failing with the given run status. */ - public JobList withStatus(RunStatus status) { - return matching(job -> job.lastStatus().map(status::equals).orElse(false)); - } - - /** Returns the subset of jobs of the given type -- most useful when negated. */ - public JobList type(Collection<? extends JobType> types) { - return matching(job -> types.contains(job.id().type())); - } - - /** Returns the subset of jobs of the given type -- most useful when negated. */ - public JobList type(JobType... types) { - return type(List.of(types)); - } - - /** Returns the subset of jobs run for the given instance. */ - public JobList instance(InstanceName... instances) { - return instance(Set.of(instances)); - } - - /** Returns the subset of jobs run for the given instance. */ - public JobList instance(Collection<InstanceName> instances) { - return matching(job -> instances.contains(job.id().application().instance())); - } - - /** Returns the subset of jobs of which are production jobs. */ - public JobList production() { - return matching(job -> job.id().type().isProduction()); - } - - /** Returns the subset of jobs which are test jobs. */ - public JobList test() { - return matching(job -> job.id().type().isTest()); - } - - /** Returns the jobs with any runs failing with non-out-of-test-capacity on the given versions — targets only for system test, everything present otherwise. */ - public JobList failingHardOn(Versions versions) { - return matching(job -> ! RunList.from(job) - .on(versions) - .matching(Run::hasFailed) - .not().matching(run -> run.status() == nodeAllocationFailure && run.id().type().environment().isTest()) - .isEmpty()); - } - - /** Returns the jobs with any runs matching the given versions — targets only for system test, everything present otherwise. */ - public JobList triggeredOn(Versions versions) { - return matching(job -> ! RunList.from(job).on(versions).isEmpty()); - } - - /** Returns the jobs with successful runs matching the given versions — targets only for system test, everything present otherwise. */ - public JobList successOn(JobType type, Versions versions) { - return matching(job -> job.id().type().equals(type) - && ! RunList.from(job) - .matching(run -> run.hasSucceeded() && run.id().type().zone().equals(type.zone())) - .on(versions) - .isEmpty()); - } - - // ----------------------------------- JobRun filtering - - /** Returns the list in a state where the next filter is for the lastTriggered run type */ - public RunFilter lastTriggered() { - return new RunFilter(JobStatus::lastTriggered); - } - - /** Returns the list in a state where the next filter is for the lastCompleted run type */ - public RunFilter lastCompleted() { - return new RunFilter(JobStatus::lastCompleted); - } - - /** Returns the list in a state where the next filter is for the lastSuccess run type */ - public RunFilter lastSuccess() { - return new RunFilter(JobStatus::lastSuccess); - } - - /** Returns the list in a state where the next filter is for the firstFailing run type */ - public RunFilter firstFailing() { - return new RunFilter(JobStatus::firstFailing); - } - - /** Allows sub-filters for runs of the indicated kind */ - public class RunFilter { - - private final Function<JobStatus, Optional<Run>> which; - - private RunFilter(Function<JobStatus, Optional<Run>> which) { - this.which = which; - } - - /** Returns the subset of jobs where the run of the indicated type exists */ - public JobList present() { - return matching(run -> true); - } - - /** Returns the runs of the indicated kind, mapped by the given function, as a list. */ - public <OtherType> List<OtherType> mapToList(Function<? super Run, OtherType> mapper) { - return present().mapToList(which.andThen(Optional::get).andThen(mapper)); - } - - /** Returns the runs of the indicated kind. */ - public List<Run> asList() { - return mapToList(Function.identity()); - } - - /** Returns the subset of jobs where the run of the indicated type ended no later than the given instant */ - public JobList endedNoLaterThan(Instant threshold) { - return matching(run -> ! run.end().orElse(Instant.MAX).isAfter(threshold)); - } - - /** Returns the subset of jobs where the run of the indicated type was on the given version */ - public JobList on(RevisionId revision) { - return matching(run -> run.versions().targetRevision().equals(revision)); - } - - /** Returns the subset of jobs where the run of the indicated type was on the given version */ - public JobList on(Version version) { - return matching(run -> run.versions().targetPlatform().equals(version)); - } - - /** Transforms the JobRun condition to a JobStatus condition, by considering only the JobRun mapped by which, and executes */ - private JobList matching(Predicate<Run> condition) { - return JobList.this.matching(job -> which.apply(job).filter(condition).isPresent()); - } - - } - - // ----------------------------------- Internal helpers - - private static boolean failingApplicationChange(JobStatus job) { - if (job.isSuccess()) return false; - if (job.lastSuccess().isEmpty()) return true; // An application which never succeeded is surely bad. - if ( ! job.firstFailing().get().versions().targetPlatform().equals(job.lastSuccess().get().versions().targetPlatform())) return false; // Version change may be to blame. - return ! job.firstFailing().get().versions().targetRevision().equals(job.lastSuccess().get().versions().targetRevision()); // Return whether there is an application change. - } - -} - diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java deleted file mode 100644 index 6a0f5e44c9e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import ai.vespa.metrics.ControllerMetrics; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; - -import java.util.Map; - -/** - * Records metrics related to deployment jobs. - * - * @author jonmv - */ -public class JobMetrics { - - public static final String start = ControllerMetrics.DEPLOYMENT_START.baseName(); - public static final String nodeAllocationFailure = ControllerMetrics.DEPLOYMENT_NODE_ALLOCATION_FAILURE.baseName(); - public static final String endpointCertificateTimeout = ControllerMetrics.DEPLOYMENT_ENDPOINT_CERTIFICATE_TIMEOUT.baseName(); - public static final String deploymentFailure = ControllerMetrics.DEPLOYMENT_DEPLOYMENT_FAILURE.baseName(); - public static final String invalidApplication = ControllerMetrics.DEPLOYMENT_INVALID_APPLICATION.baseName(); - public static final String convergenceFailure = ControllerMetrics.DEPLOYMENT_CONVERGENCE_FAILURE.baseName(); - public static final String testFailure = ControllerMetrics.DEPLOYMENT_TEST_FAILURE.baseName(); - public static final String noTests = ControllerMetrics.DEPLOYMENT_NO_TESTS.baseName(); - public static final String error = ControllerMetrics.DEPLOYMENT_ERROR.baseName(); - public static final String abort = ControllerMetrics.DEPLOYMENT_ABORT.baseName(); - public static final String cancel = ControllerMetrics.DEPLOYMENT_CANCEL.baseName(); - public static final String success = ControllerMetrics.DEPLOYMENT_SUCCESS.baseName(); - public static final String quotaExceeded = ControllerMetrics.DEPLOYMENT_QUOTA_EXCEEDED.baseName(); - - private final Metric metric; - - public JobMetrics(Metric metric) { - this.metric = metric; - } - - public void jobStarted(JobId id) { - metric.add(start, 1, metric.createContext(contextOf(id))); - } - - public void jobFinished(JobId id, RunStatus status) { - metric.add(valueOf(status), 1, metric.createContext(contextOf(id))); - } - - Map<String, String> contextOf(JobId id) { - return Map.of("applicationId", id.application().toFullString(), - "tenantName", id.application().tenant().value(), - "app", id.application().application().value() + "." + id.application().instance().value(), - "test", Boolean.toString(id.type().isTest()), - "zone", id.type().zone().value()); - } - - static String valueOf(RunStatus status) { - return switch (status) { - case nodeAllocationFailure -> nodeAllocationFailure; - case endpointCertificateTimeout -> endpointCertificateTimeout; - case invalidApplication -> invalidApplication; - case deploymentFailed -> deploymentFailure; - case installationFailed -> convergenceFailure; - case testFailure -> testFailure; - case noTests -> noTests; - case error -> error; - case cancelled -> cancel; - case aborted -> abort; - case success -> success; - case quotaExceeded -> quotaExceeded; - default -> throw new IllegalArgumentException("Unexpected run status '" + status + "'"); - }; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java deleted file mode 100644 index 1f8d2090471..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; - -import java.util.Collections; -import java.util.EnumSet; -import java.util.Set; - -import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup; -import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.report; -import static com.yahoo.vespa.hosted.controller.deployment.Step.startStagingSetup; -import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; - -/** - * Static profiles defining the {@link Step}s of a deployment job. - * - * @author jonmv - */ -public enum JobProfile { - - systemTest(EnumSet.of(deployReal, - installReal, - deployTester, - installTester, - startTests, - endTests, - copyVespaLogs, - deactivateTester, - deactivateReal, - report)), - - stagingTest(EnumSet.of(deployInitialReal, - deployTester, - installTester, - installInitialReal, - startStagingSetup, - endStagingSetup, - deployReal, - installReal, - startTests, - endTests, - copyVespaLogs, - deactivateTester, - deactivateReal, - report)), - - production(EnumSet.of(deployReal, - installReal, - report)), - - productionTest(EnumSet.of(deployTester, - installTester, - startTests, - endTests, - copyVespaLogs, - deactivateTester, - report)), - - development(EnumSet.of(deployReal, - installReal, - copyVespaLogs)), - - developmentDryRun(EnumSet.of(deployReal)); - - - private final Set<Step> steps; - - JobProfile(Set<Step> steps) { - this.steps = Collections.unmodifiableSet(steps); - } - - // TODO jonmv: Let caller decide profile, and store with run? - public static JobProfile of(JobType type) { - switch (type.environment()) { - case test: return systemTest; - case staging: return stagingTest; - case prod: return type.isTest() ? productionTest : production; - case perf: - case dev: return development; - default: throw new AssertionError("Unexpected environment '" + type.environment() + "'!"); - } - } - - /** Returns all steps in this profile, the default for which is to run only when all prerequisites are successes. */ - public Set<Step> steps() { return steps; } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java deleted file mode 100644 index 3770c9cd694..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; - -import java.util.NavigableMap; -import java.util.Objects; -import java.util.Optional; - -/** - * Aggregates information about all known runs of a given job to provide the high level status. - * - * @author jonmv - */ -public class JobStatus { - - private final JobId id; - private final NavigableMap<RunId, Run> runs; - private final Optional<Run> lastTriggered; - private final Optional<Run> lastCompleted; - private final Optional<Run> lastSuccess; - private final Optional<Run> firstFailing; - - public JobStatus(JobId id, NavigableMap<RunId, Run> runs) { - this.id = Objects.requireNonNull(id); - this.runs = Objects.requireNonNull(runs); - this.lastTriggered = runs.descendingMap().values().stream().findFirst(); - this.lastCompleted = lastCompleted(runs); - this.lastSuccess = lastSuccess(runs); - this.firstFailing = firstFailing(runs); - } - - public JobId id() { - return id; - } - - public NavigableMap<RunId, Run> runs() { - return runs; - } - - public Optional<Run> lastTriggered() { - return lastTriggered; - } - - public Optional<Run> lastCompleted() { - return lastCompleted; - } - - public Optional<Run> lastSuccess() { - return lastSuccess; - } - - public Optional<Run> firstFailing() { - return firstFailing; - } - - public Optional<RunStatus> lastStatus() { - return lastCompleted().map(Run::status); - } - - public boolean isSuccess() { - return lastCompleted.map(last -> ! last.hasFailed()).orElse(false); - } - - public boolean isRunning() { - return lastTriggered.isPresent() && ! lastTriggered.get().hasEnded(); - } - - public boolean isNodeAllocationFailure() { - return lastStatus().isPresent() && lastStatus().get() == RunStatus.nodeAllocationFailure; - } - - @Override - public String toString() { - return "JobStatus{" + - "id=" + id + - ", lastTriggered=" + lastTriggered + - ", lastCompleted=" + lastCompleted + - ", lastSuccess=" + lastSuccess + - ", firstFailing=" + firstFailing + - '}'; - } - - static Optional<Run> lastCompleted(NavigableMap<RunId, Run> runs) { - return runs.descendingMap().values().stream() - .filter(run -> run.hasEnded()) - .findFirst(); - } - - static Optional<Run> lastSuccess(NavigableMap<RunId, Run> runs) { - return runs.descendingMap().values().stream() - .filter(Run::hasSucceeded) - .findFirst(); - } - - static Optional<Run> firstFailing(NavigableMap<RunId, Run> runs) { - Run failed = null; - for (Run run : runs.descendingMap().values()) { - if ( ! run.hasEnded()) continue; - if ( ! run.hasFailed()) break; - failed = run; - } - return Optional.ofNullable(failed); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java deleted file mode 100644 index 9f471116e22..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.curator.Lock; - -/** - * @author jonmv - */ -public class LockedStep { - - private final Step step; - LockedStep(Mutex lock, Step step) { this.step = step; } - public Step get() { return step; } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java deleted file mode 100644 index a3aefa55f4e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.collections.AbstractFilteringList; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; - -import java.time.Instant; -import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; - -/** - * @author jonmv - */ -public class NodeList extends AbstractFilteringList<NodeWithServices, NodeList> { - - private final long wantedConfigGeneration; - - private NodeList(Collection<? extends NodeWithServices> items, boolean negate, long wantedConfigGeneration) { - super(items, negate, (i, n) -> new NodeList(i, n, wantedConfigGeneration)); - this.wantedConfigGeneration = wantedConfigGeneration; - } - - public static NodeList of(List<Node> nodes, List<Node> parents, ServiceConvergence services) { - var servicesByHostName = services.services().stream() - .collect(Collectors.groupingBy(service -> service.host())); - var parentsByHostName = parents.stream() - .collect(Collectors.toMap(node -> node.hostname(), node -> node)); - return new NodeList(nodes.stream() - .map(node -> new NodeWithServices(node, - parentsByHostName.get(node.parentHostname().get()), - services.wantedGeneration(), - servicesByHostName.getOrDefault(node.hostname(), List.of()))) - .toList(), - false, - services.wantedGeneration()); - } - - /** The nodes on an outdated OS. */ - public NodeList needsOsUpgrade() { - return matching(NodeWithServices::needsOsUpgrade); - } - - /** The nodes with outdated firmware. */ - public NodeList needsFirmwareUpgrade() { - return matching(NodeWithServices::needsFirmwareUpgrade); - } - - /** The nodes whose parent is down. */ - public NodeList withParentDown() { - return matching(NodeWithServices::hasParentDown); - } - - /** The nodes on an outdated platform. */ - public NodeList needsPlatformUpgrade() { - return matching(NodeWithServices::needsPlatformUpgrade); - } - - /** The nodes in need of a reboot. */ - public NodeList needsReboot() { - return matching(NodeWithServices::needsReboot); - } - - /** The nodes in need of a restart. */ - public NodeList needsRestart() { - return matching(NodeWithServices::needsRestart); - } - - /** The nodes currently allowed to be down. */ - public NodeList allowedDown() { - return matching(node -> node.isAllowedDown()); - } - - /** The nodes currently expected to be down. */ - public NodeList expectedDown() { - return matching(node -> node.isAllowedDown() || node.isNewlyProvisioned()); - } - - /** The nodes which have been suspended since before the given instant. */ - public NodeList suspendedSince(Instant instant) { - return matching(node -> node.isSuspendedSince(instant)); - } - - /** The nodes with services on outdated config generation. */ - public NodeList needsNewConfig() { - return matching(NodeWithServices::needsNewConfig); - } - - public NodeList isStateful() { - return matching(NodeWithServices::isStateful); - } - - /** The nodes that are retiring. */ - public NodeList retiring() { - return matching(node -> node.node().retired()); - } - - - /** Returns a summary of the convergence status of the nodes in this list. */ - public ConvergenceSummary summary() { - NodeList allowedDown = expectedDown(); - return new ConvergenceSummary(size(), - allowedDown.size(), - withParentDown().needsOsUpgrade().size(), - withParentDown().needsFirmwareUpgrade().size(), - needsPlatformUpgrade().size(), - allowedDown.needsPlatformUpgrade().size(), - needsReboot().size(), - allowedDown.needsReboot().size(), - needsRestart().size(), - allowedDown.needsRestart().size(), - asList().stream().mapToLong(node -> node.services().size()).sum(), - asList().stream().mapToLong(node -> node.services().stream().filter(service -> wantedConfigGeneration > service.currentGeneration()).count()).sum(), - retiring().size()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java deleted file mode 100644 index 39addbd3b63..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; - -import java.time.Instant; -import java.util.List; -import java.util.Objects; - -import static java.util.Objects.requireNonNull; - -/** - * Aggregate of a node and its services, fetched from different sources. - * - * @author jonmv - */ -public class NodeWithServices { - - private final Node node; - private final Node parent; - private final long wantedConfigGeneration; - private final List<ServiceConvergence.Status> services; - - public NodeWithServices(Node node, Node parent, long wantedConfigGeneration, List<ServiceConvergence.Status> services) { - this.node = requireNonNull(node); - this.parent = requireNonNull(parent); - if (wantedConfigGeneration <= 0) - throw new IllegalArgumentException("Wanted config generation must be positive"); - this.wantedConfigGeneration = wantedConfigGeneration; - this.services = List.copyOf(services); - } - - public Node node() { return node; } - public Node parent() { return parent; } - public long wantedConfigGeneration() { return wantedConfigGeneration; } - public List<ServiceConvergence.Status> services() { return services; } - - public boolean needsOsUpgrade() { - return parent.wantedOsVersion().isAfter(parent.currentOsVersion()); - } - - public boolean needsFirmwareUpgrade(){ - return parent.wantedFirmwareCheck() - .map(wanted -> parent.currentFirmwareCheck() - .map(wanted::isAfter) - .orElse(true)) - .orElse(false); - } - - public boolean hasParentDown() { - return parent.serviceState() == Node.ServiceState.allowedDown; - } - - public boolean needsPlatformUpgrade() { - return node.wantedVersion().isAfter(node.currentVersion()) - || ! node.wantedDockerImage().equals(node.currentDockerImage()); - } - - public boolean needsReboot() { - return node.wantedRebootGeneration() > node.rebootGeneration(); - } - - public boolean needsRestart() { - return node.wantedRestartGeneration() > node.restartGeneration(); - } - - public boolean isAllowedDown() { - return node.serviceState() == Node.ServiceState.allowedDown; - } - - public boolean isNewlyProvisioned() { - return node.currentVersion().equals(Version.emptyVersion); - } - - public boolean isSuspendedSince(Instant instant) { - return node.suspendedSince().map(instant::isAfter).orElse(false); - } - - public boolean needsNewConfig() { - return services.stream().anyMatch(service -> wantedConfigGeneration > service.currentGeneration()); - } - - public boolean isStateful() { - return node.clusterType() == Node.ClusterType.content || node.clusterType() == Node.ClusterType.combined; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - NodeWithServices that = (NodeWithServices) o; - return node.equals(that.node); - } - - @Override - public int hashCode() { - return Objects.hash(node); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntry.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntry.java deleted file mode 100644 index f3bf5b2062d..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntry.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; - -/** - * @author mortent - */ -public class RetriggerEntry { - private final JobId jobId; - private final long requiredRun; - - public RetriggerEntry(JobId jobId, long requiredRun) { - this.jobId = jobId; - this.requiredRun = requiredRun; - } - - public JobId jobId() { - return jobId; - } - - public long requiredRun() { - return requiredRun; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java deleted file mode 100644 index 8ed36215cac..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.SystemName; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * @author mortent - */ -public class RetriggerEntrySerializer { - - private static final String JOB_ID_KEY = "jobId"; - private static final String APPLICATION_ID_KEY = "applicationId"; - private static final String JOB_TYPE_KEY = "jobType"; - private static final String MIN_REQUIRED_RUN_ID_KEY = "minimumRunId"; - - public List<RetriggerEntry> fromSlime(Slime slime) { - return SlimeUtils.entriesStream(slime.get().field("entries")) - .map(this::deserializeEntry) - .toList(); - } - - public Slime toSlime(List<RetriggerEntry> entryList) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor entries = root.setArray("entries"); - entryList.forEach(e -> serializeEntry(entries, e)); - return slime; - } - - private void serializeEntry(Cursor array, RetriggerEntry entry) { - Cursor root = array.addObject(); - Cursor jobid = root.setObject(JOB_ID_KEY); - jobid.setString(APPLICATION_ID_KEY, entry.jobId().application().serializedForm()); - jobid.setString(JOB_TYPE_KEY, entry.jobId().type().serialized()); - root.setLong(MIN_REQUIRED_RUN_ID_KEY, entry.requiredRun()); - } - - private RetriggerEntry deserializeEntry(Inspector inspector) { - Inspector jobid = inspector.field(JOB_ID_KEY); - ApplicationId applicationId = ApplicationId.fromSerializedForm(require(jobid, APPLICATION_ID_KEY).asString()); - JobType jobType = JobType.ofSerialized(require(jobid, JOB_TYPE_KEY).asString()); - long minRequiredRunId = require(inspector, MIN_REQUIRED_RUN_ID_KEY).asLong(); - return new RetriggerEntry(new JobId(applicationId, jobType), minRequiredRunId); - } - - private Inspector require(Inspector inspector, String fieldName) { - Inspector field = inspector.field(fieldName); - if (!field.valid()) { - throw new IllegalStateException("Could not deserialize, field not found in json: " + fieldName); - } - return field; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java deleted file mode 100644 index 0d086aa7012..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import ai.vespa.validation.Validation; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; - -import java.util.ArrayDeque; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Deque; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.TreeMap; -import java.util.function.Predicate; - -import static ai.vespa.validation.Validation.require; -import static java.util.Collections.emptyNavigableMap; -import static java.util.function.Predicate.not; - -/** - * History of application revisions for an {@link com.yahoo.vespa.hosted.controller.Application}. - * - * @author jonmv - */ -public class RevisionHistory { - - private static final Comparator<JobId> comparator = Comparator.comparing(JobId::application).thenComparing(JobId::type); - - private final NavigableMap<RevisionId, ApplicationVersion> production; - private final NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development; - - private RevisionHistory(NavigableMap<RevisionId, ApplicationVersion> production, - NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development) { - this.production = production; - this.development = development; - } - - public static RevisionHistory empty() { - return ofRevisions(List.of(), Map.of()); - } - - public static RevisionHistory ofRevisions(Collection<ApplicationVersion> productionRevisions, - Map<JobId, ? extends Collection<ApplicationVersion>> developmentRevisions) { - NavigableMap<RevisionId, ApplicationVersion> production = new TreeMap<>(); - for (ApplicationVersion revision : productionRevisions) - production.put(revision.id(), revision); - - NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development = new TreeMap<>(comparator); - developmentRevisions.forEach((job, jobRevisions) -> { - NavigableMap<RevisionId, ApplicationVersion> revisions = development.computeIfAbsent(job, __ -> new TreeMap<>()); - for (ApplicationVersion revision : jobRevisions) - revisions.put(revision.id(), revision); - }); - - return new RevisionHistory(production, development); - } - - /** Returns a copy of this where any production revisions without packages, and older than the given one, are removed. */ - public RevisionHistory withoutOlderThan(RevisionId id) { - if (production.headMap(id).isEmpty()) return this; - NavigableMap<RevisionId, ApplicationVersion> production = new TreeMap<>(this.production); - production.headMap(id).values().removeIf(not(ApplicationVersion::hasPackage)); - return new RevisionHistory(production, development); - } - - /** Returns a copy of this without any development revisions older than the given. */ - public RevisionHistory withoutOlderThan(RevisionId id, JobId job) { - if ( ! development.containsKey(job) || development.get(job).headMap(id).isEmpty()) return this; - NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development = new TreeMap<>(this.development); - development.compute(job, (__, revisions) -> revisions.tailMap(id, true)); - return new RevisionHistory(production, development); - } - - /** Returns a copy of this with the revision added or updated. */ - public RevisionHistory with(ApplicationVersion revision) { - if (revision.id().isProduction()) { - if ( ! production.isEmpty() && revision.bundleHash().flatMap(hash -> production.lastEntry().getValue().bundleHash().map(hash::equals)).orElse(false)) - revision = revision.skipped(); - - NavigableMap<RevisionId, ApplicationVersion> production = new TreeMap<>(this.production); - production.put(revision.id(), revision); - return new RevisionHistory(production, development); - } - else { - NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development = new TreeMap<>(this.development); - NavigableMap<RevisionId, ApplicationVersion> revisions = development.compute(revision.id().job(), (__, old) -> new TreeMap<>(old != null ? old : emptyNavigableMap())); - if ( ! revisions.isEmpty()) revisions.compute(revisions.lastKey(), (__, last) -> last.withoutPackage()); - revisions.put(revision.id(), revision); - return new RevisionHistory(production, development); - } - } - - // Fallback for when an application version isn't known for the given key. - private static ApplicationVersion revisionOf(RevisionId id) { - return new ApplicationVersion(id, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), false, false, Optional.empty(), Optional.empty(), 0); - } - - /** Returns the production {@link ApplicationVersion} with this revision ID. */ - public ApplicationVersion get(RevisionId id) { - return id.isProduction() ? production.getOrDefault(id, revisionOf(id)) - : development.getOrDefault(id.job(), emptyNavigableMap()) - .getOrDefault(id, revisionOf(id)); - } - - /** Returns the last submitted production build. */ - public Optional<ApplicationVersion> last() { - return Optional.ofNullable(production.lastEntry()).map(Map.Entry::getValue); - } - - /** Returns all known production revisions we still have the package for, from oldest to newest. */ - public List<ApplicationVersion> withPackage() { - return production.values().stream() - .filter(ApplicationVersion::hasPackage) - .toList(); - } - - /** Returns the currently deployable revisions of the application. */ - public Deque<ApplicationVersion> deployable(boolean ascending) { - Deque<ApplicationVersion> versions = new ArrayDeque<>(); - for (ApplicationVersion version : withPackage()) { - if (version.isDeployable()) { - if (ascending) versions.addLast(version); - else versions.addFirst(version); - } - } - return versions; - } - - /** All known production revisions, in ascending order. */ - public List<ApplicationVersion> production() { - return List.copyOf(production.values()); - } - - /* All known development revisions, in ascending order, per job. */ - public NavigableMap<JobId, List<ApplicationVersion>> development() { - NavigableMap<JobId, List<ApplicationVersion>> copy = new TreeMap<>(comparator); - development.forEach((job, revisions) -> copy.put(job, List.copyOf(revisions.values()))); - return Collections.unmodifiableNavigableMap(copy); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java deleted file mode 100644 index 2b207e6662b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ /dev/null @@ -1,353 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.Change; - -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.util.Collections; -import java.util.EnumMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.cancelled; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static java.util.Objects.requireNonNull; - -/** - * Immutable class containing status information for a deployment job run by a {@link JobController}. - * - * @author jonmv - */ -public class Run { - - private final RunId id; - private final Map<Step, StepInfo> steps; - private final Versions versions; - private final boolean isRedeployment; - private final Instant start; - private final Optional<Instant> end; - private final Optional<Instant> sleepUntil; - private final RunStatus status; - private final long lastTestRecord; - private final Instant lastVespaLogTimestamp; - private final Optional<Instant> noNodesDownSince; - private final Optional<ConvergenceSummary> convergenceSummary; - private final Optional<X509Certificate> testerCertificate; - private final boolean dryRun; - private final Optional<CloudAccount> cloudAccount; - private final Reason reason; - - // For deserialisation only -- do not use! - public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, boolean isRedeployment, Instant start, Optional<Instant> end, - Optional<Instant> sleepUntil, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, - Optional<Instant> noNodesDownSince, Optional<ConvergenceSummary> convergenceSummary, - Optional<X509Certificate> testerCertificate, boolean dryRun, Optional<CloudAccount> cloudAccount, Reason reason) { - this.id = id; - this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); - this.versions = versions; - this.isRedeployment = isRedeployment; - this.start = start; - this.end = end; - this.sleepUntil = sleepUntil; - this.status = status; - this.lastTestRecord = lastTestRecord; - this.lastVespaLogTimestamp = lastVespaLogTimestamp; - this.noNodesDownSince = noNodesDownSince; - this.convergenceSummary = convergenceSummary; - this.testerCertificate = testerCertificate; - this.dryRun = dryRun; - this.cloudAccount = cloudAccount; - this.reason = reason; - } - - public static Run initial(RunId id, Versions versions, boolean isRedeployment, Instant now, JobProfile profile, Reason reason) { - EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class); - profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step))); - return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), - Optional.empty(), running, -1, Instant.EPOCH, Optional.empty(), Optional.empty(), - Optional.empty(), profile == JobProfile.developmentDryRun, Optional.empty(), reason); - } - - /** Returns a new Run with the status of the given completed step set accordingly. */ - public Run with(RunStatus status, LockedStep step) { - requireActive(); - StepInfo stepInfo = getRequiredStepInfo(step.get()); - if (stepInfo.status() != unfinished) - throw new IllegalStateException("Step '" + step.get() + "' can't be set to '" + status + "'" + - " -- it already completed with status '" + stepInfo.status() + "'!"); - - EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); - steps.put(step.get(), stepInfo.with(Step.Status.of(status))); - RunStatus newStatus = hasFailed() || status == running ? this.status : status; - return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, newStatus, lastTestRecord, - lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); - } - - /** Returns a new Run with a new start time*/ - public Run with(Instant startTime, LockedStep step) { - requireActive(); - StepInfo stepInfo = getRequiredStepInfo(step.get()); - if (stepInfo.status() != unfinished) - throw new IllegalStateException("Unable to set start timestamp of step " + step.get() + - ": it has already completed with status " + stepInfo.status() + "!"); - - EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); - steps.put(step.get(), stepInfo.with(startTime)); - - return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); - } - - public Run finished(Instant now) { - requireActive(); - return new Run(id, steps, versions, isRedeployment, start, Optional.of(now), sleepUntil, status == running ? success : status, - lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty(), dryRun, cloudAccount, reason); - } - - public Run aborted(boolean cancelledByHumans) { - requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, - cancelledByHumans ? cancelled : aborted, - lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, - convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); - } - - public Run reset() { - requireActive(); - Map<Step, StepInfo> reset = new EnumMap<>(steps); - reset.replaceAll((step, __) -> StepInfo.initial(step)); - return new Run(id, reset, versions, isRedeployment, start, end, sleepUntil, running, -1, lastVespaLogTimestamp, - Optional.empty(), Optional.empty(), testerCertificate, dryRun, cloudAccount, reason); - } - - public Run with(long lastTestRecord) { - requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); - } - - public Run with(Instant lastVespaLogTimestamp) { - requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); - } - - public Run noNodesDownSince(Instant noNodesDownSince) { - requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); - } - - public Run withSummary(ConvergenceSummary convergenceSummary) { - requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate, dryRun, cloudAccount, reason); - } - - public Run with(X509Certificate testerCertificate) { - requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, Optional.of(testerCertificate), dryRun, cloudAccount, reason); - } - - public Run sleepingUntil(Instant instant) { - requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, Optional.of(instant), status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); - } - - public Run with(CloudAccount account) { - requireActive(); - return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun, Optional.of(account), reason); - } - - /** Returns the id of this run. */ - public RunId id() { - return id; - } - - /** Whether this run contains this step. */ - public boolean hasStep(Step step) { - return steps.containsKey(step); - } - - /** Returns info on step, or empty if the given step is not a part of this run. */ - public Optional<StepInfo> stepInfo(Step step) { - return Optional.ofNullable(steps.get(step)); - } - - private StepInfo getRequiredStepInfo(Step step) { - return stepInfo(step).orElseThrow(() -> new IllegalArgumentException("There is no such step " + step + " for run " + id)); - } - - /** Returns status of step, or empty if the given step is not a part of this run. */ - public Optional<Step.Status> stepStatus(Step step) { - return stepInfo(step).map(StepInfo::status); - } - - /** Returns an unmodifiable view of all step information in this run. */ - public Map<Step, StepInfo> steps() { - return steps; - } - - /** Returns an unmodifiable view of the status of all steps in this run. */ - public Map<Step, Step.Status> stepStatuses() { - return Collections.unmodifiableMap(steps.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().status()))); - } - - public RunStatus status() { - return status; - } - - /** Returns the instant at which this run began. */ - public Instant start() { - return start; - } - - /** Returns the instant at which this run ended, if it has. */ - public Optional<Instant> end() { - return end; - } - - /** Returns the instant until which this should sleep. */ - public Optional<Instant> sleepUntil() { - return sleepUntil; - } - - /** Returns whether the run has failed, and should switch to its run-always steps. */ - public boolean hasFailed() { - return status != running && status != success && status != noTests; - } - - /** Returns whether the run has ended, i.e., has become inactive, and can no longer be updated. */ - public boolean hasEnded() { - return end.isPresent(); - } - - public boolean hasSucceeded() { return hasEnded() && ! hasFailed(); } - - /** Returns the target, and possibly source, versions for this run. */ - public Versions versions() { - return versions; - } - - /** Returns the sequence id of the last test record received from the tester, for the test logs of this run. */ - public long lastTestLogEntry() { - return lastTestRecord; - } - - /** Returns the timestamp of the last Vespa log record fetched and stored for this run. */ - public Instant lastVespaLogTimestamp() { - return lastVespaLogTimestamp; - } - - /** Returns since when no nodes have been allowed to be down. */ - public Optional<Instant> noNodesDownSince() { - return noNodesDownSince; - } - - /** Returns a summary of convergence status during an application deployment — staging or upgrade. */ - public Optional<ConvergenceSummary> convergenceSummary() { - return convergenceSummary; - } - - /** Returns the tester certificate for this run, or empty. */ - public Optional<X509Certificate> testerCertificate() { - return testerCertificate; - } - - /** Whether this is a automatic redeployment. */ - public boolean isRedeployment() { - return isRedeployment; - } - - /** Whether this is a dry run deployment. */ - public boolean isDryRun() { return dryRun; } - - /** Cloud account used for deployments in this run. This is set by the first deployment. */ - public Optional<CloudAccount> cloudAccount() { return cloudAccount; } - - /** The specific reason for triggering this run, if any. This should be empty for jobs triggered bvy deployment orchestration. */ - public Reason reason() { - return reason; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if ( ! (o instanceof Run)) return false; - - Run run = (Run) o; - - return id.equals(run.id); - } - - @Override - public int hashCode() { - return id.hashCode(); - } - - @Override - public String toString() { - return "RunStatus{" + - "id=" + id + - ", versions=" + versions + - ", start=" + start + - ", end=" + end + - ", status=" + status + - '}'; - } - - /** Returns the list of steps to run for this job right now, depending on whether the job has failed. */ - public List<Step> readySteps() { - return hasFailed() ? forcedSteps() : normalSteps(); - } - - /** Returns the list of unfinished steps whose prerequisites have all succeeded. */ - private List<Step> normalSteps() { - return steps.entrySet().stream() - .filter(entry -> entry.getValue().status() == unfinished - && entry.getKey().prerequisites().stream() - .allMatch(step -> steps.get(step) == null - || steps.get(step).status() == succeeded)) - .map(Map.Entry::getKey) - .toList(); - } - - /** Returns the list of not-yet-run run-always steps whose run-always prerequisites have all run. */ - private List<Step> forcedSteps() { - return steps.entrySet().stream() - .filter(entry -> entry.getValue().status() == unfinished - && entry.getKey().alwaysRun() - && entry.getKey().prerequisites().stream() - .filter(Step::alwaysRun) - .allMatch(step -> steps.get(step) == null - || steps.get(step).status() != unfinished)) - .map(Map.Entry::getKey) - .toList(); - } - - private void requireActive() { - if (hasEnded()) - throw new IllegalStateException("This run ended at " + end.get() + " -- it can't be further modified!"); - } - - public record Reason(Optional<String> reason, Optional<JobId> dependent, Optional<Change> change) { - private static final Reason empty = new Reason(Optional.empty(), Optional.empty(), Optional.empty()); - public static Reason empty() { return empty; } - public static Reason because(String reason) { return new Reason(Optional.of(reason), Optional.empty(), Optional.empty()); } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java deleted file mode 100644 index b3846dca2c0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.collections.AbstractFilteringList; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; - -import java.util.Collection; -import java.util.List; - -/** - * List for filtering deployment job {@link Run}s. - * - * @author jonmv - */ -public class RunList extends AbstractFilteringList<Run, RunList> { - - private RunList(Collection<? extends Run> items, boolean negate) { - super(items, negate, RunList::new); - } - - public static RunList from(Collection<? extends Run> runs) { - return new RunList(runs, false); - } - - public static RunList from(JobStatus job) { - return from(job.runs().descendingMap().values()); - } - - /** Returns the jobs with runs matching the given versions — targets only for system test, everything present otherwise. */ - public RunList on(Versions versions) { - return matching(run -> matchingVersions(run, versions)); - } - - /** Returns the runs with status among the given. */ - public RunList status(RunStatus... status) { - return matching(run -> List.of(status).contains(run.status())); - } - - private static boolean matchingVersions(Run run, Versions versions) { - return versions.targetsMatch(run.versions()) - && (versions.sourcesMatchIfPresent(run.versions()) || run.id().type().isSystemTest()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunLog.java deleted file mode 100644 index 371607ec1c7..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunLog.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.google.common.collect.ImmutableMap; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.OptionalLong; - -/** - * Immutable class which contains the log of a deployment job run. - * - * @author jonmv - */ -public class RunLog { - - private static final RunLog empty = RunLog.of(Collections.emptyMap()); - - private final Map<Step, List<LogEntry>> log; - private final OptionalLong lastId; - - private RunLog(OptionalLong lastId, Map<Step, List<LogEntry>> log) { - this.log = log; - this.lastId = lastId; - } - - /** Creates a RunLog which contains a deep copy of the given log. */ - public static RunLog of(Map<Step, List<LogEntry>> log) { - ImmutableMap.Builder<Step, List<LogEntry>> builder = ImmutableMap.builder(); - log.forEach((step, entries) -> { - if ( ! entries.isEmpty()) - builder.put(step, List.copyOf(entries)); - }); - OptionalLong lastId = log.values().stream() - .flatMap(List::stream) - .mapToLong(LogEntry::id) - .max(); - return new RunLog(lastId, builder.build()); - } - - /** Returns an empty RunLog. */ - public static RunLog empty() { - return empty; - } - - /** Returns the log entries for the given step, if any are recorded. */ - public List<LogEntry> get(Step step) { - return log.getOrDefault(step, Collections.emptyList()); - } - - /** Returns the id of the last log entry in this, if it has any. */ - public OptionalLong lastId() { - return lastId; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java deleted file mode 100644 index 7e1806ad9ac..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -/** - * Status of jobs run by a {@link JobController}. - * - * @author jonmv - */ -public enum RunStatus { - - /** Run is still proceeding normally, i.e., without failures. */ - running, - - /** Deployment was rejected due node allocation failure. */ - nodeAllocationFailure, - - /** Deployment of the real application was rejected because the package is faulty. */ - invalidApplication, - - /** Deployment of the real application was rejected, for other reasons. */ - deploymentFailed, - - /** Deployment timed out waiting for endpoint certificate */ - endpointCertificateTimeout, - - /** Installation of the real application timed out. */ - installationFailed, - - /** The verification tests failed. */ - testFailure, - - /** No tests, for a test job. */ - noTests, - - /** An unexpected error occurred. */ - error, - - /** Everything completed with great success! */ - success, - - /** Run was abandoned, due to job timeout or blocking a newer target for the same job. */ - aborted, - - /** Cancelled by a human being. */ - cancelled, - - /** Run should be reset to its starting state. Used for production tests. */ - reset, - - /** Deployment of the real application was rejected due to exceeding quota. */ - quotaExceeded - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java deleted file mode 100644 index e975f5874f4..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import java.util.Collection; -import java.util.List; -import java.util.stream.Stream; - -import static java.util.Comparator.reverseOrder; - -/** - * Steps that make up a deployment job. See {@link JobProfile} for preset profiles. - * - * Each step lists its prerequisites; this serves two purposes: - * - * 1. A step may only run after its prerequisites, so these define a topological order in which - * the steps can be run. Since a job profile may list only a subset of the existing steps, - * only the prerequisites of a step which are included in a run's profile will be considered. - * Under normal circumstances, a step will run only after each of its prerequisites have succeeded. - * When a run has failed, however, each of the always-run steps of the run's profile will be run, - * again in a topological order, and requiring all their always-run prerequisites to have run. - * - * 2. A step will never run concurrently with its prerequisites. This is to ensure, e.g., that relevant - * information from a failed run is stored, and that deployment does not occur after deactivation. - * - * @see JobController - * @author jonmv - */ -public enum Step { - - /** Download test-jar and assemble and deploy tester application. */ - deployTester(false), - - /** See that tester is done deploying, and is ready to serve. */ - installTester(false, deployTester), - - /** Download and deploy the initial real application, for staging tests. */ - deployInitialReal(false), - - /** See that the real application has had its nodes converge to the initial state. */ - installInitialReal(false, deployInitialReal), - - /** Ask the tester to run its staging setup. */ - startStagingSetup(false, installInitialReal, installTester), - - /** See that the staging setup is done. */ - endStagingSetup(false, startStagingSetup), - - /** Download and deploy real application, restarting services if required. */ - deployReal(false, endStagingSetup), - - /** See that real application has had its nodes converge to the wanted version and generation. */ - installReal(false, deployReal), - - /** Ask the tester to run its tests. */ - startTests(false, installReal, installTester), - - /** See that the tests are done running. */ - endTests(false, startTests), - - /** Fetch and store Vespa logs from the log server cluster of the deployment -- used for test and dev deployments. */ - copyVespaLogs(true, installReal, endTests), - - /** Delete the real application -- used for test deployments. */ - deactivateReal(true, deployInitialReal, deployReal, endTests, copyVespaLogs), - - /** Deactivate the tester. */ - deactivateTester(true, deployTester, endTests, copyVespaLogs), - - /** Report completion to the deployment orchestration machinery. */ - report(true, installReal, deactivateReal, deactivateTester); - - - private final boolean alwaysRun; - private final List<Step> prerequisites; - - Step(boolean alwaysRun, Step... prerequisites) { - this.alwaysRun = alwaysRun; - this.prerequisites = List.of(prerequisites); - } - - /** Returns whether this is a cleanup-step, and should always run, regardless of job outcome, when specified in a job. */ - public boolean alwaysRun() { return alwaysRun; } - - /** Returns all prerequisite steps for this, including transient ones, in a job profile containing the given steps. */ - public List<Step> allPrerequisites(Collection<Step> among) { - return prerequisites.stream() - .filter(among::contains) - .flatMap(pre -> Stream.concat(Stream.of(pre), - pre.allPrerequisites(among).stream())) - .sorted(reverseOrder()) - .distinct() - .toList(); - } - - - /** Returns the direct prerequisite steps that must be completed before this, assuming the job contains these steps. */ - public List<Step> prerequisites() { return prerequisites; } - - - public enum Status { - - /** Step still has unsatisfied finish criteria -- it may not even have started. */ - unfinished, - - /** Step failed and subsequent steps may not start. */ - failed, - - /** Step succeeded and subsequent steps may now start. */ - succeeded; - - public static Step.Status of(RunStatus status) { - return switch (status) { - case success -> throw new AssertionError("Unexpected run status '" + status + "'!"); - case cancelled, reset, aborted -> unfinished; - case noTests, running -> succeeded; - default -> failed; - }; - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepInfo.java deleted file mode 100644 index 60743e45434..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepInfo.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import java.time.Instant; -import java.util.Objects; -import java.util.Optional; - -/** - * Information about a step. Immutable. - * - * @author hakonhall - */ -public class StepInfo { - - private final Step step; - private final Step.Status status; - private final Optional<Instant> startTime; - - public static StepInfo initial(Step step) { return new StepInfo(step, Step.Status.unfinished, Optional.empty()); } - - public StepInfo(Step step, Step.Status status, Optional<Instant> startTime) { - this.step = step; - this.status = status; - this.startTime = startTime; - } - - public Step step() { return step; } - public Step.Status status() { return status; } - public Optional<Instant> startTime() { return startTime; } - - /** Returns a copy of this, but with the given status. */ - public StepInfo with(Step.Status status) { return new StepInfo(step, status, startTime); } - - /** Returns a copy of this, but with the given start timestamp. */ - public StepInfo with(Instant startTimestamp) { return new StepInfo(step, status, Optional.of(startTimestamp)); } - - @Override - public String toString() { - return "StepInfo{" + - "step=" + step + - ", status=" + status + - ", startTime=" + startTime + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - StepInfo stepInfo = (StepInfo) o; - return step == stepInfo.step && - status == stepInfo.status && - Objects.equals(startTime, stepInfo.startTime); - } - - @Override - public int hashCode() { - return Objects.hash(step, status, startTime); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java deleted file mode 100644 index 87df1e925f0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; - -import java.util.Optional; - -/** - * Advances a given job run by running the appropriate {@link Step}s, based on their current status. - * - * When an attempt is made to advance a given job, a lock for that job (application and type) is - * taken, and released again only when the attempt finishes. Multiple other attempts may be made in - * the meantime, but they should give up unless the lock is promptly acquired. - * - * @author jonmv - */ -public interface StepRunner { - - /** Attempts to run the given step in the given run, and returns the new status of the run, if the step completed. */ - Optional<RunStatus> run(LockedStep step, RunId id); - -} - diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Submission.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Submission.java deleted file mode 100644 index ce346f5ba74..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Submission.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; - -import java.time.Instant; -import java.util.Optional; - -import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.calculateHash; - -/** - * @author jonmv - */ -public class Submission { - - private final ApplicationPackage applicationPackage; - private final byte[] testPackage; - private final Optional<String> sourceUrl; - private final Optional<SourceRevision> source; - private final Optional<String> authorEmail; - private final Optional<String> description; - private final Instant now; - private final int risk; - - public Submission(ApplicationPackage applicationPackage, byte[] testPackage, Optional<String> sourceUrl, - Optional<SourceRevision> source, Optional<String> authorEmail, Optional<String> description, - Instant now, int risk) { - this.applicationPackage = applicationPackage; - this.testPackage = testPackage; - this.sourceUrl = sourceUrl; - this.source = source; - this.authorEmail = authorEmail; - this.description = description; - this.now = now; - this.risk = risk; - } - - public ApplicationVersion toApplicationVersion(long number) { - return ApplicationVersion.forProduction(RevisionId.forProduction(number), - source, - authorEmail, - applicationPackage.compileVersion(), - applicationPackage.deploymentSpec().majorVersion(), - applicationPackage.buildTime(), - sourceUrl, - source.map(SourceRevision::commit), - Optional.of(applicationPackage.bundleHash() + calculateHash(testPackage)), - description, - now, - risk); - } - - public ApplicationPackage applicationPackage() { return applicationPackage; } - - public byte[] testPackage() { return testPackage; } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java deleted file mode 100644 index a5a91e7cdd2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.application.Endpoint; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.time.Instant; -import java.util.List; -import java.util.Map; - -/** - * Serializes config for integration tests against Vespa deployments. - * - * @author jonmv - */ -public class TestConfigSerializer { - - private final SystemName system; - - public TestConfigSerializer(SystemName system) { - this.system = system; - } - - public Slime configSlime(ApplicationId id, - JobType type, - boolean isCI, - Version platform, - RevisionId revision, - Instant deployedAt, - Map<ZoneId, List<Endpoint>> deployments, - Map<ZoneId, List<String>> clusters) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - root.setString("application", id.serializedForm()); - root.setString("zone", type.zone().value()); - root.setString("system", system.value()); - root.setBool("isCI", isCI); - root.setString("platform", platform.toFullString()); - root.setLong("revision", revision.number()); - root.setLong("deployedAt", deployedAt.toEpochMilli()); - - // TODO jvenstad: remove when clients can be updated - Cursor endpointsObject = root.setObject("endpoints"); - deployments.forEach((zone, endpoints) -> { - Cursor endpointArray = endpointsObject.setArray(zone.value()); - for (Endpoint endpoint : endpoints) - endpointArray.addString(endpoint.url().toString()); - }); - - Cursor zoneEndpointsObject = root.setObject("zoneEndpoints"); - deployments.forEach((zone, endpoints) -> { - Cursor clusterEndpointsObject = zoneEndpointsObject.setObject(zone.value()); - for (Endpoint endpoint : endpoints) - clusterEndpointsObject.setString(endpoint.name(), endpoint.url().toString()); - }); - - if ( ! clusters.isEmpty()) { - Cursor clustersObject = root.setObject("clusters"); - clusters.forEach((zone, clusterList) -> { - Cursor clusterArray = clustersObject.setArray(zone.value()); - for (String cluster : clusterList) - clusterArray.addString(cluster); - }); - } - - return slime; - } - - /** Returns the config for the tests to run for the given job. */ - public byte[] configJson(ApplicationId id, - JobType type, - boolean isCI, - Version platform, - RevisionId revision, - Instant deployedAt, - Map<ZoneId, List<Endpoint>> deployments, - Map<ZoneId, List<String>> clusters) { - try { - return SlimeUtils.toJsonBytes(configSlime(id, type, isCI, platform, revision, deployedAt, deployments, clusters)); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java deleted file mode 100644 index 9b4fbf06e21..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.component.Version; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.Deployment; - -import java.util.Objects; -import java.util.Optional; -import java.util.function.Supplier; - -import static java.util.Objects.requireNonNull; - -/** - * Source and target versions for an application. - * - * @author jvenstad - * @author mpolden - */ -public class Versions { - - private final Version targetPlatform; - private final RevisionId targetRevision; - private final Optional<Version> sourcePlatform; - private final Optional<RevisionId> sourceRevision; - - public Versions(Version targetPlatform, RevisionId targetRevision, Optional<Version> sourcePlatform, - Optional<RevisionId> sourceRevision) { - if (sourcePlatform.isPresent() ^ sourceRevision.isPresent()) - throw new IllegalArgumentException("Sources must both be present or absent."); - - this.targetPlatform = requireNonNull(targetPlatform); - this.targetRevision = requireNonNull(targetRevision); - this.sourcePlatform = requireNonNull(sourcePlatform); - this.sourceRevision = requireNonNull(sourceRevision); - } - - /** A copy of this, without source versions. */ - public Versions withoutSources() { - return new Versions(targetPlatform, targetRevision, Optional.empty(), Optional.empty()); - } - - /** Target platform version for this */ - public Version targetPlatform() { - return targetPlatform; - } - - /** Target revision for this */ - public RevisionId targetRevision() { - return targetRevision; - } - - /** Source platform version for this */ - public Optional<Version> sourcePlatform() { - return sourcePlatform; - } - - /** Source application version for this */ - public Optional<RevisionId> sourceRevision() { - return sourceRevision; - } - - /** Returns whether source versions are present and match those of the given job other versions. */ - public boolean sourcesMatchIfPresent(Versions versions) { - return (sourcePlatform.map(targetPlatform::equals).orElse(true) || - sourcePlatform.equals(versions.sourcePlatform())) && - (sourceRevision.map(targetRevision::equals).orElse(true) || - sourceRevision.equals(versions.sourceRevision())); - } - - public boolean targetsMatch(Versions versions) { - return targetPlatform.equals(versions.targetPlatform()) && - targetRevision.equals(versions.targetRevision()); - } - - /** Returns whether this change could result in the given target versions. */ - public boolean targetsMatch(Change change) { - return change.platform().map(targetPlatform::equals).orElse(true) - && change.revision().map(targetRevision::equals).orElse(true); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if ( ! (o instanceof Versions)) return false; - Versions versions = (Versions) o; - return Objects.equals(targetPlatform, versions.targetPlatform) && - Objects.equals(targetRevision, versions.targetRevision) && - Objects.equals(sourcePlatform, versions.sourcePlatform) && - Objects.equals(sourceRevision, versions.sourceRevision); - } - - @Override - public int hashCode() { - return Objects.hash(targetPlatform, targetRevision, sourcePlatform, sourceRevision); - } - - @Override - public String toString() { - return Text.format("platform %s%s, revision %s%s", - sourcePlatform.filter(source -> ! source.equals(targetPlatform)) - .map(source -> source + " -> ").orElse(""), - targetPlatform, - sourceRevision.filter(source -> ! source.equals(targetRevision)) - .map(source -> source + " -> ").orElse(""), - targetRevision); - } - - /** Create versions using given change and application */ - public static Versions from(Change change, Application application, Optional<Version> existingPlatform, - Optional<RevisionId> existingRevision, Supplier<Version> defaultPlatformVersion) { - return new Versions(targetPlatform(application, change, existingPlatform, defaultPlatformVersion), - targetRevision(application, change, existingRevision), - existingPlatform, - existingRevision); - } - - /** Create versions using given change and application */ - public static Versions from(Change change, Application application, Optional<Deployment> deployment, Supplier<Version> defaultPlatformVersion) { - return from(change, application, deployment.map(Deployment::version), deployment.map(Deployment::revision), defaultPlatformVersion); - } - - private static Version targetPlatform(Application application, Change change, Optional<Version> existing, - Supplier<Version> defaultVersion) { - if (change.isPlatformPinned() && change.platform().isPresent()) - return change.platform().get(); - - return max(change.platform(), existing) - .orElseGet(() -> application.oldestDeployedPlatform().orElseGet(defaultVersion)); - } - - private static RevisionId targetRevision(Application application, Change change, - Optional<RevisionId> existing) { - if (change.isRevisionPinned() && change.revision().isPresent()) - return change.revision().get(); - - return change.revision() - .or(() -> existing) - .orElseGet(() -> defaultRevision(application)); - } - - private static RevisionId defaultRevision(Application application) { - return application.oldestDeployedRevision() - .or(() -> application.revisions().last().map(ApplicationVersion::id)) - .orElseThrow(() -> new IllegalStateException("no known prod revisions, but asked for one, for " + application)); - } - - private static <T extends Comparable<T>> Optional<T> max(Optional<T> o1, Optional<T> o2) { - return o1.isEmpty() ? o2 : o2.isEmpty() ? o1 : o1.get().compareTo(o2.get()) >= 0 ? o1 : o2; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java deleted file mode 100644 index 17d347bda17..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.function.Predicate; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; - -/** - * Utility class to build zipped content by adding already zipped byte content or - * adding new unzipped entries. - * - * @author freva - */ -public class ZipBuilder implements AutoCloseable { - - private final ByteArrayOutputStream byteArrayOutputStream; - private final ZipOutputStream zipOutputStream; - - public ZipBuilder(int initialSize) { - byteArrayOutputStream = new ByteArrayOutputStream(initialSize); - zipOutputStream = new ZipOutputStream(byteArrayOutputStream); - } - - public void add(byte[] zippedContent, Predicate<String> filter) { - try (ZipInputStream zin = new ZipInputStream(new ByteArrayInputStream(zippedContent))) { - for (ZipEntry entry = zin.getNextEntry(); entry != null; entry = zin.getNextEntry()) { - if ( ! filter.test(entry.getName())) continue; - zipOutputStream.putNextEntry(new ZipEntry(entry.getName())); - zin.transferTo(zipOutputStream); - zipOutputStream.closeEntry(); - } - } catch (IOException e) { - throw new UncheckedIOException("Failed to add zipped content", e); - } - } - - public void add(String entryName, byte[] content) { - try { - zipOutputStream.putNextEntry(new ZipEntry(entryName)); - zipOutputStream.write(content); - zipOutputStream.closeEntry(); - } catch (IOException e) { - throw new UncheckedIOException("Failed to add entry " + entryName, e); - } - } - - /** @return zipped byte array */ - public byte[] toByteArray() { - return byteArrayOutputStream.toByteArray(); - } - - @Override - public void close() { - try { - zipOutputStream.close(); - } catch (IOException e) { - throw new UncheckedIOException("Failed to close zip output stream", e); - } - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java deleted file mode 100644 index 4619f5d32c2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/AbstractNameServiceRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/AbstractNameServiceRequest.java deleted file mode 100644 index f4223ad90bc..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/AbstractNameServiceRequest.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.dns; - -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; - -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -/** - * @author jonmv - */ -public abstract class AbstractNameServiceRequest implements NameServiceRequest { - - private final Optional<TenantAndApplicationId> owner; - private final RecordName name; - - AbstractNameServiceRequest(Optional<TenantAndApplicationId> owner, RecordName name) { - this.owner = requireNonNull(owner); - this.name = requireNonNull(name); - } - - @Override - public RecordName name() { - return name; - } - - @Override - public Optional<TenantAndApplicationId> owner() { - return owner; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java deleted file mode 100644 index c4c76bc7954..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.dns; - -import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * Create or update a record. - * - * @author mpolden - */ -public class CreateRecord extends AbstractNameServiceRequest { - - private final Record record; - - /** DO NOT USE. Public for serialization purposes */ - public CreateRecord(Optional<TenantAndApplicationId> owner, Record record) { - super(owner, record.name()); - this.record = Objects.requireNonNull(record, "record must be non-null"); - if (record.type() != Record.Type.CNAME && record.type() != Record.Type.A) { - throw new IllegalArgumentException("Record of type " + record.type() + " is not supported: " + record); - } - } - - public Record record() { - return record; - } - - @Override - public void dispatchTo(NameService nameService) { - List<Record> records = nameService.findRecords(record.type(), record.name()); - records.forEach(r -> { - // Ensure that existing record has correct data - if (!r.data().equals(record.data())) { - nameService.updateRecord(r, record.data()); - } - }); - if (records.isEmpty()) { - nameService.createRecord(record.type(), record.name(), record.data()); - } - } - - @Override - public String toString() { - return "create record " + record; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - CreateRecord that = (CreateRecord) o; - return owner().equals(that.owner()) && record.equals(that.record); - } - - @Override - public int hashCode() { - return Objects.hash(owner(), record); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java deleted file mode 100644 index d560dbd8db9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.dns; - -import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.DirectTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * Create or update multiple records of the same type and name. - * - * @author mpolden - */ -public class CreateRecords extends AbstractNameServiceRequest { - - private final Record.Type type; - private final List<Record> records; - - /** DO NOT USE. Public for serialization purposes */ - public CreateRecords(Optional<TenantAndApplicationId> owner, List<Record> records) { - super(owner, requireOneOf(Record::name, records)); - this.type = requireOneOf(Record::type, records); - this.records = List.copyOf(Objects.requireNonNull(records, "records must be non-null")); - if (type != Record.Type.ALIAS && type != Record.Type.TXT && type != Record.Type.DIRECT) { - throw new IllegalArgumentException("Records of type " + type + " are not supported: " + records); - } - } - - public List<Record> records() { - return records; - } - - @Override - public void dispatchTo(NameService nameService) { - switch (type) { - case ALIAS -> { - var targets = records.stream().map(Record::data).map(AliasTarget::unpack).collect(Collectors.toSet()); - nameService.createAlias(name(), targets); - } - case DIRECT -> { - var targets = records.stream().map(Record::data).map(DirectTarget::unpack).collect(Collectors.toSet()); - nameService.createDirect(name(), targets); - } - case TXT -> { - var dataFields = records.stream().map(Record::data).toList(); - nameService.createTxtRecords(name(), dataFields); - } - } - } - - @Override - public String toString() { - return "create records " + records(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - CreateRecords that = (CreateRecords) o; - return owner().equals(that.owner()) && records.equals(that.records); - } - - @Override - public int hashCode() { - return Objects.hash(owner(), records); - } - - /** Find exactly one distinct value of field in given list */ - private static <T, V> T requireOneOf(Function<V, T> field, List<V> list) { - Set<T> values = list.stream().map(field).collect(Collectors.toSet()); - if (values.size() != 1) { - throw new IllegalArgumentException("Expected one distinct value, but found " + values + " in " + list); - } - return values.iterator().next(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java deleted file mode 100644 index 40d2667b9ae..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.dns; - -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.DirectTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * This adds name service requests to the {@link NameServiceQueue}. - * - * Name service requests passed to this are not immediately sent to a name service, but are instead persisted - * in a curator-backed queue. Enqueued requests are later dispatched to a {@link NameService} by - * {@link NameServiceDispatcher}. - * - * @author mpolden - */ -public class NameServiceForwarder { - - private static final Logger log = Logger.getLogger(NameServiceForwarder.class.getName()); - - private final CuratorDb db; - - public NameServiceForwarder(CuratorDb db) { - this.db = Objects.requireNonNull(db, "db must be non-null"); - } - - /** Create or update a given record */ - public void createRecord(Record record, NameServiceQueue.Priority priority, Optional<TenantAndApplicationId> owner) { - forward(new CreateRecord(owner, record), priority); - } - - /** Create or update an ALIAS record with given name and targets */ - public void createAlias(RecordName name, Set<AliasTarget> targets, NameServiceQueue.Priority priority, Optional<TenantAndApplicationId> owner) { - var records = targets.stream() - .map(target -> new Record(Record.Type.ALIAS, name, target.pack())) - .toList(); - forward(new CreateRecords(owner, records), priority); - } - - /** Create or update a DIRECT record with given name and targets */ - public void createDirect(RecordName name, Set<DirectTarget> targets, NameServiceQueue.Priority priority, Optional<TenantAndApplicationId> owner) { - var records = targets.stream() - .map(target -> new Record(Record.Type.DIRECT, name, target.pack())) - .toList(); - forward(new CreateRecords(owner, records), priority); - } - - /** Create or update a TXT record with given name and data */ - public void createTxt(RecordName name, List<RecordData> txtData, NameServiceQueue.Priority priority, Optional<TenantAndApplicationId> owner) { - var records = txtData.stream() - .map(data -> new Record(Record.Type.TXT, name, data)) - .toList(); - forward(new CreateRecords(owner, records), priority); - } - - /** Remove all records of given type and name */ - public void removeRecords(Record.Type type, RecordName name, NameServiceQueue.Priority priority, Optional<TenantAndApplicationId> owner) { - forward(new RemoveRecords(owner, type, name), priority); - } - - /** Remove all records of given type, name and data */ - public void removeRecords(Record.Type type, RecordName name, RecordData data, NameServiceQueue.Priority priority, Optional<TenantAndApplicationId> owner) { - forward(new RemoveRecords(owner, type, name, data), priority); - } - - protected void forward(NameServiceRequest request, NameServiceQueue.Priority priority) { - try (Mutex lock = db.lockNameServiceQueue()) { - NameServiceQueue queue = db.readNameServiceQueue(); - var queued = queue.requests().size(); - if (queued > NameServiceQueue.QUEUE_CAPACITY) { - log.log(Level.WARNING, "Queue is above capacity (size: " + queued + "), failing requests will be dropped. " + - "This likely means that the name service is not successfully executing requests"); - } - log.log(Level.FINE, () -> "Queueing name service request: " + request); - db.writeNameServiceQueue(queue.with(request, priority)); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java deleted file mode 100644 index 033a019f35f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.dns; - -import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.yolean.Exceptions; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.function.UnaryOperator; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * A queue of outstanding {@link NameServiceRequest}s. Requests in this have not yet been dispatched to a - * {@link NameService} and are thus not visible in DNS. - * - * This is immutable. - * - * @author mpolden - * @author jonmv - */ -public record NameServiceQueue(List<NameServiceRequest> requests) { - - public static final NameServiceQueue EMPTY = new NameServiceQueue(List.of()); - - /** - * The number of {@link NameServiceRequest}s we allow to be queued. When the queue overflows, failing requests - * are dropped in a FIFO order until the queue shrinks below this capacity. If that is not enough, the oldest - * requests will also be dropped, as needed. - */ - static final int QUEUE_CAPACITY = 400; - - private static final Logger log = Logger.getLogger(NameServiceQueue.class.getName()); - - /** DO NOT USE. Public for serialization purposes */ - public NameServiceQueue(List<NameServiceRequest> requests) { - this.requests = List.copyOf(Objects.requireNonNull(requests, "requests must be non-null")); - } - - /** Returns a copy of this containing the last n requests */ - public NameServiceQueue last(int n) { - return resize(n, (requests) -> requests.subList(requests.size() - n, requests.size())); - } - - /** Returns a copy of this containing the first n requests */ - public NameServiceQueue first(int n) { - return resize(n, (requests) -> requests.subList(0, n)); - } - - /** Returns a copy of this with given request queued according to priority */ - public NameServiceQueue with(NameServiceRequest request, Priority priority) { - List<NameServiceRequest> copy = new ArrayList<>(this.requests.size() + 1); - switch (priority) { - case normal -> { - copy.addAll(this.requests); - copy.add(request); - } - case high -> { - copy.add(request); - copy.addAll(this.requests); - } - } - return new NameServiceQueue(copy); - } - - /** Returns a copy of this without the requests present in other. Duplicates are not removed */ - public NameServiceQueue without(NameServiceQueue other) { - List<NameServiceRequest> toRemove = new ArrayList<>(other.requests); - return new NameServiceQueue(requests.stream() - .filter(request -> !toRemove.remove(request)) - .toList()); - } - - /** Returns a copy of this with given request added */ - public NameServiceQueue with(NameServiceRequest request) { - return with(request, Priority.normal); - } - - /** - * Dispatch n requests from the head of this to given name service. Requests may be re-ordered if errors are - * encountered, but are always dispatched in order within an application. - * - * @return A copy of this, without the successfully dispatched requests. - */ - public NameServiceQueue dispatchTo(NameService nameService, int n) { - requireNonNegative(n); - if (requests.isEmpty()) return this; - - LinkedList<NameServiceRequest> pending = new LinkedList<>(requests); - while (n-- > 0 && ! pending.isEmpty()) { - NameServiceRequest request = pending.poll(); - try { - request.dispatchTo(nameService); - } catch (Exception e) { - boolean dropFailingRequest = pending.size() > QUEUE_CAPACITY; - log.log(Level.WARNING, "Failed to execute " + request + ": " + Exceptions.toMessageString(e) + - ", request will " + (dropFailingRequest ? "be dropped, as queue is over capacity" - : "be moved backwards, and retried")); - if (dropFailingRequest) continue; // Drop this request and keep dispatching others - - // Move all requests with the same owner backwards as far as we can, i.e., to the back, or to the first owner-less request. - Optional<TenantAndApplicationId> owner = request.owner(); - LinkedList<NameServiceRequest> owned = new LinkedList<>(); - LinkedList<NameServiceRequest> others = new LinkedList<>(); - do { - if (request.owner().isEmpty()) { - pending.push(request); - break; // Can't modify anything past this, as owner-less requests must come in order with all others. - } - (request.owner().equals(owner) ? owned : others).offer(request); - } while ((request = pending.poll()) != null); - pending.addAll(0, owned); // Append owned requests before those we can't modify (or none), and - pending.addAll(0, others); // then append requests owned by others before that again. - } - } - - NameServiceQueue remaining = new NameServiceQueue(pending); - if (pending.size() > 2 * QUEUE_CAPACITY) { - log.log(Level.WARNING, "Queue has " + pending.size() + " entries, and must be emptying far too slowly; " + - "dropping the oldest entries past " + 2 * QUEUE_CAPACITY); - remaining = remaining.last(2 * QUEUE_CAPACITY); - } - return remaining; - } - - @Override - public String toString() { - return requests.toString(); - } - - private NameServiceQueue resize(int n, UnaryOperator<List<NameServiceRequest>> resizer) { - requireNonNegative(n); - if (requests.size() <= n) return this; - return new NameServiceQueue(resizer.apply(requests)); - } - - private static void requireNonNegative(int n) { - if (n < 0) throw new IllegalArgumentException("n must be >= 0, got " + n); - } - - /** - * Replaces the requests in {@code oldQueue} contained in this with requests in {@code newQueue}, or best effort - * amendment when not contained. - */ - public NameServiceQueue replace(NameServiceQueue oldQueue, NameServiceQueue newQueue) { - int sublistIndex = indexOf(oldQueue.requests, requests); - if (sublistIndex >= 0) { - List<NameServiceRequest> updated = new ArrayList<>(); - updated.addAll(requests.subList(0, sublistIndex)); - updated.addAll(newQueue.requests); - updated.addAll(requests.subList(sublistIndex + oldQueue.requests.size(), requests.size())); - return new NameServiceQueue(updated); - } else { - log.log(Level.WARNING, "Name service queue has changed unexpectedly; expected requests: " + - oldQueue.requests + " to be present, but that was not found in: " + requests); - // Do a best-effort amendment, where requests removed from initial to remaining, are removed, from the front, from this. - return without(oldQueue.without(newQueue)); - } - } - - /** - * Find the starting index of subList in list. I.e. the lowest index {@code i} in {@code list} so that - * {@code list.subList(i, i + subList.size()).equals(subList)}. Naïve implementation. - */ - private static <T> int indexOf(List<T> subList, List<T> list) { - for (int i = 0; i + subList.size() <= list.size(); i++) { - if (list.subList(i, i + subList.size()).equals(subList)) { - return i; - } - } - return -1; - } - - /** Priority of a request added to this */ - public enum Priority { - - /** Default priority. Request will be delivered in FIFO order */ - normal, - - /** Request is queued first. Useful for code that needs to act on effects of a request */ - high - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java deleted file mode 100644 index d86c2ce565b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.dns; - -import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; - -import java.util.Optional; - -/** - * Interface for requests to a {@link NameService}. - * - * @author mpolden - */ -public interface NameServiceRequest { - - /** The record name this request pertains to. */ - RecordName name(); - - /** The application owning this request */ - Optional<TenantAndApplicationId> owner(); - - /** Send this to given name service */ - void dispatchTo(NameService nameService); - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java deleted file mode 100644 index 0ed835f32bd..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.dns; - -import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.DirectTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * Permanently removes all matching records by type and matching either: - * - * - name and data - * - only name - * - * @author mpolden - */ -public class RemoveRecords extends AbstractNameServiceRequest { - - private final Record.Type type; - private final Optional<RecordData> data; - - public RemoveRecords(Optional<TenantAndApplicationId> owner, Record.Type type, RecordName name) { - this(owner, type, name, Optional.empty()); - } - - public RemoveRecords(Optional<TenantAndApplicationId> owner, Record.Type type, RecordName name, RecordData data) { - this(owner, type, name, Optional.of(data)); - } - - /** DO NOT USE. Public for serialization purposes */ - public RemoveRecords(Optional<TenantAndApplicationId> owner, Record.Type type, RecordName name, Optional<RecordData> data) { - super(owner, name); - this.type = Objects.requireNonNull(type, "type must be non-null"); - this.data = Objects.requireNonNull(data, "data must be non-null"); - } - - public Record.Type type() { - return type; - } - - public Optional<RecordData> data() { - return data; - } - - @Override - public void dispatchTo(NameService nameService) { - // Deletions require all records fields to match exactly, data may be incomplete even if present. To ensure - // completeness we search for the record(s) first - List<Record> completeRecords = nameService.findRecords(type, name()).stream() - .filter(record -> data.isEmpty() || matchingFqdnIn(data.get(), record)) - .toList(); - nameService.removeRecords(completeRecords); - } - - @Override - public String toString() { - return "remove records of type " + type + ", by name " + name() + - data.map(d -> " data " + d).orElse(""); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RemoveRecords that = (RemoveRecords) o; - return owner().equals(that.owner()) && type == that.type && name().equals(that.name()) && data.equals(that.data); - } - - @Override - public int hashCode() { - return Objects.hash(owner(), type, name(), data); - } - - private static boolean matchingFqdnIn(RecordData data, Record record) { - String dataValue = switch (record.type()) { - case ALIAS -> AliasTarget.unpack(record.data()).name().value(); - case DIRECT -> DirectTarget.unpack(record.data()).recordData().asString(); - default -> record.data().asString(); - }; - return fqdn(dataValue).equals(fqdn(data.asString())); - } - - private static String fqdn(String name) { - return name.endsWith(".") ? name : name + "."; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java deleted file mode 100644 index 29e251a9de3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; - -import java.time.Duration; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author jvenstad - */ -public class ApplicationMetaDataGarbageCollector extends ControllerMaintainer { - - private static final Logger log = Logger.getLogger(ApplicationMetaDataGarbageCollector.class.getName()); - - private final Duration timeToLive; - - public ApplicationMetaDataGarbageCollector(Controller controller, Duration interval) { - super(controller, interval); - this.timeToLive = controller.system().isCd() ? Duration.ofDays(7) : Duration.ofDays(365); - } - - @Override - protected double maintain() { - try { - controller().applications().applicationStore().pruneMeta(controller().clock().instant().minus(timeToLive)); - return 1.0; - } - catch (Exception e) { - log.log(Level.WARNING, "Exception pruning old application meta data", e); - return 0.0; - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java deleted file mode 100644 index d998413e675..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.ApplicationController; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.AccountId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.ApplicationSummary; -import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; -import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.util.HashMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; - -/** - * Periodically request application ownership confirmation through filing issues. - * - * When to file new issues, escalate inactive ones, etc., is handled by the enclosed OwnershipIssues. - * - * @author jonmv - */ -public class ApplicationOwnershipConfirmer extends ControllerMaintainer { - - private final OwnershipIssues ownershipIssues; - private final ApplicationController applications; - private final int shards; - - public ApplicationOwnershipConfirmer(Controller controller, Duration interval, OwnershipIssues ownershipIssues) { - this(controller, interval, ownershipIssues, 24); - } - - public ApplicationOwnershipConfirmer(Controller controller, Duration interval, OwnershipIssues ownershipIssues, int shards) { - super(controller, interval); - this.ownershipIssues = ownershipIssues; - this.applications = controller.applications(); - if (shards <= 0) throw new IllegalArgumentException("shards must be a positive number, but got " + shards); - this.shards = shards; - } - - @Override - protected double maintain() { - return ( confirmApplicationOwnerships() + - ensureConfirmationResponses() + - updateConfirmedApplicationOwners() ) - / 3; - } - - /** File an ownership issue with the owners of all applications we know about. */ - private double confirmApplicationOwnerships() { - AtomicInteger attempts = new AtomicInteger(0); - AtomicInteger failures = new AtomicInteger(0); - applications() - .withProjectId() - .withProductionDeployment() - .asList() - .stream() - .filter(application -> application.createdAt().isBefore(controller().clock().instant().minus(Duration.ofDays(90)))) - .filter(application -> isInCurrentShard(application.id())) - .forEach(application -> { - try { - attempts.incrementAndGet(); - tenantOf(application.id()).contact().flatMap(contact -> { - return ownershipIssues.confirmOwnership(application.ownershipIssueId(), - summaryOf(application.id()), - application.issueOwner().orElse(null), - application.userOwner().orElse(null), - contact); - }).ifPresent(newIssueId -> store(newIssueId, application.id())); - } - catch (RuntimeException e) { // Catch errors due to wrong data in the controller, or issues client timeout. - failures.incrementAndGet(); - log.log(Level.INFO, "Exception caught when attempting to file an issue for '" + application.id() + "': " + Exceptions.toMessageString(e)); - } - }); - return asSuccessFactorDeviation(attempts.get(), failures.get()); - } - - private boolean isInCurrentShard(TenantAndApplicationId id) { - double participants = Math.max(1, controller().curator().cluster().size()); - long ticksSinceEpoch = Math.round((controller().clock().millis() * participants / interval().toMillis())); - return (ticksSinceEpoch + id.hashCode()) % shards == 0; - } - - private ApplicationSummary summaryOf(TenantAndApplicationId application) { - var app = applications.requireApplication(application); - var metrics = new HashMap<DeploymentId, ApplicationSummary.Metric>(); - for (Instance instance : app.instances().values()) { - for (var kv : instance.deployments().entrySet()) { - var zone = kv.getKey(); - var deploymentMetrics = kv.getValue().metrics(); - metrics.put(new DeploymentId(instance.id(), zone), - new ApplicationSummary.Metric(deploymentMetrics.documentCount(), - deploymentMetrics.queriesPerSecond(), - deploymentMetrics.writesPerSecond())); - } - } - return new ApplicationSummary(app.id().defaultInstance(), app.activity().lastQueried(), app.activity().lastWritten(), - app.revisions().last().flatMap(version -> version.buildTime()), metrics); - } - - /** Escalate ownership issues which have not been closed before a defined amount of time has passed. */ - private double ensureConfirmationResponses() { - AtomicInteger attempts = new AtomicInteger(0); - AtomicInteger failures = new AtomicInteger(0); - for (Application application : applications()) - if (isInCurrentShard(application.id())) - application.ownershipIssueId().ifPresent(issueId -> { - try { - attempts.incrementAndGet(); - Tenant tenant = tenantOf(application.id()); - ownershipIssues.ensureResponse(issueId, tenant.contact()); - } - catch (RuntimeException e) { - failures.incrementAndGet(); - log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); - } - }); - return asSuccessFactorDeviation(attempts.get(), failures.get()); - } - - private double updateConfirmedApplicationOwners() { - AtomicInteger attempts = new AtomicInteger(0); - AtomicInteger failures = new AtomicInteger(0); - applications() - .withProjectId() - .withProductionDeployment() - .asList() - .stream() - .filter(application -> isInCurrentShard(application.id())) - .filter(application -> application.ownershipIssueId().isPresent()) - .forEach(application -> { - attempts.incrementAndGet(); - IssueId issueId = application.ownershipIssueId().get(); - try { - ownershipIssues.getConfirmedOwner(issueId).ifPresent(owner -> { - controller().applications().lockApplicationIfPresent(application.id(), lockedApplication -> - controller().applications().store(lockedApplication.withOwner(owner))); - }); - } - catch (RuntimeException e) { - failures.incrementAndGet(); - log.log(Level.INFO, "Exception caught when attempting to find confirmed owner of issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); - } - }); - return asSuccessFactorDeviation(attempts.get(), failures.get()); - } - - private ApplicationList applications() { - return ApplicationList.from(controller().applications().readable()); - } - - private AccountId determineAssignee(Application application) { - return application.issueOwner().orElse(null); - } - - private User determineLegacyAssignee(Application application) { - return application.userOwner().orElse(null); - } - - private Tenant tenantOf(TenantAndApplicationId applicationId) { - return controller().tenants().get(applicationId.tenant()) - .orElseThrow(() -> new IllegalStateException("No tenant found for application " + applicationId)); - } - - protected void store(IssueId issueId, TenantAndApplicationId applicationId) { - controller().applications().lockApplicationIfPresent(applicationId, application -> - controller().applications().store(application.withOwnershipIssueId(issueId))); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java deleted file mode 100644 index b6f73d6e5e3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import ai.vespa.metrics.ControllerMetrics; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; -import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Update archive access permissions with roles from tenants - * - * @author andreer - */ -public class ArchiveAccessMaintainer extends ControllerMaintainer { - - private static final String bucketCountMetricName = ControllerMetrics.ARCHIVE_BUCKET_COUNT.baseName(); - - private final CuratorArchiveBucketDb archiveBucketDb; - private final ArchiveService archiveService; - private final ZoneRegistry zoneRegistry; - private final Metric metric; - - public ArchiveAccessMaintainer(Controller controller, Metric metric, Duration interval) { - super(controller, interval); - this.archiveBucketDb = controller.archiveBucketDb(); - this.archiveService = controller.serviceRegistry().archiveService(); - this.zoneRegistry = controller().zoneRegistry(); - this.metric = metric; - } - - @Override - protected double maintain() { - // Count buckets - so we can alert if we get close to the AWS account limit of 1000 - zoneRegistry.zonesIncludingSystem().all().zones().forEach(z -> - metric.set(bucketCountMetricName, archiveBucketDb.buckets(z.getVirtualId()).vespaManaged().size(), - metric.createContext(Map.of( - "zone", z.getVirtualId().value(), - "cloud", z.getCloudName().value())))); - - zoneRegistry.zonesIncludingSystem().controllerUpgraded().zones().forEach(z -> { - ZoneId zoneId = z.getVirtualId(); - try { - var tenantArchiveAccessRoles = cloudTenantArchiveExternalAccessRoles(); - var buckets = archiveBucketDb.buckets(zoneId).vespaManaged(); - archiveService.updatePolicies(zoneId, buckets, tenantArchiveAccessRoles); - } catch (Exception e) { - throw new RuntimeException("Failed to maintain archive access in " + zoneId.value(), e); - } - }); - - return 1.0; - } - - private Map<TenantName, ArchiveAccess> cloudTenantArchiveExternalAccessRoles() { - List<Tenant> tenants = controller().tenants().asList(); - return tenants.stream() - .filter(t -> t instanceof CloudTenant) - .map(t -> (CloudTenant) t) - .collect(Collectors.toUnmodifiableMap( - Tenant::name, CloudTenant::archiveAccess)); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java deleted file mode 100644 index 8913d6e7166..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.ApplicationController; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ArchiveUris; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.logging.Level; -import java.util.stream.Stream; - -/** - * Updates archive URIs for tenants in all zones. - * - * @author freva - */ -public class ArchiveUriUpdater extends ControllerMaintainer { - - private static final Set<TenantName> INFRASTRUCTURE_TENANTS = Set.of(SystemApplication.TENANT); - - private final ApplicationController applications; - private final NodeRepository nodeRepository; - private final CuratorArchiveBucketDb archiveBucketDb; - private final ZoneRegistry zoneRegistry; - - public ArchiveUriUpdater(Controller controller, Duration interval) { - super(controller, interval); - this.applications = controller.applications(); - this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.archiveBucketDb = controller.archiveBucketDb(); - this.zoneRegistry = controller.zoneRegistry(); - } - - @Override - protected double maintain() { - Map<ZoneId, Set<TenantName>> tenantsByZone = new HashMap<>(); - Map<ZoneId, Set<CloudAccount>> accountsByZone = new HashMap<>(); - - controller().zoneRegistry().zonesIncludingSystem().reachable().zones().forEach(zone -> { - tenantsByZone.put(zone.getVirtualId(), new HashSet<>(INFRASTRUCTURE_TENANTS)); - accountsByZone.put(zone.getVirtualId(), new HashSet<>()); - }); - - for (var application : applications.asList()) { - for (var instance : application.instances().values()) { - for (var deployment : instance.deployments().values()) { - if (zoneRegistry.isExclave(deployment.cloudAccount())) accountsByZone.get(deployment.zone()).add(deployment.cloudAccount()); - else tenantsByZone.get(deployment.zone()).add(instance.id().tenant()); - } - } - } - - int failures = 0; - for (ZoneId zone : tenantsByZone.keySet()) { - try { - ArchiveUris zoneArchiveUris = nodeRepository.getArchiveUris(zone); - - Stream.of( - // Tenant URIs that need to be added or updated - tenantsByZone.get(zone).stream() - .flatMap(tenant -> archiveBucketDb.archiveUriFor(zone, tenant, true) - .filter(uri -> !uri.equals(zoneArchiveUris.tenantArchiveUris().get(tenant))) - .map(uri -> ArchiveUriUpdate.setArchiveUriFor(tenant, uri)) - .stream()), - // Account URIs that need to be added or updated - accountsByZone.get(zone).stream() - .flatMap(account -> archiveBucketDb.archiveUriFor(zone, account, true) - .filter(uri -> !uri.equals(zoneArchiveUris.accountArchiveUris().get(account))) - .map(uri -> ArchiveUriUpdate.setArchiveUriFor(account, uri)) - .stream()), - // Tenant URIs that need to be deleted - zoneArchiveUris.tenantArchiveUris().keySet().stream() - .filter(tenant -> !tenantsByZone.get(zone).contains(tenant)) - .map(ArchiveUriUpdate::deleteArchiveUriFor), - // Account URIs that need to be deleted - zoneArchiveUris.accountArchiveUris().keySet().stream() - .filter(account -> !accountsByZone.get(zone).contains(account)) - .map(ArchiveUriUpdate::deleteArchiveUriFor)) - .flatMap(s -> s) - .forEach(update -> nodeRepository.updateArchiveUri(zone, update)); - } catch (Exception e) { - log.log(Level.WARNING, "Failed to update archive URI in " + zone + ". Retrying in " + interval() + ". Error: " + - Exceptions.toMessageString(e)); - failures++; - } - } - - return asSuccessFactorDeviation(tenantsByZone.size(), failures); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java deleted file mode 100644 index 02cf7a85445..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.artifact.Artifact; -import com.yahoo.vespa.hosted.controller.api.integration.artifact.ArtifactRegistry; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.time.Instant; -import java.util.Comparator; -import java.util.List; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static com.yahoo.yolean.Exceptions.uncheck; -import static java.util.logging.Level.FINE; -import static java.util.logging.Level.INFO; - -/** - * Periodically expire unused artifacts, e.g. container images and RPMs. Artifacts with a version that is - * present in config-models-*.xml are never expired (in cd/publiccd we also consider the model versions in main/public). - * - * @author mpolden - */ -public class ArtifactExpirer extends ControllerMaintainer { - - private static final Logger log = Logger.getLogger(ArtifactExpirer.class.getName()); - - private static final Duration MIN_AGE = Duration.ofDays(14); - - private final Path configModelPath; - - public ArtifactExpirer(Controller controller, Duration interval) { - this(controller, interval, Paths.get(Defaults.getDefaults().underVespaHome("conf/configserver-app/"))); - } - - public ArtifactExpirer(Controller controller, Duration interval, Path configModelPath) { - super(controller, interval); - this.configModelPath = configModelPath; - } - - @Override - protected double maintain() { - VersionStatus versionStatus = controller().readVersionStatus(); - return controller().clouds().stream() - .flatMapToDouble(cloud -> - controller().serviceRegistry().artifactRegistry(cloud).stream() - .mapToDouble(artifactRegistry -> maintain(versionStatus, cloud, artifactRegistry))) - .average() - .orElse(1); - } - - private double maintain(VersionStatus versionStatus, CloudName cloudName, ArtifactRegistry artifactRegistry) { - try { - Instant now = controller().clock().instant(); - List<Artifact> artifactsToExpire = artifactRegistry.list().stream() - .filter(artifact -> isExpired(artifact, now, versionStatus, modelVersionsInUse())) - .toList(); - if (!artifactsToExpire.isEmpty()) { - log.log(INFO, "Expiring " + artifactsToExpire.size() + " artifacts in " + cloudName + ": " + artifactsToExpire); - artifactRegistry.deleteAll(artifactsToExpire); - } - return 0; - } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to expire artifacts in " + cloudName + ". Will retry in " + interval(), e); - return 1; - } - } - - /** Returns whether given artifact is expired */ - private boolean isExpired(Artifact artifact, Instant now, VersionStatus versionStatus, Set<Version> versionsInUse) { - List<VespaVersion> versions = versionStatus.versions(); - versionsInUse.addAll(versions.stream().map(VespaVersion::versionNumber).collect(Collectors.toSet())); - - if (versionsInUse.contains(artifact.version())) return false; - if (versionStatus.isActive(artifact.version())) return false; - if (artifact.createdAt().isAfter(now.minus(MIN_AGE))) return false; - - Version maxVersion = versions.stream().map(VespaVersion::versionNumber).max(Comparator.naturalOrder()).get(); - if (artifact.version().isAfter(maxVersion)) return false; // A future version - - return true; - } - - /** Model versions in use in this system, and, if this is a CD system, in the main/public system */ - private Set<Version> modelVersionsInUse() { - var system = controller().system(); - var versions = versionsForSystem(system); - - if (system == SystemName.PublicCd) - versions.addAll(versionsForSystem(SystemName.Public)); - else if (system == SystemName.cd) - versions.addAll(versionsForSystem(SystemName.main)); - - log.log(FINE, "model versions in use: " + versions); - return versions; - } - - private Set<Version> versionsForSystem(SystemName systemName) { - var versions = readConfigModelVersionsForSystem(systemName.name().toLowerCase()); - log.log(FINE, "model versions in use in " + systemName.name() + ": " + versions); - return versions; - } - - private Set<Version> readConfigModelVersionsForSystem(String systemName) { - List<String> lines = uncheck(() -> Files.readAllLines(configModelPath.resolve("config-models-" + systemName + ".xml"))); - var stringToMatch = "id='VespaModelFactory."; - return lines.stream() - .filter(line -> line.contains(stringToMatch)) - .map(line -> { - var start = line.indexOf(stringToMatch) + stringToMatch.length(); - int end = line.indexOf("'", start); - return line.substring(start, end); - }) - .map(Version::fromString) - .collect(Collectors.toSet()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java deleted file mode 100644 index 92aaacaa1f0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java +++ /dev/null @@ -1,321 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.application.api.Bcp; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.ApplicationController; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.ApplicationPatch; -import com.yahoo.vespa.hosted.controller.application.Deployment; - -import java.time.Duration; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.logging.Level; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * This computes, for every application deployment - * - the current fraction of the application's global traffic it receives. - * - the max fraction it can possibly receive, given its BCP group membership. - * - for each cluster in the deployment, average statistics from the other members in the group. - * - * These values are sent to a config server of each region where it is consumed by autoscaling. - * - * It depends on the traffic metrics collected by DeploymentMetricsMaintainer. - * - * @author bratseth - */ -public class BcpGroupUpdater extends ControllerMaintainer { - - private final ApplicationController applications; - private final NodeRepository nodeRepository; - private final Double successFactorBaseline; - - public BcpGroupUpdater(Controller controller, Duration duration, Double successFactorBaseline) { - super(controller, duration, successFactorBaseline); - this.applications = controller.applications(); - this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.successFactorBaseline = successFactorBaseline; - } - - public BcpGroupUpdater(Controller controller, Duration duration) { - this(controller, duration, 1.0); - } - - @Override - protected double maintain() { - Exception lastException = null; - int attempts = 0; - int failures = 0; - var metrics = collectClusterMetrics(); - for (var application : applications.asList()) { - for (var instance : application.instances().values()) { - for (var deployment : instance.productionDeployments().values()) { - if (shuttingDown()) return 0.0; - try { - attempts++; - var bcpGroups = BcpGroup.groupsFrom(instance, application.deploymentSpec()); - var patch = new ApplicationPatch(); - addTrafficShare(deployment, bcpGroups, patch); - addBcpGroupInfo(deployment.zone().region(), metrics.get(instance.id()), bcpGroups, patch); - - StringBuilder patchAsStringBuilder = new StringBuilder("Patch of instance ").append(instance.id().serializedForm()).append(": ") - .append("\n\tcurrentReadShare: ") - .append(patch.currentReadShare) - .append("\n\tmaxReadShare: ") - .append(patch.maxReadShare); - for (Map.Entry<String, ApplicationPatch.ClusterPatch> entry : patch.clusters.entrySet()) { - String key = entry.getKey(); - ApplicationPatch.ClusterPatch value = entry.getValue(); - patchAsStringBuilder.append("\n\tbcpGroupInfo for ").append(key).append(": ") - .append("\n\t\tcpuCostPerQuery: ") - .append(value.bcpGroupInfo.cpuCostPerQuery) - .append("\n\t\tqueryRate: ") - .append(value.bcpGroupInfo.queryRate) - .append("\n\t\tgrowthRateHeadroom: ") - .append(value.bcpGroupInfo.growthRateHeadroom); - } - log.log(Level.FINER, patchAsStringBuilder.toString()); - nodeRepository.patchApplication(deployment.zone(), instance.id(), patch); - } - catch (Exception e) { - // Some failures due to locked applications are expected and benign - failures++; - lastException = e; - } - } - } - } - double successFactorDeviation = asSuccessFactorDeviation(attempts, failures); - if ( successFactorDeviation == -successFactorBaseline ) - log.log(Level.WARNING, "Could not update traffic share on any applications", lastException); - else if ( successFactorDeviation < 0 ) - log.log(Level.FINE, "Could not update traffic share on all applications", lastException); - return successFactorDeviation; - } - - /** Adds deployment traffic share to the given patch. */ - private void addTrafficShare(Deployment deployment, List<BcpGroup> bcpGroups, ApplicationPatch patch) { - // maxReadShare / currentReadShare = how much additional traffic must the zone be able to handle - double currentReadShare = 0; // How much of the total traffic of the group(s) this is a member of does this deployment receive - double maxReadShare = 0; // How much of the total traffic of the group(s) this is a member of might this deployment receive if a member of the group fails - for (BcpGroup group : bcpGroups) { - if ( ! group.contains(deployment.zone().region())) continue; - - double deploymentQps = deployment.metrics().queriesPerSecond(); - double groupQps = group.totalQps(); - double fraction = group.fraction(deployment.zone().region()); - currentReadShare += groupQps == 0 ? 0 : fraction * deploymentQps / groupQps; - maxReadShare += group.size() == 1 - ? currentReadShare - : groupQps != 0 - ? fraction * (deploymentQps + group.maxQpsExcluding(deployment.zone().region()) / (group.size() - 1)) / groupQps - : 0; - } - patch.currentReadShare = currentReadShare; - patch.maxReadShare = maxReadShare; - } - - private Map<ApplicationId, Map<ClusterSpec.Id, ClusterDeploymentMetrics>> collectClusterMetrics() { - Map<ApplicationId, Map<ClusterSpec.Id, ClusterDeploymentMetrics>> metrics = new HashMap<>(); - for (var deploymentEntry : new HashMap<>(controller().applications().deploymentInfo()).entrySet()) { - if ( ! deploymentEntry.getKey().zoneId().environment().isProduction()) continue; - var appEntry = metrics.computeIfAbsent(deploymentEntry.getKey().applicationId(), __ -> new HashMap<>()); - for (var clusterEntry : deploymentEntry.getValue().clusters().entrySet()) { - var clusterMetrics = appEntry.computeIfAbsent(clusterEntry.getKey(), __ -> new ClusterDeploymentMetrics()); - clusterMetrics.put(deploymentEntry.getKey().zoneId().region(), - new DeploymentMetrics(clusterEntry.getValue().target().metrics().queryRate(), - clusterEntry.getValue().target().metrics().growthRateHeadroom(), - clusterEntry.getValue().target().metrics().cpuCostPerQuery())); - } - } - return metrics; - } - - /** Adds bcp group info to the given patch, for any clusters where we have information. */ - private void addBcpGroupInfo(RegionName regionToUpdate, Map<ClusterSpec.Id, ClusterDeploymentMetrics> metrics, - List<BcpGroup> bcpGroups, ApplicationPatch patch) { - if (metrics == null) return; - for (var clusterEntry : metrics.entrySet()) { - addClusterBcpGroupInfo(clusterEntry.getKey(), clusterEntry.getValue(), regionToUpdate, bcpGroups, patch); - } - } - - private void addClusterBcpGroupInfo(ClusterSpec.Id id, ClusterDeploymentMetrics metrics, - RegionName regionToUpdate, List<BcpGroup> bcpGroups, ApplicationPatch patch) { - var weightedSumOfMaxMetrics = DeploymentMetrics.empty(); - double sumOfCompleteMemberships = 0; - for (BcpGroup bcpGroup : bcpGroups) { - if ( ! bcpGroup.contains(regionToUpdate)) continue; - var groupMetrics = metrics.subsetOf(bcpGroup); - if ( ! groupMetrics.isCompleteExcluding(regionToUpdate, bcpGroup)) continue; - var max = groupMetrics.maxQueryRateExcluding(regionToUpdate, bcpGroup); - if (max.isEmpty()) continue; - - weightedSumOfMaxMetrics = weightedSumOfMaxMetrics.add(max.get().multipliedBy(bcpGroup.fraction(regionToUpdate))); - sumOfCompleteMemberships += bcpGroup.fraction(regionToUpdate); - } - if (sumOfCompleteMemberships > 0) - patch.clusters.put(id.value(), weightedSumOfMaxMetrics.dividedBy(sumOfCompleteMemberships).asClusterPatch()); - } - - /** - * A set of regions which will take over traffic from each other if one of them fails. - * Each region will take an equal share (modulated by fraction) of the failing region's traffic. - * - * A regions membership in a group may be partial, represented by a fraction [0, 1], - * in which case the other regions will collectively only take that fraction of the failing regions traffic, - * and symmetrically, the region will only take its fraction of its share of traffic of any other failing region. - */ - private static class BcpGroup { - - /** The instance which has this group. */ - private final Instance instance; - - /** Regions in this group, with their fractions. */ - private final Map<RegionName, Double> regions; - - /** Creates a group of a subset of the deployments in this instance. */ - private BcpGroup(Instance instance, Map<RegionName, Double> regions) { - this.instance = instance; - this.regions = regions; - } - - /** Returns the sum of the fractional memberships of this. */ - double size() { - return regions.values().stream().mapToDouble(f -> f).sum(); - } - - Set<RegionName> regions() { return regions.keySet(); } - - double fraction(RegionName region) { - return regions.getOrDefault(region, 0.0); - } - - boolean contains(RegionName region) { - return regions.containsKey(region); - } - - double totalQps() { - return instance.productionDeployments().values().stream() - .mapToDouble(i -> i.metrics().queriesPerSecond()).sum(); - } - - double maxQpsExcluding(RegionName region) { - return instance.productionDeployments().values().stream() - .filter(d -> ! d.zone().region().equals(region)) - .mapToDouble(d -> d.metrics().queriesPerSecond() * fraction(d.zone().region())) - .max() - .orElse(0); - } - - private static Bcp bcpOf(InstanceName instanceName, DeploymentSpec deploymentSpec) { - var instanceSpec = deploymentSpec.instance(instanceName); - if (instanceSpec.isEmpty()) return Bcp.empty(); - return instanceSpec.get().bcp(); - } - - private static Map<RegionName, Double> regionsFrom(Instance instance) { - return instance.productionDeployments().values().stream() - .collect(Collectors.toMap(deployment -> deployment.zone().region(), __ -> 1.0)); - } - - private static Map<RegionName, Double> regionsFrom(Bcp.Group groupSpec) { - return groupSpec.members().stream() - .collect(Collectors.toMap(member -> member.region(), member -> member.fraction())); - } - - static List<BcpGroup> groupsFrom(Instance instance, DeploymentSpec deploymentSpec) { - Bcp bcp = bcpOf(instance.name(), deploymentSpec); - if (bcp.isEmpty()) - return List.of(new BcpGroup(instance, regionsFrom(instance))); - return bcp.groups().stream().map(groupSpec -> new BcpGroup(instance, regionsFrom(groupSpec))).toList(); - } - - } - - record ApplicationClusterKey(ApplicationId application, ClusterSpec.Id cluster) { } - - static class ClusterDeploymentMetrics { - - private final Map<RegionName, DeploymentMetrics> deploymentMetrics; - - public ClusterDeploymentMetrics() { - this.deploymentMetrics = new ConcurrentHashMap<>(); - } - - public ClusterDeploymentMetrics(Map<RegionName, DeploymentMetrics> deploymentMetrics) { - this.deploymentMetrics = new ConcurrentHashMap<>(deploymentMetrics); - } - - void put(RegionName region, DeploymentMetrics metrics) { - deploymentMetrics.put(region, metrics); - } - - ClusterDeploymentMetrics subsetOf(BcpGroup group) { - Map<RegionName, DeploymentMetrics> filteredMetrics = new HashMap<>(); - for (var entry : deploymentMetrics.entrySet()) { - if (group.contains(entry.getKey())) - filteredMetrics.put(entry.getKey(), entry.getValue()); - } - return new ClusterDeploymentMetrics(filteredMetrics); - } - - /** Returns whether this has deployment metrics for each of the deployments in the given instance. */ - boolean isCompleteExcluding(RegionName regionToExclude, BcpGroup bcpGroup) { - return regionsExcluding(regionToExclude, bcpGroup).allMatch(region -> deploymentMetrics.containsKey(region)); - } - - /** Returns the metrics with the max query rate among the given instance, if any. */ - Optional<DeploymentMetrics> maxQueryRateExcluding(RegionName regionToExclude, BcpGroup bcpGroup) { - return regionsExcluding(regionToExclude, bcpGroup) - .map(region -> deploymentMetrics.get(region)) - .max(Comparator.comparingDouble(m -> m.queryRate)); - } - - private Stream<RegionName> regionsExcluding(RegionName regionToExclude, BcpGroup bcpGroup) { - return bcpGroup.regions().stream() - .filter(region -> ! region.equals(regionToExclude)); - } - - } - - /** Metrics for a given application, cluster and deployment. */ - record DeploymentMetrics(double queryRate, double growthRateHeadroom, double cpuCostPerQuery) { - - public ApplicationPatch.ClusterPatch asClusterPatch() { - return new ApplicationPatch.ClusterPatch(new ApplicationPatch.BcpGroupInfo(queryRate, growthRateHeadroom, cpuCostPerQuery)); - } - - DeploymentMetrics dividedBy(double d) { - return new DeploymentMetrics(queryRate / d, growthRateHeadroom / d, cpuCostPerQuery / d); - } - - DeploymentMetrics multipliedBy(double m) { - return new DeploymentMetrics(queryRate * m, growthRateHeadroom * m, cpuCostPerQuery * m); - } - - DeploymentMetrics add(DeploymentMetrics other) { - return new DeploymentMetrics(queryRate + other.queryRate, - growthRateHeadroom + other.growthRateHeadroom, - cpuCostPerQuery + other.cpuCostPerQuery); - } - - public static DeploymentMetrics empty() { return new DeploymentMetrics(0, 0, 0); } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java deleted file mode 100644 index 426abb16549..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.Controller; - -import java.time.Duration; -import java.util.EnumSet; - -/** - * @author olaa - */ -public class BillingDatabaseMaintainer extends ControllerMaintainer { - - public BillingDatabaseMaintainer(Controller controller, Duration interval) { - super(controller, interval, null, EnumSet.of(SystemName.PublicCd)); - } - - @Override - protected double maintain() { - controller().serviceRegistry().billingDatabase().maintain(); - return 0.0; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java deleted file mode 100644 index 7434fce31bf..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.LockedTenant; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Bill; -import com.yahoo.vespa.hosted.controller.api.integration.billing.BillStatus; -import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; -import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClient; -import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporter; -import com.yahoo.vespa.hosted.controller.api.integration.billing.InvoiceUpdate; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -public class BillingReportMaintainer extends ControllerMaintainer { - - private final BillingReporter reporter; - private final BillingController billing; - private final BillingDatabaseClient databaseClient; - - private final PlanRegistry plans; - - public BillingReportMaintainer(Controller controller, Duration interval) { - super(controller, interval, null, Set.of(SystemName.Public, SystemName.PublicCd)); - reporter = controller.serviceRegistry().billingReporter(); - billing = controller.serviceRegistry().billingController(); - databaseClient = controller.serviceRegistry().billingDatabase(); - plans = controller.serviceRegistry().planRegistry(); - } - - @Override - protected double maintain() { - maintainTenants(); - - var updates = maintainInvoices(); - log.fine("Updated invoices: " + updates); - - return 0.0; - } - - private void maintainTenants() { - var tenants = cloudTenants(); - var tenantNames = List.copyOf(tenants.keySet()); - var billableTenants = billableTenants(tenantNames); - - billableTenants.forEach(tenant -> { - controller().tenants().lockIfPresent(tenant, LockedTenant.Cloud.class, locked -> { - var ref = reporter.maintainTenant(locked.get()); - if (locked.get().billingReference().isEmpty() || ! locked.get().billingReference().get().equals(ref)) { - controller().tenants().store(locked.with(ref)); - } - }); - }); - } - - List<InvoiceUpdate> maintainInvoices() { - var updates = new ArrayList<InvoiceUpdate>(); - - var tenants = cloudTenants(); - var billsNeedingMaintenance = databaseClient.readBills().stream() - .filter(bill -> bill.getExportedId().isPresent()) - .filter(exported -> ! exported.status().isFinal()) - .toList(); - - for (var bill : billsNeedingMaintenance) { - var exportedId = bill.getExportedId().orElseThrow(); - var update = reporter.maintainInvoice(tenants.get(bill.tenant()), bill); - switch (update.type()) { - case UNMODIFIED -> log.finer(() ->invoiceMessage(bill.id(), exportedId) + " was not modified"); - case MODIFIED -> log.fine(invoiceMessage(bill.id(), exportedId) + " was updated with " + update.itemsUpdate().get()); - case UNMODIFIABLE -> { - // This check is needed to avoid setting the status multiple times - if (bill.status() != BillStatus.FROZEN) { - log.fine(() -> invoiceMessage(bill.id(), exportedId) + " is now unmodifiable"); - databaseClient.setStatus(bill.id(), "system", BillStatus.FROZEN); - } - } - case REMOVED -> { - log.fine(() -> invoiceMessage(bill.id(), exportedId) + " has been deleted in the external system"); - // Reset the exportedId to null, so that we don't maintain it again - databaseClient.setExportedInvoiceId(bill.id(), null); - } - case PAID -> { - log.fine(() -> invoiceMessage(bill.id(), exportedId) + " has been paid in the external system"); - databaseClient.setStatus(bill.id(), "system", BillStatus.SUCCESSFUL); - } - case VOIDED -> { - log.fine(() -> invoiceMessage(bill.id(), exportedId) + " has been voided in the external system"); - databaseClient.setStatus(bill.id(), "system", BillStatus.VOID); - } - } - updates.add(update); - } - return updates; - } - - private String invoiceMessage(Bill.Id billId, String invoiceId) { - return "Invoice '" + invoiceId + "' for bill '" + billId.value() + "'"; - } - - private Map<TenantName, CloudTenant> cloudTenants() { - return controller().tenants().asList() - .stream() - .filter(CloudTenant.class::isInstance) - .map(CloudTenant.class::cast) - .collect(Collectors.toMap( - Tenant::name, - Function.identity())); - } - - private List<Plan> billablePlans() { - return plans.all().stream() - .filter(Plan::isBilled) - .toList(); - } - - private List<TenantName> billableTenants(List<TenantName> tenants) { - return billablePlans().stream() - .flatMap(p -> billing.tenantsWithPlan(tenants, p.id()).stream()) - .toList(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java deleted file mode 100644 index 5e6e495e473..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import ai.vespa.metrics.ControllerMetrics; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.jdisc.Metric; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.IntFlag; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.StringFlag; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; -import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; -import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; -import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Manages a pool of ready-to-use endpoint certificates. - * - * @author andreer - */ -public class CertificatePoolMaintainer extends ControllerMaintainer { - - private static final Logger log = Logger.getLogger(CertificatePoolMaintainer.class.getName()); - - private final CuratorDb curator; - private final SecretStore secretStore; - private final EndpointCertificateProvider endpointCertificateProvider; - private final Metric metric; - private final Controller controller; - private final IntFlag certPoolSize; - private final StringFlag endpointCertificateAlgo; - private final BooleanFlag useAlternateCertProvider; - - public CertificatePoolMaintainer(Controller controller, Metric metric, Duration interval) { - super(controller, interval); - this.controller = controller; - this.secretStore = controller.secretStore(); - this.certPoolSize = PermanentFlags.CERT_POOL_SIZE.bindTo(controller.flagSource()); - this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); - this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); - this.curator = controller.curator(); - this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); - this.metric = metric; - } - - protected double maintain() { - try { - moveRequestedCertsToReady(); - List<UnassignedCertificate> certificatePool = curator.readUnassignedCertificates(); - - // Create metric for available certificates in the pool as a fraction of configured size - int poolSize = certPoolSize.value(); - long available = certificatePool.stream().filter(c -> c.state() == UnassignedCertificate.State.ready).count(); - metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? ((double)available/poolSize) : 1.0), metric.createContext(Map.of())); - - if (certificatePool.size() < poolSize) { - provisionCertificate(); - } - } catch (Exception e) { - log.log(Level.SEVERE, "Failed to maintain certificate pool", e); - return 1.0; - } - return 0.0; - } - - private void moveRequestedCertsToReady() { - try (Mutex lock = controller.curator().lockCertificatePool()) { - for (UnassignedCertificate cert : curator.readUnassignedCertificates()) { - if (cert.state() == UnassignedCertificate.State.ready) continue; - try { - OptionalInt maxKeyVersion = secretStore.listSecretVersions(cert.certificate().keyName()).stream().mapToInt(i -> i).max(); - OptionalInt maxCertVersion = secretStore.listSecretVersions(cert.certificate().certName()).stream().mapToInt(i -> i).max(); - if (maxKeyVersion.isPresent() && maxCertVersion.equals(maxKeyVersion)) { - curator.writeUnassignedCertificate(cert.withState(UnassignedCertificate.State.ready)); - log.log(Level.INFO, "Readied certificate %s".formatted(cert.id())); - } - } catch (SecretNotFoundException s) { - // Likely because the certificate is very recently provisioned - ignore till next time - should we log? - log.log(Level.INFO, "Cannot ready certificate %s yet, will retry in %s".formatted(cert.id(), interval())); - } - } - } - } - - private void provisionCertificate() { - try (Mutex lock = controller.curator().lockCertificatePool()) { - Set<String> existingNames = controller.curator().readUnassignedCertificates().stream().map(UnassignedCertificate::id).collect(Collectors.toSet()); - - curator.readAssignedCertificates().stream() - .map(AssignedCertificate::certificate) - .map(EndpointCertificate::generatedId) - .forEach(id -> id.ifPresent(existingNames::add)); - - String id = generateId(); - while (existingNames.contains(id)) id = generateId(); - List<String> dnsNames = wildcardDnsNames(id); - EndpointCertificate cert = endpointCertificateProvider.requestCaSignedCertificate( - "preprovisioned.%s".formatted(id), - dnsNames, - Optional.empty(), - endpointCertificateAlgo.value(), - useAlternateCertProvider.value()).withGeneratedId(id); - - UnassignedCertificate certificate = new UnassignedCertificate(cert, UnassignedCertificate.State.requested); - curator.writeUnassignedCertificate(certificate); - } - } - - private List<String> wildcardDnsNames(String id) { - DeploymentId defaultDeployment = new DeploymentId(ApplicationId.defaultId(), ZoneId.defaultId()); - return controller.routing().certificateDnsNames(defaultDeployment, // Not used for non-legacy names - DeploymentSpec.empty, // Not used for non-legacy names - id, - false); - } - - private String generateId() { - return GeneratedEndpoint.createPart(controller.random(true)); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java deleted file mode 100644 index 51720806371..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java +++ /dev/null @@ -1,267 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * @author smorgrav - */ -public class ChangeManagementAssessor { - - private final NodeRepository nodeRepository; - - public ChangeManagementAssessor(NodeRepository nodeRepository) { - this.nodeRepository = nodeRepository; - } - - public Assessment assessment(List<String> impactedHostnames, ZoneId zone) { - return assessmentInner(impactedHostnames, nodeRepository.list(zone, NodeFilter.all()), zone); - } - - Assessment assessmentInner(List<String> impactedHostnames, List<Node> allNodes, ZoneId zone) { - List<String> impactedParentHosts = toParentHosts(impactedHostnames, allNodes); - // Group impacted application nodes by parent host - Map<Node, List<Node>> prParentHost = allNodes.stream() - .filter(node -> node.state() == Node.State.active) //TODO look at more states? - .filter(node -> impactedParentHosts.contains(node.parentHostname().map(HostName::value).orElse(""))) - .collect(Collectors.groupingBy(node -> - allNodes.stream() - .filter(parent -> parent.hostname().equals(node.parentHostname().get())) - .findFirst().orElseThrow() - )); - - // Group nodes pr cluster - Map<Cluster, List<Node>> prCluster = prParentHost.values() - .stream() - .flatMap(Collection::stream) - .collect(Collectors.groupingBy(ChangeManagementAssessor::clusterKey)); - - var tenantHosts = prParentHost.keySet().stream() - .filter(node -> node.type() == NodeType.host) - .map(node -> node.hostname()) - .toList(); - - boolean allHostsReplacable = tenantHosts.isEmpty() || nodeRepository.isReplaceable(zone, tenantHosts); - - // Report assessment pr cluster - var clusterAssessments = prCluster.entrySet().stream().map((entry) -> { - Cluster cluster = entry.getKey(); - List<Node> nodes = entry.getValue(); - - long[] totalStats = clusterStats(cluster, allNodes); - long[] impactedStats = clusterStats(cluster, nodes); - - ClusterAssessment assessment = new ClusterAssessment(); - assessment.app = cluster.getApp(); - assessment.zone = zone.value(); - assessment.cluster = cluster.getClusterType() + ":" + cluster.getClusterId(); - assessment.clusterSize = totalStats[0]; - assessment.clusterImpact = impactedStats[0]; - assessment.groupsTotal = totalStats[1]; - assessment.groupsImpact = impactedStats[1]; - - - // TODO check upgrade policy - assessment.upgradePolicy = "na"; - // TODO do some heuristic on suggestion action - assessment.suggestedAction = allHostsReplacable ? "Retire all hosts" : "nothing"; - // TODO do some heuristic on impact - assessment.impact = getImpact(cluster, impactedStats, totalStats); - - return assessment; - }).toList(); - - var hostAssessments = prParentHost.entrySet().stream().map((entry) -> { - HostAssessment hostAssessment = new HostAssessment(); - hostAssessment.hostName = entry.getKey().hostname().value(); - hostAssessment.switchName = entry.getKey().switchHostname().orElse(null); - hostAssessment.numberOfChildren = entry.getValue().size(); - - //TODO: Some better heuristic for what's considered problematic - hostAssessment.numberOfProblematicChildren = (int) entry.getValue().stream() - .mapToInt(node -> prCluster.get(clusterKey(node)).size()) - .filter(i -> i > 1) - .count(); - - return hostAssessment; - }).toList(); - - return new Assessment(clusterAssessments, hostAssessments); - } - - private List<String> toParentHosts(List<String> impactedHostnames, List<Node> allNodes) { - return impactedHostnames.stream() - .flatMap(hostname -> - allNodes.stream() - .filter(node -> List.of(NodeType.config, NodeType.proxy, NodeType.host).contains(node.type())) - .filter(node -> hostname.equals(node.hostname().value()) || hostname.equals(node.parentHostname().map(HostName::value).orElse(""))) - .map(node -> { - if (node.type() == NodeType.host) - return node.hostname().value(); - return node.parentHostname().get().value(); - }).findFirst().stream() - ) - .toList(); - } - - private static Cluster clusterKey(Node node) { - if (node.owner().isEmpty()) - return Cluster.EMPTY; - String appId = node.owner().get().serializedForm(); - return new Cluster(node.clusterType(), node.clusterId(), appId, node.type()); - } - - private static long[] clusterStats(Cluster cluster, List<Node> containerNodes) { - List<Node> clusterNodes = containerNodes.stream().filter(node -> cluster.equals(clusterKey(node))).toList(); - long groups = clusterNodes.stream().map(Node::group).distinct().count(); - return new long[] { clusterNodes.size(), groups}; - } - - private String getImpact(Cluster cluster, long[] impactedStats, long[] totalStats) { - switch (cluster.getNodeType()) { - case tenant: - return getTenantImpact(cluster, impactedStats, totalStats); - case proxy: - return getProxyImpact(impactedStats[0], totalStats[0]); - case config: - return getConfigServerImpact(impactedStats[0]); - default: - return "Unkown impact"; - } - } - - private String getTenantImpact(Cluster cluster, long[] impactedStats, long[] totalStats) { - switch (cluster.getClusterType()) { - case container: - return getContainerImpact(impactedStats[0], totalStats[0]); - case content: - case combined: - return getContentImpact(totalStats[1] > 1, impactedStats[0], impactedStats[1]); - default: - return "Unknown impact"; - } - } - - private String getProxyImpact(long impactedNodes, long totalNodes) { - int impact = (int) (100.0 * impactedNodes / totalNodes); - return impact + "% of routing nodes impacted. Consider reprovisioning if too many"; - } - - private String getConfigServerImpact(long impactedNodes) { - if (impactedNodes == 1) { - return "Acceptable impact"; - } - return "Large impact. Consider reprovisioning one or more config servers"; - } - - private String getContainerImpact(long impactedNodes, long totalNodes) { - if ((double) impactedNodes / totalNodes <= 0.1) { - return "Impact not larger than upgrade policy"; - } - return "Impact larger than upgrade policy"; - } - - private String getContentImpact(boolean isGrouped, long impactedNodes, long impactedGroups) { - if ((isGrouped && impactedGroups == 1) || impactedNodes == 1) - return "Impact not larger than upgrade policy"; - return "Impact larger than upgrade policy"; - } - - - public static class Assessment { - List<ClusterAssessment> clusterAssessments; - List<HostAssessment> hostAssessments; - - Assessment(List<ClusterAssessment> clusterAssessments, List<HostAssessment> hostAssessments) { - this.clusterAssessments = clusterAssessments; - this.hostAssessments = hostAssessments; - } - - public List<ClusterAssessment> getClusterAssessments() { - return clusterAssessments; - } - - public List<HostAssessment> getHostAssessments() { - return hostAssessments; - } - } - - public static class ClusterAssessment { - public String app; - public String zone; - public String cluster; - public long clusterImpact; - public long clusterSize; - public long groupsImpact; - public long groupsTotal; - public String upgradePolicy; - public String suggestedAction; - public String impact; - } - - public static class HostAssessment { - public String hostName; - public String switchName; - public int numberOfChildren; - public int numberOfProblematicChildren; - } - - private static class Cluster { - private Node.ClusterType clusterType; - private String clusterId; - private String app; - private NodeType nodeType; - - public final static Cluster EMPTY = new Cluster(Node.ClusterType.unknown, "na", "na", NodeType.tenant); - - public Cluster(Node.ClusterType clusterType, String clusterId, String app, NodeType nodeType) { - this.clusterType = clusterType; - this.clusterId = clusterId; - this.app = app; - this.nodeType = nodeType; - } - - public Node.ClusterType getClusterType() { - return clusterType; - } - - public String getClusterId() { - return clusterId; - } - - public String getApp() { - return app; - } - - public NodeType getNodeType() { - return nodeType; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Cluster cluster = (Cluster) o; - return Objects.equals(clusterType, cluster.clusterType) && - Objects.equals(clusterId, cluster.clusterId) && - Objects.equals(app, cluster.app); - } - - @Override - public int hashCode() { - return Objects.hash(clusterType, clusterId, app); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java deleted file mode 100644 index 9f687249f38..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestSource; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.time.Duration; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * @author olaa - */ -public class ChangeRequestMaintainer extends ControllerMaintainer { - - private final Logger logger = Logger.getLogger(ChangeRequestMaintainer.class.getName()); - private final ChangeRequestClient changeRequestClient; - private final CuratorDb curator; - private final NodeRepository nodeRepository; - private final SystemName system; - - public ChangeRequestMaintainer(Controller controller, Duration interval) { - super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); - this.changeRequestClient = controller.serviceRegistry().changeRequestClient(); - this.curator = controller.curator(); - this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.system = controller.system(); - } - - - @Override - protected double maintain() { - var currentChangeRequests = pruneOldChangeRequests(); - var changeRequests = changeRequestClient.getChangeRequests(currentChangeRequests); - - logger.fine(() -> "Found requests: " + changeRequests); - storeChangeRequests(changeRequests); - - return 1.0; - } - - private void storeChangeRequests(List<ChangeRequest> changeRequests) { - var existingChangeRequests = curator.readChangeRequests() - .stream() - .collect(Collectors.toMap(ChangeRequest::getId, Function.identity())); - - var hostsByZone = hostsByZone(); - // Create or update requests in curator - try (var lock = curator.lockChangeRequests()) { - changeRequests.forEach(changeRequest -> { - var optionalZone = inferZone(changeRequest, hostsByZone); - optionalZone.ifPresentOrElse(zone -> { - var vcmr = existingChangeRequests - .getOrDefault(changeRequest.getId(), new VespaChangeRequest(changeRequest, zone)) - .withSource(changeRequest.getChangeRequestSource()) - .withImpact(changeRequest.getImpact()) - .withApproval(changeRequest.getApproval()); - logger.fine(() -> "Storing " + vcmr); - curator.writeChangeRequest(vcmr); - }, - () -> approveChangeRequest(changeRequest)); - }); - } - } - - // Deletes closed change requests older than 7 days, returns the current list of requests - private List<ChangeRequest> pruneOldChangeRequests() { - List<ChangeRequest> currentChangeRequests = new ArrayList<>(); - - try (var lock = curator.lockChangeRequests()) { - for (var changeRequest : curator.readChangeRequests()) { - if (shouldDeleteChangeRequest(changeRequest.getChangeRequestSource())) { - curator.deleteChangeRequest(changeRequest); - } else { - currentChangeRequests.add(changeRequest); - } - } - } - return currentChangeRequests; - } - - private Map<ZoneId, List<String>> hostsByZone() { - return controller().zoneRegistry() - .zones() - .reachable() - .in(Environment.prod) - .ids() - .stream() - .collect(Collectors.toMap( - zone -> zone, - zone -> nodeRepository.list(zone, NodeFilter.all()) - .stream() - .map(node -> node.hostname().value()) - .toList() - )); - } - - private Optional<ZoneId> inferZone(ChangeRequest changeRequest, Map<ZoneId, List<String>> hostsByZone) { - return hostsByZone.entrySet().stream() - .filter(entry -> !Collections.disjoint(entry.getValue(), changeRequest.getImpactedHosts())) - .map(Map.Entry::getKey) - .findFirst(); - } - - private boolean shouldDeleteChangeRequest(ChangeRequestSource source) { - return source.isClosed() && - source.plannedStartTime() - .plus(Duration.ofDays(7)) - .isBefore(ZonedDateTime.now()); - } - - private void approveChangeRequest(ChangeRequest changeRequest) { - if (system.equals(SystemName.main) && - changeRequest.getApproval() == ChangeRequest.Approval.REQUESTED) { - logger.info("Approving " + changeRequest.getChangeRequestSource().id()); - changeRequestClient.approveChangeRequest(changeRequest); - } - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java deleted file mode 100644 index fedfea792f3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.time.Duration; -import java.util.List; -import java.util.Set; -import java.util.logging.Logger; - -import static java.util.logging.Level.WARNING; - -/** - * Verifies the cloud accounts that may be used by a given user have applied the enclave template - * and extracts the version of the applied template. - * - * All maintainers that operate on external cloud accounts should use the list on the Tenant instance - * maintained by this class rather than the cloud-accounts feature flag. - * - * The template version can be used to determine if new features can be enabled for the cloud account. - * - * @author freva - */ -public class CloudAccountVerifier extends ControllerMaintainer { - - private static final Logger logger = Logger.getLogger(CloudAccountVerifier.class.getName()); - - CloudAccountVerifier(Controller controller, Duration interval) { - super(controller, interval, null, Set.of(SystemName.PublicCd, SystemName.Public)); - } - - @Override - protected double maintain() { - int attempts = 0, failures = 0; - for (Tenant tenant : controller().tenants().asList()) { - try { - attempts++; - List<CloudAccountInfo> cloudAccountInfos = controller().applications().accountsOf(tenant.name()).stream() - .flatMap(account -> controller().serviceRegistry() - .archiveService() - .getEnclaveTemplateVersion(account) - .map(version -> new CloudAccountInfo(account, version)) - .stream()) - .toList(); - controller().tenants().updateCloudAccounts(tenant.name(), cloudAccountInfos); - } catch (RuntimeException e) { - logger.log(WARNING, "Failed to verify cloud accounts for tenant " + tenant.name(), e); - failures++; - } - } - return asSuccessFactorDeviation(attempts, failures); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java deleted file mode 100644 index 73204fb1655..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; - -public class CloudDatabaseMaintainer extends ControllerMaintainer { - - public CloudDatabaseMaintainer(Controller controller, Duration interval) { - super(controller, interval); - } - - @Override - protected double maintain() { - try { - var tenants = controller().tenants().asList().stream().map(Tenant::name).toList(); - controller().serviceRegistry().billingController().updateCache(tenants); - } catch (Exception e) { - log.warning("Could not update cloud database cache: " + Exceptions.toMessageString(e)); - return 1.0; - } - return 0.0; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java deleted file mode 100644 index 1e261f78db3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.ListFlag; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; -import com.yahoo.vespa.hosted.controller.notification.MailTemplating; -import com.yahoo.vespa.hosted.controller.notification.Notification; -import com.yahoo.vespa.hosted.controller.notification.NotificationSource; -import com.yahoo.vespa.hosted.controller.persistence.TrialNotifications; -import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import java.util.function.Predicate; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.EXPIRED; -import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.EXPIRES_IMMEDIATELY; -import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.MID_CHECK_IN; -import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.SIGNED_UP; -import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.UNKNOWN; - -/** - * Expires unused tenants from Vespa Cloud. - * - * @author ogronnesby - */ -public class CloudTrialExpirer extends ControllerMaintainer { - private static final Logger log = Logger.getLogger(CloudTrialExpirer.class.getName()); - - private static final Duration nonePlanAfter = Duration.ofDays(14); - private static final Duration tombstoneAfter = Duration.ofDays(91); - private final ListFlag<String> extendedTrialTenants; - private final BooleanFlag cloudTrialNotificationEnabled; - - public CloudTrialExpirer(Controller controller, Duration interval) { - super(controller, interval, null, SystemName.allOf(SystemName::isPublic)); - this.extendedTrialTenants = PermanentFlags.EXTENDED_TRIAL_TENANTS.bindTo(controller().flagSource()); - this.cloudTrialNotificationEnabled = Flags.CLOUD_TRIAL_NOTIFICATIONS.bindTo(controller().flagSource()); - } - - @Override - protected double maintain() { - var a = tombstoneNonePlanTenants(); - var b = moveInactiveTenantsToNonePlan(); - var c = notifyTenants(); - return (a ? 0.0 : -(1D/3)) + (b ? 0.0 : -(1D/3) + (c ? 0.0 : -(1D/3))); - } - - private boolean moveInactiveTenantsToNonePlan() { - var idleTrialTenants = controller().tenants().asList().stream() - .filter(this::tenantIsCloudTenant) - .filter(this::tenantIsNotExemptFromExpiry) - .filter(this::tenantHasNoDeployments) - .filter(this::tenantHasTrialPlan) - .filter(tenantReadersNotLoggedIn(nonePlanAfter)) - .toList(); - - if (! idleTrialTenants.isEmpty()) { - var tenants = idleTrialTenants.stream().map(Tenant::name).map(TenantName::value).collect(Collectors.joining(", ")); - log.info("Setting tenants to 'none' plan: " + tenants); - } - - return setPlanNone(idleTrialTenants); - } - - private boolean tombstoneNonePlanTenants() { - var idleOldPlanTenants = controller().tenants().asList().stream() - .filter(this::tenantIsCloudTenant) - .filter(this::tenantIsNotExemptFromExpiry) - .filter(this::tenantHasNoDeployments) - .filter(this::tenantHasNonePlan) - .filter(tenantReadersNotLoggedIn(tombstoneAfter)) - .toList(); - - if (! idleOldPlanTenants.isEmpty()) { - var tenants = idleOldPlanTenants.stream().map(Tenant::name).map(TenantName::value).collect(Collectors.joining(", ")); - log.info("Setting tenants as tombstoned: " + tenants); - } - - return tombstoneTenants(idleOldPlanTenants); - } - - /* - * Trial plan notification states. Transition to a new state triggers a notification/email - * - SIGNED_UP: Tenant has signed up for trial - * - MID_CHECK_IN: Tenant is halfway through trial (7 days) - * - EXPIRES_IMMEDIATELY: Tenant has 1 day left of trial - * - EXPIRED: Tenant has expired - */ - private boolean notifyTenants() { - try { - var currentStatus = controller().curator().readTrialNotifications() - .map(TrialNotifications::tenants).orElse(List.of()); - log.fine(() -> "Current: %s".formatted(currentStatus)); - var currentStatusByTenant = new HashMap<TenantName, TrialNotifications.Status>(); - currentStatus.forEach(status -> currentStatusByTenant.put(status.tenant(), status)); - var updatedStatus = new ArrayList<TrialNotifications.Status>(); - var now = controller().clock().instant(); - - for (var tenant : controller().tenants().asList()) { - - var status = currentStatusByTenant.get(tenant.name()); - var state = status == null ? UNKNOWN : status.state(); - var plan = controller().serviceRegistry().billingController().getPlan(tenant.name()).value(); - var ageInDays = Duration.between(tenant.createdAt(), now).toDays(); - - // TODO Replace stubs with proper email content stored in templates. - - var enabled = cloudTrialNotificationEnabled.with(FetchVector.Dimension.TENANT_ID, tenant.name().value()).value(); - if (!enabled) { - if (status != null) updatedStatus.add(status); - } else if (!List.of("none", "trial").contains(plan)) { - // Ignore tenants that are on a paid plan and skip from inclusion in updated data structure - } else if (status == null && "trial".equals(plan) && ageInDays <= 1) { - updatedStatus.add(updatedStatus(tenant, now, SIGNED_UP)); - notifySignup(tenant); - } else if ("none".equals(plan) && !List.of(EXPIRED).contains(state)) { - updatedStatus.add(updatedStatus(tenant, now, EXPIRED)); - notifyExpired(tenant); - } else if ("trial".equals(plan) && ageInDays >= 13 - && !List.of(EXPIRES_IMMEDIATELY, EXPIRED).contains(state)) { - updatedStatus.add(updatedStatus(tenant, now, EXPIRES_IMMEDIATELY)); - notifyExpiresImmediately(tenant); - } else if ("trial".equals(plan) && ageInDays >= 7 - && !List.of(MID_CHECK_IN, EXPIRES_IMMEDIATELY, EXPIRED).contains(state)) { - updatedStatus.add(updatedStatus(tenant, now, MID_CHECK_IN)); - notifyMidCheckIn(tenant); - } else { - updatedStatus.add(status); - } - } - log.fine(() -> "Updated: %s".formatted(updatedStatus)); - controller().curator().writeTrialNotifications(new TrialNotifications(updatedStatus)); - return true; - } catch (Exception e) { - log.log(Level.WARNING, "Failed to process trial notifications", e); - return false; - } - } - - private void notifySignup(Tenant tenant) { - var consoleMsg = "Welcome to Vespa Cloud trial! [Manage plan](%s)".formatted(billingUrl(tenant)); - queueNotification(tenant, consoleMsg, "Welcome to Vespa Cloud", MailTemplating.Template.TRIAL_SIGNED_UP); - } - - private void notifyMidCheckIn(Tenant tenant) { - var consoleMsg = "You're halfway through the **14 day** trial period. [Manage plan](%s)".formatted(billingUrl(tenant)); - queueNotification(tenant, consoleMsg, "How is your Vespa Cloud trial going?", MailTemplating.Template.TRIAL_MIDWAY_CHECKIN); - } - - private void notifyExpiresImmediately(Tenant tenant) { - var consoleMsg = "Your Vespa Cloud trial expires **tomorrow**. [Manage plan](%s)".formatted(billingUrl(tenant)); - queueNotification(tenant, consoleMsg, "Your Vespa Cloud trial expires tomorrow", MailTemplating.Template.TRIAL_EXPIRES_IMMEDIATELY); - } - - private void notifyExpired(Tenant tenant) { - var consoleMsg = "Your Vespa Cloud trial has expired. [Upgrade plan](%s)".formatted(billingUrl(tenant)); - queueNotification(tenant, consoleMsg, "Your Vespa Cloud trial has expired", MailTemplating.Template.TRIAL_EXPIRED); - } - - private void queueNotification(Tenant tenant, String consoleMsg, String emailSubject, MailTemplating.Template template) { - var mail = Optional.of(Notification.MailContent.fromTemplate(MailTemplating.Template.DEFAULT_MAIL_CONTENT) - .subject(emailSubject) - .with("mailMessageTemplate", template.getId()) - .with("mailTitle", emailSubject) - .with("consoleLink", controller().serviceRegistry().consoleUrls().tenantOverview(tenant.name())) - .build()); - var source = NotificationSource.from(tenant.name()); - // Remove previous notification to ensure new notification is sent by email - controller().notificationsDb().removeNotification(source, Notification.Type.account); - controller().notificationsDb().setNotification( - source, Notification.Type.account, Notification.Level.info, consoleMsg, List.of(), mail); - } - - private String billingUrl(Tenant t) { return controller().serviceRegistry().consoleUrls().tenantBilling(t.name()); } - - private static TrialNotifications.Status updatedStatus(Tenant t, Instant i, TrialNotifications.State s) { - return new TrialNotifications.Status(t.name(), s, i); - } - - private boolean tenantIsCloudTenant(Tenant tenant) { - return tenant.type() == Tenant.Type.cloud; - } - - private Predicate<Tenant> tenantReadersNotLoggedIn(Duration duration) { - // returns true if no user has logged in to the tenant after (now - duration) - return (Tenant tenant) -> { - var timeLimit = controller().clock().instant().minus(duration); - return tenant.lastLoginInfo().get(LastLoginInfo.UserLevel.user) - .map(instant -> instant.isBefore(timeLimit)) - .orElse(false); - }; - } - - private boolean tenantHasTrialPlan(Tenant tenant) { - var planId = controller().serviceRegistry().billingController().getPlan(tenant.name()); - return "trial".equals(planId.value()); - } - - private boolean tenantHasNonePlan(Tenant tenant) { - var planId = controller().serviceRegistry().billingController().getPlan(tenant.name()); - return "none".equals(planId.value()); - } - - private boolean tenantIsNotExemptFromExpiry(Tenant tenant) { - return !extendedTrialTenants.value().contains(tenant.name().value()); - } - - private boolean tenantHasNoDeployments(Tenant tenant) { - return controller().applications().asList(tenant.name()).stream() - .flatMap(app -> app.instances().values().stream()) - .mapToLong(instance -> instance.deployments().values().size()) - .sum() == 0; - } - - private boolean setPlanNone(List<Tenant> tenants) { - var success = true; - for (var tenant : tenants) { - try { - controller().serviceRegistry().billingController().setPlan(tenant.name(), PlanId.from("none"), false, false); - } catch (RuntimeException e) { - log.info("Could not change plan for " + tenant.name() + ": " + e.getMessage()); - success = false; - } - } - return success; - } - - private boolean tombstoneTenants(List<Tenant> tenants) { - var success = true; - for (var tenant : tenants) { - success &= deleteApplicationsWithNoDeployments(tenant); - log.fine("Tombstoning empty tenant: " + tenant.name()); - try { - controller().tenants().delete(tenant.name(), Optional.empty(), false); - } catch (RuntimeException e) { - log.info("Could not tombstone tenant " + tenant.name() + ": " + e.getMessage()); - success = false; - } - } - return success; - } - - private boolean deleteApplicationsWithNoDeployments(Tenant tenant) { - // this method only removes applications with no active deployments in them - var success = true; - for (var application : controller().applications().asList(tenant.name())) { - try { - log.fine("Removing empty application: " + application.id()); - controller().applications().deleteApplication(application.id(), Optional.empty()); - } catch (RuntimeException e) { - log.info("Could not removing application " + application.id() + ": " + e.getMessage()); - success = false; - } - } - return success; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java deleted file mode 100644 index e0db7780fbb..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.LockedTenant; -import com.yahoo.vespa.hosted.controller.TenantController; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; -import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.util.Optional; -import java.util.function.Predicate; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static java.util.logging.Level.FINE; - -/** - * Periodically fetch and store contact information for tenants. - * - * @author mpolden - */ -public class ContactInformationMaintainer extends ControllerMaintainer { - - private static final Logger log = Logger.getLogger(ContactInformationMaintainer.class.getName()); - - private final ContactRetriever contactRetriever; - - public ContactInformationMaintainer(Controller controller, Duration interval, Double successFactorBaseline) { - super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic)), successFactorBaseline); - this.contactRetriever = controller.serviceRegistry().contactRetriever(); - } - - @Override - protected double maintain() { - TenantController tenants = controller().tenants(); - int attempts = 0; - int failures = 0; - for (Tenant tenant : tenants.asList()) { - log.log(FINE, () -> "Updating contact information for " + tenant); - try { - attempts++; - switch (tenant.type()) { - case athenz: - tenants.lockIfPresent(tenant.name(), LockedTenant.Athenz.class, lockedTenant -> { - Contact contact = contactRetriever.getContact(lockedTenant.get().propertyId()); - log.log(FINE, () -> "Contact found for " + tenant + " was " + - (Optional.of(contact).equals(tenant.contact()) ? "un" : "") + "changed"); - tenants.store(lockedTenant.with(contact)); - }); - break; - case cloud: - break; - default: - throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); - } - } catch (Exception e) { - failures++; - log.log(Level.WARNING, "Failed to update contact information for " + tenant + ": " + - Exceptions.toMessageString(e) + ". Retrying in " + - interval()); - } - } - return asSuccessFactorDeviation(attempts, failures); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java deleted file mode 100644 index 3bc9126f835..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.concurrent.maintenance.JobMetrics; -import com.yahoo.concurrent.maintenance.Maintainer; -import com.yahoo.config.provision.SystemName; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.hosted.controller.Controller; - -import java.time.Duration; -import java.util.EnumSet; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -/** - * A maintainer is some job which runs at a fixed interval to perform some maintenance task in the controller. - * - * @author bratseth - */ -public abstract class ControllerMaintainer extends Maintainer { - - private final Controller controller; - - /** The systems in which this maintainer should run */ - private final Set<SystemName> activeSystems; - - - public ControllerMaintainer(Controller controller, Duration interval) { - this(controller, interval, null, EnumSet.allOf(SystemName.class), 1.0); - } - - public ControllerMaintainer(Controller controller, Duration interval, Double successFactorBaseline) { - this(controller, interval, null, EnumSet.allOf(SystemName.class), successFactorBaseline); - } - - public ControllerMaintainer(Controller controller, Duration interval, String name, Set<SystemName> activeSystems) { - this(controller, interval, name, activeSystems, 1.0); - } - - public ControllerMaintainer(Controller controller, Duration interval, String name, Set<SystemName> activeSystems, double successFactorBaseline) { - super(name, interval, controller.clock(), controller.jobControl(), - new ControllerJobMetrics(controller.metric()), controller.curator().cluster(), true, successFactorBaseline); - this.controller = controller; - this.activeSystems = Set.copyOf(Objects.requireNonNull(activeSystems)); - } - - protected Controller controller() { return controller; } - - @Override - public void run() { - if (!activeSystems.contains(controller.system())) return; - super.run(); - } - - private static class ControllerJobMetrics extends JobMetrics { - - private final Metric metric; - - public ControllerJobMetrics(Metric metric) { - this.metric = metric; - } - - @Override - public void completed(String job, double successFactorDeviation, long durationMs) { - metric.set("maintenance.successFactorDeviation", successFactorDeviation, metric.createContext(Map.of("job", job))); - metric.set("maintenance.duration", durationMs, metric.createContext(Map.of("job", job))); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java deleted file mode 100644 index 8d45fcb8878..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.component.annotation.Inject; -import com.yahoo.concurrent.maintenance.Maintainer; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; - -import java.time.Duration; -import java.time.temporal.TemporalUnit; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.stream.Collectors; - -import static java.time.temporal.ChronoUnit.HOURS; -import static java.time.temporal.ChronoUnit.MINUTES; -import static java.time.temporal.ChronoUnit.SECONDS; - -/** - * Maintenance jobs of the controller. - * Each maintenance job is a singleton instance of its implementing class, created and owned by this, - * and running its own dedicated thread. - * - * @author bratseth - */ -public class ControllerMaintenance extends AbstractComponent { - - private final Upgrader upgrader; - private final OsUpgradeScheduler osUpgradeScheduler; - private final List<Maintainer> maintainers = new CopyOnWriteArrayList<>(); - - @Inject - @SuppressWarnings("unused") // instantiated by Dependency Injection - public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement, AthenzClientFactory athenzClientFactory) { - Intervals intervals = new Intervals(controller.system()); - SuccessFactorBaseline successFactorBaseline = new SuccessFactorBaseline(controller.system()); - upgrader = new Upgrader(controller, intervals.defaultInterval); - osUpgradeScheduler = new OsUpgradeScheduler(controller, intervals.osUpgradeScheduler); - maintainers.add(upgrader); - maintainers.add(osUpgradeScheduler); - maintainers.addAll(osUpgraders(controller, intervals.osUpgrader)); - maintainers.add(new DeploymentExpirer(controller, intervals.defaultInterval)); - maintainers.add(new DeploymentInfoMaintainer(controller, intervals.deploymentInfoMaintainer, successFactorBaseline.deploymentInfoMaintainerBaseline)); - maintainers.add(new DeploymentUpgrader(controller, intervals.defaultInterval)); - maintainers.add(new DeploymentIssueReporter(controller, controller.serviceRegistry().deploymentIssues(), intervals.defaultInterval)); - maintainers.add(new MetricsReporter(controller, metric, athenzClientFactory.createZmsClient())); - maintainers.add(new OutstandingChangeDeployer(controller, intervals.outstandingChangeDeployer)); - maintainers.add(new VersionStatusUpdater(controller, intervals.versionStatusUpdater)); - maintainers.add(new ReadyJobsTrigger(controller, intervals.readyJobsTrigger)); - maintainers.add(new DeploymentMetricsMaintainer(controller, intervals.deploymentMetricsMaintainer, successFactorBaseline.deploymentMetricsMaintainerBaseline)); - maintainers.add(new ApplicationOwnershipConfirmer(controller, intervals.applicationOwnershipConfirmer, controller.serviceRegistry().ownershipIssues())); - maintainers.add(new SystemUpgrader(controller, intervals.systemUpgrader)); - maintainers.add(new JobRunner(controller, intervals.jobRunner)); - maintainers.add(new OsVersionStatusUpdater(controller, intervals.osVersionStatusUpdater)); - maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer, successFactorBaseline.contactInformationMaintainerBaseline)); - maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher)); - maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer())); - maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().resourceDatabase())); - maintainers.add(new ResourceTagMaintainer(controller, intervals.resourceTagMaintainer, controller.serviceRegistry().resourceTagger())); - maintainers.add(new ApplicationMetaDataGarbageCollector(controller, intervals.applicationMetaDataGarbageCollector)); - maintainers.add(new ArtifactExpirer(controller, intervals.containerImageExpirer)); - maintainers.add(new HostInfoUpdater(controller, intervals.hostInfoUpdater)); - maintainers.add(new ReindexingTriggerer(controller, intervals.reindexingTriggerer)); - maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer)); - maintainers.add(new BcpGroupUpdater(controller, intervals.trafficFractionUpdater, successFactorBaseline.trafficFractionUpdater)); - maintainers.add(new ArchiveUriUpdater(controller, intervals.archiveUriUpdater)); - maintainers.add(new ArchiveAccessMaintainer(controller, metric, intervals.archiveAccessMaintainer)); - maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer)); - maintainers.add(new TenantRoleCleanupMaintainer(controller, intervals.tenantRoleMaintainer)); - maintainers.add(new ChangeRequestMaintainer(controller, intervals.changeRequestMaintainer)); - maintainers.add(new VcmrMaintainer(controller, intervals.vcmrMaintainer, metric)); - maintainers.add(new CloudDatabaseMaintainer(controller, intervals.defaultInterval)); - maintainers.add(new CloudTrialExpirer(controller, intervals.defaultInterval)); - maintainers.add(new RetriggerMaintainer(controller, intervals.retriggerMaintainer)); - maintainers.add(new UserManagementMaintainer(controller, intervals.userManagementMaintainer, controller.serviceRegistry().roleMaintainer())); - maintainers.add(new BillingDatabaseMaintainer(controller, intervals.billingDatabaseMaintainer)); - maintainers.add(new MeteringMonitorMaintainer(controller, intervals.meteringMonitorMaintainer, controller.serviceRegistry().resourceDatabase(), metric)); - maintainers.add(new EnclaveAccessMaintainer(controller, intervals.defaultInterval)); - maintainers.add(new CertificatePoolMaintainer(controller, metric, intervals.certificatePoolMaintainer)); - maintainers.add(new BillingReportMaintainer(controller, intervals.billingReportMaintainer)); - maintainers.add(new CloudAccountVerifier(controller, intervals.cloudAccountVerifier)); - maintainers.add(new DataPlaneTokenRedeployer(controller, intervals.dataPlaneTokenRedeployer)); - } - - public Upgrader upgrader() { return upgrader; } - - public OsUpgradeScheduler osUpgradeScheduler() { return osUpgradeScheduler; } - - @Override - public void deconstruct() { - maintainers.forEach(Maintainer::shutdown); - maintainers.forEach(Maintainer::awaitShutdown); - } - - /** Create one OS upgrader per cloud found in the zone registry of controller */ - private static List<OsUpgrader> osUpgraders(Controller controller, Duration interval) { - return controller.zoneRegistry().zones().controllerUpgraded().zones().stream() - .map(ZoneApi::getCloudName) - .distinct() - .sorted() - .map(cloud -> new OsUpgrader(controller, interval, cloud)) - .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); - } - - private static class Intervals { - - private static final Duration MAX_CD_INTERVAL = Duration.ofHours(1); - - private final SystemName system; - - private final Duration defaultInterval; - private final Duration deploymentInfoMaintainer; - private final Duration outstandingChangeDeployer; - private final Duration versionStatusUpdater; - private final Duration readyJobsTrigger; - private final Duration deploymentMetricsMaintainer; - private final Duration applicationOwnershipConfirmer; - private final Duration systemUpgrader; - private final Duration jobRunner; - private final Duration osVersionStatusUpdater; - private final Duration osUpgrader; - private final Duration osUpgradeScheduler; - private final Duration contactInformationMaintainer; - private final Duration nameServiceDispatcher; - private final Duration costReportMaintainer; - private final Duration resourceMeterMaintainer; - private final Duration resourceTagMaintainer; - private final Duration applicationMetaDataGarbageCollector; - private final Duration containerImageExpirer; - private final Duration hostInfoUpdater; - private final Duration reindexingTriggerer; - private final Duration endpointCertificateMaintainer; - private final Duration trafficFractionUpdater; - private final Duration archiveUriUpdater; - private final Duration archiveAccessMaintainer; - private final Duration tenantRoleMaintainer; - private final Duration changeRequestMaintainer; - private final Duration vcmrMaintainer; - private final Duration retriggerMaintainer; - private final Duration userManagementMaintainer; - private final Duration billingDatabaseMaintainer; - private final Duration meteringMonitorMaintainer; - private final Duration certificatePoolMaintainer; - private final Duration billingReportMaintainer; - private final Duration cloudAccountVerifier; - private final Duration dataPlaneTokenRedeployer; - - public Intervals(SystemName system) { - this.system = Objects.requireNonNull(system); - this.defaultInterval = duration(system.isCd() ? 1 : 5, MINUTES); - this.deploymentInfoMaintainer = duration(system.isCd() ? 1 : 10, MINUTES); - this.outstandingChangeDeployer = duration(3, MINUTES); - this.versionStatusUpdater = duration(3, MINUTES); - this.readyJobsTrigger = duration(1, MINUTES); - this.deploymentMetricsMaintainer = duration(10, MINUTES); - this.applicationOwnershipConfirmer = duration(3, HOURS); - this.systemUpgrader = duration(2, MINUTES); - this.jobRunner = duration(system.isCd() ? 45 : 90, SECONDS); - this.osVersionStatusUpdater = duration(2, MINUTES); - this.osUpgrader = duration(1, MINUTES); - this.osUpgradeScheduler = duration(15, MINUTES); - this.contactInformationMaintainer = duration(12, HOURS); - this.nameServiceDispatcher = duration(10, SECONDS); - this.costReportMaintainer = duration(2, HOURS); - this.resourceMeterMaintainer = duration(3, MINUTES); - this.resourceTagMaintainer = duration(30, MINUTES); - this.applicationMetaDataGarbageCollector = duration(12, HOURS); - this.containerImageExpirer = duration(12, HOURS); - this.hostInfoUpdater = duration(12, HOURS); - this.reindexingTriggerer = duration(1, HOURS); - this.endpointCertificateMaintainer = duration(1, HOURS); - this.trafficFractionUpdater = duration(5, MINUTES); - this.archiveUriUpdater = duration(5, MINUTES); - this.archiveAccessMaintainer = duration(10, MINUTES); - this.tenantRoleMaintainer = duration(5, MINUTES); - this.changeRequestMaintainer = duration(1, HOURS); - this.vcmrMaintainer = duration(1, HOURS); - this.retriggerMaintainer = duration(1, MINUTES); - this.userManagementMaintainer = duration(12, HOURS); - this.billingDatabaseMaintainer = duration(5, MINUTES); - this.meteringMonitorMaintainer = duration(30, MINUTES); - this.certificatePoolMaintainer = duration(15, MINUTES); - this.billingReportMaintainer = duration(60, MINUTES); - this.cloudAccountVerifier = duration(10, MINUTES); - this.dataPlaneTokenRedeployer = duration(1, MINUTES); - } - - private Duration duration(long amount, TemporalUnit unit) { - Duration duration = Duration.of(amount, unit); - if (system.isCd() && duration.compareTo(MAX_CD_INTERVAL) > 0) { - return MAX_CD_INTERVAL; // Ensure that maintainer is given enough time to run in CD - } - return duration; - } - - } - - private static class SuccessFactorBaseline { - - private final Double deploymentMetricsMaintainerBaseline; - private final Double trafficFractionUpdater; - private final Double deploymentInfoMaintainerBaseline; - private final Double contactInformationMaintainerBaseline; - - public SuccessFactorBaseline(SystemName system) { - Objects.requireNonNull(system); - this.deploymentMetricsMaintainerBaseline = 0.90; - this.trafficFractionUpdater = system.isCd() ? 0.5 : 0.65; - this.deploymentInfoMaintainerBaseline = system.isCd() ? 0.5 : 0.95; - this.contactInformationMaintainerBaseline = 0.95; - } - - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java deleted file mode 100644 index af8248c399c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; -import com.yahoo.vespa.hosted.controller.metric.CostCalculator; - -import java.time.Clock; -import java.time.Duration; -import java.util.EnumSet; - -/** - * Periodically calculate and store cost allocation for properties. - * - * @author ldalves - * @author andreer - */ -public class CostReportMaintainer extends ControllerMaintainer { - - private final CostReportConsumer consumer; - private final NodeRepository nodeRepository; - private final Clock clock; - - public CostReportMaintainer(Controller controller, Duration interval, CostReportConsumer costReportConsumer) { - super(controller, interval, null, EnumSet.of(SystemName.main)); - this.consumer = costReportConsumer; - this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.clock = controller.clock(); - } - - @Override - protected double maintain() { - var csv = CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations()); - consumer.consume(csv); - return 0.0; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DataPlaneTokenRedeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DataPlaneTokenRedeployer.java deleted file mode 100644 index e9d2dc0714b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DataPlaneTokenRedeployer.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; - -import java.time.Duration; - -/** - * @author jonmv - */ -public class DataPlaneTokenRedeployer extends ControllerMaintainer { - - public DataPlaneTokenRedeployer(Controller controller, Duration interval) { - super(controller, interval); - } - - @Override - protected double maintain() { - controller().dataplaneTokenService().triggerTokenChangeDeployments(); - return 0; - } - - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java deleted file mode 100644 index aea23e6def8..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.util.Optional; -import java.util.logging.Level; - -/** - * Expires instances in zones that have configured expiration using TimeToLive. - * - * @author mortent - * @author bratseth - */ -public class DeploymentExpirer extends ControllerMaintainer { - - public DeploymentExpirer(Controller controller, Duration interval) { - super(controller, interval); - } - - @Override - protected double maintain() { - int attempts = 0; - int failures = 0; - for (Application application : controller().applications().readable()) { - for (Instance instance : application.instances().values()) - for (Deployment deployment : instance.deployments().values()) { - if (!isExpired(deployment, instance.id())) continue; - - try { - log.log(Level.INFO, "Expiring deployment of " + instance.id() + " in " + deployment.zone()); - attempts++; - controller().applications().deactivate(instance.id(), deployment.zone()); - } catch (Exception e) { - failures++; - log.log(Level.WARNING, "Could not expire " + deployment + " of " + instance + - ": " + Exceptions.toMessageString(e) + ". Retrying in " + - interval()); - } - } - } - return asSuccessFactorDeviation(attempts, failures); - } - - /** Returns whether given deployment has expired according to its TTL */ - private boolean isExpired(Deployment deployment, ApplicationId instance) { - if (deployment.zone().environment().isProduction()) return false; // Never expire production deployments - - Optional<Duration> ttl = controller().zoneRegistry().getDeploymentTimeToLive(deployment.zone()); - if (ttl.isEmpty()) return false; - - return controller().jobController().lastDeploymentStart(instance, deployment) - .plus(ttl.get()).isBefore(controller().clock().instant()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java deleted file mode 100644 index 7b4ed9e1e98..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.util.Collection; -import java.util.Map; - -/** - * This pulls application deployment information from the node repo on all config servers, - * and stores it in memory in controller.applications().deploymentInfo(). - * - * @author bratseth - */ -public class DeploymentInfoMaintainer extends ControllerMaintainer { - - private final NodeRepository nodeRepository; - - public DeploymentInfoMaintainer(Controller controller, Duration duration, Double successFactorBaseline) { - super(controller, duration, successFactorBaseline); - this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - } - - @Override - protected double maintain() { - int attempts = 0; - int failures = 0; - outer: - for (var application : controller().applications().idList()) { - for (var instance : controller().applications().getApplication(application).map(Application::instances).orElse(Map.of()).values()) { - for (var deployment : instanceDeployments(instance)) { - if (shuttingDown()) break outer; - attempts++; - if ( ! updateDeploymentInfo(deployment)) - failures++; - } - } - } - return asSuccessFactorDeviation(attempts, failures); - } - - private Collection<DeploymentId> instanceDeployments(Instance instance) { - return instance.deployments().keySet().stream() - .filter(zoneId -> ! zoneId.environment().isTest()) - .map(zoneId -> new DeploymentId(instance.id(), zoneId)) - .toList(); - } - - private boolean updateDeploymentInfo(DeploymentId id) { - try { - controller().applications().deploymentInfo().put(id, nodeRepository.getApplication(id.zoneId(), id.applicationId())); - return true; - } - catch (ConfigServerException e) { - log.info("Could not retrieve deployment info for " + id + ": " + Exceptions.toMessageString(e)); - return false; - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java deleted file mode 100644 index ae9eb1dc2b5..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; -import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; - -import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.broken; - -/** - * Maintenance job which files issues for tenants when they have jobs which fails continuously - * and escalates issues which are not handled in a timely manner. - * - * @author jonmv - */ -public class DeploymentIssueReporter extends ControllerMaintainer { - - static final Duration maxFailureAge = Duration.ofDays(2); - static final Duration maxInactivity = Duration.ofDays(4); - static final Duration upgradeGracePeriod = Duration.ofHours(2); - - private final DeploymentIssues deploymentIssues; - - DeploymentIssueReporter(Controller controller, DeploymentIssues deploymentIssues, Duration maintenanceInterval) { - super(controller, maintenanceInterval); - this.deploymentIssues = deploymentIssues; - } - - @Override - protected double maintain() { - return ( maintainDeploymentIssues(applications()) + - maintainPlatformIssue(applications()) + - escalateInactiveDeploymentIssues(applications())) - / 3; - } - - /** Returns the applications to maintain issue status for. */ - private List<Application> applications() { - return ApplicationList.from(controller().applications().readable()) - .withProjectId() - .matching(appliaction -> appliaction.deploymentSpec().steps().stream().anyMatch(step -> step.concerns(Environment.prod))) - .asList(); - } - - /** - * File issues for applications which have failed deployment for longer than maxFailureAge - * and store the issue id for the filed issues. Also, clear the issueIds of applications - * where deployment has not failed for this amount of time. - */ - private double maintainDeploymentIssues(List<Application> applications) { - List<TenantAndApplicationId> failingApplications = controller().jobController().deploymentStatuses(ApplicationList.from(applications)) - .matching(status -> ! status.jobSteps().isEmpty()) - .failingApplicationChangeSince(controller().clock().instant().minus(maxFailureAge)) - .mapToList(status -> status.application().id()); - - for (Application application : applications) - if (failingApplications.contains(application.id())) - fileDeploymentIssueFor(application); - else - store(application.id(), null); - return 0.0; - } - - /** - * When the confidence for the system version is BROKEN, file an issue listing the - * applications that have been failing the upgrade to the system version for - * longer than the set grace period, or update this list if the issue already exists. - */ - private double maintainPlatformIssue(List<Application> applications) { - if (controller().system() == SystemName.cd) - return 0.0; - - VersionStatus versionStatus = controller().readVersionStatus(); - Version systemVersion = controller().systemVersion(versionStatus); - - if (versionStatus.version(systemVersion).confidence() != broken) - return 0.0; - - DeploymentStatusList statuses = controller().jobController().deploymentStatuses(ApplicationList.from(applications)); - if (statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant().minus(upgradeGracePeriod)).isEmpty()) - return 0.0; - - List<ApplicationId> failingApplications = statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant()) - .mapToList(status -> status.application().id().defaultInstance()); - - // TODO jonmv: Send only tenant and application, here and elsewhere in this. - deploymentIssues.fileUnlessOpen(failingApplications, systemVersion); - return 0.0; - } - - private Tenant ownerOf(TenantAndApplicationId applicationId) { - return controller().tenants().get(applicationId.tenant()) - .orElseThrow(() -> new IllegalStateException("No tenant found for application " + applicationId)); - } - - /** File an issue for applicationId, if it doesn't already have an open issue associated with it. */ - private void fileDeploymentIssueFor(Application application) { - try { - Tenant tenant = ownerOf(application.id()); - tenant.contact().ifPresent(contact -> { - Optional<IssueId> ourIssueId = application.deploymentIssueId(); - IssueId issueId = deploymentIssues.fileUnlessOpen(ourIssueId, application.id().defaultInstance(), - application.issueOwner().orElse(null), application.userOwner().orElse(null), - contact); - store(application.id(), issueId); - }); - } - catch (RuntimeException e) { // Catch errors due to wrong data in the controller, or issues client timeout. - log.log(Level.INFO, "Exception caught when attempting to file an issue for '" + application.id() + "': " + Exceptions.toMessageString(e)); - } - } - - /** Escalate issues for which there has been no activity for a certain amount of time. */ - private double escalateInactiveDeploymentIssues(Collection<Application> applications) { - AtomicInteger attempts = new AtomicInteger(0); - AtomicInteger failures = new AtomicInteger(0); - applications.forEach(application -> application.deploymentIssueId().ifPresent(issueId -> { - try { - attempts.incrementAndGet(); - Tenant tenant = ownerOf(application.id()); - deploymentIssues.escalateIfInactive(issueId, - maxInactivity, - tenant.type() == Tenant.Type.athenz ? tenant.contact() : Optional.empty()); - } - catch (RuntimeException e) { - failures.incrementAndGet(); - log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); - } - })); - return asSuccessFactorDeviation(attempts.get(), failures.get()); - } - - private void store(TenantAndApplicationId id, IssueId issueId) { - controller().applications().lockApplicationIfPresent(id, application -> - controller().applications().store(application.withDeploymentIssueId(issueId))); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java deleted file mode 100644 index df1f793914e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.ApplicationController; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Retrieves deployment metrics such as QPS and document count over the config server API - * and updates application objects in the controller with this info. - * - * @author smorgrav - * @author mpolden - */ -public class DeploymentMetricsMaintainer extends ControllerMaintainer { - - private static final Logger log = Logger.getLogger(DeploymentMetricsMaintainer.class.getName()); - - private static final int applicationsToUpdateInParallel = 10; - - private final ApplicationController applications; - - public DeploymentMetricsMaintainer(Controller controller, Duration duration, Double successFactorBaseline) { - super(controller, duration, successFactorBaseline); - this.applications = controller.applications(); - } - - public DeploymentMetricsMaintainer(Controller controller, Duration duration) { - this(controller, duration, 1.0); - } - - @Override - protected double maintain() { - AtomicInteger failures = new AtomicInteger(0); - AtomicInteger attempts = new AtomicInteger(0); - AtomicReference<Exception> lastException = new AtomicReference<>(null); - - // Run parallel stream inside a custom ForkJoinPool so that we can control the number of threads used - ForkJoinPool pool = new ForkJoinPool(applicationsToUpdateInParallel); - pool.submit(() -> - applications.readable().parallelStream().forEach(application -> { - for (Instance instance : application.instances().values()) - for (Deployment deployment : instance.deployments().values()) { - attempts.incrementAndGet(); - try { - DeploymentId deploymentId = new DeploymentId(instance.id(), deployment.zone()); - List<ClusterMetrics> clusterMetrics = controller().serviceRegistry().configServer().getDeploymentMetrics(deploymentId); - Instant now = controller().clock().instant(); - applications.lockApplicationIfPresent(application.id(), locked -> { - Deployment existingDeployment = locked.get().require(instance.name()).deployments().get(deployment.zone()); - if (existingDeployment == null) return; // Deployment removed since we started collecting metrics - DeploymentMetrics newMetrics = updateDeploymentMetrics(existingDeployment.metrics(), clusterMetrics).at(now); - applications.store(locked.with(instance.name(), - lockedInstance -> lockedInstance.with(existingDeployment.zone(), newMetrics) - .recordActivityAt(now, existingDeployment.zone()))); - - ApplicationReindexing applicationReindexing = controller().serviceRegistry().configServer().getReindexing(deploymentId); - controller().notificationsDb().setDeploymentMetricsNotifications(deploymentId, clusterMetrics, applicationReindexing); - }); - } catch (Exception e) { - failures.incrementAndGet(); - lastException.set(e); - } - } - }) - ); - pool.shutdown(); - try { - Duration timeout = Duration.ofMinutes(30); - if (!pool.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS)) { - log.log(Level.WARNING, "Could not shut down metrics collection thread pool within " + timeout); - } - if (lastException.get() != null) { - log.log(Level.WARNING, - Text.format("Could not gather metrics for %d/%d deployments. Retrying in %s. Last error: %s", - failures.get(), - attempts.get(), - interval(), - Exceptions.toMessageString(lastException.get()))); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - return asSuccessFactorDeviation(attempts.get(), failures.get()); - } - - static DeploymentMetrics updateDeploymentMetrics(DeploymentMetrics current, List<ClusterMetrics> metrics) { - return current - .withQueriesPerSecond(metrics.stream().flatMap(m -> m.queriesPerSecond().stream()).mapToDouble(Double::doubleValue).sum()) - .withWritesPerSecond(metrics.stream().flatMap(m -> m.feedPerSecond().stream()).mapToDouble(Double::doubleValue).sum()) - .withDocumentCount(metrics.stream().flatMap(m -> m.documentCount().stream()).mapToLong(Double::longValue).sum()) - .withQueryLatencyMillis(weightedAverageLatency(metrics, ClusterMetrics::queriesPerSecond, ClusterMetrics::queryLatency)) - .withWriteLatencyMillis(weightedAverageLatency(metrics, ClusterMetrics::feedPerSecond, ClusterMetrics::feedLatency)); - } - - private static double weightedAverageLatency(List<ClusterMetrics> metrics, - Function<ClusterMetrics, Optional<Double>> rateExtractor, - Function<ClusterMetrics, Optional<Double>> latencyExtractor) { - double rateSum = metrics.stream().flatMap(m -> rateExtractor.apply(m).stream()).mapToDouble(Double::longValue).sum(); - if (rateSum == 0) return 0.0; - - double weightedLatency = metrics.stream() - .flatMap(m -> latencyExtractor.apply(m).flatMap(l -> rateExtractor.apply(m).map(r -> l * r)).stream()) - .mapToDouble(Double::doubleValue) - .sum(); - - return weightedLatency / rateSum; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java deleted file mode 100644 index 270c388d73c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.Environment; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.Versions; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.time.Instant; -import java.util.Comparator; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; - -/** - * Upgrades instances in manually deployed zones to the system version, at a convenient time. - * - * @author jonmv - */ -public class DeploymentUpgrader extends ControllerMaintainer { - - public DeploymentUpgrader(Controller controller, Duration interval) { - super(controller, interval); - } - - @Override - protected double maintain() { - AtomicInteger attempts = new AtomicInteger(); - AtomicInteger failures = new AtomicInteger(); - - Version targetPlatform = null; // Upgrade to the newest non-broken, deployable version. - VersionStatus versionStatus = controller().readVersionStatus(); - for (VespaVersion platform : versionStatus.deployableVersions()) - if (platform.confidence().equalOrHigherThan(VespaVersion.Confidence.normal)) - targetPlatform = platform.versionNumber(); - - if (targetPlatform == null) - return 0; - - for (Application application : controller().applications().readable()) - for (Instance instance : application.instances().values()) - for (Deployment deployment : instance.deployments().values()) - try { - JobId job = new JobId(instance.id(), JobType.deploymentTo(deployment.zone())); - if ( ! deployment.zone().environment().isManuallyDeployed()) continue; - - Run last = controller().jobController().last(job).get(); - Versions target = new Versions(targetPlatform, last.versions().targetRevision(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetRevision())); - if ( ! last.hasEnded()) continue; - ApplicationVersion devVersion = application.revisions().get(last.versions().targetRevision()); - if (devVersion.compileVersion() - .map(version -> controller().applications().versionCompatibility(instance.id()).refuse(version, target.targetPlatform())) - .orElse(false)) continue; - if ( devVersion.allowedMajor().isPresent() - && devVersion.allowedMajor().get() < targetPlatform.getMajor()) continue; - - if ( ! deployment.version().isBefore(target.targetPlatform())) continue; - if ( ! isLikelyNightFor(job)) continue; - if (deployment.zone().environment() == Environment.perf && ! isIdleOrOutdated(deployment, job)) continue; - - log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); - attempts.incrementAndGet(); - controller().jobController().start(instance.id(), JobType.deploymentTo(deployment.zone()), target, true, Run.Reason.because("automated upgrade")); - } catch (Exception e) { - failures.incrementAndGet(); - log.log(Level.WARNING, "Failed upgrading " + deployment + " of " + instance + - ": " + Exceptions.toMessageString(e) + ". Retrying in " + - interval()); - } - return asSuccessFactorDeviation(attempts.get(), failures.get()); - } - - /** Returns whether query and feed metrics are ~zero, or currently platform has been deployed for a week. */ - private boolean isIdleOrOutdated(Deployment deployment, JobId job) { - if (deployment.metrics().queriesPerSecond() <= 1 && deployment.metrics().writesPerSecond() <= 1) return true; - return controller().jobController().runs(job).descendingMap().values().stream() - .takeWhile(run -> run.versions().targetPlatform().equals(deployment.version())) - .anyMatch(run -> run.start().isBefore(controller().clock().instant().minus(Duration.ofDays(7)))); - } - - private boolean isLikelyNightFor(JobId job) { - int hour = hourOf(controller().clock().instant()); - int[] runStarts = controller().jobController().jobStarts(job).stream() - .mapToInt(DeploymentUpgrader::hourOf) - .toArray(); - int localNight = mostLikelyWeeHour(runStarts); - return Math.abs(hour - localNight) <= 1; - } - - static int mostLikelyWeeHour(int[] starts) { - double weight = 1; - double[] buckets = new double[24]; - for (int start : starts) - buckets[start] += weight *= 0.8; // Weight more recent deployments higher. - - int best = -1; - double min = Double.MAX_VALUE; - for (int i = 12; i < 36; i++) { - double sum = 0; - for (int j = -12; j < 12; j++) - sum += buckets[(i + j) % 24] / (Math.abs(j) + 1); - - if (sum < min) { - min = sum; - best = i; - } - } - return (best + 2) % 24; - } - - private static int hourOf(Instant instant) { - return (int) (instant.toEpochMilli() / 3_600_000 % 24); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java deleted file mode 100644 index 8fd9dc919fb..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.time.Duration; -import java.util.HashSet; -import java.util.Set; -import java.util.logging.Logger; - -import static java.util.logging.Level.WARNING; - -public class EnclaveAccessMaintainer extends ControllerMaintainer { - - private static final Logger logger = Logger.getLogger(EnclaveAccessMaintainer.class.getName()); - - EnclaveAccessMaintainer(Controller controller, Duration interval) { - super(controller, interval, null, Set.of(SystemName.PublicCd, SystemName.Public)); - } - - @Override - protected double maintain() { - try { - return controller().serviceRegistry().enclaveAccessService().allowAccessFor(externalAccounts()); - } catch (RuntimeException e) { - logger.log(WARNING, "Failed sharing resources with enclave", e); - return 1.0; - } - } - - private Set<CloudAccount> externalAccounts() { - Set<CloudAccount> accounts = new HashSet<>(); - for (Tenant tenant : controller().tenants().asList()) - tenant.cloudAccounts().forEach(accountInfo -> accounts.add(accountInfo.cloudAccount())); - - return accounts; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java deleted file mode 100644 index e3e3e347c04..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.google.common.collect.Sets; -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.transaction.Mutex; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateDetails; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateRequest; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.secrets.EndpointSecretManager; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; -import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates. - * <p> - * See also class EndpointCertificates, which provisions, reprovisions and validates certificates on deploy - * - * @author andreer - */ -public class EndpointCertificateMaintainer extends ControllerMaintainer { - - private static final Logger log = Logger.getLogger(EndpointCertificateMaintainer.class.getName()); - - private final DeploymentTrigger deploymentTrigger; - private final Clock clock; - private final CuratorDb curator; - private final SecretStore secretStore; - private final EndpointSecretManager endpointSecretManager; - private final EndpointCertificateProvider endpointCertificateProvider; - final Comparator<EligibleJob> oldestFirst = Comparator.comparing(e -> e.deployment.at()); - - @Inject - public EndpointCertificateMaintainer(Controller controller, Duration interval) { - super(controller, interval); - this.deploymentTrigger = controller.applications().deploymentTrigger(); - this.clock = controller.clock(); - this.secretStore = controller.secretStore(); - this.endpointSecretManager = controller.serviceRegistry().secretManager(); - this.curator = controller().curator(); - this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); - } - - @Override - protected double maintain() { - try { - // In order of importance - deployRefreshedCertificates(); - updateRefreshedCertificates(); - deleteUnusedCertificates(); - deleteOrReportUnmanagedCertificates(); - } catch (Exception e) { - log.log(Level.SEVERE, "Exception caught while maintaining endpoint certificates", e); - return 1.0; - } - return 0.0; - } - - private void updateRefreshedCertificates() { - curator.readAssignedCertificates().forEach(assignedCertificate -> { - // Look for and use refreshed certificate - var latestAvailableVersion = latestVersionInSecretStore(assignedCertificate.certificate()); - if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > assignedCertificate.certificate().version()) { - var refreshedCertificateMetadata = assignedCertificate.certificate() - .withVersion(latestAvailableVersion.getAsInt()) - .withLastRefreshed(clock.instant().getEpochSecond()); - - try (Mutex lock = lock(assignedCertificate.application())) { - if (unchanged(assignedCertificate, lock)) { - try (NestedTransaction transaction = new NestedTransaction()) { - curator.writeAssignedCertificate(assignedCertificate.with(refreshedCertificateMetadata), transaction); // Certificate not validated here, but on deploy. - transaction.commit(); - } - } - } - } - }); - } - - private boolean unchanged(AssignedCertificate assignedCertificate, @SuppressWarnings("unused") Mutex lock) { - return Optional.of(assignedCertificate).equals(curator.readAssignedCertificate(assignedCertificate.application(), assignedCertificate.instance())); - } - - record EligibleJob(Deployment deployment, ApplicationId applicationId, JobType job) {} - - /** - * If it's been four days since the cert has been refreshed, re-trigger prod deployment jobs (one at a time). - */ - private void deployRefreshedCertificates() { - var now = clock.instant(); - var eligibleJobs = new ArrayList<EligibleJob>(); - - curator.readAssignedCertificates().forEach(assignedCertificate -> - assignedCertificate.certificate().lastRefreshed().ifPresent(lastRefreshTime -> { - Instant refreshTime = Instant.ofEpochSecond(lastRefreshTime); - if (now.isAfter(refreshTime.plus(4, ChronoUnit.DAYS))) { - if (assignedCertificate.instance().isPresent()) { - ApplicationId applicationId = assignedCertificate.application().instance(assignedCertificate.instance().get()); - controller().applications().getInstance(applicationId) - .ifPresent(instance -> instance.productionDeployments().forEach((zone, deployment) -> { - if (deployment.at().isBefore(refreshTime)) { - JobType job = JobType.deploymentTo(zone); - eligibleJobs.add(new EligibleJob(deployment, applicationId, job)); - } - })); - } else { - // This is an application-wide certificate. Trigger all instances - controller().applications().getApplication(assignedCertificate.application()).ifPresent(application -> { - application.instances().forEach((ignored, i) -> { - i.productionDeployments().forEach((zone, deployment) -> { - if (deployment.at().isBefore(refreshTime)) { - JobType job = JobType.deploymentTo(zone); - eligibleJobs.add(new EligibleJob(deployment, i.id(), job)); - } - }); - }); - }); - } - } - })); - - eligibleJobs.stream() - .min(oldestFirst) - .ifPresent(e -> { - deploymentTrigger.reTrigger(e.applicationId, e.job, "re-triggered by EndpointCertificateMaintainer"); - log.info("Re-triggering deployment job " + e.job.jobName() + " for instance " + - e.applicationId.serializedForm() + " to roll out refreshed endpoint certificate"); - }); - } - - private OptionalInt latestVersionInSecretStore(EndpointCertificate originalCertificateMetadata) { - try { - var certVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.certName())); - var keyVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.keyName())); - return Sets.intersection(certVersions, keyVersions).stream().mapToInt(Integer::intValue).max(); - } catch (SecretNotFoundException s) { - return OptionalInt.empty(); // Likely because the certificate is very recently provisioned - keep current version - } - } - - private void deleteUnusedCertificates() { - var oneMonthAgo = clock.instant().minus(30, ChronoUnit.DAYS); - curator.readAssignedCertificates().forEach(assignedCertificate -> { - EndpointCertificate certificate = assignedCertificate.certificate(); - var lastRequested = Instant.ofEpochSecond(certificate.lastRequested()); - if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(assignedCertificate.application())) { - try (Mutex lock = lock(assignedCertificate.application())) { - if (unchanged(assignedCertificate, lock)) { - log.log(Level.INFO, "Cert for app " + asString(assignedCertificate.application(), assignedCertificate.instance()) - + " has not been requested in a month and app has no deployments, deleting from provider, ZK and secret store"); - endpointCertificateProvider.deleteCertificate(certificate.rootRequestId()); - curator.removeAssignedCertificate(assignedCertificate.application(), assignedCertificate.instance()); - endpointSecretManager.deleteSecret(certificate.certName()); - endpointSecretManager.deleteSecret(certificate.keyName()); - } - } - } - }); - } - - private Mutex lock(TenantAndApplicationId application) { - return curator.lock(application); - } - - private boolean hasNoDeployments(TenantAndApplicationId application) { - Optional<Application> app = controller().applications().getApplication(application); - if (app.isEmpty()) return true; - for (var instance : app.get().instances().values()) { - if (!instance.deployments().isEmpty()) return false; - } - return true; - } - - private void deleteOrReportUnmanagedCertificates() { - List<EndpointCertificateRequest> requests = endpointCertificateProvider.listCertificates(); - List<AssignedCertificate> assignedCertificates = curator.readAssignedCertificates(); - - List<String> leafRequestIds = assignedCertificates.stream().map(AssignedCertificate::certificate).flatMap(m -> m.leafRequestId().stream()).toList(); - List<String> rootRequestIds = assignedCertificates.stream().map(AssignedCertificate::certificate).map(EndpointCertificate::rootRequestId).toList(); - List<UnassignedCertificate> unassignedCertificates = curator.readUnassignedCertificates(); - List<String> certPoolRootIds = unassignedCertificates.stream().map(p -> p.certificate().leafRequestId()).flatMap(Optional::stream).toList(); - List<String> certPoolLeafIds = unassignedCertificates.stream().map(p -> p.certificate().rootRequestId()).toList(); - - var managedIds = new HashSet<String>(); - managedIds.addAll(leafRequestIds); - managedIds.addAll(rootRequestIds); - managedIds.addAll(certPoolRootIds); - managedIds.addAll(certPoolLeafIds); - - for (var request : requests) { - if (!managedIds.contains(request.requestId())) { - - // It could just be a refresh we're not aware of yet. See if it matches the cert/keyname of any known cert - EndpointCertificateDetails unknownCertDetails = endpointCertificateProvider.certificateDetails(request.requestId()); - boolean matchFound = false; - for (AssignedCertificate assignedCertificate : assignedCertificates) { - if (assignedCertificate.certificate().certName().equals(unknownCertDetails.certKeyKeyname())) { - matchFound = true; - try (Mutex lock = lock(assignedCertificate.application())) { - if (unchanged(assignedCertificate, lock)) { - log.log(Level.INFO, "Cert for app " + asString(assignedCertificate.application(), assignedCertificate.instance()) - + " has a new leafRequestId " + unknownCertDetails.requestId() + ", updating in ZK"); - try (NestedTransaction transaction = new NestedTransaction()) { - EndpointCertificate updated = assignedCertificate.certificate().withLeafRequestId(Optional.of(unknownCertDetails.requestId())); - curator.writeAssignedCertificate(assignedCertificate.with(updated), transaction); - transaction.commit(); - } - } - break; - } - } - } - if (!matchFound) { - // The certificate is not known - however it could be in the process of being requested by us or another controller. - // So we only delete if it was requested more than 7 days ago. - if (Instant.parse(request.createTime()).isBefore(Instant.now().minus(7, ChronoUnit.DAYS))) { - log.log(Level.INFO, String.format("Deleting unmaintained certificate with request_id %s and SANs %s", - request.requestId(), - request.dnsNames().stream().map(EndpointCertificateRequest.DnsNameStatus::dnsName).collect(Collectors.joining(", ")))); - endpointCertificateProvider.deleteCertificate(request.requestId()); - } - } - } - } - } - - private static String asString(TenantAndApplicationId application, Optional<InstanceName> instanceName) { - return application.toString() + instanceName.map(name -> "." + name.value()).orElse(""); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java deleted file mode 100644 index 5d6e60ee0bf..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; -import com.yahoo.vespa.hosted.controller.api.integration.entity.NodeEntity; - -import java.time.Duration; -import java.util.EnumSet; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - * Ensures that the host information for all hosts is up to date. - * - * @author mpolden - * @author bjormel - */ -public class HostInfoUpdater extends ControllerMaintainer { - - private static final Logger LOG = Logger.getLogger(HostInfoUpdater.class.getName()); - private static final Pattern HOST_PATTERN = Pattern.compile("^(proxy|cfg|controller)host(.+)$"); - - private final NodeRepository nodeRepository; - - public HostInfoUpdater(Controller controller, Duration interval) { - super(controller, interval, null, EnumSet.of(SystemName.cd, SystemName.main)); - this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - } - - @Override - protected double maintain() { - Map<String, NodeEntity> nodeEntities = controller().serviceRegistry().entityService().listNodes().stream() - .collect(Collectors.toMap(NodeEntity::hostname, - Function.identity())); - int hostsUpdated = 0; - try { - for (var zone : controller().zoneRegistry().zones().controllerUpgraded().all().ids()) { - for (var node : nodeRepository.list(zone, NodeFilter.all())) { - if (!node.type().isHost()) continue; - NodeEntity nodeEntity = nodeEntities.get(registeredHostnameOf(node)); - if (nodeEntity == null) continue; - - boolean updatedHost = false; - Optional<String> modelName = modelNameOf(nodeEntity); - if (modelName.isPresent() && !modelName.equals(node.modelName())) { - nodeRepository.updateModel(zone, node.hostname().value(), modelName.get()); - updatedHost = true; - } - - Optional<String> switchHostname = nodeEntity.switchHostname(); - if (switchHostname.isPresent() && !switchHostname.equals(node.switchHostname())) { - nodeRepository.updateSwitchHostname(zone, node.hostname().value(), switchHostname.get()); - updatedHost = true; - } - - if (updatedHost) { - hostsUpdated++; - } - } - } - } finally { - if (hostsUpdated > 0) { - LOG.info("Updated information for " + hostsUpdated + " hosts(s)"); - } - } - return 0.0; - } - - private static Optional<String> modelNameOf(NodeEntity nodeEntity) { - if (nodeEntity.manufacturer().isEmpty() || nodeEntity.model().isEmpty()) return Optional.empty(); - return Optional.of(nodeEntity.manufacturer().get() + " " + nodeEntity.model().get()); - } - - /** Returns the hostname that given host is registered under in the {@link EntityService} */ - private static String registeredHostnameOf(Node host) { - String hostname = host.hostname().value(); - if (!host.type().isHost()) return hostname; - Matcher matcher = HOST_PATTERN.matcher(hostname); - if (!matcher.matches()) return hostname; - return matcher.replaceFirst("$1$2"); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java deleted file mode 100644 index 97bb709d423..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.NodeSlice; -import com.yahoo.config.provision.zone.UpgradePolicy; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.versions.VersionTarget; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.util.Comparator; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Base class for maintainers that upgrade zone infrastructure. - * - * @author mpolden - */ -public abstract class InfrastructureUpgrader<TARGET extends VersionTarget> extends ControllerMaintainer { - - private static final Logger log = Logger.getLogger(InfrastructureUpgrader.class.getName()); - - protected final UpgradePolicy upgradePolicy; - private final List<SystemApplication> managedApplications; - - public InfrastructureUpgrader(Controller controller, Duration interval, UpgradePolicy upgradePolicy, - List<SystemApplication> managedApplications, String name) { - super(controller, interval, name, EnumSet.allOf(SystemName.class)); - this.upgradePolicy = upgradePolicy; - this.managedApplications = List.copyOf(Objects.requireNonNull(managedApplications)); - } - - @Override - protected double maintain() { - return target().map(target -> upgradeAll(target, managedApplications)) - .orElse(0.0); - } - - /** Deploy a list of system applications until they converge on the given version */ - private double upgradeAll(TARGET target, List<SystemApplication> applications) { - int attempts = 0; - int failures = 0; - // Invert zone order if we're downgrading - UpgradePolicy policy = target.downgrade() ? upgradePolicy.inverted() : upgradePolicy; - for (UpgradePolicy.Step step : policy.steps()) { - boolean converged = true; - for (ZoneApi zone : step.zones()) { - try { - attempts++; - converged &= upgradeAll(target, applications, zone, step.nodeSlice()); - } catch (UnreachableNodeRepositoryException e) { - failures++; - converged = false; - log.warning(Text.format("%s: Failed to communicate with node repository in %s, continuing with next parallel zone: %s", - this, zone, Exceptions.toMessageString(e))); - } catch (Exception e) { - failures++; - converged = false; - log.warning(Text.format("%s: Failed to upgrade zone: %s, continuing with next parallel zone: %s", - this, zone, Exceptions.toMessageString(e))); - } - } - if (!converged) { - break; - } - } - return asSuccessFactorDeviation(attempts, failures); - } - - /** Returns whether all applications have converged to the target version in zone */ - private boolean upgradeAll(TARGET target, List<SystemApplication> applications, ZoneApi zone, NodeSlice nodeSlice) { - Map<SystemApplication, Set<SystemApplication>> dependenciesByApplication = new HashMap<>(); - if (target.downgrade()) { // Invert dependencies when we're downgrading - for (var application : applications) { - dependenciesByApplication.computeIfAbsent(application, k -> new HashSet<>()); - for (var dependency : application.dependencies()) { - dependenciesByApplication.computeIfAbsent(dependency, k -> new HashSet<>()) - .add(application); - } - } - } else { - applications.forEach(app -> dependenciesByApplication.put(app, Set.copyOf(app.dependencies()))); - } - boolean converged = true; - for (var kv : dependenciesByApplication.entrySet()) { - SystemApplication application = kv.getKey(); - Set<SystemApplication> dependencies = kv.getValue(); - boolean allConverged = dependencies.stream().allMatch(app -> convergedOn(target, app, zone, nodeSlice)); - if (allConverged) { - if (changeTargetTo(target, application, zone)) { - upgrade(target, application, zone); - } - converged &= convergedOn(target, application, zone, nodeSlice); - } - } - return converged; - } - - /** Returns whether target version for application in zone should be changed */ - protected abstract boolean changeTargetTo(TARGET target, SystemApplication application, ZoneApi zone); - - /** Upgrade component to target version. Implementation should be idempotent */ - protected abstract void upgrade(TARGET target, SystemApplication application, ZoneApi zone); - - /** Returns whether application has converged to target version in zone */ - protected abstract boolean convergedOn(TARGET target, SystemApplication application, ZoneApi zone, NodeSlice nodeSlice); - - /** Returns the version target for the component upgraded by this, if any */ - protected abstract Optional<TARGET> target(); - - /** Returns whether the upgrader should expect given node to upgrade */ - protected abstract boolean expectUpgradeOf(Node node, SystemApplication application, ZoneApi zone); - - /** - * Find the version currently used by a slice of nodes, in given zone. If no such slice exists, - * the lowest (or highest, when downgrading) overall version is returned. - */ - protected final Optional<Version> versionOf(NodeSlice nodeSlice, ZoneApi zone, SystemApplication application, - Function<Node, Version> versionField, boolean downgrading) { - try { - Map<Version, Long> nodeCountByVersion = controller().serviceRegistry().configServer() - .nodeRepository() - .list(zone.getVirtualId(), NodeFilter.all().applications(application.id())) - .stream() - .filter(node -> expectUpgradeOf(node, application, zone)) - .collect(Collectors.groupingBy(versionField, - Collectors.counting())); - long totalNodes = nodeCountByVersion.values().stream().reduce(Long::sum).orElse(0L); - Set<Version> versionsOfMatchingSlices = new HashSet<>(); - for (var kv : nodeCountByVersion.entrySet()) { - long nodesOnVersion = kv.getValue(); - if (nodeSlice.satisfiedBy(nodesOnVersion, totalNodes)) { - versionsOfMatchingSlices.add(kv.getKey()); - } - } - if (!versionsOfMatchingSlices.isEmpty()) { - return downgrading - ? versionsOfMatchingSlices.stream().min(Comparator.naturalOrder()) - : versionsOfMatchingSlices.stream().max(Comparator.naturalOrder()); - } - return downgrading - ? nodeCountByVersion.keySet().stream().max(Comparator.naturalOrder()) - : nodeCountByVersion.keySet().stream().min(Comparator.naturalOrder()); - } catch (Exception e) { - throw new UnreachableNodeRepositoryException(Text.format("Failed to get version for %s in %s: %s", - application.id(), zone, - Exceptions.toMessageString(e))); - } - } - - private static class UnreachableNodeRepositoryException extends RuntimeException { - private UnreachableNodeRepositoryException(String reason) { - super(reason); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java deleted file mode 100644 index 0f482b1a015..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import ai.vespa.metrics.ControllerMetrics; -import com.yahoo.concurrent.DaemonThreadFactory; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner; -import com.yahoo.vespa.hosted.controller.deployment.JobController; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.Step; -import com.yahoo.vespa.hosted.controller.deployment.StepRunner; - -import java.time.Duration; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Advances the set of {@link Run}s for a {@link JobController}. - * - * @author jonmv - */ -public class JobRunner extends ControllerMaintainer { - - public static final Duration jobTimeout = Duration.ofDays(1).plusHours(1); - private static final Logger log = Logger.getLogger(JobRunner.class.getName()); - - private final JobController jobs; - private final ExecutorService executors; - private final StepRunner runner; - private final Metrics metrics; - - public JobRunner(Controller controller, Duration duration) { - this(controller, duration, Executors.newFixedThreadPool(32, new DaemonThreadFactory("job-runner-")), - new InternalStepRunner(controller)); - } - - public JobRunner(Controller controller, Duration duration, ExecutorService executors, StepRunner runner) { - this(controller, duration, executors, runner, new Metrics(controller.metric(), Duration.ofMillis(100))); - } - - JobRunner(Controller controller, Duration duration, ExecutorService executors, StepRunner runner, Metrics metrics) { - super(controller, duration); - this.jobs = controller.jobController(); - this.jobs.setRunner(this::advance); - this.executors = executors; - this.runner = runner; - this.metrics = metrics; - } - - @Override - protected double maintain() { - execute(() -> jobs.active().forEach(this::advance)); - jobs.collectGarbage(); - return 1.0; - } - - @Override - public void shutdown() { - super.shutdown(); - metrics.shutdown(); - executors.shutdown(); - } - - @Override - public void awaitShutdown() { - super.awaitShutdown(); - try { - if ( ! executors.awaitTermination(40, TimeUnit.SECONDS)) { - executors.shutdownNow(); - if ( ! executors.awaitTermination(10, TimeUnit.SECONDS)) - throw new IllegalStateException("Failed shutting down " + JobRunner.class.getName()); - } - } - catch (InterruptedException e) { - log.log(Level.WARNING, "Interrupted during shutdown of " + JobRunner.class.getName(), e); - Thread.currentThread().interrupt(); - } - } - - public void advance(Run run) { - if ( ! jobs.isDisabled(run.id().job())) advance(run.id()); - } - - /** Advances each of the ready steps for the given run, or marks it as finished, and stashes it. Public for testing. */ - public void advance(RunId id) { - jobs.locked(id, run -> { - if ( ! run.hasFailed() - && controller().clock().instant().isAfter(run.sleepUntil().orElse(run.start()).plus(jobTimeout))) - execute(() -> { - jobs.abort(run.id(), "job timeout of " + jobTimeout + " reached", false); - advance(run.id()); - }); - else if (run.readySteps().isEmpty()) - execute(() -> finish(run.id())); - else if (run.hasFailed() || run.sleepUntil().map(sleepUntil -> ! sleepUntil.isAfter(controller().clock().instant())).orElse(true)) - run.readySteps().forEach(step -> execute(() -> advance(run.id(), step))); - - return null; - }); - } - - private void finish(RunId id) { - try { - jobs.finish(id); - if ( ! id.type().environment().isManuallyDeployed()) - controller().applications().deploymentTrigger().notifyOfCompletion(id.application()); - } - catch (TimeoutException e) { - // One of the steps are still being run — that's ok, we'll try to finish the run again later. - } - catch (Exception e) { - log.log(Level.WARNING, "Exception finishing " + id, e); - } - } - - /** Attempts to advance the status of the given step, for the given run. */ - private void advance(RunId id, Step step) { - try { - AtomicBoolean changed = new AtomicBoolean(false); - jobs.locked(id.application(), id.type(), step, lockedStep -> { - jobs.locked(id, run -> { - if ( ! run.readySteps().contains(step)) { - changed.set(true); - return run; // Someone may have updated the run status, making this step obsolete, so we bail out. - } - - if (run.stepInfo(lockedStep.get()).orElseThrow().startTime().isEmpty()) - run = run.with(controller().clock().instant(), lockedStep); - - return run; - }); - - if ( ! changed.get()) { - runner.run(lockedStep, id).ifPresent(status -> { - jobs.update(id, status, lockedStep); - changed.set(true); - }); - } - }); - if (changed.get()) - jobs.active(id).ifPresent(this::advance); - } - catch (TimeoutException e) { - // Something else is already advancing this step, or a prerequisite -- try again later! - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Exception attempting to advance " + step + " of " + id, e); - } - } - - private void execute(Runnable task) { - metrics.queued.incrementAndGet(); - executors.execute(() -> { - metrics.queued.decrementAndGet(); - metrics.active.incrementAndGet(); - try { task.run(); } - finally { metrics.active.decrementAndGet(); } - }); - } - - static class Metrics { - - private final AtomicInteger queued = new AtomicInteger(); - private final AtomicInteger active = new AtomicInteger(); - private final ScheduledExecutorService reporter = Executors.newSingleThreadScheduledExecutor(new DaemonThreadFactory("job-runner-metrics-")); - private final Metric metric; - private final Metric.Context context; - - Metrics(Metric metric, Duration interval) { - this.metric = metric; - this.context = metric.createContext(Map.of()); - reporter.scheduleAtFixedRate(this::report, interval.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS); - } - - void report() { - metric.set(ControllerMetrics.DEPLOYMENT_JOBS_QUEUED.baseName(), queued.get(), context); - metric.set(ControllerMetrics.DEPLOYMENT_JOBS_ACTIVE.baseName(), active.get(), context); - } - - void shutdown() { - reporter.shutdown(); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java deleted file mode 100644 index 396ec1ec6f9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; - -import java.time.Duration; -import java.util.Collections; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Reports discrepancies between currently deployed applications and - * recently stored metering data in ResourceDatabaseClient. - * - * @author olaa - */ -public class MeteringMonitorMaintainer extends ControllerMaintainer { - - private final ResourceDatabaseClient resourceDatabaseClient; - private final Metric metric; - - protected static final String METERING_AGE_METRIC_NAME = "metering.age.seconds"; - private static final Logger logger = Logger.getLogger(MeteringMonitorMaintainer.class.getName()); - - public MeteringMonitorMaintainer(Controller controller, Duration interval, ResourceDatabaseClient resourceDatabaseClient, Metric metric) { - super(controller, interval, null, SystemName.allOf(SystemName::isPublic)); - this.resourceDatabaseClient = resourceDatabaseClient; - this.metric = metric; - } - - @Override - protected double maintain() { - var activeDeployments = activeDeployments(); - var lastSnapshotTime = resourceDatabaseClient.getOldestSnapshotTimestamp(activeDeployments); - var age = controller().clock().instant().getEpochSecond() - lastSnapshotTime.getEpochSecond(); - metric.set(METERING_AGE_METRIC_NAME, age, metric.createContext(Collections.emptyMap())); - return 1; - } - - private Set<DeploymentId> activeDeployments() { - return controller().applications().asList() - .stream() - .flatMap(app -> app.instances().values().stream()) - .flatMap(this::toProdDeployments) - .collect(Collectors.toSet()); - } - - private Stream<DeploymentId> toProdDeployments(Instance instance) { - return instance.deployments() - .keySet() - .stream() - .filter(deployment -> deployment.environment().isProduction()) - .map(deployment -> new DeploymentId(instance.id(), deployment)); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java deleted file mode 100644 index 6f070cbba84..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ /dev/null @@ -1,388 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import ai.vespa.metrics.ConfigServerMetrics; -import ai.vespa.metrics.ControllerMetrics; -import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.athenz.client.zms.ZmsClient; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; -import com.yahoo.vespa.hosted.controller.deployment.JobList; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; -import com.yahoo.vespa.hosted.controller.versions.NodeVersion; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; - -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * This calculates and reports system-wide metrics based on data from a {@link Controller}. - * - * @author mortent - * @author mpolden - */ -public class MetricsReporter extends ControllerMaintainer { - - public static final String TENANT_METRIC = ControllerMetrics.BILLING_TENANTS.baseName(); - public static final String DEPLOYMENT_FAIL_METRIC = ControllerMetrics.DEPLOYMENT_FAILURE_PERCENTAGE.baseName(); - public static final String DEPLOYMENT_AVERAGE_DURATION = ControllerMetrics.DEPLOYMENT_AVERAGE_DURATION.baseName(); - public static final String DEPLOYMENT_FAILING_UPGRADES = ControllerMetrics.DEPLOYMENT_FAILING_UPGRADES.baseName(); - public static final String DEPLOYMENT_BUILD_AGE_SECONDS = ControllerMetrics.DEPLOYMENT_BUILD_AGE_SECONDS.baseName(); - public static final String DEPLOYMENT_WARNINGS = ControllerMetrics.DEPLOYMENT_WARNINGS.baseName(); - public static final String DEPLOYMENT_OVERDUE_UPGRADE = ControllerMetrics.DEPLOYMENT_OVERDUE_UPGRADE_SECONDS.baseName(); - public static final String OS_CHANGE_DURATION = ControllerMetrics.DEPLOYMENT_OS_CHANGE_DURATION.baseName(); - public static final String PLATFORM_CHANGE_DURATION = ControllerMetrics.DEPLOYMENT_PLATFORM_CHANGE_DURATION.baseName(); - public static final String OS_NODE_COUNT = ControllerMetrics.DEPLOYMENT_NODE_COUNT_BY_OS_VERSION.baseName(); - public static final String PLATFORM_NODE_COUNT = ControllerMetrics.DEPLOYMENT_NODE_COUNT_BY_PLATFORM_VERSION.baseName(); - public static final String BROKEN_SYSTEM_VERSION = ControllerMetrics.DEPLOYMENT_BROKEN_SYSTEM_VERSION.baseName(); - public static final String REMAINING_ROTATIONS = ControllerMetrics.REMAINING_ROTATIONS.baseName(); - public static final String NAME_SERVICE_REQUESTS_QUEUED = ControllerMetrics.DNS_QUEUED_REQUESTS.baseName(); - public static final String OPERATION_PREFIX = "operation."; - public static final String ZMS_QUOTA_USAGE = ControllerMetrics.ZMS_QUOTA_USAGE.baseName(); - - private final Metric metric; - private final Clock clock; - private final ZmsClient zmsClient; - - // Keep track of reported node counts for each version - private final ConcurrentHashMap<NodeCountKey, Long> nodeCounts = new ConcurrentHashMap<>(); - - public MetricsReporter(Controller controller, Metric metric, ZmsClient zmsClient) { - super(controller, Duration.ofMinutes(1)); // use fixed rate for metrics - this.metric = metric; - this.clock = controller.clock(); - this.zmsClient = zmsClient; - } - - @Override - public double maintain() { - reportDeploymentMetrics(); - reportRemainingRotations(); - reportQueuedNameServiceRequests(); - VersionStatus versionStatus = controller().readVersionStatus(); - reportInfrastructureUpgradeMetrics(versionStatus); - reportAuditLog(); - reportBrokenSystemVersion(versionStatus); - reportTenantMetrics(); - reportZmsQuotaMetrics(); - return 0.0; - } - - private void reportBrokenSystemVersion(VersionStatus versionStatus) { - Version systemVersion = controller().systemVersion(versionStatus); - VespaVersion.Confidence confidence = versionStatus.version(systemVersion).confidence(); - int isBroken = confidence == VespaVersion.Confidence.broken ? 1 : 0; - metric.set(BROKEN_SYSTEM_VERSION, isBroken, metric.createContext(Map.of())); - } - - private void reportAuditLog() { - AuditLog log = controller().auditLogger().readLog(); - HashMap<String, HashMap<String, Integer>> metricCounts = new HashMap<>(); - - for (AuditLog.Entry entry : log.entries()) { - String[] resource = entry.resource().split("/"); - if((resource.length > 1) && (resource[1] != null)) { - String api = resource[1]; - String operationMetric = OPERATION_PREFIX + api; - HashMap<String, Integer> dimension = metricCounts.get(operationMetric); - if (dimension != null) { - Integer count = dimension.get(entry.principal()); - if (count != null) { - dimension.replace(entry.principal(), ++count); - } else { - dimension.put(entry.principal(), 1); - } - - } else { - dimension = new HashMap<>(); - dimension.put(entry.principal(),1); - metricCounts.put(operationMetric, dimension); - } - } - } - for (String operationMetric : metricCounts.keySet()) { - for (String userDimension : metricCounts.get(operationMetric).keySet()) { - metric.set(operationMetric, (metricCounts.get(operationMetric)).get(userDimension), metric.createContext(Map.of("operator", userDimension))); - } - } - } - - private void reportInfrastructureUpgradeMetrics(VersionStatus versionStatus) { - Map<NodeVersion, Duration> osChangeDurations = osChangeDurations(); - Map<NodeVersion, Duration> platformChangeDurations = platformChangeDurations(versionStatus); - reportChangeDurations(osChangeDurations, OS_CHANGE_DURATION); - reportChangeDurations(platformChangeDurations, PLATFORM_CHANGE_DURATION); - reportNodeCount(osChangeDurations.keySet(), OS_NODE_COUNT); - reportNodeCount(platformChangeDurations.keySet(), PLATFORM_NODE_COUNT); - } - - private void reportRemainingRotations() { - try (RotationLock lock = controller().routing().rotations().lock()) { - int availableRotations = controller().routing().rotations().availableRotations(lock).size(); - metric.set(REMAINING_ROTATIONS, availableRotations, metric.createContext(Map.of())); - } - } - - private void reportDeploymentMetrics() { - ApplicationList applications = ApplicationList.from(controller().applications().readable()) - .withProductionDeployment(); - DeploymentStatusList deployments = controller().jobController().deploymentStatuses(applications); - - metric.set(DEPLOYMENT_FAIL_METRIC, deploymentFailRatio(deployments) * 100, metric.createContext(Map.of())); - - averageDeploymentDurations(deployments, clock.instant()).forEach((instance, duration) -> { - metric.set(DEPLOYMENT_AVERAGE_DURATION, duration.toSeconds(), metric.createContext(dimensions(instance))); - }); - - deploymentsFailingUpgrade(deployments).forEach((instance, failingJobs) -> { - metric.set(DEPLOYMENT_FAILING_UPGRADES, failingJobs, metric.createContext(dimensions(instance))); - }); - - deploymentWarnings(deployments).forEach((instance, warnings) -> { - metric.set(DEPLOYMENT_WARNINGS, warnings, metric.createContext(dimensions(instance))); - }); - - overdueUpgradeDurationByInstance(deployments).forEach((instance, overduePeriod) -> { - metric.set(DEPLOYMENT_OVERDUE_UPGRADE, overduePeriod.toSeconds(), metric.createContext(dimensions(instance))); - }); - - for (Application application : applications.asList()) - application.revisions().last() - .flatMap(ApplicationVersion::buildTime) - .ifPresent(buildTime -> metric.set(DEPLOYMENT_BUILD_AGE_SECONDS, - controller().clock().instant().getEpochSecond() - buildTime.getEpochSecond(), - metric.createContext(dimensions(application.id().defaultInstance())))); - } - - private Map<ApplicationId, Duration> overdueUpgradeDurationByInstance(DeploymentStatusList deployments) { - Instant now = clock.instant(); - Map<ApplicationId, Duration> overdueUpgrades = new HashMap<>(); - for (var deploymentStatus : deployments) { - for (var kv : deploymentStatus.instanceJobs().entrySet()) { - ApplicationId instance = kv.getKey(); - JobList jobs = kv.getValue(); - boolean upgradeRunning = !jobs.production().upgrading().isEmpty(); - DeploymentInstanceSpec instanceSpec = deploymentStatus.application().deploymentSpec().requireInstance(instance.instance()); - Duration overdueDuration = upgradeRunning ? overdueUpgradeDuration(now, instanceSpec) : Duration.ZERO; - overdueUpgrades.put(instance, overdueDuration); - } - } - return Collections.unmodifiableMap(overdueUpgrades); - } - - /** Returns how long an upgrade has been running inside a block window */ - static Duration overdueUpgradeDuration(Instant upgradingAt, DeploymentInstanceSpec instanceSpec) { - Optional<Instant> lastOpened = Optional.empty(); // When the upgrade window most recently opened - Instant oneWeekAgo = upgradingAt.minus(Duration.ofDays(7)); - Duration step = Duration.ofHours(1); - for (Instant instant = upgradingAt.truncatedTo(ChronoUnit.HOURS); !instanceSpec.canUpgradeAt(instant); instant = instant.minus(step)) { - if (!instant.isAfter(oneWeekAgo)) { // Wrapped around, the entire week is being blocked - lastOpened = Optional.empty(); - break; - } - lastOpened = Optional.of(instant); - } - if (lastOpened.isEmpty()) return Duration.ZERO; - return Duration.between(lastOpened.get(), upgradingAt); - } - - private void reportQueuedNameServiceRequests() { - metric.set(NAME_SERVICE_REQUESTS_QUEUED, controller().curator().readNameServiceQueue().requests().size(), - metric.createContext(Map.of())); - } - - private void reportNodeCount(Set<NodeVersion> nodeVersions, String metricName) { - Map<NodeCountKey, Long> newNodeCounts = nodeVersions.stream() - .collect(Collectors.groupingBy(nodeVersion -> { - return new NodeCountKey(metricName, - nodeVersion.currentVersion(), - nodeVersion.zone()); - }, Collectors.counting())); - nodeCounts.putAll(newNodeCounts); - nodeCounts.forEach((key, count) -> { - if (newNodeCounts.containsKey(key)) { - // Version is still present: Update the metric. - metric.set(metricName, count, metric.createContext(dimensions(key.zone, key.version))); - } else if (key.metricName.equals(metricName)) { - // Version is no longer present, but has been previously reported: Set it to zero. - metric.set(metricName, 0, metric.createContext(dimensions(key.zone, key.version))); - } - }); - } - - private void reportChangeDurations(Map<NodeVersion, Duration> changeDurations, String metricName) { - changeDurations.forEach((nodeVersion, duration) -> { - metric.set(metricName, duration.toSeconds(), metric.createContext(dimensions(nodeVersion.hostname(), nodeVersion.zone()))); - }); - } - - private void reportTenantMetrics() { - if (! controller().system().isPublic()) return; - - var planCounter = new TreeMap<String, Integer>(); - - controller().tenants().asList().forEach(tenant -> { - var planId = controller().serviceRegistry().billingController().getPlan(tenant.name()); - planCounter.merge(planId.value(), 1, Integer::sum); - }); - - planCounter.forEach((planId, count) -> { - var context = metric.createContext(Map.of("plan", planId)); - metric.set(TENANT_METRIC, count, context); - }); - } - - private void reportZmsQuotaMetrics() { - var quota = zmsClient.getQuotaUsage(); - reportZmsQuota("subdomains", quota.getSubdomainUsage()); - reportZmsQuota("services", quota.getServiceUsage()); - reportZmsQuota("policies", quota.getPolicyUsage()); - reportZmsQuota("roles", quota.getRoleUsage()); - reportZmsQuota("groups", quota.getGroupUsage()); - } - - private void reportZmsQuota(String resourceType, double usage) { - var context = metric.createContext(Map.of("resourceType", resourceType)); - metric.set(ZMS_QUOTA_USAGE, usage, context); - } - - private Map<NodeVersion, Duration> platformChangeDurations(VersionStatus versionStatus) { - return changeDurations(versionStatus.versions(), VespaVersion::nodeVersions); - } - - private Map<NodeVersion, Duration> osChangeDurations() { - return changeDurations(controller().os().status().versions().values(), Function.identity()); - } - - private <V> Map<NodeVersion, Duration> changeDurations(Collection<V> versions, Function<V, List<NodeVersion>> versionsGetter) { - var now = clock.instant(); - var durations = new HashMap<NodeVersion, Duration>(); - for (var version : versions) { - for (var nodeVersion : versionsGetter.apply(version)) { - durations.put(nodeVersion, nodeVersion.changeDuration(now)); - } - } - return durations; - } - - private static double deploymentFailRatio(DeploymentStatusList statuses) { - return statuses.asList().stream() - .mapToInt(status -> status.hasFailures() ? 1 : 0) - .average().orElse(0); - } - - private static Map<ApplicationId, Duration> averageDeploymentDurations(DeploymentStatusList statuses, Instant now) { - return statuses.asList().stream() - .flatMap(status -> status.instanceJobs().entrySet().stream()) - .collect(Collectors.toUnmodifiableMap(entry -> entry.getKey(), - entry -> averageDeploymentDuration(entry.getValue(), now))); - } - - private static Map<ApplicationId, Integer> deploymentsFailingUpgrade(DeploymentStatusList statuses) { - return statuses.asList().stream() - .flatMap(status -> status.instanceJobs().entrySet().stream()) - .collect(Collectors.toUnmodifiableMap(entry -> entry.getKey(), - entry -> deploymentsFailingUpgrade(entry.getValue()))); - } - - private static int deploymentsFailingUpgrade(JobList jobs) { - return jobs.failingHard().not().failingApplicationChange().size(); - } - - private static Duration averageDeploymentDuration(JobList jobs, Instant now) { - List<Duration> jobDurations = jobs.lastTriggered() - .mapToList(run -> Duration.between(run.start(), run.end().orElse(now))); - return jobDurations.stream() - .reduce(Duration::plus) - .map(totalDuration -> totalDuration.dividedBy(jobDurations.size())) - .orElse(Duration.ZERO); - } - - private static Map<ApplicationId, Integer> deploymentWarnings(DeploymentStatusList statuses) { - return statuses.asList().stream() - .flatMap(status -> status.application().instances().values().stream()) - .collect(Collectors.toMap(Instance::id, a -> maxWarningCountOf(a.deployments().values()))); - } - - private static int maxWarningCountOf(Collection<Deployment> deployments) { - return deployments.stream() - .map(Deployment::metrics) - .map(DeploymentMetrics::warnings) - .map(Map::values) - .flatMap(Collection::stream) - .max(Integer::compareTo) - .orElse(0); - } - - private static Map<String, String> dimensions(ApplicationId application) { - return Map.of("tenantName", application.tenant().value(), - "app", application.application().value() + "." + application.instance().value(), - "applicationId", application.toFullString()); - } - - private static Map<String, String> dimensions(HostName hostname, ZoneId zone) { - return Map.of("host", hostname.value(), - "zone", zone.value()); - } - - private static Map<String, String> dimensions(ZoneId zone, Version currentVersion) { - return Map.of("zone", zone.value(), - "currentVersion", currentVersion.toFullString()); - } - - private static class NodeCountKey { - - private final String metricName; - private final Version version; - private final ZoneId zone; - - public NodeCountKey(String metricName, Version version, ZoneId zone) { - this.metricName = metricName; - this.version = version; - this.zone = zone; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - NodeCountKey that = (NodeCountKey) o; - return metricName.equals(that.metricName) && - version.equals(that.version) && - zone.equals(that.zone); - } - - @Override - public int hashCode() { - return Objects.hash(metricName, version, zone); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java deleted file mode 100644 index 3ee9650d4ca..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.logging.Level; - -/** - * This dispatches requests from {@link NameServiceQueue} to a {@link NameService}. Successfully dispatched requests are - * removed from the queue. - * - * @author mpolden - */ -public class NameServiceDispatcher extends ControllerMaintainer { - - private final Clock clock; - private final CuratorDb db; - private final NameService nameService; - - NameServiceDispatcher(Controller controller, NameService nameService, Duration interval) { - super(controller, interval); - this.clock = controller.clock(); - this.db = controller.curator(); - this.nameService = nameService; - } - - public NameServiceDispatcher(Controller controller, Duration interval) { - this(controller, controller.serviceRegistry().nameService(), interval); - } - - @Override - protected double maintain() { - // Dispatch 1 request per second on average. Note that this is not entirely accurate because a NameService - // implementation may need to perform multiple API-specific requests to execute a single NameServiceRequest - int requestCount = trueIntervalInSeconds(); - final NameServiceQueue initial; - try (var lock = db.lockNameServiceQueue()) { - initial = db.readNameServiceQueue(); - } - if (initial.requests().isEmpty() || requestCount == 0) return 1.0; - - Instant instant = clock.instant(); - NameServiceQueue remaining = initial.dispatchTo(nameService, requestCount); - NameServiceQueue dispatched = initial.without(remaining); - - if (!dispatched.requests().isEmpty()) { - Level logLevel = controller().system().isCd() ? Level.INFO : Level.FINE; - log.log(logLevel, () -> "Dispatched name service request(s) in " + - Duration.between(instant, clock.instant()) + - ": " + dispatched); - } - - try (var lock = db.lockNameServiceQueue()) { - db.writeNameServiceQueue(db.readNameServiceQueue().replace(initial, remaining)); - } - return dispatched.requests().size() / (double) Math.min(requestCount, initial.requests().size()); - } - - /** The true interval at which this runs in this cluster */ - private int trueIntervalInSeconds() { - return (int) interval().dividedBy(db.cluster().size()).getSeconds(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java deleted file mode 100644 index a712c4f35d9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.OsRelease; -import com.yahoo.vespa.hosted.controller.versions.OsVersion; -import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; -import com.yahoo.yolean.Exceptions; - -import java.time.DayOfWeek; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDate; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.util.Objects; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Automatically schedule upgrades to the next OS version. - * - * @author mpolden - */ -public class OsUpgradeScheduler extends ControllerMaintainer { - - private static final Logger LOG = Logger.getLogger(OsUpgradeScheduler.class.getName()); - - public OsUpgradeScheduler(Controller controller, Duration interval) { - super(controller, interval); - } - - @Override - protected double maintain() { - Instant now = controller().clock().instant(); - int attempts = 0; - int failures = 0; - for (var cloud : controller().clouds()) { - Optional<Change> change = changeIn(cloud, now, false); - if (change.isEmpty()) continue; - try { - attempts++; - controller().os().upgradeTo(change.get().osVersion().version(), cloud, false, false); - } catch (IllegalArgumentException e) { - failures++; - LOG.log(Level.WARNING, "Failed to schedule OS upgrade: " + Exceptions.toMessageString(e) + - ". Retrying in " + interval()); - } - } - return asSuccessFactorDeviation(attempts, failures); - } - - /** - * Returns the next OS version change - * - * @param cloud The cloud where the change will be deployed - * @param now Current time - * @param future Whether to return a change that cannot be scheduled now - */ - public Optional<Change> changeIn(CloudName cloud, Instant now, boolean future) { - Optional<OsVersionTarget> currentTarget = controller().os().target(cloud); - if (currentTarget.isEmpty()) return Optional.empty(); - if (upgradingToNewMajor(cloud)) return Optional.empty(); // Skip further upgrades until major version upgrade is complete - - Version currentVersion = currentTarget.get().version(); - Change change = releaseIn(cloud).change(currentVersion, now); - if (!change.osVersion().version().isAfter(currentVersion)) return Optional.empty(); - if (!future && !change.scheduleAt(now)) return Optional.empty(); - - boolean certified = certified(change); - if (!future && !certified) return Optional.empty(); - return Optional.of(change.withCertified(certified)); - } - - private boolean certified(Change change) { - boolean certified = controller().os().certified(change.osVersion()); - if (!certified) { - LOG.log(Level.WARNING, "Want to schedule " + change + ", but this change is not certified for " + - "the current system version"); - } - return certified; - } - - private boolean upgradingToNewMajor(CloudName cloud) { - return controller().os().status().versionsIn(cloud).stream() - .filter(version -> !version.isEmpty()) // Ignore empty/unknown versions - .map(Version::getMajor) - .distinct() - .count() > 1; - } - - private Release releaseIn(CloudName cloud) { - boolean useTaggedRelease = controller().zoneRegistry().zones().all().dynamicallyProvisioned().in(cloud) - .zones().isEmpty(); - if (useTaggedRelease) { - return new TaggedRelease(controller().system(), cloud, controller().serviceRegistry().artifactRepository()); - } - return new CalendarVersionedRelease(controller().system(), cloud); - } - - private static boolean canTriggerAt(Instant instant, boolean isCd) { - ZonedDateTime dateTime = instant.atZone(ZoneOffset.UTC); - int hourOfDay = dateTime.getHour(); - int dayOfWeek = dateTime.getDayOfWeek().getValue(); - // Upgrade can only be scheduled between 07:00 (02:00 in CD systems) and 12:59 UTC, Monday-Thursday - int startHour = isCd ? 2 : 7; - return hourOfDay >= startHour && hourOfDay <= 12 && dayOfWeek < 5; - } - - /** Returns the earliest time, at or after instant, an upgrade can be scheduled */ - private static Instant schedulingInstant(Instant instant, SystemName system) { - ChronoUnit schedulingResolution = ChronoUnit.HOURS; - while (!canTriggerAt(instant, system.isCd())) { - instant = instant.truncatedTo(schedulingResolution) - .plus(schedulingResolution.getDuration()); - } - return instant; - } - - /** Returns the remaining cool-down period relative to releaseAge */ - private static Duration remainingCooldownOf(Duration cooldown, Duration releaseAge) { - return releaseAge.compareTo(cooldown) < 0 ? cooldown.minus(releaseAge) : Duration.ZERO; - } - - private interface Release { - - /** The next available change of this release at given instant */ - Change change(Version currentVersion, Instant instant); - - } - - /** OS version change and the earliest time it can be scheduled */ - public record Change(OsVersion osVersion, Instant scheduleAt, boolean certified) { - - public Change { - Objects.requireNonNull(osVersion); - Objects.requireNonNull(scheduleAt); - } - - public Change withCertified(boolean certified) { - return new Change(osVersion, scheduleAt, certified); - } - - /** Returns whether this can be scheduled at given instant */ - public boolean scheduleAt(Instant instant) { - return !instant.isBefore(scheduleAt); - } - - } - - /** OS release based on a tag */ - private record TaggedRelease(SystemName system, CloudName cloud, ArtifactRepository artifactRepository) implements Release { - - public TaggedRelease { - Objects.requireNonNull(system); - Objects.requireNonNull(cloud); - Objects.requireNonNull(artifactRepository); - } - - @Override - public Change change(Version currentVersion, Instant instant) { - OsRelease release = artifactRepository.osRelease(currentVersion.getMajor(), OsRelease.Tag.latest); - Duration cooldown = remainingCooldownOf(cooldown(), release.age(instant)); - Instant scheduleAt = schedulingInstant(instant.plus(cooldown), system); - return new Change(new OsVersion(release.version(), cloud), scheduleAt, false); - } - - /** The cool-down period that must pass before a release can be used */ - private Duration cooldown() { - return system.isCd() ? Duration.ofDays(1) : Duration.ZERO; - } - - } - - /** OS release based on calendar-versioning */ - record CalendarVersionedRelease(SystemName system, CloudName cloud) implements Release { - - /** A fixed point in time which the release schedule is calculated from */ - private static final Instant START_OF_SCHEDULE = LocalDate.of(2022, 1, 1) - .atStartOfDay() - .toInstant(ZoneOffset.UTC); - - /** The approximate time that should elapse between versions */ - private static final Duration SCHEDULING_STEP = Duration.ofDays(60); - - /** The day of week new releases are published */ - private static final DayOfWeek RELEASE_DAY = DayOfWeek.TUESDAY; - - /** How far into release day we should wait before triggering. This is to give the new release some time to propagate */ - private static final Duration COOLDOWN = Duration.ofHours(6); - - public CalendarVersionedRelease { - Objects.requireNonNull(system); - } - - @Override - public Change change(Version currentVersion, Instant instant) { - CalendarVersion version = findVersion(instant, currentVersion); - Instant predicted = instant; - while (!version.version().isAfter(currentVersion)) { - predicted = predicted.plus(Duration.ofDays(1)); - version = findVersion(predicted, currentVersion); - } - Duration cooldown = remainingCooldownOf(COOLDOWN, version.age(instant)); - Instant schedulingInstant = schedulingInstant(instant.plus(cooldown), system); - return new Change(new OsVersion(version.version(), cloud), schedulingInstant, false); - } - - /** Find the most recent version available according to the scheduling step, relative to now */ - static CalendarVersion findVersion(Instant now, Version currentVersion) { - Instant candidate = START_OF_SCHEDULE; - while (!candidate.plus(SCHEDULING_STEP).isAfter(now)) { - candidate = candidate.plus(SCHEDULING_STEP); - } - LocalDate date = LocalDate.ofInstant(candidate, ZoneOffset.UTC); - while (date.getDayOfWeek() != RELEASE_DAY) { - date = date.minusDays(1); - } - return CalendarVersion.from(date, currentVersion); - } - - record CalendarVersion(Version version, LocalDate date) { - - private static final DateTimeFormatter CALENDAR_VERSION_PATTERN = DateTimeFormatter.ofPattern("yyyyMMdd"); - - private static CalendarVersion from(LocalDate date, Version currentVersion) { - String qualifier = date.format(CALENDAR_VERSION_PATTERN); - return new CalendarVersion(new Version(currentVersion.getMajor(), - currentVersion.getMinor(), - currentVersion.getMicro(), - qualifier), - date); - } - - /** Returns the age of this at given instant */ - private Duration age(Instant instant) { - return Duration.between(date.atStartOfDay().toInstant(ZoneOffset.UTC), instant); - } - - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java deleted file mode 100644 index 25a0abbce90..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.zone.NodeSlice; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; - -import java.time.Duration; -import java.util.Optional; -import java.util.Set; -import java.util.logging.Logger; - -/** - * Trigger OS upgrade of zones in the system, according to the current OS version target. - * - * Target OS version is set per cloud, and an instance of this exists per cloud in the system. - * - * {@link OsUpgradeScheduler} may update the target automatically in supported clouds. - * - * @author mpolden - */ -public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { - - private static final Logger log = Logger.getLogger(OsUpgrader.class.getName()); - - private static final Set<Node.State> upgradableNodeStates = Set.of( - Node.State.ready, - Node.State.active, - Node.State.reserved - ); - - private final CloudName cloud; - - public OsUpgrader(Controller controller, Duration interval, CloudName cloud) { - super(controller, interval, controller.zoneRegistry().osUpgradePolicy(cloud), SystemApplication.all(), name(cloud)); - this.cloud = cloud; - } - - @Override - protected void upgrade(OsVersionTarget target, SystemApplication application, ZoneApi zone) { - log.info(Text.format((target.downgrade() ? "Downgrading" : "Upgrading") + " OS of %s to version %s in %s in cloud %s", application.id(), - target.osVersion().version().toFullString(), - zone.getVirtualId(), zone.getCloudName())); - controller().serviceRegistry().configServer().nodeRepository().upgradeOs(zone.getVirtualId(), application.nodeType(), - target.osVersion().version(), - target.downgrade()); - } - - @Override - protected boolean convergedOn(OsVersionTarget target, SystemApplication application, ZoneApi zone, NodeSlice nodeSlice) { - Version currentVersion = versionOf(nodeSlice, zone, application, Node::currentOsVersion, target.downgrade()).orElse(target.version()); - return satisfiedBy(currentVersion, target); - } - - @Override - protected boolean expectUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { - return cloud.equals(zone.getCloudName()) && // Cloud is managed by this upgrader - application.shouldUpgradeOs() && // Application should upgrade in this cloud - canUpgrade(node, false); - } - - @Override - protected Optional<OsVersionTarget> target() { - // Return target if we have nodes in this cloud on the wrong version, or if we're downgrading a zone which does - // not support downgrading all nodes - return controller().os().target(cloud) - .filter(target -> (target.downgrade() && !downgradingSupported()) || - controller().os().status().nodesIn(cloud).stream() - .anyMatch(node -> !satisfiedBy(node.currentVersion(), target))); - } - - @Override - protected boolean changeTargetTo(OsVersionTarget target, SystemApplication application, ZoneApi zone) { - if (!application.shouldUpgradeOs()) return false; - return controller().serviceRegistry().configServer().nodeRepository() - .targetVersionsOf(zone.getVirtualId()) - .osVersion(application.nodeType()) - .map(currentVersion -> !currentVersion.equals(target.version())) - .orElse(true); - } - - private boolean satisfiedBy(Version version, OsVersionTarget target) { - if (target.downgrade() && downgradingSupported()) { - // When downgrading we want an exact version if the cloud supports downgrades - return version.equals(target.osVersion().version()); - } - // Otherwise, matching or later version is fine - return !version.isBefore(target.osVersion().version()); - } - - private boolean downgradingSupported() { - return !controller().zoneRegistry().zones().all().dynamicallyProvisioned().in(cloud).zones().isEmpty(); - } - - /** Returns whether node currently allows upgrades */ - public static boolean canUpgrade(Node node, boolean includeDeferring) { - return (includeDeferring || !node.deferOsUpgrade()) && upgradableNodeStates.contains(node.state()); - } - - private static String name(CloudName cloud) { - return capitalize(cloud.value()) + OsUpgrader.class.getSimpleName(); // Prefix maintainer name with cloud name - } - - private static String capitalize(String s) { - if (s.isEmpty()) { - return s; - } - char firstLetter = Character.toUpperCase(s.charAt(0)); - if (s.length() > 1) { - return firstLetter + s.substring(1).toLowerCase(); - } - return String.valueOf(firstLetter); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java deleted file mode 100644 index 2fd92970bc9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.util.logging.Level; - -/** - * @author mpolden - */ -public class OsVersionStatusUpdater extends ControllerMaintainer { - - public OsVersionStatusUpdater(Controller controller, Duration interval) { - super(controller, interval); - } - - @Override - protected double maintain() { - try { - OsVersionStatus newStatus = OsVersionStatus.compute(controller()); - controller().os().updateStatus(newStatus); - controller().os().removeStaleCertifications(newStatus); - return 0.0; - } catch (Exception e) { - log.log(Level.WARNING, "Failed to compute OS version status: " + Exceptions.toMessageString(e) + - ". Retrying in " + interval()); - } - return 1.0; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java deleted file mode 100644 index 6c414e44a96..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.util.logging.Logger; - -/** - * Deploys application changes which have been postponed due to an ongoing upgrade, or a block window. - * - * @author bratseth - */ -public class OutstandingChangeDeployer extends ControllerMaintainer { - - private static final Logger logger = Logger.getLogger(OutstandingChangeDeployer.class.getName()); - - public OutstandingChangeDeployer(Controller controller, Duration interval) { - super(controller, interval); - } - - @Override - protected double maintain() { - double ok = 0, total = 0; - for (Application application : ApplicationList.from(controller().applications().readable()) - .withProjectId() - .withJobs() - .asList()) - try { - ++total; - controller().applications().deploymentTrigger().triggerNewRevision(application.id()); - ++ok; - } - catch (RuntimeException e) { - logger.info("Failed triggering new revision for " + application + ": " + Exceptions.toMessageString(e)); - } - return total > 0 ? ok / total : 1; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java deleted file mode 100644 index e35bb139142..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.TriggerResult; - -import java.time.Duration; - -/** - * Trigger ready deployment jobs. This drives jobs through each application's deployment pipeline. - * - * @author bratseth - */ -public class ReadyJobsTrigger extends ControllerMaintainer { - - public ReadyJobsTrigger(Controller controller, Duration interval) { - super(controller, interval); - } - - @Override - public double maintain() { - TriggerResult result = controller().applications().deploymentTrigger().triggerReadyJobs(); - long total = result.triggered() + result.failed(); - return total == 0 ? 1 : (double) result.triggered() / (result.triggered() + result.failed()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java deleted file mode 100644 index 0668f8c481c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.time.Instant; -import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Periodically triggers reindexing for all hosted Vespa applications. - * - * Since reindexing is meant to be a background effort, exactly when things are triggered is not critical, - * and a hash of id of each deployment is used to spread triggering out across the reindexing period. - * Only deployments within a window of opportunity of two maintainer periods are considered in each run. - * Reindexing is triggered for a deployment if it was last triggered more than half a period ago, and - * if no reindexing is currently ongoing. This means an application may skip reindexing during a period - * if it happens to reindex, e.g., a particular document type in its window of opportunity. This is fine. - * - * @author jonmv - */ -public class ReindexingTriggerer extends ControllerMaintainer { - - static final Duration reindexingPeriod = Duration.ofDays(91); // 13 weeks — four times a year. - static final double speed = 0.2; // Careful reindexing, as this is supposed to be a background operation. - - private static final Logger log = Logger.getLogger(ReindexingTriggerer.class.getName()); - - public ReindexingTriggerer(Controller controller, Duration duration) { - super(controller, duration); - } - - @Override - protected double maintain() { - try { - Instant now = controller().clock().instant(); - for (Application application : controller().applications().asList()) - application.productionDeployments().forEach((name, deployments) -> { - ApplicationId id = application.id().instance(name); - for (Deployment deployment : deployments) - if ( inWindowOfOpportunity(now, id, deployment.zone()) - && reindexingIsReady(controller().applications().applicationReindexing(id, deployment.zone()), now)) - controller().applications().reindex(id, deployment.zone(), List.of(), List.of(), true, speed, - "bakground reindexing, to account for changes in built-in linguistics components"); - }); - return 0.0; - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to trigger reindexing: " + Exceptions.toMessageString(e)); - return 1.0; - } - } - - static boolean inWindowOfOpportunity(Instant now, ApplicationId id, ZoneId zone) { - long dayOfPeriodToTrigger = Math.floorMod((id.serializedForm() + zone.value()).hashCode(), 65); // 13 weeks a 5 week days. - long weekOfPeriodToTrigger = dayOfPeriodToTrigger / 5; - long dayOfWeekToTrigger = dayOfPeriodToTrigger % 5; - long daysSinceFirstMondayAfterEpoch = Instant.EPOCH.plus(Duration.ofDays(4)).until(now, ChronoUnit.DAYS); // EPOCH was a Thursday. - long weekOfPeriod = (daysSinceFirstMondayAfterEpoch / 7) % 13; // 7 days to a calendar week, 13 weeks to the period. - long dayOfWeek = daysSinceFirstMondayAfterEpoch % 7; - long hourOfTrondheimTime = ZonedDateTime.ofInstant(now, java.time.ZoneId.of("Europe/Oslo")).getHour(); - - return weekOfPeriod == weekOfPeriodToTrigger - && dayOfWeek == dayOfWeekToTrigger - && 8 <= hourOfTrondheimTime && hourOfTrondheimTime < 12; - } - - static boolean reindexingIsReady(ApplicationReindexing reindexing, Instant now) { - return reindexing.clusters().values().stream().flatMap(cluster -> cluster.ready().values().stream()) - .allMatch(status -> status.readyAt().map(now.minus(reindexingPeriod.dividedBy(2))::isAfter).orElse(true) - && (status.startedAt().isEmpty() || status.endedAt().isPresent())); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java deleted file mode 100644 index 5cadd13309b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import ai.vespa.metrics.ControllerMetrics; -import com.yahoo.concurrent.UncheckedTimeoutException; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.ApplicationController; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.yolean.Exceptions; - -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeoutException; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.stream.Collector; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Creates a {@link ResourceSnapshot} per application, which is then passed on to a MeteringClient - * - * @author olaa - */ -public class ResourceMeterMaintainer extends ControllerMaintainer { - - /** - * Checks if the node is in some state where it is in active use by the tenant, - * and not transitioning out of use, in a failed state, etc. - */ - private static final Set<Node.State> METERABLE_NODE_STATES = EnumSet.of( - Node.State.reserved, // an application will soon use this node - Node.State.active, // an application is currently using this node - Node.State.inactive // an application is not using it, but it is reserved for being re-introduced or decommissioned - ); - - private final ApplicationController applications; - private final NodeRepository nodeRepository; - private final ResourceDatabaseClient resourceClient; - private final CuratorDb curator; - private final SystemName systemName; - private final Metric metric; - private final Clock clock; - - private static final String METERING_LAST_REPORTED = ControllerMetrics.METERING_LAST_REPORTED.baseName(); - private static final String METERING_TOTAL_REPORTED = ControllerMetrics.METERING_TOTAL_REPORTED.baseName(); - private static final int METERING_REFRESH_INTERVAL_SECONDS = 1800; - - @SuppressWarnings("WeakerAccess") - public ResourceMeterMaintainer(Controller controller, - Duration interval, - Metric metric, - ResourceDatabaseClient resourceClient) { - super(controller, interval); - this.applications = controller.applications(); - this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.resourceClient = resourceClient; - this.curator = controller.curator(); - this.systemName = controller.serviceRegistry().zoneRegistry().system(); - this.metric = metric; - this.clock = controller.clock(); - } - - @Override - protected double maintain() { - Collection<ResourceSnapshot> resourceSnapshots; - try { - resourceSnapshots = getAllResourceSnapshots(); - } catch (Exception e) { - log.log(Level.WARNING, "Failed to collect resource snapshots. Retrying in " + interval() + ". Error: " + - Exceptions.toMessageString(e)); - return 1.0; - } - - if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots); - if (systemName.isPublic()) reportAllScalingEvents(); - updateDeploymentCost(resourceSnapshots); - return 0.0; - } - - void updateDeploymentCost(Collection<ResourceSnapshot> resourceSnapshots) { - resourceSnapshots.stream() - .collect(Collectors.groupingBy(snapshot -> TenantAndApplicationId.from(snapshot.getApplicationId()), - Collectors.groupingBy(snapshot -> snapshot.getApplicationId().instance()))) - .forEach(this::updateDeploymentCost); - } - - private void updateDeploymentCost(TenantAndApplicationId tenantAndApplication, Map<InstanceName, List<ResourceSnapshot>> snapshotsByInstance) { - try { - applications.lockApplicationIfPresent(tenantAndApplication, locked -> { - for (InstanceName instanceName : locked.get().instances().keySet()) { - Map<ZoneId, Double> deploymentCosts = snapshotsByInstance.getOrDefault(instanceName, List.of()).stream() - .collect(Collectors.toUnmodifiableMap( - ResourceSnapshot::getZoneId, - snapshot -> cost(snapshot.resources(), systemName), - Double::sum)); - locked = locked.with(instanceName, i -> i.withDeploymentCosts(deploymentCosts)); - updateCostMetrics(tenantAndApplication.instance(instanceName), deploymentCosts); - } - applications.store(locked); - }); - } catch (UncheckedTimeoutException ignored) { - // Will be retried on next maintenance, avoid throwing so we can update the other apps instead - } - } - - private void reportResourceSnapshots(Collection<ResourceSnapshot> resourceSnapshots) { - resourceClient.writeResourceSnapshots(resourceSnapshots); - - updateMeteringMetrics(resourceSnapshots); - - try (var lock = curator.lockMeteringRefreshTime()) { - if (needsRefresh(curator.readMeteringRefreshTime())) { - resourceClient.refreshMaterializedView(); - curator.writeMeteringRefreshTime(clock.millis()); - } - } catch (TimeoutException ignored) { - // If it's locked, it means we're currently refreshing - } - } - - private List<ResourceSnapshot> getAllResourceSnapshots() { - return controller().zoneRegistry().zones() - .reachable().zones().stream() - .map(ZoneApi::getId) - .map(zoneId -> createResourceSnapshotsFromNodes(zoneId, nodeRepository.list(zoneId, NodeFilter.all()))) - .flatMap(Collection::stream) - .toList(); - } - - private Stream<Instance> mapApplicationToInstances(Application application) { - return application.instances().values().stream(); - } - - private Stream<DeploymentId> mapInstanceToDeployments(Instance instance) { - return instance.deployments().keySet().stream() - .filter(zoneId -> !zoneId.environment().isTest()) - .map(zoneId -> new DeploymentId(instance.id(), zoneId)); - } - - private Stream<Map.Entry<ClusterId, List<Cluster.ScalingEvent>>> mapDeploymentToClusterScalingEvent(DeploymentId deploymentId) { - try { - // TODO: get Application from controller.applications().deploymentInfo() - return nodeRepository.getApplication(deploymentId.zoneId(), deploymentId.applicationId()) - .clusters().entrySet().stream() - .map(cluster -> Map.entry(new ClusterId(deploymentId, cluster.getKey()), cluster.getValue().scalingEvents())); - } catch (Exception e) { - log.info("Could not retrieve scaling events for " + deploymentId + ": " + e.getMessage()); - return Stream.empty(); - } - } - - private void reportAllScalingEvents() { - var clusters = controller().applications().asList().stream() - .flatMap(this::mapApplicationToInstances) - .flatMap(this::mapInstanceToDeployments) - .flatMap(this::mapDeploymentToClusterScalingEvent) - .collect(Collectors.toMap( - Map.Entry::getKey, - Map.Entry::getValue - )); - - for (var cluster : clusters.entrySet()) { - resourceClient.writeScalingEvents(cluster.getKey(), cluster.getValue()); - } - } - - private Collection<ResourceSnapshot> createResourceSnapshotsFromNodes(ZoneId zoneId, List<Node> nodes) { - return nodes.stream() - .filter(this::unlessNodeOwnerIsSystemApplication) - .filter(this::isNodeStateMeterable) - .filter(this::isClusterTypeMeterable) - .collect(groupSnapshots(zoneId)) - .values() - .stream() - .toList(); - } - - private boolean unlessNodeOwnerIsSystemApplication(Node node) { - return node.owner() - .map(owner -> !owner.tenant().equals(SystemApplication.TENANT)) - .orElse(false); - } - - private boolean isNodeStateMeterable(Node node) { - return METERABLE_NODE_STATES.contains(node.state()); - } - - private boolean isClusterTypeMeterable(Node node) { - return node.clusterType() != Node.ClusterType.admin; // log servers and shared cluster controllers - } - - private boolean needsRefresh(long lastRefreshTimestamp) { - return clock.instant() - .minusSeconds(METERING_REFRESH_INTERVAL_SECONDS) - .isAfter(Instant.ofEpochMilli(lastRefreshTimestamp)); - } - - public static double cost(ClusterResources clusterResources, SystemName systemName) { - var totalResources = clusterResources.nodeResources().multipliedBy(clusterResources.nodes()); - return cost(totalResources, systemName); - } - - private static double cost(NodeResources resources, SystemName systemName) { - // Divide cost by 3 in non-public zones to show approx. AWS equivalent cost - double costDivisor = systemName.isPublic() ? 1.0 : 3.0; - return Math.round(resources.cost() * 100.0 / costDivisor) / 100.0; - } - - private void updateMeteringMetrics(Collection<ResourceSnapshot> resourceSnapshots) { - metric.set(METERING_LAST_REPORTED, clock.millis() / 1000, metric.createContext(Collections.emptyMap())); - // total metered resource usage, for alerting on drastic changes - metric.set(METERING_TOTAL_REPORTED, - resourceSnapshots.stream() - .mapToDouble(r -> r.resources().vcpu() + r.resources().memoryGb() + r.resources().diskGb()).sum(), - metric.createContext(Collections.emptyMap())); - - resourceSnapshots.forEach(snapshot -> { - var context = getMetricContext(snapshot); - metric.set("metering.vcpu", snapshot.resources().vcpu(), context); - metric.set("metering.memoryGB", snapshot.resources().memoryGb(), context); - metric.set("metering.diskGB", snapshot.resources().diskGb(), context); - }); - } - - private void updateCostMetrics(ApplicationId applicationId, Map<ZoneId, Double> deploymentCost) { - deploymentCost.forEach((zoneId, cost) -> { - var context = getMetricContext(applicationId, zoneId); - metric.set("metering.cost.hourly", cost, context); - }); - } - - private Metric.Context getMetricContext(ApplicationId applicationId, ZoneId zoneId) { - return metric.createContext(Map.of( - "tenantName", applicationId.tenant().value(), - "applicationId", applicationId.toFullString(), - "zoneId", zoneId.value() - )); - } - - private Metric.Context getMetricContext(ResourceSnapshot snapshot) { - return metric.createContext(Map.of( - "tenantName", snapshot.getApplicationId().tenant().value(), - "applicationId", snapshot.getApplicationId().toFullString(), - "zoneId", snapshot.getZoneId().value(), - "architecture", snapshot.resources().architecture() - )); - } - - private Collector<Node, ?, Map<ResourceKey, ResourceSnapshot>> groupSnapshots(ZoneId zoneId) { - return Collectors.collectingAndThen( - Collectors.groupingBy( - (Node node) -> new ResourceKey(node.owner().get(), node.resources().architecture(), node.wantedVersion().getMajor(), node.cloudAccount()), - Collectors.toList()), - convertNodeListToResourceSnapshot(zoneId)); - } - - private Function<Map<ResourceKey, List<Node>>, Map<ResourceKey, ResourceSnapshot>> convertNodeListToResourceSnapshot(ZoneId zoneId) { - return nodesByMajor -> { - return nodesByMajor.entrySet().stream() - .collect(Collectors.toMap( - entry -> entry.getKey(), - entry -> ResourceSnapshot.from(entry.getValue(), clock.instant(), zoneId))); - }; - } - - private record ResourceKey( - ApplicationId applicationId, - NodeResources.Architecture architecture, - int majorVersion, - CloudAccount account) {} -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java deleted file mode 100644 index a0c94c0b9a7..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import org.apache.hc.client5.http.ConnectTimeoutException; - -import java.time.Duration; -import java.util.Map; -import java.util.Optional; -import java.util.logging.Level; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger.INFRASTRUCTURE_APPLICATION; - -/** - * @author olaa - */ -public class ResourceTagMaintainer extends ControllerMaintainer { - - private final ResourceTagger resourceTagger; - - public ResourceTagMaintainer(Controller controller, Duration interval, ResourceTagger resourceTagger) { - super(controller, interval); - this.resourceTagger = resourceTagger; - } - - @Override - public double maintain() { - controller().zoneRegistry().zones() - .reachable() - .in(CloudName.AWS) - .zones().forEach(zone -> { - Map<HostName, ApplicationId> applicationOfHosts = getTenantOfParentHosts(zone.getId()); - int taggedResources = resourceTagger.tagResources(zone, applicationOfHosts); - if (taggedResources > 0) - log.log(Level.INFO, "Tagged " + taggedResources + " resources in " + zone.getId()); - }); - return 0.0; - } - - private Map<HostName, ApplicationId> getTenantOfParentHosts(ZoneId zoneId) { - try { - return controller().serviceRegistry().configServer().nodeRepository() - .list(zoneId, NodeFilter.all()) - .stream() - .filter(node -> node.type().isHost()) - .collect(Collectors.toMap( - Node::hostname, - node -> ownerApplicationId(node.type(), node.exclusiveTo(), node.exclusiveToClusterType()), - (node1, node2) -> node1 - )); - } catch (Exception e) { - if (e.getCause() instanceof ConnectTimeoutException) { - // Usually transient - try again later - log.warning("Unable to retrieve hosts from " + zoneId.value()); - return Map.of(); - } - throw e; - } - } - - // Must be the same as CloudHostProvisioner::ownerApplicationId - private static ApplicationId ownerApplicationId(NodeType hostType, Optional<ApplicationId> exclusiveTo, Optional<Node.ClusterType> exclusiveToClusterType) { - if (hostType != NodeType.host) return INFRASTRUCTURE_APPLICATION; - return exclusiveTo.orElseGet(() -> - ApplicationId.from("hosted-vespa", "shared-host", exclusiveToClusterType.map(Node.ClusterType::name).orElse("default"))); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java deleted file mode 100644 index 3cbd7b3e0e6..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.deployment.RetriggerEntry; -import com.yahoo.vespa.hosted.controller.deployment.Run; - -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Trigger any jobs that are marked for re-triggering to effectuate some other change, e.g. a change in access to a - * deployment's nodes. - * - * @author tokle - */ -public class RetriggerMaintainer extends ControllerMaintainer { - - private static final Logger logger = Logger.getLogger(RetriggerMaintainer.class.getName()); - - public RetriggerMaintainer(Controller controller, Duration interval) { - super(controller, interval); - } - - @Override - protected double maintain() { - try (var lock = controller().curator().lockDeploymentRetriggerQueue()) { - List<RetriggerEntry> retriggerEntries = controller().curator().readRetriggerEntries(); - - // Trigger all jobs that still need triggering and is not running - retriggerEntries.stream() - .filter(this::needsTrigger) - .filter(entry -> readyToTrigger(entry.jobId())) - .forEach(entry -> controller().applications().deploymentTrigger().reTrigger(entry.jobId().application(), entry.jobId().type(), - "re-triggered by " + getClass().getSimpleName())); - - // Remove all jobs that has succeeded with the required job run and persist the list - List<RetriggerEntry> remaining = retriggerEntries.stream() - .filter(this::needsTrigger) - .toList(); - controller().curator().writeRetriggerEntries(remaining); - } catch (Exception e) { - logger.log(Level.WARNING, "Exception while triggering jobs", e); - return 1.0; - } - return 0.0; - } - - /** Returns true if a job is ready to run, i.e. is currently not running */ - private boolean readyToTrigger(JobId jobId) { - Optional<Run> existingRun = controller().jobController().active(jobId.application()).stream() - .filter(run -> run.id().type().equals(jobId.type())) - .findFirst(); - return existingRun.isEmpty(); - } - - /** Returns true of job needs triggering. I.e. the job has not run since the queue item was last run */ - private boolean needsTrigger(RetriggerEntry entry) { - return controller().jobController().lastCompleted(entry.jobId()) - .filter(run -> run.id().number() < entry.requiredRun()) - .isPresent(); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java deleted file mode 100644 index c31f81497e6..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.zone.NodeSlice; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.hosted.controller.versions.VespaVersionTarget; - -import java.time.Duration; -import java.util.Optional; -import java.util.Set; -import java.util.logging.Logger; - -/** - * Maintenance job which upgrades system applications. - * - * @author mpolden - */ -public class SystemUpgrader extends InfrastructureUpgrader<VespaVersionTarget> { - - private static final Logger log = Logger.getLogger(SystemUpgrader.class.getName()); - - private static final Set<Node.State> upgradableNodeStates = Set.of(Node.State.active, Node.State.reserved); - - public SystemUpgrader(Controller controller, Duration interval) { - super(controller, interval, controller.zoneRegistry().upgradePolicy(), SystemApplication.notController(), null); - } - - @Override - protected void upgrade(VespaVersionTarget target, SystemApplication application, ZoneApi zone) { - log.info(Text.format("Deploying %s on %s in %s", application.id(), target, zone.getId())); - controller().applications().deploy(application, zone.getId(), target.version(), target.downgrade()); - } - - @Override - protected boolean convergedOn(VespaVersionTarget target, SystemApplication application, ZoneApi zone, NodeSlice nodeSlice) { - Optional<Version> currentVersion = versionOf(nodeSlice, zone, application, Node::currentVersion, target.downgrade()); - // Skip application convergence check if there are no nodes belonging to the application in the zone - if (currentVersion.isEmpty()) return true; - - return currentVersion.get().equals(target.version()) && - application.configConvergedIn(zone.getId(), controller(), Optional.of(target.version())); - } - - @Override - protected boolean expectUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { - return eligibleForUpgrade(node); - } - - @Override - protected Optional<VespaVersionTarget> target() { - VersionStatus status = controller().readVersionStatus(); - Optional<VespaVersion> target = status.controllerVersion() - .filter(version -> { - Version systemVersion = status.systemVersion() - .map(VespaVersion::versionNumber) - .orElse(Version.emptyVersion); - return version.versionNumber().isAfter(systemVersion); - }) - .filter(version -> version.confidence() != VespaVersion.Confidence.broken); - boolean downgrade = target.isPresent() && target.get().confidence() == VespaVersion.Confidence.aborted; - if (downgrade) { - target = status.systemVersion(); - } - return target.map(VespaVersion::versionNumber) - .map(version -> new VespaVersionTarget(version, downgrade)); - } - - @Override - protected boolean changeTargetTo(VespaVersionTarget target, SystemApplication application, ZoneApi zone) { - if (application.hasApplicationPackage()) { - // For applications with package we do not have a zone-wide version target. This means that we must check - // the wanted version of each node. - boolean zoneHasSharedRouting = controller().zoneRegistry().routingMethod(zone.getId()).isShared(); - return versionOf(NodeSlice.ALL, zone, application, Node::wantedVersion, target.downgrade()) - .map(wantedVersion -> !wantedVersion.equals(target.version())) - .orElse(zoneHasSharedRouting); // Always upgrade if zone uses shared routing, but has no nodes allocated yet - } - return controller().serviceRegistry().configServer().nodeRepository() - .targetVersionsOf(zone.getId()) - .vespaVersion(application.nodeType()) - .map(wantedVersion -> !wantedVersion.equals(target.version())) - .orElse(true); // Always set target if there are no nodes - } - - /** Returns whether node in application should be upgraded by this */ - public static boolean eligibleForUpgrade(Node node) { - return upgradableNodeStates.contains(node.state()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java deleted file mode 100644 index 5539c62be98..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.time.Duration; - -public class TenantRoleCleanupMaintainer extends ControllerMaintainer { - - public TenantRoleCleanupMaintainer(Controller controller, Duration interval) { - super(controller, interval); - } - - @Override - protected double maintain() { - var roleService = controller().serviceRegistry().roleService(); - - var deletedTenants = controller().tenants().asList(true).stream() - .filter(tenant -> tenant.type() == Tenant.Type.deleted) - .map(Tenant::name) - .toList(); - roleService.cleanupRoles(deletedTenants); - - if (controller().system().isPublic()) { - controller().serviceRegistry().tenantSecretService().cleanupSecretStores(deletedTenants); - } - - return 0.0; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java deleted file mode 100644 index f76b7634c62..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.time.Instant; -import java.util.Comparator; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class TenantRoleMaintainer extends ControllerMaintainer { - - private static final Logger log = Logger.getLogger(TenantRoleMaintainer.class.getName()); - - public TenantRoleMaintainer(Controller controller, Duration tenantRoleMaintainer) { - super(controller, tenantRoleMaintainer); - } - - @Override - protected double maintain() { - var roleService = controller().serviceRegistry().roleService(); - var tenants = controller().tenants().asList().stream() - .sorted(Comparator.comparing(Tenant::tenantRolesLastMaintained)) - .limit(5) - .toList(); - - double ok = 0, attempts = 0, total = 0; - // Create separate athenz service for all tenants - for (Tenant tenant : tenants) { - ++attempts; - try { - roleService.createTenantRole(tenant); - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to create role for " + tenant.name() + ": " + Exceptions.toMessageString(e)); - } - ++ok; - } - total += attempts == 0 ? 1 : ok / attempts; - - total += roleService.maintainRoles(tenants.stream().map(Tenant::name).toList()); - - // Update last maintained timestamp - var updated = controller().clock().instant(); - for (Tenant tenant : tenants) controller().tenants().updateLastTenantRolesMaintained(tenant.name(), updated); - - return total * 0.5; - } - - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java deleted file mode 100644 index dceb3921061..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.InstanceList; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; - -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.function.UnaryOperator; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; - -/** - * Maintenance job which schedules applications for Vespa version upgrade - * - * @author bratseth - * @author mpolden - */ -public class Upgrader extends ControllerMaintainer { - - private static final Logger log = Logger.getLogger(Upgrader.class.getName()); - - private final CuratorDb curator; - private final Random random; - - public Upgrader(Controller controller, Duration interval) { - this(controller, interval, controller.random(false)); - } - - Upgrader(Controller controller, Duration interval, Random random) { - super(controller, interval); - this.curator = controller.curator(); - this.random = random; - } - - /** - * Schedule application upgrades. Note that this implementation must be idempotent. - */ - @Override - public double maintain() { - // Determine target versions for each upgrade policy - VersionStatus versionStatus = controller().readVersionStatus(); - cancelBrokenUpgrades(versionStatus); - - DeploymentStatusList deploymentStatuses = deploymentStatuses(versionStatus); - for (UpgradePolicy policy : UpgradePolicy.values()) - updateTargets(versionStatus, deploymentStatuses, policy); - - return 0.0; - } - - private DeploymentStatusList deploymentStatuses(VersionStatus versionStatus) { - return controller().jobController().deploymentStatuses(ApplicationList.from(controller().applications().readable()) - .withProjectId() - .withJobs(), - versionStatus); - } - - /** Returns a list of all production application instances, except those which are pinned, which we should not manipulate here. */ - private InstanceList instances(DeploymentStatusList deploymentStatuses) { - return InstanceList.from(deploymentStatuses) - .withDeclaredJobs() - .shuffle(random) - .byIncreasingDeployedVersion() - .unpinned(); - } - - private void cancelBrokenUpgrades(VersionStatus versionStatus) { - // Cancel upgrades to broken targets (let other ongoing upgrades complete to avoid starvation) - InstanceList instances = instances(deploymentStatuses(controller().readVersionStatus())); - for (VespaVersion version : versionStatus.versions()) { - if (version.confidence() == Confidence.broken) - cancelUpgradesOf(instances.upgradingTo(version.versionNumber()).not().with(UpgradePolicy.canary), - version.versionNumber() + " is broken"); - } - } - - private void updateTargets(VersionStatus versionStatus, DeploymentStatusList deploymentStatuses, UpgradePolicy policy) { - InstanceList instances = instances(deploymentStatuses); - InstanceList remaining = instances.with(policy); - Instant failureThreshold = controller().clock().instant().minus(DeploymentTrigger.maxFailingRevisionTime); - Set<ApplicationId> failingRevision = InstanceList.from(deploymentStatuses.failingApplicationChangeSince(failureThreshold)).asSet(); - - List<Version> targetAndNewer = new ArrayList<>(); - UnaryOperator<InstanceList> cancellationCriterion = policy == UpgradePolicy.canary ? i -> i.not().upgradingTo(targetAndNewer) - : i -> i.failing() - .not().upgradingTo(targetAndNewer); - - Map<ApplicationId, Version> targets = new LinkedHashMap<>(); - for (Version version : DeploymentStatus.targetsForPolicy(versionStatus, controller().systemVersion(versionStatus), policy)) { - targetAndNewer.add(version); - InstanceList eligible = eligibleForVersion(remaining, version, versionStatus); - InstanceList outdated = cancellationCriterion.apply(eligible); - cancelUpgradesOf(outdated.upgrading(), "Upgrading to outdated versions"); - - // Prefer the newest target for each instance. - remaining = remaining.not().matching(eligible.asList()::contains) - .not().hasCompleted(Change.of(version)); - for (ApplicationId id : outdated.and(eligible.not().upgrading())) - targets.put(id, version); - } - - int numberToUpgrade = policy == UpgradePolicy.canary ? instances.size() : numberOfApplicationsToUpgrade(); - for (ApplicationId id : instances.matching(targets.keySet()::contains)) { - if (failingRevision.contains(id)) { - log.log(Level.INFO, "Cancelling failing revision for " + id); - controller().applications().deploymentTrigger().cancelChange(id, ChangesToCancel.APPLICATION); - } - - if (controller().applications().requireInstance(id).change().isEmpty()) { - log.log(Level.INFO, "Triggering upgrade to " + targets.get(id) + " for " + id); - controller().applications().deploymentTrigger().forceChange(id, Change.of(targets.get(id))); - --numberToUpgrade; - } - if (numberToUpgrade <= 0) break; - } - } - - private InstanceList eligibleForVersion(InstanceList instances, Version version, VersionStatus versionStatus) { - Change change = Change.of(version); - return instances.not().failingOn(version) - .allowingMajorVersion(version.getMajor(), versionStatus) - .compatibleWithPlatform(version, controller().applications()::versionCompatibility) - .not().hasCompleted(change) // Avoid rescheduling change for instances without production steps. - .onLowerVersionThan(version) - .canUpgradeAt(version, controller().clock().instant()); - } - - private void cancelUpgradesOf(InstanceList instances, String reason) { - instances = instances.unpinned(); - if (instances.isEmpty()) return; - log.info("Cancelling upgrading of " + instances.asList() + " instances: " + reason); - for (ApplicationId instance : instances.asList()) - controller().applications().deploymentTrigger().cancelChange(instance, PLATFORM); - } - - /** Returns the number of applications to upgrade in this run */ - private int numberOfApplicationsToUpgrade() { - return numberOfApplicationsToUpgrade(interval().dividedBy(Math.max(1, controller().curator().cluster().size())).toMillis(), - controller().clock().millis(), - upgradesPerMinute()); - } - - /** Returns the number of applications to upgrade in the interval containing now */ - static int numberOfApplicationsToUpgrade(long intervalMillis, long nowMillis, double upgradesPerMinute) { - long intervalStart = Math.round(nowMillis / (double) intervalMillis) * intervalMillis; - double upgradesPerMilli = upgradesPerMinute / 60_000; - long upgradesAtStart = (long) (intervalStart * upgradesPerMilli); - long upgradesAtEnd = (long) ((intervalStart + intervalMillis) * upgradesPerMilli); - return (int) (upgradesAtEnd - upgradesAtStart); - } - - /** Returns number of upgrades per minute */ - public double upgradesPerMinute() { - return curator.readUpgradesPerMinute(); - } - - /** Sets the number of upgrades per minute */ - public void setUpgradesPerMinute(double n) { - if (n < 0) - throw new IllegalArgumentException("Upgrades per minute must be >= 0, got " + n); - curator.writeUpgradesPerMinute(n); - } - - /** Override confidence for given version. This will cause the computed confidence to be ignored */ - public void overrideConfidence(Version version, Confidence confidence) { - if (confidence == Confidence.aborted && !version.isAfter(controller().readSystemVersion())) { - throw new IllegalArgumentException("Cannot override confidence to " + confidence + - " for version " + version.toFullString() + - ": Version may be in use by applications"); - } - try (Mutex lock = curator.lockConfidenceOverrides()) { - Map<Version, Confidence> overrides = new LinkedHashMap<>(curator.readConfidenceOverrides()); - overrides.put(version, confidence); - curator.writeConfidenceOverrides(overrides); - } - } - - /** Returns all confidence overrides */ - public Map<Version, Confidence> confidenceOverrides() { - return curator.readConfidenceOverrides(); - } - - /** Remove confidence override for given version */ - public void removeConfidenceOverride(Version version) { - controller().removeConfidenceOverride(version::equals); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java deleted file mode 100644 index 0f39ef7d0f0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; - -import java.time.Duration; -import java.util.Optional; -import java.util.logging.Logger; -import java.util.stream.Collectors; - - -/** - * Maintains user management resources. - * For now, ensures there's no discrepnacy between expected tenant/application roles and auth0/athenz roles - * - * @author olaa - */ -public class UserManagementMaintainer extends ControllerMaintainer { - - private final RoleMaintainer roleMaintainer; - private static final Logger logger = Logger.getLogger(UserManagementMaintainer.class.getName()); - - public UserManagementMaintainer(Controller controller, Duration interval, RoleMaintainer roleMaintainer) { - super(controller, interval); - this.roleMaintainer = roleMaintainer; - } - - @Override - protected double maintain() { - var tenants = controller().tenants().asList(); - var applications = controller().applications().idList() - .stream() - .map(appId -> ApplicationId.from(appId.tenant(), appId.application(), InstanceName.defaultName())) - .toList(); - roleMaintainer.deleteLeftoverRoles(tenants, applications); - - if (!controller().system().isPublic()) { - roleMaintainer.tenantsToDelete(tenants) - .forEach(tenant -> { - logger.warning(tenant.name() + " has a non-existing Athenz domain. Deleting"); - controller().applications().asList(tenant.name()) - .forEach(application -> controller().applications().deleteApplication(application.id(), Optional.empty())); - controller().tenants().delete(tenant.name(), Optional.empty(), false); - }); - } - - return 0.0; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java deleted file mode 100644 index b0d7a0c47e9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ /dev/null @@ -1,399 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.jdisc.Metric; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest.Impact; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction.State; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VcmrReport; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest.Status; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.time.Instant; -import java.time.ZonedDateTime; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Predicate; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * - * Maintains status and execution of Vespa CMRs. - * - * Currently, this retires all affected tenant hosts if zone capacity allows it. - * - * @author olaa - */ -public class VcmrMaintainer extends ControllerMaintainer { - - private static final Logger LOG = Logger.getLogger(VcmrMaintainer.class.getName()); - private static final int DAYS_TO_RETIRE = 2; - private static final Duration ALLOWED_POSTPONEMENT_TIME = Duration.ofDays(7); - protected static final String TRACKED_CMRS_METRIC = "cmr.tracked"; - - private final CuratorDb curator; - private final NodeRepository nodeRepository; - private final ChangeRequestClient changeRequestClient; - private final SystemName system; - private final Metric metric; - - public VcmrMaintainer(Controller controller, Duration interval, Metric metric) { - super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); - this.curator = controller.curator(); - this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.changeRequestClient = controller.serviceRegistry().changeRequestClient(); - this.system = controller.system(); - this.metric = metric; - } - - @Override - protected double maintain() { - var changeRequests = curator.readChangeRequests() - .stream() - .filter(shouldUpdate()).toList(); - - var nodesByZone = nodesByZone(); - - changeRequests.forEach(changeRequest -> { - var nodes = impactedNodes(nodesByZone, changeRequest); - var nextActions = getNextActions(nodes, changeRequest); - var status = getStatus(nextActions, changeRequest); - - try (var lock = curator.lockChangeRequests()) { - // Read the vcmr again, in case the source status has been updated - curator.readChangeRequest(changeRequest.getId()) - .ifPresent(vcmr -> { - var updatedVcmr = vcmr.withActionPlan(nextActions) - .withStatus(status); - curator.writeChangeRequest(updatedVcmr); - if (nodes.keySet().size() == 1) - approveChangeRequest(updatedVcmr); - }); - } - }); - updateMetrics(); - return 0.0; - } - - /** - * Status is based on: - * 1. Whether the source has reportedly closed the request - * 2. Whether any host requires operator action - * 3. Whether any host is pending/started/finished retirement - */ - private Status getStatus(List<HostAction> nextActions, VespaChangeRequest changeRequest) { - if (changeRequest.getChangeRequestSource().isClosed()) { - return Status.COMPLETED; - } - - var byActionState = nextActions.stream().collect(Collectors.groupingBy(HostAction::getState, Collectors.counting())); - - if (byActionState.getOrDefault(State.REQUIRES_OPERATOR_ACTION, 0L) > 0) { - return Status.REQUIRES_OPERATOR_ACTION; - } - - if (byActionState.getOrDefault(State.OUT_OF_SYNC, 0L) > 0) { - return Status.OUT_OF_SYNC; - } - - if (byActionState.getOrDefault(State.RETIRING, 0L) > 0) { - return Status.IN_PROGRESS; - } - - if (Set.of(State.RETIRED, State.NONE).containsAll(byActionState.keySet())) { - return Status.READY; - } - - if (byActionState.getOrDefault(State.PENDING_RETIREMENT, 0L) > 0) { - return Status.PENDING_ACTION; - } - - return Status.NOOP; - } - - private List<HostAction> getNextActions(Map<ZoneId, List<Node>> nodesByZone, VespaChangeRequest changeRequest) { - return nodesByZone.entrySet() - .stream() - .flatMap(entry -> { - var zone = entry.getKey(); - var nodes = entry.getValue(); - if (nodes.isEmpty()) { - return Stream.empty(); - } - var spareCapacity = hasSpareCapacity(zone, nodes); - var impactedProxyCount = nodes.stream() - .filter(node -> node.type() == NodeType.proxy) - .count(); - return nodes.stream().map(node -> nextAction(zone, node, changeRequest, spareCapacity, impactedProxyCount)); - }).toList(); - - } - - // Get the superset of impacted hosts by looking at impacted switches - private Map<ZoneId, List<Node>> impactedNodes(Map<ZoneId, List<Node>> nodesByZone, VespaChangeRequest changeRequest) { - return nodesByZone.entrySet() - .stream() - .filter(entry -> entry.getValue().stream().anyMatch(isImpacted(changeRequest))) // Skip zones without impacted nodes - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> entry.getValue().stream().filter(isImpacted(changeRequest)).toList() - )); - } - - private Optional<HostAction> getPreviousAction(Node node, VespaChangeRequest changeRequest) { - return changeRequest.getHostActionPlan() - .stream() - .filter(hostAction -> hostAction.getHostname().equals(node.hostname().value())) - .findFirst(); - } - - private HostAction nextAction(ZoneId zoneId, Node node, VespaChangeRequest changeRequest, boolean spareCapacity, long impactedProxyCount) { - var hostAction = getPreviousAction(node, changeRequest) - .orElse(new HostAction(node.hostname().value(), State.NONE, Instant.now())); - - if (changeRequest.getChangeRequestSource().isClosed()) { - LOG.fine(() -> changeRequest.getChangeRequestSource().id() + " is closed, recycling " + node.hostname()); - recycleNode(zoneId, node, hostAction); - removeReport(zoneId, changeRequest, node); - return hostAction.withState(State.COMPLETE); - } - - if (isLowImpact(changeRequest)) - return hostAction; - - if (shouldAddReport(node, changeRequest.getChangeRequestSource().id(), hostAction)) - addReport(zoneId, changeRequest, node); - - if (isOutOfSync(node, hostAction)) - return hostAction.withState(State.OUT_OF_SYNC); - - if (isPostponed(changeRequest, hostAction)) { - LOG.fine(() -> changeRequest.getChangeRequestSource().id() + " is postponed, recycling " + node.hostname()); - recycleNode(zoneId, node, hostAction); - return hostAction.withState(State.PENDING_RETIREMENT); - } - - if (!spareCapacity) { - return hostAction.withState(State.REQUIRES_OPERATOR_ACTION); - } - - if (node.type() != NodeType.host) { - if (node.type() == NodeType.proxy && impactedProxyCount == 1) - return hostAction.withState(State.READY); - return hostAction.withState(State.REQUIRES_OPERATOR_ACTION); - } - - if (shouldRetire(changeRequest, hostAction)) { - if (!wantToRetireRecursive(zoneId, node)) { - LOG.info(Text.format("Retiring %s due to %s", node.hostname().value(), changeRequest.getChangeRequestSource().id())); - // TODO: Remove try/catch once retirement is stabilized - try { - setWantToRetire(zoneId, node, true); - } catch (Exception e) { - LOG.warning("Failed to retire host " + node.hostname() + ": " + Exceptions.toMessageString(e)); - // Will retry next maintenance run - return hostAction; - } - } - return hostAction.withState(State.RETIRING); - } - - if (hasRetired(node, hostAction)) { - LOG.fine(() -> node.hostname() + " has retired"); - return hostAction.withState(State.RETIRED); - } - - if (pendingRetirement(node, hostAction)) { - LOG.fine(() -> node.hostname() + " is pending retirement"); - return hostAction.withState(State.PENDING_RETIREMENT); - } - - if (isFailed(node)) { - return hostAction.withState(State.NONE); - } - - return hostAction; - } - - // Determines if a host and all its children are retiring - private boolean wantToRetireRecursive(ZoneId zoneId, Node node) { - var children = nodeRepository.list(zoneId, NodeFilter.all().parentHostnames(node.hostname())); - return node.wantToRetire() && - children.stream().allMatch(Node::wantToRetire); - } - - // Dirty host iff the parked host was retired by this maintainer - private void recycleNode(ZoneId zoneId, Node node, HostAction hostAction) { - if (hostAction.getState() == State.RETIRED && - node.state() == Node.State.parked) { - LOG.info("Setting " + node.hostname() + " to dirty"); - nodeRepository.setState(zoneId, Node.State.dirty, node.hostname().value()); - } - if (hostAction.getState() == State.RETIRING && node.wantToRetire()) { - try { - setWantToRetire(zoneId, node, false); - } catch (Exception ignored) {} - } - } - - private boolean isPostponed(VespaChangeRequest changeRequest, HostAction action) { - return List.of(State.RETIRED, State.RETIRING).contains(action.getState()) && - changeRequest.getChangeRequestSource().plannedStartTime() - .minus(ALLOWED_POSTPONEMENT_TIME) - .isAfter(ZonedDateTime.now()); - } - - private boolean shouldRetire(VespaChangeRequest changeRequest, HostAction action) { - return action.getState() == State.PENDING_RETIREMENT && - getRetirementStartTime(changeRequest.getChangeRequestSource().plannedStartTime()) - .isBefore(ZonedDateTime.now()); - } - - private boolean hasRetired(Node node, HostAction hostAction) { - return List.of(State.RETIRING, State.REQUIRES_OPERATOR_ACTION).contains(hostAction.getState()) && - node.state() == Node.State.parked; - } - - private boolean pendingRetirement(Node node, HostAction action) { - return List.of(State.NONE, State.REQUIRES_OPERATOR_ACTION).contains(action.getState()) - && node.state() == Node.State.active; - } - - private boolean shouldAddReport(Node node, String vcmrId, HostAction previousAction) { - var vcmrReport = VcmrReport.fromReports(node.reports()); - var hasReport = vcmrReport.getVcmrs().stream().map(VcmrReport.Vcmr::id).anyMatch(id -> id.equals(vcmrId)); - // Don't add report if none exists and this is not initial assessment - // Presumably removed manually by operator. - if (!hasReport && previousAction.getState() != State.NONE) - return false; - return true; - } - - // Determines if node state is unexpected based on previous action taken - private boolean isOutOfSync(Node node, HostAction action) { - return action.getState() == State.RETIRED && node.state() != Node.State.parked || - action.getState() == State.RETIRING && !node.wantToRetire(); - } - - private boolean isFailed(Node node) { - return node.state() == Node.State.failed || - node.state() == Node.State.breakfixed; - } - - private Map<ZoneId, List<Node>> nodesByZone() { - return controller().zoneRegistry() - .zones() - .reachable() - .in(Environment.prod) - .ids() - .stream() - .collect(Collectors.toMap( - zone -> zone, - zone -> nodeRepository.list(zone, NodeFilter.all()) - )); - } - - private Predicate<Node> isImpacted(VespaChangeRequest changeRequest) { - return node -> changeRequest.getImpactedHosts().contains(node.hostname().value()) || - node.switchHostname() - .map(switchHostname -> changeRequest.getImpactedSwitches().contains(switchHostname)) - .orElse(false); - } - private Predicate<VespaChangeRequest> shouldUpdate() { - return changeRequest -> changeRequest.getStatus() != Status.COMPLETED; - } - - private boolean isLowImpact(VespaChangeRequest changeRequest) { - return !List.of(Impact.HIGH, Impact.VERY_HIGH) - .contains(changeRequest.getImpact()); - } - - private boolean hasSpareCapacity(ZoneId zoneId, List<Node> nodes) { - var tenantHosts = nodes.stream() - .filter(node -> node.type() == NodeType.host) - .map(Node::hostname) - .toList(); - - return tenantHosts.isEmpty() || - nodeRepository.isReplaceable(zoneId, tenantHosts); - } - - private void setWantToRetire(ZoneId zoneId, Node node, boolean wantToRetire) { - nodeRepository.retire(zoneId, node.hostname().value(), wantToRetire, false); - } - - private void approveChangeRequest(VespaChangeRequest changeRequest) { - if (!system.equals(SystemName.main)) - return; - if (changeRequest.getStatus() == Status.REQUIRES_OPERATOR_ACTION) - return; - if (changeRequest.getApproval() != ChangeRequest.Approval.REQUESTED) - return; - - LOG.info("Approving " + changeRequest.getChangeRequestSource().id()); - changeRequestClient.approveChangeRequest(changeRequest); - } - - private void removeReport(ZoneId zoneId, VespaChangeRequest changeRequest, Node node) { - var report = VcmrReport.fromReports(node.reports()); - - if (report.removeVcmr(changeRequest.getChangeRequestSource().id())) { - updateReport(zoneId, node, report); - } - } - - private void addReport(ZoneId zoneId, VespaChangeRequest changeRequest, Node node) { - var report = VcmrReport.fromReports(node.reports()); - - if (report.addVcmr(changeRequest.getChangeRequestSource())) { - updateReport(zoneId, node, report); - } - } - - private void updateReport(ZoneId zoneId, Node node, VcmrReport report) { - LOG.fine(() -> Text.format("Updating report for %s: %s", node.hostname(), report)); - nodeRepository.updateReports(zoneId, node.hostname().value(), report.toNodeReports()); - } - - // Calculate wanted retirement start time, ignoring weekends - // protected for testing - protected ZonedDateTime getRetirementStartTime(ZonedDateTime plannedStartTime) { - var time = plannedStartTime; - var days = 0; - while (days < DAYS_TO_RETIRE) { - time = time.minusDays(1); - if (time.getDayOfWeek().getValue() < 6) days++; - } - return time; - } - - private void updateMetrics() { - var cmrsByStatus = curator.readChangeRequests() - .stream() - .collect(Collectors.groupingBy(VespaChangeRequest::getStatus)); - - for (var status : Status.values()) { - var count = cmrsByStatus.getOrDefault(status, List.of()).size(); - metric.set(TRACKED_CMRS_METRIC, count, metric.createContext(Map.of("status", status.name()))); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java deleted file mode 100644 index 721819522f5..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.util.logging.Level; - -import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.aborted; -import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.broken; -import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.high; -import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.legacy; -import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.low; -import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.normal; - -/** - * This maintenance job periodically updates the version status. - * Since the version status is expensive to compute and does not need to be perfectly up to date, - * we do not want to recompute it each time it is accessed. - * - * @author bratseth - */ -public class VersionStatusUpdater extends ControllerMaintainer { - - public VersionStatusUpdater(Controller controller, Duration interval) { - super(controller, interval); - } - - @Override - protected double maintain() { - try { - VersionStatus newStatus = VersionStatus.compute(controller()); - controller().updateVersionStatus(newStatus); - newStatus.systemVersion().ifPresent(version -> { - controller().serviceRegistry().systemMonitor().reportSystemVersion(version.versionNumber(), - convert(version.confidence())); - }); - return 0.0; - } catch (Exception e) { - log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + - ". Retrying in " + interval()); - } - return 1.0; - } - - static SystemMonitor.Confidence convert(VespaVersion.Confidence confidence) { - return switch (confidence) { - case aborted -> aborted; - case broken -> broken; - case low -> low; - case legacy -> legacy; - case normal -> normal; - case high -> high; - }; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java deleted file mode 100644 index f8eed5804bb..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ApplicationMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ApplicationMetrics.java deleted file mode 100644 index 79d196b07fa..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ApplicationMetrics.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.metric; - -/** - * Application metrics aggregated across all deployments. - * - * @author bratseth - */ -public class ApplicationMetrics { - - private final double queryServiceQuality; - private final double writeServiceQuality; - - public ApplicationMetrics(double queryServiceQuality, double writeServiceQuality) { - this.queryServiceQuality = queryServiceQuality; - this.writeServiceQuality = writeServiceQuality; - } - - /** - * Returns the quality of service for queries as a number between 1 (perfect) and 0 (none) - */ - public double queryServiceQuality() { - return queryServiceQuality; - } - - /** - * Returns the quality of service for writes as a number between 1 (perfect) and 0 (none) - */ - public double writeServiceQuality() { - return writeServiceQuality; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java deleted file mode 100644 index 23b81ebcd34..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.metric; - -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.Property; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.time.Clock; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * @author ldalves - */ -public class CostCalculator { - - private static final double SELF_HOSTED_DISCOUNT = .5; - - public static String resourceShareByPropertyToCsv(NodeRepository nodeRepository, - Controller controller, - Clock clock, - Map<Property, ResourceAllocation> fixedAllocations) { - - var date = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")).format(clock.instant()); - - // Group properties by tenant name - Map<TenantName, Property> propertyByTenantName = controller.tenants().asList().stream() - .filter(AthenzTenant.class::isInstance) - .collect(Collectors.toMap(Tenant::name, - tenant -> ((AthenzTenant) tenant).property())); - - // Sum up allocations - Map<Property, ResourceAllocation> allocationByProperty = new HashMap<>(); - var nodes = controller.zoneRegistry().zones() - .reachable().in(Environment.prod).in(CloudName.YAHOO).zones().stream() - .flatMap(zone -> uncheck(() -> nodeRepository.list(zone.getId(), NodeFilter.all()).stream())) - .filter(node -> node.owner().isPresent() && !node.owner().get().tenant().equals(SystemApplication.TENANT)) - .toList(); - var totalAllocation = ResourceAllocation.ZERO; - for (var node : nodes) { - Property property = propertyByTenantName.get(node.owner().get().tenant()); - if (property == null) continue; - var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO); - var nodeAllocation = new ResourceAllocation(node.resources().vcpu(), node.resources().memoryGb(), node.resources().diskGb(), node.resources().architecture()); - allocationByProperty.put(property, allocation.plus(nodeAllocation)); - totalAllocation = totalAllocation.plus(nodeAllocation); - } - - // Add fixed allocations from config - for (var kv : fixedAllocations.entrySet()) { - var property = kv.getKey(); - var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO); - var discountedFixedAllocation = kv.getValue().multiply(SELF_HOSTED_DISCOUNT); - allocationByProperty.put(property, allocation.plus(discountedFixedAllocation)); - totalAllocation = totalAllocation.plus(discountedFixedAllocation); - } - - return toCsv(allocationByProperty, date, totalAllocation); - } - - private static String toCsv(Map<Property, ResourceAllocation> resourceShareByProperty, String date, ResourceAllocation totalResourceAllocation) { - String header = "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n"; - String entries = resourceShareByProperty.entrySet().stream() - .sorted((Comparator.comparingDouble(entry -> entry.getValue().usageFraction(totalResourceAllocation)))) - .map(propertyEntry -> { - ResourceAllocation r = propertyEntry.getValue(); - return Stream.of(date, propertyEntry.getKey(), r.getCpuCores(), r.getMemoryGb(), r.getDiskGb(), r.usageFraction(totalResourceAllocation)) - .map(Object::toString).collect(Collectors.joining(",")); - }) - .collect(Collectors.joining("\n")); - return header + entries; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java deleted file mode 100644 index bed053d592f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.notification; - -import java.util.Objects; - -/** - * Contains formatted text that can be displayed to a user to give extra information and pointers for a given - * Notification. - * - * @author enygaard - */ -public record FormattedNotification(Notification notification, String prettyType, String messagePrefix, String uri) { - - public FormattedNotification { - Objects.requireNonNull(prettyType); - Objects.requireNonNull(messagePrefix); - Objects.requireNonNull(uri); - Objects.requireNonNull(notification); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java deleted file mode 100644 index c54791f511e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.notification; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; -import com.yahoo.vespa.hosted.controller.tenant.PendingMailVerification; -import com.yahoo.yolean.Exceptions; -import org.apache.velocity.VelocityContext; -import org.apache.velocity.app.Velocity; -import org.apache.velocity.app.VelocityEngine; -import org.apache.velocity.runtime.resource.loader.StringResourceLoader; -import org.apache.velocity.runtime.resource.util.StringResourceRepository; -import org.apache.velocity.tools.generic.EscapeTool; - -import java.io.StringWriter; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Map; -import java.util.Optional; - -/** - * @author bjorncs - */ -public class MailTemplating { - - public enum Template { - MAIL("mail"), DEFAULT_MAIL_CONTENT("default-mail-content"), NOTIFICATION_MESSAGE("notification-message"), - MAIL_VERIFICATION("mail-verification"), TRIAL_SIGNED_UP("trial-signed-up"), TRIAL_MIDWAY_CHECKIN("trial-midway-checkin"), - TRIAL_EXPIRES_IMMEDIATELY("trial-expires-immediately"), TRIAL_EXPIRED("trial-expired") - ; - - public static Optional<Template> fromId(String id) { - return Arrays.stream(values()).filter(t -> t.id.equals(id)).findAny(); - } - - private final String id; - - Template(String id) { this.id = id; } - - public String getId() { return id; } - } - - private final VelocityEngine velocity; - private final EscapeTool escapeTool = new EscapeTool(); - private final ConsoleUrls consoleUrls; - - public MailTemplating(ConsoleUrls consoleUrls) { - this.velocity = createTemplateEngine(); - this.consoleUrls = consoleUrls; - } - - public String generateDefaultMailHtml(Template mailBodyTemplate, Map<String, Object> params, TenantName tenant) { - var ctx = createVelocityContext(); - ctx.put("accountNotificationLink", consoleUrls.tenantNotifications(tenant)); - ctx.put("privacyPolicyLink", "https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html"); - ctx.put("termsOfServiceLink", consoleUrls.termsOfService()); - ctx.put("supportLink", consoleUrls.support()); - ctx.put("mailBodyTemplate", mailBodyTemplate.getId()); - params.forEach(ctx::put); - return render(ctx, Template.MAIL); - } - - public String generateMailVerificationHtml(PendingMailVerification pmf) { - var ctx = createVelocityContext(); - ctx.put("verifyLink", consoleUrls.verifyEmail(pmf.getVerificationCode())); - ctx.put("email", pmf.getMailAddress()); - return render(ctx, Template.MAIL_VERIFICATION); - } - - public String escapeHtml(String s) { return escapeTool.html(s); } - - private VelocityContext createVelocityContext() { - var ctx = new VelocityContext(); - ctx.put("esc", escapeTool); - return ctx; - } - - private String render(VelocityContext ctx, Template template) { - var writer = new StringWriter(); - // Ignoring return value - implementation either returns 'true' or throws, never 'false' - velocity.mergeTemplate(template.getId(), StandardCharsets.UTF_8.name(), ctx, writer); - return writer.toString(); - } - - private static VelocityEngine createTemplateEngine() { - var v = new VelocityEngine(); - v.setProperty(Velocity.RESOURCE_LOADERS, "string"); - v.setProperty(Velocity.RESOURCE_LOADER + ".string.class", StringResourceLoader.class.getName()); - v.setProperty(Velocity.RESOURCE_LOADER + ".string.repository.static", "false"); - v.init(); - var repo = (StringResourceRepository) v.getApplicationAttribute(StringResourceLoader.REPOSITORY_NAME_DEFAULT); - Arrays.stream(Template.values()).forEach(t -> registerTemplate(repo, t.getId())); - return v; - } - - private static void registerTemplate(StringResourceRepository repo, String name) { - var templateStr = Exceptions.uncheck(() -> { - var in = MailTemplating.class.getResourceAsStream("/mail/%s.vm".formatted(name)); - return new String(in.readAllBytes()); - }); - repo.putStringResource(name, templateStr); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MissingOptionalException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MissingOptionalException.java deleted file mode 100644 index 50e4cd40af7..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MissingOptionalException.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.notification; - -/** - * Used to signal that an expected value was not present when creating NotificationContent - * - * @author enygaard - */ -class MissingOptionalException extends RuntimeException { - private final String field; - public MissingOptionalException(String field) { - super(field + " was expected but not present"); - this.field = field; - } - - public String field() { - return field; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java deleted file mode 100644 index 897e0be2d22..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.notification; - -import java.time.Instant; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.SortedMap; -import java.util.TreeMap; - -/** - * Represents an event that we want to notify the tenant about. The message(s) should be short - * and only describe event details: the final presentation will prefix the message with general - * information from other metadata in this notification (e.g. links to relevant console views - * and/or relevant documentation. - * - * @author freva - */ -public record Notification(Instant at, Notification.Type type, Notification.Level level, NotificationSource source, - String title, List<String> messages, Optional<MailContent> mailContent) { - - public Notification(Instant at, Type type, Level level, NotificationSource source, String title, List<String> messages) { - this(at, type, level, source, title, messages, Optional.empty()); - } - - public Notification(Instant at, Type type, Level level, NotificationSource source, List<String> messages) { - this(at, type, level, source, "", messages); - } - - public Notification { - Objects.requireNonNull(at, "at cannot be null"); - Objects.requireNonNull(type, "type cannot be null"); - Objects.requireNonNull(level, "level cannot be null"); - Objects.requireNonNull(source, "source cannot be null"); - Objects.requireNonNull(title, "title cannot be null"); - messages = List.copyOf(Objects.requireNonNull(messages, "messages cannot be null")); - - // Allowing empty title temporarily until all notifications have a title - // if (title.isBlank()) throw new IllegalArgumentException("title cannot be empty"); - if (messages.isEmpty() && title.isBlank()) throw new IllegalArgumentException("messages cannot be empty when title is empty"); - - Objects.requireNonNull(mailContent); - } - - public enum Level { - // Must be ordered in order of importance - info, warning, error - } - - public enum Type { - - /** Related to contents of application package, e.g., usage of deprecated features/syntax */ - applicationPackage, - - /** Related to contents of application package detectable by the controller on submission */ - submission, - - /** Related to contents of application test package, e.g., mismatch between deployment spec and provided tests */ - testPackage, - - /** Related to deployment of application, e.g., system test failure, node allocation failure, internal errors, etc. */ - deployment, - - /** Application cluster is (near) external feed blocked */ - feedBlock, - - /** Application cluster is reindexing document(s) */ - reindex, - - /** Account, e.g. expiration of trial plan */ - account, - } - - public static class MailContent { - private final MailTemplating.Template template; - private final SortedMap<String, Object> values; - private final String subject; - - private MailContent(Builder b) { - template = Objects.requireNonNull(b.template); - values = new TreeMap<>(b.values); - subject = b.subject; - } - - public MailTemplating.Template template() { return template; } - public SortedMap<String, Object> values() { return Collections.unmodifiableSortedMap(values); } - public Optional<String> subject() { return Optional.ofNullable(subject); } - - public static Builder fromTemplate(MailTemplating.Template template) { return new Builder(template); } - - public static class Builder { - private final MailTemplating.Template template; - private final Map<String, Object> values = new HashMap<>(); - private String subject; - - private Builder(MailTemplating.Template template) { - this.template = template; - } - - public Builder with(String name, String value) { values.put(name, value); return this; } - public Builder with(String name, Collection<String> items) { values.put(name, List.copyOf(items)); return this; } - public Builder subject(String s) { this.subject = s; return this; } - public MailContent build() { return new MailContent(this); } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - MailContent that = (MailContent) o; - return Objects.equals(template, that.template) && Objects.equals(values, that.values) && Objects.equals(subject, that.subject); - } - - @Override - public int hashCode() { - return Objects.hash(template, values, subject); - } - - @Override - public String toString() { - return "MailContent{" + - "template='" + template + '\'' + - ", values=" + values + - ", subject='" + subject + '\'' + - '}'; - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java deleted file mode 100644 index e9b38f7a122..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.notification; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; - -import java.util.Objects; -import java.util.Optional; - -import static com.yahoo.vespa.hosted.controller.notification.Notifier.notificationLink; - -/** - * Created a NotificationContent for a given Notification. - * - * The formatter will create specific summary, message start and URI for a given Notification. - * - * @author enygaard - */ -public class NotificationFormatter { - private final ConsoleUrls consoleUrls; - - public NotificationFormatter(ConsoleUrls consoleUrls) { - this.consoleUrls = Objects.requireNonNull(consoleUrls); - } - - public FormattedNotification format(Notification n) { - return switch (n.type()) { - case applicationPackage, submission -> applicationPackage(n); - case deployment -> deployment(n); - case testPackage -> testPackage(n); - case reindex -> reindex(n); - case feedBlock -> feedBlock(n); - default -> new FormattedNotification(n, n.type().name(), "", consoleUrls.tenantOverview(n.source().tenant())); - }; - } - - private FormattedNotification applicationPackage(Notification n) { - var source = n.source(); - var application = requirePresent(source.application(), "application"); - var message = Text.format("Application package for %s%s has %s", - application, - source.instance().map(instance -> "." + instance.value()).orElse(""), - levelText(n.level(), n.messages().size())); - return new FormattedNotification(n, "Application package", message, notificationLink(consoleUrls, n.source())); - } - - private FormattedNotification deployment(Notification n) { - var source = n.source(); - var message = Text.format("%s for %s.%s has %s", - jobText(source), - requirePresent(source.application(), "application"), - requirePresent(source.instance(), "instance"), - levelText(n.level(), n.messages().size())); - return new FormattedNotification(n,"Deployment", message, notificationLink(consoleUrls, n.source())); - } - - private FormattedNotification testPackage(Notification n) { - var source = n.source(); - var application = requirePresent(source.application(), "application"); - var message = Text.format("There %s with tests for %s%s", - n.messages().size() > 1 ? "are problems" : "is a problem", - application, - source.instance().map(i -> "."+i).orElse("")); - return new FormattedNotification(n, "Test package", message, notificationLink(consoleUrls, n.source())); - } - - private FormattedNotification reindex(Notification n) { - var message = Text.format("%s is reindexing", clusterInfo(n.source())); - var application = requirePresent(n.source().application(), "application"); - var instance = requirePresent(n.source().instance(), "instance"); - var clusterId = requirePresent(n.source().clusterId(), "clusterId"); - var zone = requirePresent(n.source().zoneId(), "zoneId"); - return new FormattedNotification(n, "Reindex", message, - consoleUrls.clusterReindexing(ApplicationId.from(n.source().tenant(), application, instance), zone, clusterId)); - } - - private FormattedNotification feedBlock(Notification n) { - String type = n.level() == Notification.Level.warning ? "Nearly feed blocked" : "Feed blocked"; - var message = Text.format("%s is %s", clusterInfo(n.source()), type.toLowerCase()); - return new FormattedNotification(n, type, message, notificationLink(consoleUrls, n.source())); - } - - private String jobText(NotificationSource source) { - var jobType = requirePresent(source.jobType(), "jobType"); - var zone = jobType.zone(); - var runNumber = source.runNumber().orElseThrow(() -> new MissingOptionalException("runNumber")); - switch (zone.environment().value()) { - case "production": - return Text.format("Deployment job #%d to %s", runNumber, zone.region()); - case "test": - return Text.format("Test job #%d to %s", runNumber, zone.region()); - case "dev": - case "perf": - return Text.format("Deployment job #%d to %s.%s", runNumber, zone.environment().value(), zone.region().value()); - } - switch (jobType.jobName()) { - case "system-test": - case "staging-test": - } - return Text.format("%s #%d", jobType.jobName(), runNumber); - } - - private String levelText(Notification.Level level, int count) { - return switch (level) { - case error -> "failed"; - case warning -> count > 1 ? Text.format("%d warnings", count) : "a warning"; - default -> count > 1 ? Text.format("%d messages", count) : "a message"; - }; - } - - private String clusterInfo(NotificationSource source) { - var application = requirePresent(source.application(), "application"); - var instance = requirePresent(source.instance(), "instance"); - var zone = requirePresent(source.zoneId(), "zoneId"); - var clusterId = requirePresent(source.clusterId(), "clusterId"); - return Text.format("Cluster %s in %s.%s for %s.%s", - clusterId.value(), - zone.environment(), zone.region(), - application, instance); - } - - - private static <T> T requirePresent(Optional<T> optional, String field) { - return optional.orElseThrow(() -> new MissingOptionalException(field)); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationSource.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationSource.java deleted file mode 100644 index 72d3dd933aa..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationSource.java +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.notification; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; - -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; - -/** - * Denotes the source of the notification. - * - * @author freva - */ -public class NotificationSource { - private final TenantName tenant; - private final Optional<ApplicationName> application; - private final Optional<InstanceName> instance; - private final Optional<ZoneId> zoneId; - private final Optional<ClusterSpec.Id> clusterId; - private final Optional<JobType> jobType; - private final OptionalLong runNumber; - - public NotificationSource(TenantName tenant, Optional<ApplicationName> application, Optional<InstanceName> instance, - Optional<ZoneId> zoneId, Optional<ClusterSpec.Id> clusterId, Optional<JobType> jobType, OptionalLong runNumber) { - this.tenant = Objects.requireNonNull(tenant, "tenant cannot be null"); - this.application = Objects.requireNonNull(application, "application cannot be null"); - this.instance = Objects.requireNonNull(instance, "instance cannot be null"); - this.zoneId = Objects.requireNonNull(zoneId, "zoneId cannot be null"); - this.clusterId = Objects.requireNonNull(clusterId, "clusterId cannot be null"); - this.jobType = Objects.requireNonNull(jobType, "jobType cannot be null"); - this.runNumber = Objects.requireNonNull(runNumber, "runNumber cannot be null"); - - if (instance.isPresent() && application.isEmpty()) - throw new IllegalArgumentException("Application name must be present with instance name"); - if (zoneId.isPresent() && instance.isEmpty()) - throw new IllegalArgumentException("Instance name must be present with zone ID"); - if (clusterId.isPresent() && zoneId.isEmpty()) - throw new IllegalArgumentException("Zone ID must be present with cluster ID"); - if (clusterId.isPresent() && jobType.isPresent()) - throw new IllegalArgumentException("Cannot set both cluster ID and job type"); - if (jobType.isPresent() && instance.isEmpty()) - throw new IllegalArgumentException("Instance name must be present with job type"); - if (jobType.isPresent() != runNumber.isPresent()) - throw new IllegalArgumentException(Text.format("Run number (%s) must be 1-to-1 with job type (%s)", - runNumber.isPresent() ? "present" : "missing", jobType.map(i -> "present").orElse("missing"))); - } - - - public TenantName tenant() { return tenant; } - public Optional<ApplicationName> application() { return application; } - public Optional<InstanceName> instance() { return instance; } - public Optional<ZoneId> zoneId() { return zoneId; } - public Optional<ClusterSpec.Id> clusterId() { return clusterId; } - public Optional<JobType> jobType() { return jobType; } - public OptionalLong runNumber() { return runNumber; } - - /** - * Returns true iff this source contains the given source. A source contains the other source if - * all the set fields in this source are equal to the given source, while the fields not set - * in this source are ignored. - */ - public boolean contains(NotificationSource other) { - return tenant.equals(other.tenant) && - (application.isEmpty() || application.equals(other.application)) && - (instance.isEmpty() || instance.equals(other.instance)) && - (zoneId.isEmpty() || zoneId.equals(other.zoneId)) && - (clusterId.isEmpty() || clusterId.equals(other.clusterId)) && - (jobType.isEmpty() || jobType.equals(other.jobType)); // Do not consider run number (it's unique!) - } - - /** - * Returns whether this source from a production deployment or deployment related to prod deployment (e.g. to - * staging zone), or if this is at tenant or application level - */ - public boolean isProduction() { - return ! zoneId.map(ZoneId::environment) - .or(() -> jobType.map(JobType::environment)) - .map(Environment::isManuallyDeployed) - .orElse(false); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - NotificationSource that = (NotificationSource) o; - return tenant.equals(that.tenant) && application.equals(that.application) && instance.equals(that.instance) && - zoneId.equals(that.zoneId) && clusterId.equals(that.clusterId) && jobType.equals(that.jobType); // Do not consider run number (it's unique!) - } - - @Override - public int hashCode() { - return Objects.hash(tenant, application, instance, zoneId, clusterId, jobType, runNumber); - } - - @Override - public String toString() { - return "NotificationSource{" + - "tenant=" + tenant + - application.map(application -> ", application=" + application.value()).orElse("") + - instance.map(instance -> ", instance=" + instance.value()).orElse("") + - zoneId.map(zoneId -> ", zone=" + zoneId.value()).orElse("") + - clusterId.map(clusterId -> ", clusterId=" + clusterId.value()).orElse("") + - jobType.map(jobType -> ", job=" + jobType.jobName() + "#" + runNumber.getAsLong()).orElse("") + - '}'; - } - - private static NotificationSource from(TenantName tenant, ApplicationName application, InstanceName instance, ZoneId zoneId, - ClusterSpec.Id clusterId, JobType jobType, Long runNumber) { - return new NotificationSource(tenant, Optional.ofNullable(application), Optional.ofNullable(instance), Optional.ofNullable(zoneId), - Optional.ofNullable(clusterId), Optional.ofNullable(jobType), runNumber == null ? OptionalLong.empty() : OptionalLong.of(runNumber)); - } - - public static NotificationSource from(TenantName tenantName) { - return from(tenantName, null, null, null, null, null, null); - } - - public static NotificationSource from(TenantAndApplicationId id) { - return from(id.tenant(), id.application(), null, null, null, null, null); - } - - public static NotificationSource from(ApplicationId app) { - return from(app.tenant(), app.application(), app.instance(), null, null, null, null); - } - - public static NotificationSource from(DeploymentId deploymentId) { - ApplicationId app = deploymentId.applicationId(); - return from(app.tenant(), app.application(), app.instance(), deploymentId.zoneId(), null, null, null); - } - - public static NotificationSource from(DeploymentId deploymentId, ClusterSpec.Id clusterId) { - ApplicationId app = deploymentId.applicationId(); - return from(app.tenant(), app.application(), app.instance(), deploymentId.zoneId(), clusterId, null, null); - } - - public static NotificationSource from(RunId runId) { - ApplicationId app = runId.application(); - return from(app.tenant(), app.application(), app.instance(), null, null, runId.job().type(), runId.number()); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java deleted file mode 100644 index e279e4feacd..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.notification; - -import com.yahoo.collections.Pair; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.TenantName; -import com.yahoo.text.Text; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.notification.Notification.MailContent; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.time.Clock; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing.Cluster; -import static com.yahoo.vespa.hosted.controller.notification.Notification.Level; -import static com.yahoo.vespa.hosted.controller.notification.Notification.Type; -import static com.yahoo.vespa.hosted.controller.notification.Notifier.notificationLink; - -/** - * Adds, updates and removes tenant notifications in ZK - * - * @author freva - */ -public class NotificationsDb { - - private static final Logger log = Logger.getLogger(NotificationsDb.class.getName()); - - private final Clock clock; - private final CuratorDb curatorDb; - private final Notifier notifier; - private final ConsoleUrls consoleUrls; - - public NotificationsDb(Controller controller) { - this(controller.clock(), controller.curator(), controller.notifier(), controller.serviceRegistry().consoleUrls()); - } - - NotificationsDb(Clock clock, CuratorDb curatorDb, Notifier notifier, ConsoleUrls consoleUrls) { - this.clock = clock; - this.curatorDb = curatorDb; - this.notifier = notifier; - this.consoleUrls = consoleUrls; - } - - public List<TenantName> listTenantsWithNotifications() { - return curatorDb.listTenantsWithNotifications(); - } - - public List<Notification> listNotifications(NotificationSource source, boolean productionOnly) { - return curatorDb.readNotifications(source.tenant()).stream() - .filter(notification -> source.contains(notification.source()) && (!productionOnly || notification.source().isProduction())) - .toList(); - } - - public void setSubmissionNotification(TenantAndApplicationId tenantApp, String message) { - NotificationSource source = NotificationSource.from(tenantApp); - String title = "Application package for [%s](%s) has a warning".formatted( - tenantApp.application().value(), notificationLink(consoleUrls, source)); - setNotification(source, Type.submission, Level.warning, title, List.of(message), Optional.empty()); - } - - public void setApplicationPackageNotification(NotificationSource source, List<String> messages) { - String title = "Application package for [%s%s](%s) has %s".formatted( - source.application().get().value(), source.instance().map(i -> "." + i.value()).orElse(""), notificationLink(consoleUrls, source), - messages.size() == 1 ? "a warning" : "warnings"); - setNotification(source, Type.applicationPackage, Level.warning, title, messages, Optional.empty()); - } - - public void setTestPackageNotification(TenantAndApplicationId tenantApp, List<String> messages) { - NotificationSource source = NotificationSource.from(tenantApp); - String title = "There %s with tests for [%s](%s)".formatted( - messages.size() == 1 ? "is a problem" : "are problems", tenantApp.application().value(), - notificationLink(consoleUrls, source)); - setNotification(source, Type.testPackage, Level.warning, title, messages, Optional.empty()); - } - - public void setDeploymentNotification(RunId runId, String message) { - String description, linkText; - if (runId.type().isProduction()) { - description = runId.type().isTest() ? "Test job " : "Deployment job "; - linkText = "#" + runId.number() + " to " + runId.type().zone().region().value(); - } else if (runId.type().isTest()) { - description = ""; - linkText = (runId.type().isStagingTest() ? "Staging" : "System") + " test #" + runId.number(); - } else if (runId.type().isDeployment()) { - description = "Deployment job "; - linkText = "#" + runId.number() + " to " + runId.type().zone().value(); - } else throw new IllegalStateException("Unexpected job type " + runId.type()); - NotificationSource source = NotificationSource.from(runId); - String title = "%s[%s](%s) for application **%s.%s** has failed".formatted( - description, linkText, notificationLink(consoleUrls, source), runId.application().application().value(), runId.application().instance().value()); - setNotification(source, Type.deployment, Level.error, title, List.of(message), Optional.empty()); - } - - /** - * Add a notification with given source and type. If a notification with same source and type - * already exists, it'll be replaced by this one instead. - */ - public void setNotification(NotificationSource source, Type type, Level level, String title, List<String> messages, - Optional<MailContent> mailContent) { - Optional<Notification> changed = Optional.empty(); - try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { - var existingNotifications = curatorDb.readNotifications(source.tenant()); - List<Notification> notifications = existingNotifications.stream() - .filter(notification -> !source.equals(notification.source()) || type != notification.type()) - .collect(Collectors.toCollection(ArrayList::new)); - var notification = new Notification(clock.instant(), type, level, source, title, messages, mailContent); - if (!notificationExists(notification, existingNotifications, false)) { - changed = Optional.of(notification); - } - notifications.add(notification); - curatorDb.writeNotifications(source.tenant(), notifications); - } - changed.ifPresent(c -> { - log.fine(() -> "New notification %s".formatted(c)); - notifier.dispatch(c); - }); - } - - /** Remove the notification with the given source and type */ - public void removeNotification(NotificationSource source, Type type) { - try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { - List<Notification> initial = curatorDb.readNotifications(source.tenant()); - List<Notification> filtered = initial.stream() - .filter(notification -> !source.equals(notification.source()) || type != notification.type()) - .toList(); - if (initial.size() > filtered.size()) - curatorDb.writeNotifications(source.tenant(), filtered); - } - } - - /** Remove all notifications for this source or sources contained by this source */ - public void removeNotifications(NotificationSource source) { - try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { - if (source.application().isEmpty()) { // Source is tenant - curatorDb.deleteNotifications(source.tenant()); - return; - } - - List<Notification> initial = curatorDb.readNotifications(source.tenant()); - List<Notification> filtered = initial.stream() - .filter(notification -> !source.contains(notification.source())) - .toList(); - if (initial.size() > filtered.size()) - curatorDb.writeNotifications(source.tenant(), filtered); - } - } - - /** - * Updates notifications based on deployment metrics (e.g. feed blocked and reindexing progress) for the given - * deployment based on current cluster metrics. - * Will clear notifications of any cluster not reporting the metrics or whose metrics indicate feed is not blocked - * or reindexing no longer in progress. Will set notification for clusters: - * - that are (Level.error) or are nearly (Level.warning) feed blocked, - * - that are (Level.info) currently reindexing at least 1 document type. - */ - public void setDeploymentMetricsNotifications(DeploymentId deploymentId, List<ClusterMetrics> clusterMetrics, ApplicationReindexing applicationReindexing) { - Instant now = clock.instant(); - List<Notification> changed = List.of(); - List<Notification> newNotifications = Stream.concat( - clusterMetrics.stream().map(metric -> createFeedBlockNotification(consoleUrls, deploymentId, metric.getClusterId(), now, metric)), - applicationReindexing.clusters().entrySet().stream().map(entry -> - createReindexNotification(consoleUrls, deploymentId, entry.getKey(), now, entry.getValue()))) - .flatMap(Optional::stream) - .toList(); - - NotificationSource deploymentSource = NotificationSource.from(deploymentId); - try (Mutex lock = curatorDb.lockNotifications(deploymentSource.tenant())) { - List<Notification> initial = curatorDb.readNotifications(deploymentSource.tenant()); - List<Notification> updated = Stream.concat( - initial.stream() - .filter(notification -> - // Filter out old feed block notifications and reindex for this deployment - (notification.type() != Type.feedBlock && notification.type() != Type.reindex) || - !deploymentSource.contains(notification.source())), - // ... and add the new notifications for this deployment - newNotifications.stream()) - .toList(); - if (!initial.equals(updated)) { - curatorDb.writeNotifications(deploymentSource.tenant(), updated); - } - changed = newNotifications.stream().filter(n -> !notificationExists(n, initial, true)).toList(); - } - notifier.dispatch(changed, deploymentSource); - } - - private boolean notificationExists(Notification notification, List<Notification> existing, boolean mindHigherLevel) { - // Be conservative for now, only dispatch notifications if they are from new source or with new type. - // the message content and level is ignored for now - boolean exists = existing.stream() - .anyMatch(e -> notification.source().contains(e.source()) && notification.type().equals(e.type()) && - (!mindHigherLevel || notification.level().ordinal() <= e.level().ordinal())); - log.fine(() -> "%s in %s == %b".formatted(notification, existing, exists)); - return exists; - } - - private static Optional<Notification> createFeedBlockNotification(ConsoleUrls consoleUrls, DeploymentId deployment, String clusterId, Instant at, ClusterMetrics metric) { - Optional<Pair<Level, String>> memoryStatus = - resourceUtilToFeedBlockStatus("memory", metric.memoryUtil(), metric.memoryFeedBlockLimit()); - Optional<Pair<Level, String>> diskStatus = - resourceUtilToFeedBlockStatus("disk", metric.diskUtil(), metric.diskFeedBlockLimit()); - if (memoryStatus.isEmpty() && diskStatus.isEmpty()) return Optional.empty(); - - NotificationSource source = NotificationSource.from(deployment, ClusterSpec.Id.from(clusterId)); - // Find the max among levels - Level level = Stream.of(memoryStatus, diskStatus) - .flatMap(status -> status.stream().map(Pair::getFirst)) - .max(Comparator.comparing(Enum::ordinal)).get(); - String title = "Cluster [%s](%s) in **%s** for **%s.%s** is %sfeed blocked".formatted( - clusterId, notificationLink(consoleUrls, source), deployment.zoneId().value(), deployment.applicationId().application().value(), - deployment.applicationId().instance().value(), level == Level.warning ? "nearly " : ""); - List<String> messages = Stream.concat(memoryStatus.stream(), diskStatus.stream()) - .filter(status -> status.getFirst() == level) // Do not mix message from different levels - .map(Pair::getSecond) - .toList(); - - return Optional.of(new Notification(at, Type.feedBlock, level, source, title, messages)); - } - - private static Optional<Notification> createReindexNotification(ConsoleUrls consoleUrls, DeploymentId deployment, String clusterId, Instant at, Cluster cluster) { - NotificationSource source = NotificationSource.from(deployment, ClusterSpec.Id.from(clusterId)); - String title = "Cluster [%s](%s) in **%s** for **%s.%s** is [reindexing](https://docs.vespa.ai/en/operations/reindexing.html)".formatted( - clusterId, consoleUrls.clusterReindexing(deployment.applicationId(), deployment.zoneId(), source.clusterId().get()), - deployment.zoneId().value(), deployment.applicationId().application().value(), deployment.applicationId().instance().value()); - List<String> messages = cluster.ready().entrySet().stream() - .filter(entry -> entry.getValue().progress().isPresent()) - .map(entry -> Text.format("document type '%s'%s (%.1f%% done)", - entry.getKey(), entry.getValue().cause().map(s -> " " + s).orElse(""), 100 * entry.getValue().progress().get())) - .sorted() - .toList(); - if (messages.isEmpty()) return Optional.empty(); - return Optional.of(new Notification(at, Type.reindex, Level.info, source, title, messages)); - } - - /** - * Returns a feed block summary for the given resource: the notification level and - * notification message for the given resource utilization wrt. given resource limit. - * If utilization is well below the limit, Optional.empty() is returned. - */ - private static Optional<Pair<Level, String>> resourceUtilToFeedBlockStatus( - String resource, Optional<Double> util, Optional<Double> feedBlockLimit) { - if (util.isEmpty() || feedBlockLimit.isEmpty()) return Optional.empty(); - double utilRelativeToLimit = util.get() / feedBlockLimit.get(); - if (utilRelativeToLimit < 0.95) return Optional.empty(); - - String message = Text.format("%s (usage: %.1f%%, feed block limit: %.1f%%)", - resource, 100 * util.get(), 100 * feedBlockLimit.get()); - if (utilRelativeToLimit < 1) return Optional.of(new Pair<>(Level.warning, message)); - return Optional.of(new Pair<>(Level.error, message)); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java deleted file mode 100644 index f27e69c4636..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.notification; - -import com.google.common.annotations.VisibleForTesting; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.text.Text; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; -import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; - -import java.util.Collection; -import java.util.List; -import java.util.Objects; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - * Notifier is responsible for dispatching user notifications to their chosen Contact points. - * - * @author enygaard - */ -public class Notifier { - private final CuratorDb curatorDb; - private final Mailer mailer; - private final FlagSource flagSource; - private final ConsoleUrls consoleUrls; - private final NotificationFormatter formatter; - private final MailTemplating mailTemplating; - - private static final Logger log = Logger.getLogger(Notifier.class.getName()); - - // Minimal url pattern matcher to detect hardcoded URLs in Notification messages - private static final Pattern urlPattern = Pattern.compile("https://[\\w\\d./]+"); - - public Notifier(CuratorDb curatorDb, ConsoleUrls consoleUrls, Mailer mailer, FlagSource flagSource) { - this.curatorDb = Objects.requireNonNull(curatorDb); - this.mailer = Objects.requireNonNull(mailer); - this.flagSource = Objects.requireNonNull(flagSource); - this.consoleUrls = Objects.requireNonNull(consoleUrls); - this.formatter = new NotificationFormatter(consoleUrls); - this.mailTemplating = new MailTemplating(consoleUrls); - } - - public void dispatch(List<Notification> notifications, NotificationSource source) { - if (!dispatchEnabled(source) || skipSource(source)) { - return; - } - if (notifications.isEmpty()) { - return; - } - var tenant = curatorDb.readTenant(source.tenant()); - tenant.stream().forEach(t -> { - if (t instanceof CloudTenant ct) { - ct.info().contacts().all().stream() - .filter(c -> c.audiences().contains(TenantContacts.Audience.NOTIFICATIONS)) - .collect(Collectors.groupingBy(TenantContacts.Contact::type, Collectors.toList())) - .forEach((type, contacts) -> notifications.forEach(n -> dispatch(n, type, contacts))); - } - }); - } - - public void dispatch(Notification notification) { - dispatch(List.of(notification), notification.source()); - } - - private boolean dispatchEnabled(NotificationSource source) { - return PermanentFlags.NOTIFICATION_DISPATCH_FLAG.bindTo(flagSource) - .with(FetchVector.Dimension.TENANT_ID, source.tenant().value()) - .value(); - } - - private boolean skipSource(NotificationSource source) { - // Do not dispatch notification for dev and perf environments - return source.zoneId() - .map(z -> z.environment()) - .map(e -> e == Environment.dev || e == Environment.perf) - .orElse(false); - } - - private void dispatch(Notification notification, TenantContacts.Type type, Collection<? extends TenantContacts.Contact> contacts) { - switch (type) { - case EMAIL -> dispatch(notification, contacts.stream().map(c -> (TenantContacts.EmailContact) c).toList()); - default -> throw new IllegalArgumentException("Unknown TenantContacts type " + type.name()); - } - } - - private void dispatch(Notification notification, Collection<TenantContacts.EmailContact> contacts) { - try { - log.fine(() -> "Sending notification " + notification + " to " + - contacts.stream().map(c -> c.email().getEmailAddress()).toList()); - var content = formatter.format(notification); - var verifiedContacts = contacts.stream() - .filter(c -> c.email().isVerified()).map(c -> c.email().getEmailAddress()).toList(); - if (verifiedContacts.isEmpty()) { - log.fine(() -> "None of the %d contact(s) are verified - skipping delivery of %s".formatted(contacts.size(), notification)); - return; - } - mailer.send(mailOf(content, verifiedContacts)); - } catch (MailerException e) { - log.log(Level.SEVERE, "Failed sending email", e); - } catch (MissingOptionalException e) { - log.log(Level.WARNING, "Missing value in required field '" + e.field() + "' for notification type: " + notification.type(), e); - } - } - - public Mail mailOf(FormattedNotification content, Collection<String> recipients) { - var notification = content.notification(); - var subject = content.notification().mailContent().flatMap(Notification.MailContent::subject) - .orElseGet(() -> Text.format( - "[%s] %s Vespa Notification for %s", notification.level().toString().toUpperCase(), - content.prettyType(), applicationIdSource(notification.source()))); - var html = generateHtml(content); - return new Mail(recipients, subject, "", html); - } - - private String generateHtml(FormattedNotification content) { - var mailContent = content.notification().mailContent().orElseGet(() -> generateContentFromMessages(content)); - return mailTemplating.generateDefaultMailHtml(mailContent.template(), mailContent.values(), content.notification().source().tenant()); - } - - private Notification.MailContent generateContentFromMessages(FormattedNotification f) { - var items = f.notification().messages().stream().map(m -> capitalise(linkify(mailTemplating.escapeHtml(m)))).toList(); - return Notification.MailContent.fromTemplate(MailTemplating.Template.DEFAULT_MAIL_CONTENT) - .with("mailMessageTemplate", "notification-message") - .with("mailTitle", "Vespa Cloud Notifications") - .with("notificationHeader", f.messagePrefix()) - .with("notificationItems", items) - .with("consoleLink", notificationLink(consoleUrls, f.notification().source())) - .build(); - } - - @VisibleForTesting - static String linkify(String text) { - return urlPattern.matcher(text).replaceAll((res) -> String.format("<a href=\"%s\">%s</a>", res.group(), res.group())); - } - - private String applicationIdSource(NotificationSource source) { - StringBuilder sb = new StringBuilder(); - sb.append(source.tenant().value()); - source.application().ifPresent(applicationName -> sb.append(".").append(applicationName.value())); - source.instance().ifPresent(instanceName -> sb.append(".").append(instanceName.value())); - return sb.toString(); - } - - static String notificationLink(ConsoleUrls consoleUrls, NotificationSource source) { - if (source.application().isEmpty()) return consoleUrls.tenantOverview(source.tenant()); - if (source.instance().isEmpty()) return consoleUrls.prodApplicationOverview(source.tenant(), source.application().get()); - - ApplicationId application = ApplicationId.from(source.tenant(), source.application().get(), source.instance().get()); - if (source.jobType().isPresent()) - return consoleUrls.deploymentRun(new RunId(application, source.jobType().get(), source.runNumber().getAsLong())); - if (source.clusterId().isPresent()) - return consoleUrls.clusterOverview(application, source.zoneId().get(), source.clusterId().get()); - return consoleUrls.instanceOverview(application, source.zoneId().map(ZoneId::environment).orElse(Environment.prod)); - } - - private static String capitalise(String m) { - return m.substring(0, 1).toUpperCase() + m.substring(1); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java deleted file mode 100644 index 22d10386d7f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * The root package of the controller - * - * @author bratseth - */ -@ExportPackage -package com.yahoo.vespa.hosted.controller; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java deleted file mode 100644 index 07fac67100f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ /dev/null @@ -1,575 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.security.KeyUtils; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.api.integration.organization.AccountId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.application.AssignedRotation; -import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; -import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.QuotaUsage; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; -import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; - -import java.security.PublicKey; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.toMap; - -/** - * Serializes {@link Application}s to/from slime. - * This class is multithread safe. - * - * @author jonmv - * @author mpolden - */ -public class ApplicationSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - // Application fields - private static final String idField = "id"; - private static final String createdAtField = "createdAt"; - private static final String deploymentSpecField = "deploymentSpecField"; - private static final String validationOverridesField = "validationOverrides"; - private static final String instancesField = "instances"; - private static final String deployingField = "deployingField"; - private static final String projectIdField = "projectId"; - private static final String versionsField = "versions"; - private static final String prodVersionsField = "prodVersions"; - private static final String devVersionsField = "devVersions"; - private static final String platformPinnedField = "pinned"; - private static final String revisionPinnedField = "revisionPinned"; - private static final String deploymentIssueField = "deploymentIssueId"; - private static final String ownershipIssueIdField = "ownershipIssueId"; - private static final String userOwnerField = "confirmedOwner"; - private static final String issueOwnerField = "confirmedOwnerId"; - private static final String majorVersionField = "majorVersion"; - private static final String writeQualityField = "writeQuality"; - private static final String queryQualityField = "queryQuality"; - private static final String pemDeployKeysField = "pemDeployKeys"; - private static final String assignedRotationClusterField = "clusterId"; - private static final String assignedRotationRotationField = "rotationId"; - private static final String assignedRotationRegionsField = "regions"; - private static final String versionField = "version"; - - // Instance fields - private static final String instanceNameField = "instanceName"; - private static final String deploymentsField = "deployments"; - private static final String deploymentJobsField = "deploymentJobs"; // TODO jonmv: clean up serialisation format - private static final String assignedRotationsField = "assignedRotations"; - private static final String assignedRotationEndpointField = "endpointId"; - - // Deployment fields - private static final String zoneField = "zone"; - private static final String cloudAccountField = "cloudAccount"; - private static final String environmentField = "environment"; - private static final String regionField = "region"; - private static final String deployTimeField = "deployTime"; - private static final String applicationBuildNumberField = "applicationBuildNumber"; - private static final String applicationPackageRevisionField = "applicationPackageRevision"; - private static final String sourceRevisionField = "sourceRevision"; - private static final String repositoryField = "repositoryField"; - private static final String branchField = "branchField"; - private static final String commitField = "commitField"; - private static final String descriptionField = "description"; - private static final String submittedAtField = "submittedAt"; - private static final String riskField = "risk"; - private static final String authorEmailField = "authorEmailField"; - private static final String deployedDirectlyField = "deployedDirectly"; - private static final String obsoleteAtField = "obsoleteAt"; - private static final String hasPackageField = "hasPackage"; - private static final String shouldSkipField = "shouldSkip"; - private static final String compileVersionField = "compileVersion"; - private static final String allowedMajorField = "allowedMajor"; - private static final String buildTimeField = "buildTime"; - private static final String sourceUrlField = "sourceUrl"; - private static final String bundleHashField = "bundleHash"; - private static final String lastQueriedField = "lastQueried"; - private static final String lastWrittenField = "lastWritten"; - private static final String lastQueriesPerSecondField = "lastQueriesPerSecond"; - private static final String lastWritesPerSecondField = "lastWritesPerSecond"; - private static final String dataPlaneTokensField = "dataPlaneTokens"; - private static final String tokenIdField = "id"; - private static final String tokenUpdatedField = "updated"; - - // DeploymentJobs fields - private static final String jobStatusField = "jobStatus"; - - // JobStatus field - private static final String jobTypeField = "jobType"; - private static final String pausedUntilField = "pausedUntil"; - - // Deployment metrics fields - private static final String deploymentMetricsField = "metrics"; - private static final String deploymentMetricsQPSField = "queriesPerSecond"; - private static final String deploymentMetricsWPSField = "writesPerSecond"; - private static final String deploymentMetricsDocsField = "documentCount"; - private static final String deploymentMetricsQueryLatencyField = "queryLatencyMillis"; - private static final String deploymentMetricsWriteLatencyField = "writeLatencyMillis"; - private static final String deploymentMetricsUpdateTime = "lastUpdated"; - private static final String deploymentMetricsWarningsField = "warnings"; - - // RotationStatus fields - private static final String rotationStatusField = "rotationStatus2"; - private static final String rotationIdField = "rotationId"; - private static final String lastUpdatedField = "lastUpdated"; - private static final String rotationStateField = "state"; - private static final String statusField = "status"; - - // Quota usage fields - private static final String quotaUsageRateField = "quotaUsageRate"; - - private static final String deploymentCostField = "cost"; - - // ------------------ Serialization - - public Slime toSlime(Application application) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString(idField, application.id().serialized()); - root.setLong(createdAtField, application.createdAt().toEpochMilli()); - root.setString(deploymentSpecField, application.deploymentSpec().xmlForm()); - root.setString(validationOverridesField, application.validationOverrides().xmlForm()); - application.projectId().ifPresent(projectId -> root.setLong(projectIdField, projectId)); - application.deploymentIssueId().ifPresent(jiraIssueId -> root.setString(deploymentIssueField, jiraIssueId.value())); - application.ownershipIssueId().ifPresent(issueId -> root.setString(ownershipIssueIdField, issueId.value())); - application.userOwner().ifPresent(owner -> root.setString(userOwnerField, owner.username())); - application.issueOwner().ifPresent(owner -> root.setString(issueOwnerField, owner.value())); - application.majorVersion().ifPresent(majorVersion -> root.setLong(majorVersionField, majorVersion)); - root.setDouble(queryQualityField, application.metrics().queryServiceQuality()); - root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); - deployKeysToSlime(application.deployKeys(), root.setArray(pemDeployKeysField)); - revisionsToSlime(application.revisions(), root.setArray(prodVersionsField), root.setArray(devVersionsField)); - instancesToSlime(application, root.setArray(instancesField)); - return slime; - } - - private void instancesToSlime(Application application, Cursor array) { - for (Instance instance : application.instances().values()) { - Cursor instanceObject = array.addObject(); - instanceObject.setString(instanceNameField, instance.name().value()); - deploymentsToSlime(instance.deployments().values(), instanceObject.setArray(deploymentsField)); - toSlime(instance.jobPauses(), instanceObject.setObject(deploymentJobsField)); - assignedRotationsToSlime(instance.rotations(), instanceObject); - toSlime(instance.rotationStatus(), instanceObject.setArray(rotationStatusField)); - toSlime(instance.change(), instanceObject, deployingField); - } - } - - private void deployKeysToSlime(Set<PublicKey> deployKeys, Cursor array) { - deployKeys.forEach(key -> array.addString(KeyUtils.toPem(key))); - } - - private void deploymentsToSlime(Collection<Deployment> deployments, Cursor array) { - for (Deployment deployment : deployments) - deploymentToSlime(deployment, array.addObject()); - } - - private void deploymentToSlime(Deployment deployment, Cursor object) { - zoneIdToSlime(deployment.zone(), object.setObject(zoneField)); - if (!deployment.cloudAccount().isUnspecified()) object.setString(cloudAccountField, deployment.cloudAccount().value()); - object.setString(versionField, deployment.version().toString()); - object.setLong(deployTimeField, deployment.at().toEpochMilli()); - toSlime(deployment.revision(), object.setObject(applicationPackageRevisionField)); - deploymentMetricsToSlime(deployment.metrics(), object); - deployment.activity().lastQueried().ifPresent(instant -> object.setLong(lastQueriedField, instant.toEpochMilli())); - deployment.activity().lastWritten().ifPresent(instant -> object.setLong(lastWrittenField, instant.toEpochMilli())); - deployment.activity().lastQueriesPerSecond().ifPresent(value -> object.setDouble(lastQueriesPerSecondField, value)); - deployment.activity().lastWritesPerSecond().ifPresent(value -> object.setDouble(lastWritesPerSecondField, value)); - object.setDouble(quotaUsageRateField, deployment.quota().rate()); - deployment.cost().ifPresent(cost -> object.setDouble(deploymentCostField, cost)); - Cursor dataPlaneTokensArray = object.setArray(dataPlaneTokensField); - deployment.dataPlaneTokens().forEach((id, updated) -> { - Cursor tokenObject = dataPlaneTokensArray.addObject(); - tokenObject.setString(tokenIdField, id.value()); - tokenObject.setLong(tokenUpdatedField, updated.toEpochMilli()); - }); - } - - private void deploymentMetricsToSlime(DeploymentMetrics metrics, Cursor object) { - Cursor root = object.setObject(deploymentMetricsField); - root.setDouble(deploymentMetricsQPSField, metrics.queriesPerSecond()); - root.setDouble(deploymentMetricsWPSField, metrics.writesPerSecond()); - root.setDouble(deploymentMetricsDocsField, metrics.documentCount()); - root.setDouble(deploymentMetricsQueryLatencyField, metrics.queryLatencyMillis()); - root.setDouble(deploymentMetricsWriteLatencyField, metrics.writeLatencyMillis()); - metrics.instant().ifPresent(instant -> root.setLong(deploymentMetricsUpdateTime, instant.toEpochMilli())); - if (!metrics.warnings().isEmpty()) { - Cursor warningsObject = root.setObject(deploymentMetricsWarningsField); - metrics.warnings().forEach((warning, count) -> warningsObject.setLong(warning.name(), count)); - } - } - - private void zoneIdToSlime(ZoneId zone, Cursor object) { - object.setString(environmentField, zone.environment().value()); - object.setString(regionField, zone.region().value()); - } - - private void revisionsToSlime(RevisionHistory revisions, Cursor revisionsArray, Cursor devRevisionsArray) { - revisionsToSlime(revisions.production(), revisionsArray); - revisions.development().forEach((job, devRevisions) -> { - Cursor devRevisionsObject = devRevisionsArray.addObject(); - devRevisionsObject.setString(instanceNameField, job.application().instance().value()); - devRevisionsObject.setString(jobTypeField, job.type().serialized()); - revisionsToSlime(devRevisions, devRevisionsObject.setArray(versionsField)); - }); - } - - private void revisionsToSlime(Iterable<ApplicationVersion> revisions, Cursor revisionsArray) { - revisions.forEach(version -> toSlime(version, revisionsArray.addObject())); - } - - private void toSlime(RevisionId revision, Cursor object) { - object.setLong(applicationBuildNumberField, revision.number()); - object.setBool(deployedDirectlyField, ! revision.isProduction()); - } - - private void toSlime(ApplicationVersion applicationVersion, Cursor object) { - object.setLong(applicationBuildNumberField, applicationVersion.buildNumber()); - applicationVersion.source().ifPresent(source -> toSlime(source, object.setObject(sourceRevisionField))); - applicationVersion.authorEmail().ifPresent(email -> object.setString(authorEmailField, email)); - applicationVersion.compileVersion().ifPresent(version -> object.setString(compileVersionField, version.toString())); - applicationVersion.allowedMajor().ifPresent(major -> object.setLong(allowedMajorField, major)); - applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli())); - applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url)); - applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit)); - object.setBool(deployedDirectlyField, applicationVersion.isDeployedDirectly()); - applicationVersion.obsoleteAt().ifPresent(at -> object.setLong(obsoleteAtField, at.toEpochMilli())); - object.setBool(hasPackageField, applicationVersion.hasPackage()); - object.setBool(shouldSkipField, applicationVersion.shouldSkip()); - applicationVersion.description().ifPresent(description -> object.setString(descriptionField, description)); - applicationVersion.submittedAt().ifPresent(at -> object.setLong(submittedAtField, at.toEpochMilli())); - if (applicationVersion.risk() != 0) object.setLong(riskField, applicationVersion.risk()); - applicationVersion.bundleHash().ifPresent(bundleHash -> object.setString(bundleHashField, bundleHash)); - } - - private void toSlime(SourceRevision sourceRevision, Cursor object) { - object.setString(repositoryField, sourceRevision.repository()); - object.setString(branchField, sourceRevision.branch()); - object.setString(commitField, sourceRevision.commit()); - } - - private void toSlime(Map<JobType, Instant> jobPauses, Cursor cursor) { - Cursor jobStatusArray = cursor.setArray(jobStatusField); - jobPauses.forEach((type, until) -> { - Cursor jobPauseObject = jobStatusArray.addObject(); - jobPauseObject.setString(jobTypeField, type.serialized()); - jobPauseObject.setLong(pausedUntilField, until.toEpochMilli()); - }); - } - - private void toSlime(Change deploying, Cursor parentObject, String fieldName) { - if (deploying.isEmpty()) return; - - Cursor object = parentObject.setObject(fieldName); - if (deploying.platform().isPresent()) - object.setString(versionField, deploying.platform().get().toString()); - if (deploying.revision().isPresent()) - toSlime(deploying.revision().get(), object); - if (deploying.isPlatformPinned()) - object.setBool(platformPinnedField, true); - if (deploying.isRevisionPinned()) - object.setBool(revisionPinnedField, true); - } - - private void toSlime(RotationStatus status, Cursor array) { - status.asMap().forEach((rotationId, targets) -> { - Cursor rotationObject = array.addObject(); - rotationObject.setString(rotationIdField, rotationId.asString()); - rotationObject.setLong(lastUpdatedField, targets.lastUpdated().toEpochMilli()); - Cursor statusArray = rotationObject.setArray(statusField); - targets.asMap().forEach((zone, state) -> { - Cursor statusObject = statusArray.addObject(); - zoneIdToSlime(zone, statusObject); - statusObject.setString(rotationStateField, state.name()); - }); - }); - } - - private void assignedRotationsToSlime(List<AssignedRotation> rotations, Cursor parent) { - var rotationsArray = parent.setArray(assignedRotationsField); - for (var rotation : rotations) { - var object = rotationsArray.addObject(); - object.setString(assignedRotationEndpointField, rotation.endpointId().id()); - object.setString(assignedRotationRotationField, rotation.rotationId().asString()); - object.setString(assignedRotationClusterField, rotation.clusterId().value()); - var regionsArray = object.setArray(assignedRotationRegionsField); - for (var region : rotation.regions()) { - regionsArray.addString(region.value()); - } - } - } - - // ------------------ Deserialization - - public Application fromSlime(byte[] data) { - return fromSlime(SlimeUtils.jsonToSlime(data)); - } - - private Application fromSlime(Slime slime) { - Inspector root = slime.get(); - - TenantAndApplicationId id = TenantAndApplicationId.fromSerialized(root.field(idField).asString()); - Instant createdAt = SlimeUtils.instant(root.field(createdAtField)); - DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(root.field(deploymentSpecField).asString(), false); - ValidationOverrides validationOverrides = ValidationOverrides.fromXml(root.field(validationOverridesField).asString()); - Optional<IssueId> deploymentIssueId = SlimeUtils.optionalString(root.field(deploymentIssueField)).map(IssueId::from); - Optional<IssueId> ownershipIssueId = SlimeUtils.optionalString(root.field(ownershipIssueIdField)).map(IssueId::from); - Optional<User> userOwner = SlimeUtils.optionalString(root.field(userOwnerField)).map(User::from); - Optional<AccountId> issueOwner = SlimeUtils.optionalString(root.field(issueOwnerField)).map(AccountId::new); - OptionalInt majorVersion = SlimeUtils.optionalInteger(root.field(majorVersionField)); - ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), - root.field(writeQualityField).asDouble()); - Set<PublicKey> deployKeys = deployKeysFromSlime(root.field(pemDeployKeysField)); - List<Instance> instances = instancesFromSlime(id, root.field(instancesField)); - OptionalLong projectId = SlimeUtils.optionalLong(root.field(projectIdField)); - RevisionHistory revisions = revisionsFromSlime(root.field(prodVersionsField), root.field(devVersionsField), id); - - return new Application(id, createdAt, deploymentSpec, validationOverrides, - deploymentIssueId, ownershipIssueId, userOwner, issueOwner, majorVersion, metrics, - deployKeys, projectId, revisions, instances); - } - - private RevisionHistory revisionsFromSlime(Inspector prodVersionsArray, Inspector devVersionsArray, TenantAndApplicationId id) { - List<ApplicationVersion> revisions = revisionsFromSlime(prodVersionsArray, null); - Map<JobId, List<ApplicationVersion>> devRevisions = new HashMap<>(); - devVersionsArray.traverse((ArrayTraverser) (__, devRevisionsObject) -> { - JobId job = jobIdFromSlime(id, devRevisionsObject); - devRevisions.put(job, revisionsFromSlime(devRevisionsObject.field(versionsField), job)); - }); - - return RevisionHistory.ofRevisions(revisions, devRevisions); - } - - private JobId jobIdFromSlime(TenantAndApplicationId base, Inspector idObject) { - return new JobId(base.instance(idObject.field(instanceNameField).asString()), - JobType.ofSerialized(idObject.field(jobTypeField).asString())); - } - - private List<ApplicationVersion> revisionsFromSlime(Inspector versionsArray, JobId job) { - List<ApplicationVersion> revisions = new ArrayList<>(); - versionsArray.traverse((ArrayTraverser) (__, revisionObject) -> revisions.add(applicationVersionFromSlime(revisionObject, job))); - return revisions; - } - - private List<Instance> instancesFromSlime(TenantAndApplicationId id, Inspector field) { - List<Instance> instances = new ArrayList<>(); - field.traverse((ArrayTraverser) (name, object) -> { - InstanceName instanceName = InstanceName.from(object.field(instanceNameField).asString()); - List < Deployment > deployments = deploymentsFromSlime(object.field(deploymentsField), id.instance(instanceName)); - Map<JobType, Instant> jobPauses = jobPausesFromSlime(object.field(deploymentJobsField)); - List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(object); - RotationStatus rotationStatus = rotationStatusFromSlime(object); - Change change = changeFromSlime(object.field(deployingField)); - instances.add(new Instance(id.instance(instanceName), - deployments, - jobPauses, - assignedRotations, - rotationStatus, - change)); - }); - return instances; - } - - private Set<PublicKey> deployKeysFromSlime(Inspector array) { - Set<PublicKey> keys = new LinkedHashSet<>(); - array.traverse((ArrayTraverser) (__, key) -> keys.add(KeyUtils.fromPemEncodedPublicKey(key.asString()))); - return keys; - } - - private List<Deployment> deploymentsFromSlime(Inspector array, ApplicationId id) { - List<Deployment> deployments = new ArrayList<>(); - array.traverse((ArrayTraverser) (int i, Inspector item) -> deployments.add(deploymentFromSlime(item, id))); - return deployments; - } - - private Deployment deploymentFromSlime(Inspector deploymentObject, ApplicationId id) { - ZoneId zone = zoneIdFromSlime(deploymentObject.field(zoneField)); - return new Deployment(zone, - SlimeUtils.optionalString(deploymentObject.field(cloudAccountField)).map(CloudAccount::from).orElse(CloudAccount.empty), - revisionFromSlime(deploymentObject.field(applicationPackageRevisionField), new JobId(id, JobType.deploymentTo(zone))), - Version.fromString(deploymentObject.field(versionField).asString()), - SlimeUtils.instant(deploymentObject.field(deployTimeField)), - deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)), - DeploymentActivity.create(SlimeUtils.optionalInstant(deploymentObject.field(lastQueriedField)), - SlimeUtils.optionalInstant(deploymentObject.field(lastWrittenField)), - SlimeUtils.optionalDouble(deploymentObject.field(lastQueriesPerSecondField)), - SlimeUtils.optionalDouble(deploymentObject.field(lastWritesPerSecondField))), - QuotaUsage.create(SlimeUtils.optionalDouble(deploymentObject.field(quotaUsageRateField))), - SlimeUtils.optionalDouble(deploymentObject.field(deploymentCostField)), - SlimeUtils.entriesStream(deploymentObject.field(dataPlaneTokensField)) - .collect(toMap(entry -> TokenId.of(entry.field(tokenIdField).asString()), - entry -> Instant.ofEpochMilli(entry.field(tokenUpdatedField).asLong())))); - } - - private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) { - Optional<Instant> instant = SlimeUtils.optionalInstant(object.field(deploymentMetricsUpdateTime)); - return new DeploymentMetrics(object.field(deploymentMetricsQPSField).asDouble(), - object.field(deploymentMetricsWPSField).asDouble(), - object.field(deploymentMetricsDocsField).asDouble(), - object.field(deploymentMetricsQueryLatencyField).asDouble(), - object.field(deploymentMetricsWriteLatencyField).asDouble(), - instant, - deploymentWarningsFrom(object.field(deploymentMetricsWarningsField))); - } - - private Map<DeploymentMetrics.Warning, Integer> deploymentWarningsFrom(Inspector object) { - Map<DeploymentMetrics.Warning, Integer> warnings = new HashMap<>(); - object.traverse((ObjectTraverser) (name, value) -> warnings.put(DeploymentMetrics.Warning.valueOf(name), - (int) value.asLong())); - return Collections.unmodifiableMap(warnings); - } - - private RotationStatus rotationStatusFromSlime(Inspector parentObject) { - var object = parentObject.field(rotationStatusField); - var statusMap = new LinkedHashMap<RotationId, RotationStatus.Targets>(); - object.traverse((ArrayTraverser) (idx, statusObject) -> statusMap.put(new RotationId(statusObject.field(rotationIdField).asString()), - new RotationStatus.Targets( - singleRotationStatusFromSlime(statusObject.field(statusField)), - SlimeUtils.instant(statusObject.field(lastUpdatedField))))); - return RotationStatus.from(statusMap); - } - - private Map<ZoneId, RotationState> singleRotationStatusFromSlime(Inspector object) { - if (!object.valid()) { - return Collections.emptyMap(); - } - Map<ZoneId, RotationState> rotationStatus = new LinkedHashMap<>(); - object.traverse((ArrayTraverser) (idx, statusObject) -> { - var zone = zoneIdFromSlime(statusObject); - var status = RotationState.valueOf(statusObject.field(rotationStateField).asString()); - rotationStatus.put(zone, status); - }); - return Collections.unmodifiableMap(rotationStatus); - } - - private ZoneId zoneIdFromSlime(Inspector object) { - return ZoneId.from(object.field(environmentField).asString(), object.field(regionField).asString()); - } - - private RevisionId revisionFromSlime(Inspector object, JobId job) { - long build = object.field(applicationBuildNumberField).asLong(); - boolean production = object.field(deployedDirectlyField).valid() // TODO jonmv: remove after migration - && build > 0 - && ! object.field(deployedDirectlyField).asBool(); - return production ? RevisionId.forProduction(build) : RevisionId.forDevelopment(build, job); - } - - private ApplicationVersion applicationVersionFromSlime(Inspector object, JobId job) { - RevisionId id = revisionFromSlime(object, job); - Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); - Optional<String> authorEmail = SlimeUtils.optionalString(object.field(authorEmailField)); - Optional<Version> compileVersion = SlimeUtils.optionalString(object.field(compileVersionField)).map(Version::fromString); - Optional<Integer> allowedMajor = SlimeUtils.optionalInteger(object.field(allowedMajorField)).stream().boxed().findFirst(); - Optional<Instant> buildTime = SlimeUtils.optionalInstant(object.field(buildTimeField)); - Optional<String> sourceUrl = SlimeUtils.optionalString(object.field(sourceUrlField)); - Optional<String> commit = SlimeUtils.optionalString(object.field(commitField)); - Optional<Instant> obsoleteAt = SlimeUtils.optionalInstant(object.field(obsoleteAtField)); - boolean hasPackage = object.field(hasPackageField).asBool(); - boolean shouldSkip = object.field(shouldSkipField).asBool(); - Optional<String> description = SlimeUtils.optionalString(object.field(descriptionField)); - Optional<Instant> submittedAt = SlimeUtils.optionalInstant(object.field(submittedAtField)); - int risk = (int) object.field(riskField).asLong(); - Optional<String> bundleHash = SlimeUtils.optionalString(object.field(bundleHashField)); - - return new ApplicationVersion(id, sourceRevision, authorEmail, compileVersion, allowedMajor, buildTime, - sourceUrl, commit, bundleHash, obsoleteAt, hasPackage, shouldSkip, description, submittedAt, risk); - } - - private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) { - if ( ! object.valid()) return Optional.empty(); - return Optional.of(new SourceRevision(object.field(repositoryField).asString(), - object.field(branchField).asString(), - object.field(commitField).asString())); - } - - private Map<JobType, Instant> jobPausesFromSlime(Inspector object) { - Map<JobType, Instant> jobPauses = new HashMap<>(); - object.field(jobStatusField).traverse((ArrayTraverser) (__, jobPauseObject) -> - jobPauses.put(JobType.ofSerialized(jobPauseObject.field(jobTypeField).asString()), - SlimeUtils.instant(jobPauseObject.field(pausedUntilField)))); - return jobPauses; - } - - private Change changeFromSlime(Inspector object) { - if ( ! object.valid()) return Change.empty(); - Inspector versionFieldValue = object.field(versionField); - Change change = Change.empty(); - if (versionFieldValue.valid()) - change = Change.of(Version.fromString(versionFieldValue.asString())); - if (object.field(applicationBuildNumberField).valid()) - change = change.with(revisionFromSlime(object, null)); - if (object.field(platformPinnedField).asBool()) - change = change.withPlatformPin(); - if (object.field(revisionPinnedField).asBool()) - change = change.withRevisionPin(); - return change; - } - - private List<AssignedRotation> assignedRotationsFromSlime(Inspector root) { - var assignedRotations = new LinkedHashMap<EndpointId, AssignedRotation>(); - root.field(assignedRotationsField).traverse((ArrayTraverser) (i, inspector) -> { - var clusterId = new ClusterSpec.Id(inspector.field(assignedRotationClusterField).asString()); - var endpointId = EndpointId.of(inspector.field(assignedRotationEndpointField).asString()); - var rotationId = new RotationId(inspector.field(assignedRotationRotationField).asString()); - var regions = new LinkedHashSet<RegionName>(); - inspector.field(assignedRotationRegionsField).traverse((ArrayTraverser) (j, regionInspector) -> { - regions.add(RegionName.from(regionInspector.asString())); - }); - assignedRotations.putIfAbsent(endpointId, new AssignedRotation(clusterId, endpointId, rotationId, regions)); - }); - - return List.copyOf(assignedRotations.values()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java deleted file mode 100644 index 40a3e35cb25..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.TenantName; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBuckets; -import com.yahoo.vespa.hosted.controller.api.integration.archive.TenantManagedArchiveBucket; -import com.yahoo.vespa.hosted.controller.api.integration.archive.VespaManagedArchiveBucket; - -import java.util.Set; -import java.util.stream.Collectors; - -/** - * (de)serializes tenant/bucket mappings for a zone - * - * @author andreer - */ -public class ArchiveBucketsSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private final static String vespaManagedBucketsFieldName = "buckets"; - private final static String tenantManagedBucketsFieldName = "tenantManagedBuckets"; - private final static String bucketNameFieldName = "bucketName"; - private final static String keyArnFieldName = "keyArn"; - private final static String tenantsFieldName = "tenantIds"; - private final static String accountFieldName = "account"; - private final static String updatedAtFieldName = "updatedAt"; - - public static Slime toSlime(ArchiveBuckets archiveBuckets) { - Slime slime = new Slime(); - Cursor rootObject = slime.setObject(); - - Cursor vespaBucketsArray = rootObject.setArray(vespaManagedBucketsFieldName); - archiveBuckets.vespaManaged().forEach(bucket -> { - Cursor cursor = vespaBucketsArray.addObject(); - cursor.setString(bucketNameFieldName, bucket.bucketName()); - cursor.setString(keyArnFieldName, bucket.keyArn()); - Cursor tenants = cursor.setArray(tenantsFieldName); - bucket.tenants().forEach(tenantName -> tenants.addString(tenantName.value())); - }); - - Cursor tenantBucketsArray = rootObject.setArray(tenantManagedBucketsFieldName); - archiveBuckets.tenantManaged().forEach(bucket -> { - Cursor cursor = tenantBucketsArray.addObject(); - cursor.setString(bucketNameFieldName, bucket.bucketName()); - cursor.setString(accountFieldName, bucket.cloudAccount().value()); - cursor.setLong(updatedAtFieldName, bucket.updatedAt().toEpochMilli()); - }); - - return slime; - } - - public static ArchiveBuckets fromSlime(Slime slime) { - Inspector inspector = slime.get(); - return new ArchiveBuckets( - SlimeUtils.entriesStream(inspector.field(vespaManagedBucketsFieldName)) - .map(ArchiveBucketsSerializer::vespaManagedArchiveBucketFromInspector) - .collect(Collectors.toUnmodifiableSet()), - SlimeUtils.entriesStream(inspector.field(tenantManagedBucketsFieldName)) - .map(ArchiveBucketsSerializer::tenantManagedArchiveBucketFromInspector) - .collect(Collectors.toUnmodifiableSet())); - } - - private static VespaManagedArchiveBucket vespaManagedArchiveBucketFromInspector(Inspector inspector) { - Set<TenantName> tenants = SlimeUtils.entriesStream(inspector.field(tenantsFieldName)) - .map(i -> TenantName.from(i.asString())) - .collect(Collectors.toUnmodifiableSet()); - - return new VespaManagedArchiveBucket( - inspector.field(bucketNameFieldName).asString(), - inspector.field(keyArnFieldName).asString()) - .withTenants(tenants); - } - - private static TenantManagedArchiveBucket tenantManagedArchiveBucketFromInspector(Inspector inspector) { - return new TenantManagedArchiveBucket( - inspector.field(bucketNameFieldName).asString(), - CloudAccount.from(inspector.field(accountFieldName).asString()), - SlimeUtils.instant(inspector.field(updatedAtFieldName))); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java deleted file mode 100644 index 92766ed4506..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -/** - * Slime serializer for {@link AuditLog}. - * - * @author mpolden - */ -public class AuditLogSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String entriesField = "entries"; - private static final String atField = "at"; - private static final String principalField = "principal"; - private static final String methodField = "method"; - private static final String resourceField = "resource"; - private static final String dataField = "data"; - private static final String clientField = "client"; - - public Slime toSlime(AuditLog log) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor entryArray = root.setArray(entriesField); - log.entries().forEach(entry -> { - Cursor entryObject = entryArray.addObject(); - entryObject.setLong(atField, entry.at().toEpochMilli()); - entryObject.setString(clientField, asString(entry.client())); - entryObject.setString(principalField, entry.principal()); - entryObject.setString(methodField, asString(entry.method())); - entryObject.setString(resourceField, entry.resource()); - entry.data().ifPresent(data -> entryObject.setString(dataField, data)); - }); - return slime; - } - public AuditLog fromSlime(Slime slime) { - List<AuditLog.Entry> entries = new ArrayList<>(); - Cursor root = slime.get(); - root.field(entriesField).traverse((ArrayTraverser) (i, entryObject) -> { - entries.add(new AuditLog.Entry( - SlimeUtils.instant(entryObject.field(atField)), - SlimeUtils.optionalString(entryObject.field(clientField)) - .map(AuditLogSerializer::clientFrom) - .orElse(AuditLog.Entry.Client.other), - entryObject.field(principalField).asString(), - methodFrom(entryObject.field(methodField)), - entryObject.field(resourceField).asString(), - SlimeUtils.optionalString(entryObject.field(dataField)) - .map(s -> s.getBytes(StandardCharsets.UTF_8)) - .orElseGet(() -> new byte[0]) - )); - }); - return new AuditLog(entries); - } - - private static String asString(AuditLog.Entry.Method method) { - return switch (method) { - case POST -> "POST"; - case PATCH -> "PATCH"; - case PUT -> "PUT"; - case DELETE -> "DELETE"; - }; - } - - private static AuditLog.Entry.Method methodFrom(Inspector field) { - return switch (field.asString()) { - case "POST" -> AuditLog.Entry.Method.POST; - case "PATCH" -> AuditLog.Entry.Method.PATCH; - case "PUT" -> AuditLog.Entry.Method.PUT; - case "DELETE" -> AuditLog.Entry.Method.DELETE; - default -> throw new IllegalArgumentException("Unknown serialized value '" + field.asString() + "'"); - }; - } - - private static String asString(AuditLog.Entry.Client client) { - return switch (client) { - case console -> "console"; - case cli -> "cli"; - case hv -> "hv"; - case other -> "other"; - }; - } - - private static AuditLog.Entry.Client clientFrom(String s) { - return switch (s) { - case "console" -> AuditLog.Entry.Client.console; - case "cli" -> AuditLog.Entry.Client.cli; - case "hv" -> AuditLog.Entry.Client.hv; - case "other" -> AuditLog.Entry.Client.other; - default -> throw new IllegalArgumentException("Unknown serialized value '" + s + "'"); - }; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java deleted file mode 100644 index 9e202ea30f2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; -import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; -import com.yahoo.vespa.hosted.controller.deployment.RunLog; -import com.yahoo.vespa.hosted.controller.deployment.Step; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * Stores logs in bite-sized chunks using a {@link CuratorDb}, and flushes to a - * {@link com.yahoo.vespa.hosted.controller.api.integration.RunDataStore} when the log is final. - * - * @author jonmv - */ -public class BufferedLogStore { - - private static final int defaultChunkSize = 1 << 13; // 8kb per node stored in ZK. - private static final int defaultMaxLogSize = 1 << 26; // 64Mb max per run. - - private final int chunkSize; - private final int maxLogSize; - private final CuratorDb buffer; - private final RunDataStore store; - private final LogSerializer logSerializer = new LogSerializer(); - - public BufferedLogStore(CuratorDb buffer, RunDataStore store) { - this(defaultChunkSize, defaultMaxLogSize, buffer, store); - } - - BufferedLogStore(int chunkSize, int maxLogSize, CuratorDb buffer, RunDataStore store) { - this.chunkSize = chunkSize; - this.maxLogSize = maxLogSize; - this.buffer = buffer; - this.store = store; - } - - /** Appends to the log of the given, active run, reassigning IDs as counted here, and converting to Vespa log levels. */ - public void append(ApplicationId id, JobType type, Step step, List<LogEntry> entries, boolean forceLog) { - if (entries.isEmpty()) - return; - - // Start a new chunk if the previous one is full, or if none have been written yet. - // The id of a chunk is the id of the first entry in it. - long lastEntryId = buffer.readLastLogEntryId(id, type).orElse(-1L); - long lastChunkId = buffer.getLogChunkIds(id, type).max().orElse(0); - long numberOfChunks = Math.max(1, buffer.getLogChunkIds(id, type).count()); - if (numberOfChunks > maxLogSize / chunkSize && ! forceLog) - return; // Max size exceeded — store no more. - - byte[] emptyChunk = "[]".getBytes(); - byte[] lastChunk = buffer.readLog(id, type, lastChunkId).filter(chunk -> chunk.length > 0).orElse(emptyChunk); - - long sizeLowerBound = lastChunk.length; - Map<Step, List<LogEntry>> log = logSerializer.fromJson(lastChunk, -1); - List<LogEntry> stepEntries = log.computeIfAbsent(step, __ -> new ArrayList<>()); - for (LogEntry entry : entries) { - if (sizeLowerBound > chunkSize) { - buffer.writeLog(id, type, lastChunkId, logSerializer.toJson(log)); - buffer.writeLastLogEntryId(id, type, lastEntryId); - lastChunkId = lastEntryId + 1; - if (++numberOfChunks > maxLogSize / chunkSize && ! forceLog) { - log = Map.of(step, List.of(new LogEntry(++lastEntryId, - entry.at(), - LogEntry.Type.warning, - "Max log size of " + (maxLogSize >> 20) + - "Mb exceeded; further user entries are discarded."))); - break; - } - log = new HashMap<>(); - log.put(step, stepEntries = new ArrayList<>()); - sizeLowerBound = 2; - } - stepEntries.add(new LogEntry(++lastEntryId, entry.at(), entry.type(), entry.message())); - sizeLowerBound += entry.message().length(); - } - buffer.writeLog(id, type, lastChunkId, logSerializer.toJson(log)); - buffer.writeLastLogEntryId(id, type, lastEntryId); - } - - /** Reads all log entries after the given threshold, from the buffered log, i.e., for an active run. */ - public RunLog readActive(ApplicationId id, JobType type, long after) { - return buffer.readLastLogEntryId(id, type).orElse(-1L) <= after - ? RunLog.empty() - : RunLog.of(readChunked(id, type, after)); - } - - /** Reads all log entries after the given threshold, from the stored log, i.e., for a finished run. */ - public Optional<RunLog> readFinished(RunId id, long after) { - return store.get(id).map(json -> RunLog.of(logSerializer.fromJson(json, after))); - } - - /** Writes the buffered log of the now finished run to the long-term store, and clears the buffer. */ - public void flush(RunId id) { - store.put(id, logSerializer.toJson(readChunked(id.application(), id.type(), -1))); - buffer.deleteLog(id.application(), id.type()); - } - - /** Deletes the logs for the given run, if already moved to storage. */ - public void delete(RunId id) { - store.delete(id); - } - - /** Deletes all logs in permanent storage for the given application. */ - public void delete(ApplicationId id) { - store.delete(id); - } - - private Map<Step, List<LogEntry>> readChunked(ApplicationId id, JobType type, long after) { - long[] chunkIds = buffer.getLogChunkIds(id, type).toArray(); - int firstChunk = chunkIds.length; - while (firstChunk > 0 && chunkIds[--firstChunk] > after + 1); - return logSerializer.fromJson(Arrays.stream(chunkIds, firstChunk, chunkIds.length) - .mapToObj(chunkId -> buffer.readLog(id, type, chunkId)) - .flatMap(Optional::stream) - .toList(), - after); - } - - public Optional<String> readTestReports(RunId id) { - return store.getTestReport(id).map(bytes -> "[" + new String(bytes, UTF_8) + "]"); - } - - public void writeTestReport(RunId id, TestReport report) { - byte[] bytes = report.toJson().getBytes(UTF_8); - Optional<byte[]> existing = store.getTestReport(id); - if (existing.isPresent()) { - byte[] aggregate = new byte[existing.get().length + 1 + bytes.length]; - System.arraycopy(existing.get(), 0, aggregate, 0, existing.get().length); - aggregate[existing.get().length] = ','; - System.arraycopy(bytes, 0, aggregate, existing.get().length + 1, bytes.length); - bytes = aggregate; - } - store.putTestReport(id, bytes); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CertifiedOsVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CertifiedOsVersionSerializer.java deleted file mode 100644 index f3b3cb0a1bf..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CertifiedOsVersionSerializer.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.CloudName; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.versions.CertifiedOsVersion; -import com.yahoo.vespa.hosted.controller.versions.OsVersion; - -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -/** - * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.CertifiedOsVersion}. - * - * @author mpolden - */ -public class CertifiedOsVersionSerializer { - - private static final String versionField = "version"; - private static final String cloudField = "cloud"; - private static final String vespaVersionField = "vespaVersion"; - - public Slime toSlime(Set<CertifiedOsVersion> versions) { - Slime slime = new Slime(); - Cursor array = slime.setArray(); - for (var version : versions) { - Cursor root = array.addObject(); - root.setString(versionField, version.osVersion().version().toFullString()); - root.setString(cloudField, version.osVersion().cloud().value()); - root.setString(vespaVersionField, version.vespaVersion().toFullString()); - } - return slime; - } - - public Set<CertifiedOsVersion> fromSlime(Slime slime) { - Cursor array = slime.get(); - Set<CertifiedOsVersion> certifiedOsVersions = new HashSet<>(); - array.traverse((ArrayTraverser) (idx, object) -> certifiedOsVersions.add( - new CertifiedOsVersion(new OsVersion(Version.fromString(object.field(versionField).asString()), - CloudName.from(object.field(cloudField).asString())), - Version.fromString(object.field(vespaVersionField).asString()))) - ); - return Collections.unmodifiableSet(certifiedOsVersions); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ChangeRequestSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ChangeRequestSerializer.java deleted file mode 100644 index f43be77b82c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ChangeRequestSerializer.java +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestSource; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; - -import java.time.Instant; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.List; - -/** - * @author olaa - */ -public class ChangeRequestSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String ID_FIELD = "id"; - private static final String SOURCE_FIELD = "source"; - private static final String SOURCE_SYSTEM_FIELD = "system"; - private static final String STATUS_FIELD = "status"; - private static final String URL_FIELD = "url"; - private static final String ZONE_FIELD = "zoneId"; - private static final String START_TIME_FIELD = "plannedStartTime"; - private static final String END_TIME_FIELD = "plannedEndTime"; - private static final String APPROVAL_FIELD = "approval"; - private static final String IMPACT_FIELD = "impact"; - private static final String IMPACTED_HOSTS_FIELD = "impactedHosts"; - private static final String IMPACTED_SWITCHES_FIELD = "impactedSwitches"; - private static final String ACTION_PLAN_FIELD = "actionPlan"; - private static final String HOST_FIELD = "hostname"; - private static final String ACTION_STATE_FIELD = "state"; - private static final String LAST_UPDATED_FIELD = "lastUpdated"; - private static final String HOSTS_FIELD = "hosts"; - private static final String CATEGORY_FIELD = "category"; - - - public static VespaChangeRequest fromSlime(Slime slime) { - var inspector = slime.get(); - var id = inspector.field(ID_FIELD).asString(); - var zoneId = ZoneId.from(inspector.field(ZONE_FIELD).asString()); - var changeRequestSource = readChangeRequestSource(inspector.field(SOURCE_FIELD)); - var actionPlan = readHostActionPlan(inspector.field(ACTION_PLAN_FIELD)); - var status = VespaChangeRequest.Status.valueOf(inspector.field(STATUS_FIELD).asString()); - var impact = ChangeRequest.Impact.valueOf(inspector.field(IMPACT_FIELD).asString()); - var approval = ChangeRequest.Approval.valueOf(inspector.field(APPROVAL_FIELD).asString()); - var category = inspector.field(CATEGORY_FIELD).valid() ? - inspector.field(CATEGORY_FIELD).asString() : "Unknown"; - - var impactedHosts = new ArrayList<String>(); - inspector.field(IMPACTED_HOSTS_FIELD) - .traverse((ArrayTraverser) (i, hostname) -> impactedHosts.add(hostname.asString())); - var impactedSwitches = new ArrayList<String>(); - inspector.field(IMPACTED_SWITCHES_FIELD) - .traverse((ArrayTraverser) (i, switchName) -> impactedSwitches.add(switchName.asString())); - - return new VespaChangeRequest( - id, - changeRequestSource, - impactedSwitches, - impactedHosts, - approval, - impact, - status, - actionPlan, - zoneId); - } - - public static Slime toSlime(VespaChangeRequest changeRequest) { - var slime = new Slime(); - writeChangeRequest(slime.setObject(), changeRequest); - return slime; - } - - public static void writeChangeRequest(Cursor cursor, VespaChangeRequest changeRequest) { - cursor.setString(ID_FIELD, changeRequest.getId()); - cursor.setString(STATUS_FIELD, changeRequest.getStatus().name()); - cursor.setString(IMPACT_FIELD, changeRequest.getImpact().name()); - cursor.setString(APPROVAL_FIELD, changeRequest.getApproval().name()); - cursor.setString(ZONE_FIELD, changeRequest.getZoneId().value()); - writeChangeRequestSource(cursor.setObject(SOURCE_FIELD), changeRequest.getChangeRequestSource()); - writeActionPlan(cursor.setObject(ACTION_PLAN_FIELD), changeRequest); - - var impactedHosts = cursor.setArray(IMPACTED_HOSTS_FIELD); - changeRequest.getImpactedHosts().forEach(impactedHosts::addString); - var impactedSwitches = cursor.setArray(IMPACTED_SWITCHES_FIELD); - changeRequest.getImpactedSwitches().forEach(impactedSwitches::addString); - } - - private static void writeActionPlan(Cursor cursor, VespaChangeRequest changeRequest) { - var hostsCursor = cursor.setArray(HOSTS_FIELD); - - changeRequest.getHostActionPlan().forEach(action -> { - var actionCursor = hostsCursor.addObject(); - actionCursor.setString(HOST_FIELD, action.getHostname()); - actionCursor.setString(ACTION_STATE_FIELD, action.getState().name()); - actionCursor.setString(LAST_UPDATED_FIELD, action.getLastUpdated().toString()); - }); - - // TODO: Add action plan per application - } - - private static void writeChangeRequestSource(Cursor cursor, ChangeRequestSource source) { - cursor.setString(SOURCE_SYSTEM_FIELD, source.system()); - cursor.setString(ID_FIELD, source.id()); - cursor.setString(URL_FIELD, source.url()); - cursor.setString(START_TIME_FIELD, source.plannedStartTime().toString()); - cursor.setString(END_TIME_FIELD, source.plannedEndTime().toString()); - cursor.setString(STATUS_FIELD, source.status().name()); - cursor.setString(CATEGORY_FIELD, source.category()); - } - - public static ChangeRequestSource readChangeRequestSource(Inspector inspector) { - var category = inspector.field(CATEGORY_FIELD).valid() ? - inspector.field(CATEGORY_FIELD).asString() : "Unknown"; - return new ChangeRequestSource( - inspector.field(SOURCE_SYSTEM_FIELD).asString(), - inspector.field(ID_FIELD).asString(), - inspector.field(URL_FIELD).asString(), - ChangeRequestSource.Status.valueOf(inspector.field(STATUS_FIELD).asString()), - ZonedDateTime.parse(inspector.field(START_TIME_FIELD).asString()), - ZonedDateTime.parse(inspector.field(END_TIME_FIELD).asString()), - category - ); - } - - public static List<HostAction> readHostActionPlan(Inspector inspector) { - if (!inspector.valid()) - return List.of(); - - var actionPlan = new ArrayList<HostAction>(); - inspector.field(HOSTS_FIELD).traverse((ArrayTraverser) (index, hostObject) -> - actionPlan.add( - new HostAction( - hostObject.field(HOST_FIELD).asString(), - HostAction.State.valueOf(hostObject.field(ACTION_STATE_FIELD).asString()), - Instant.parse(hostObject.field(LAST_UPDATED_FIELD).asString()) - ) - ) - ); - return actionPlan; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java deleted file mode 100644 index 91e12b9cb15..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.component.Version; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; - -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * Serializer for {@link Confidence} overrides. - * - * @author mpolden - */ -public class ConfidenceOverrideSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private final static String overridesField = "overrides"; - - public Slime toSlime(Map<Version, Confidence> overrides) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor object = root.setObject(overridesField); - overrides.forEach((version, confidence) -> object.setString(version.toString(), confidence.name())); - return slime; - } - - public Map<Version, Confidence> fromSlime(Slime slime) { - Cursor root = slime.get(); - Cursor overridesObject = root.field(overridesField); - Map<Version, Confidence> overrides = new LinkedHashMap<>(); - overridesObject.traverse((ObjectTraverser) (name, value) -> { - overrides.put(Version.fromString(name), Confidence.valueOf(value.asString())); - }); - return Collections.unmodifiableMap(overrides); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java deleted file mode 100644 index f19d7f68b3d..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.component.Version; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; - -/** - * Serializer for {@link ControllerVersion}. - * - * @author mpolden - */ -public class ControllerVersionSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String VERSION_FIELD = "version"; - private static final String COMMIT_SHA_FIELD = "commitSha"; - private static final String COMMIT_DATE_FIELD = "commitDate"; - - public Slime toSlime(ControllerVersion controllerVersion) { - var slime = new Slime(); - var root = slime.setObject(); - root.setString(VERSION_FIELD, controllerVersion.version().toFullString()); - root.setString(COMMIT_SHA_FIELD, controllerVersion.commitSha()); - root.setLong(COMMIT_DATE_FIELD, controllerVersion.commitDate().toEpochMilli()); - return slime; - } - - public ControllerVersion fromSlime(Slime slime) { - var root = slime.get(); - var version = Version.fromString(root.field(VERSION_FIELD).asString()); - var commitSha = root.field(COMMIT_SHA_FIELD).asString(); - var commitDate = SlimeUtils.instant(root.field(COMMIT_DATE_FIELD)); - return new ControllerVersion(version, commitSha, commitDate); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java deleted file mode 100644 index cef62438a53..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ /dev/null @@ -1,948 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.collections.Pair; -import com.yahoo.component.Version; -import com.yahoo.component.annotation.Inject; -import com.yahoo.concurrent.UncheckedTimeoutException; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec.Id; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.path.Path; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.transaction.Mutex; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.transaction.CuratorOperation; -import com.yahoo.vespa.curator.transaction.CuratorOperations; -import com.yahoo.vespa.curator.transaction.CuratorTransaction; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; -import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBuckets; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneTokenVersions; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; -import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; -import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; -import com.yahoo.vespa.hosted.controller.deployment.RetriggerEntry; -import com.yahoo.vespa.hosted.controller.deployment.RetriggerEntrySerializer; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.Step; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; -import com.yahoo.vespa.hosted.controller.notification.Notification; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; -import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; -import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; -import com.yahoo.vespa.hosted.controller.tenant.PendingMailVerification; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.versions.CertifiedOsVersion; -import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; -import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.ByteBuffer; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Optional; -import java.util.Set; -import java.util.SortedSet; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.LongStream; - -import static java.util.stream.Collectors.collectingAndThen; - -/** - * Curator backed database for storing the persistence state of controllers. This maps controller specific operations - * to general curator operations. - * - * @author bratseth - * @author mpolden - * @author jonmv - */ -public class CuratorDb { - - private static final Logger log = Logger.getLogger(CuratorDb.class.getName()); - private static final Duration deployLockTimeout = Duration.ofMinutes(30); - private static final Duration defaultLockTimeout = Duration.ofMinutes(5); - private static final Duration defaultTryLockTimeout = Duration.ofSeconds(1); - - private static final Path root = Path.fromString("/controller/v1"); - - private static final Path lockRoot = root.append("locks"); - - private static final Path tenantRoot = root.append("tenants"); - private static final Path applicationRoot = root.append("applications"); - private static final Path jobRoot = root.append("jobs"); - private static final Path controllerRoot = root.append("controllers"); - private static final Path routingPoliciesRoot = root.append("routingPolicies"); - private static final Path dnsChallengesRoot = root.append("dnsChallenges"); - private static final Path zoneRoutingPoliciesRoot = root.append("zoneRoutingPolicies"); - private static final Path endpointCertificateRoot = root.append("applicationCertificates"); - private static final Path archiveBucketsRoot = root.append("archiveBuckets"); - private static final Path changeRequestsRoot = root.append("changeRequests"); - private static final Path notificationsRoot = root.append("notifications"); - private static final Path supportAccessRoot = root.append("supportAccess"); - private static final Path mailVerificationRoot = root.append("mailVerification"); - private static final Path dataPlaneTokenRoot = root.append("dataplaneTokens"); - private static final Path certificatePoolRoot = root.append("certificatePool"); - private static final Path trialNotificationsRoot = root.append("trialNotifications"); - - private final NodeVersionSerializer nodeVersionSerializer = new NodeVersionSerializer(); - private final VersionStatusSerializer versionStatusSerializer = new VersionStatusSerializer(nodeVersionSerializer); - private final ControllerVersionSerializer controllerVersionSerializer = new ControllerVersionSerializer(); - private final ConfidenceOverrideSerializer confidenceOverrideSerializer = new ConfidenceOverrideSerializer(); - private final TenantSerializer tenantSerializer = new TenantSerializer(); - private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); - private final OsVersionTargetSerializer osVersionTargetSerializer = new OsVersionTargetSerializer(osVersionSerializer); - private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer, nodeVersionSerializer); - private final CertifiedOsVersionSerializer certifiedOsVersionSerializer = new CertifiedOsVersionSerializer(); - private final RoutingPolicySerializer routingPolicySerializer = new RoutingPolicySerializer(); - private final ZoneRoutingPolicySerializer zoneRoutingPolicySerializer = new ZoneRoutingPolicySerializer(routingPolicySerializer); - private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer(); - private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer(); - private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); - private final RunSerializer runSerializer = new RunSerializer(); - private final RetriggerEntrySerializer retriggerEntrySerializer = new RetriggerEntrySerializer(); - private final NotificationsSerializer notificationsSerializer = new NotificationsSerializer(); - private final DnsChallengeSerializer dnsChallengeSerializer = new DnsChallengeSerializer(); - private final UnassignedCertificateSerializer unassignedCertificateSerializer = new UnassignedCertificateSerializer(); - - private final Curator curator; - private final Duration tryLockTimeout; - - // For each application id (path), store the ZK node version and its deserialised data - update when version changes. - // This will grow to keep all applications in memory, but this should be OK - private final Map<Path, Pair<Integer, Application>> cachedApplications = new ConcurrentHashMap<>(); - - // For each job id (path), store the ZK node version and its deserialised data - update when version changes. - private final Map<Path, Pair<Integer, NavigableMap<RunId, Run>>> cachedHistoricRuns = new ConcurrentHashMap<>(); - - // Store the ZK node version and its deserialised data - update when version changes. - private final AtomicReference<Pair<Integer, VersionStatus>> cachedVersionStatus = new AtomicReference<>(); - - @Inject - public CuratorDb(Curator curator) { - this(curator, defaultTryLockTimeout); - } - - CuratorDb(Curator curator, Duration tryLockTimeout) { - this.curator = curator; - this.tryLockTimeout = tryLockTimeout; - } - - /** Returns all hostnames configured to be part of this ZooKeeper cluster */ - public List<String> cluster() { - return Arrays.stream(curator.zooKeeperEnsembleConnectionSpec().split(",")) - .filter(hostAndPort -> !hostAndPort.isEmpty()) - .map(hostAndPort -> hostAndPort.split(":")[0]) - .toList(); - } - - // -------------- Locks --------------------------------------------------- - - public Mutex lock(TenantName name) { - return curator.lock(lockRoot.append("tenants").append(name.value()), defaultLockTimeout.multipliedBy(2)); - } - - public Mutex lock(TenantAndApplicationId id) { - return curator.lock(lockRoot.append("applications").append(id.tenant().value() + ":" + - id.application().value()), - defaultLockTimeout.multipliedBy(2)); - } - - public Mutex lockForDeployment(ApplicationId id, ZoneId zone) { - return curator.lock(lockRoot.append("instances").append(id.serializedForm() + ":" + zone.environment().value() + - ":" + zone.region().value()), - deployLockTimeout); - } - - public Mutex lock(ApplicationId id, JobType type) { - return curator.lock(lockRoot.append("jobs").append(id.serializedForm() + ":" + type.jobName()), - defaultLockTimeout); - } - - public Mutex lock(ApplicationId id, JobType type, Step step) throws TimeoutException { - return tryLock(lockRoot.append("steps").append(id.serializedForm() + ":" + type.jobName() + ":" + step.name())); - } - - public Mutex lockRotations() { - return curator.lock(lockRoot.append("rotations"), defaultLockTimeout); - } - - public Mutex lockConfidenceOverrides() { - return curator.lock(lockRoot.append("confidenceOverrides"), defaultLockTimeout); - } - - public Mutex lockMaintenanceJob(String jobName) { - try { - return tryLock(lockRoot.append("maintenanceJobLocks").append(jobName)); - } catch (TimeoutException e) { - throw new UncheckedTimeoutException(e); - } - } - - public Mutex lockProvisionState(String provisionStateId) { - return curator.lock(lockRoot.append("provisioning").append("states").append(provisionStateId), Duration.ofSeconds(1)); - } - - public Mutex lockOsVersions() { - return curator.lock(lockRoot.append("osTargetVersion"), defaultLockTimeout); - } - - public Mutex lockOsVersionStatus() { - return curator.lock(lockRoot.append("osVersionStatus"), defaultLockTimeout); - } - - public Mutex lockCertifiedOsVersions() { - return curator.lock(lockRoot.append("certifiedOsVersions"), defaultLockTimeout); - } - - public Mutex lockRoutingPolicies() { - return curator.lock(lockRoot.append("routingPolicies"), defaultLockTimeout); - } - - public Mutex lockAuditLog() { - return curator.lock(lockRoot.append("auditLog"), defaultLockTimeout); - } - - public Mutex lockNameServiceQueue() { - return curator.lock(lockRoot.append("nameServiceQueue"), defaultLockTimeout); - } - - public Mutex lockMeteringRefreshTime() throws TimeoutException { - return tryLock(lockRoot.append("meteringRefreshTime")); - } - - public Mutex lockArchiveBuckets(ZoneId zoneId) { - return curator.lock(lockRoot.append("archiveBuckets").append(zoneId.value()), defaultLockTimeout); - } - - public Mutex lockChangeRequests() { - return curator.lock(lockRoot.append("changeRequests"), defaultLockTimeout); - } - - public Mutex lockNotifications(TenantName tenantName) { - return curator.lock(lockRoot.append("notifications").append(tenantName.value()), defaultLockTimeout); - } - - public Mutex lockSupportAccess(DeploymentId deploymentId) { - return curator.lock(lockRoot.append("supportAccess").append(deploymentId.dottedString()), defaultLockTimeout); - } - - public Mutex lockDeploymentRetriggerQueue() { - return curator.lock(lockRoot.append("deploymentRetriggerQueue"), defaultLockTimeout); - } - - public Mutex lockPendingMailVerification(String verificationCode) { - return curator.lock(lockRoot.append("pendingMailVerification").append(verificationCode), defaultLockTimeout); - } - - public Mutex lockCertificatePool() { - return curator.lock(lockRoot.append("certificatePool"), defaultLockTimeout); - } - - // -------------- Helpers ------------------------------------------ - - /** Try locking with a low timeout, meaning it is OK to fail lock acquisition. - * - * Useful for maintenance jobs, where there is no point in running the jobs back to back. - */ - private Mutex tryLock(Path path) throws TimeoutException { - try { - return curator.lock(path, tryLockTimeout); - } - catch (UncheckedTimeoutException e) { - throw new TimeoutException(e.getMessage()); - } - } - - private <T> Optional<T> read(Path path, Function<byte[], T> mapper) { - return curator.getData(path).filter(data -> data.length > 0).map(mapper); - } - - private Optional<Slime> readSlime(Path path) { - return read(path, SlimeUtils::jsonToSlime); - } - - private static byte[] asJson(Slime slime) { - try { - return SlimeUtils.toJsonBytes(slime); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - // -------------- Deployment orchestration -------------------------------- - - public double readUpgradesPerMinute() { - return read(upgradesPerMinutePath(), ByteBuffer::wrap).map(ByteBuffer::getDouble).orElse(0.125); - } - - public void writeUpgradesPerMinute(double n) { - curator.set(upgradesPerMinutePath(), ByteBuffer.allocate(Double.BYTES).putDouble(n).array()); - } - - public void writeVersionStatus(VersionStatus status) { - curator.set(versionStatusPath(), asJson(versionStatusSerializer.toSlime(status))); - } - - public VersionStatus readVersionStatus() { - Path path = versionStatusPath(); - return curator.getStat(path) - .map(stat -> cachedVersionStatus.updateAndGet(old -> - old != null && old.getFirst() == stat.getVersion() - ? old - : new Pair<>(stat.getVersion(), read(path, bytes -> versionStatusSerializer.fromSlime(SlimeUtils.jsonToSlime(bytes))).get())).getSecond()) - .orElseGet(VersionStatus::empty); - } - - public void writeConfidenceOverrides(Map<Version, VespaVersion.Confidence> overrides) { - curator.set(confidenceOverridesPath(), asJson(confidenceOverrideSerializer.toSlime(overrides))); - } - - public Map<Version, VespaVersion.Confidence> readConfidenceOverrides() { - return readSlime(confidenceOverridesPath()).map(confidenceOverrideSerializer::fromSlime) - .orElseGet(Collections::emptyMap); - } - - public void writeControllerVersion(HostName hostname, ControllerVersion version) { - curator.set(controllerPath(hostname.value()), asJson(controllerVersionSerializer.toSlime(version))); - } - - public ControllerVersion readControllerVersion(HostName hostname) { - return readSlime(controllerPath(hostname.value())) - .map(controllerVersionSerializer::fromSlime) - .orElse(ControllerVersion.CURRENT); - } - - // OS upgrades - - public void writeOsVersionTargets(SortedSet<OsVersionTarget> versions) { - curator.set(osVersionTargetsPath(), asJson(osVersionTargetSerializer.toSlime(versions))); - } - - public Set<OsVersionTarget> readOsVersionTargets() { - return readSlime(osVersionTargetsPath()).map(osVersionTargetSerializer::fromSlime).orElseGet(Collections::emptySet); - } - - public void writeOsVersionStatus(OsVersionStatus status) { - curator.set(osVersionStatusPath(), asJson(osVersionStatusSerializer.toSlime(status))); - } - - public OsVersionStatus readOsVersionStatus() { - return readSlime(osVersionStatusPath()).map(osVersionStatusSerializer::fromSlime).orElse(OsVersionStatus.empty); - } - - public void writeCertifiedOsVersions(Set<CertifiedOsVersion> certifiedOsVersions) { - curator.set(certifiedOsVersionsPath(), asJson(certifiedOsVersionSerializer.toSlime(certifiedOsVersions))); - } - - public Set<CertifiedOsVersion> readCertifiedOsVersions() { - return readSlime(certifiedOsVersionsPath()).map(certifiedOsVersionSerializer::fromSlime).orElseGet(Set::of); - } - - // -------------- Tenant -------------------------------------------------- - - public void writeTenant(Tenant tenant) { - curator.set(tenantPath(tenant.name()), asJson(tenantSerializer.toSlime(tenant))); - } - - public Optional<Tenant> readTenant(TenantName name) { - return readSlime(tenantPath(name)).map(tenantSerializer::tenantFrom); - } - - public List<Tenant> readTenants() { - return readTenantNames().stream() - .map(this::readTenant) - .flatMap(Optional::stream) - .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); - } - - public List<TenantName> readTenantNames() { - return curator.getChildren(tenantRoot).stream() - .map(TenantName::from) - .toList(); - } - - public void removeTenant(TenantName name) { - curator.delete(tenantPath(name)); - } - - // -------------- Applications --------------------------------------------- - - public void writeApplication(Application application) { - curator.set(applicationPath(application.id()), asJson(applicationSerializer.toSlime(application))); - } - - public Optional<Application> readApplication(TenantAndApplicationId application) { - Path path = applicationPath(application); - return curator.getStat(path) - .map(stat -> cachedApplications.compute(path, (__, old) -> - old != null && old.getFirst() == stat.getVersion() - ? old - : new Pair<>(stat.getVersion(), read(path, applicationSerializer::fromSlime).get())).getSecond()); - } - - public List<Application> readApplications(boolean canFail) { - return readApplications(ignored -> true, canFail); - } - - public List<Application> readApplications(TenantName name) { - return readApplications(application -> application.tenant().equals(name), false); - } - - private List<Application> readApplications(Predicate<TenantAndApplicationId> applicationFilter, boolean canFail) { - var applicationIds = readApplicationIds(); - var applications = new ArrayList<Application>(applicationIds.size()); - for (var id : applicationIds) { - if (!applicationFilter.test(id)) continue; - try { - readApplication(id).ifPresent(applications::add); - } catch (Exception e) { - if (canFail) { - log.log(Level.SEVERE, "Failed to read application '" + id + "', this must be fixed through " + - "manual intervention", e); - } else { - throw e; - } - } - } - return Collections.unmodifiableList(applications); - } - - public List<TenantAndApplicationId> readApplicationIds() { - return curator.getChildren(applicationRoot).stream() - .map(TenantAndApplicationId::fromSerialized) - .sorted() - .toList(); - } - - public void removeApplication(TenantAndApplicationId id) { - curator.delete(applicationPath(id)); - } - - // -------------- Job Runs ------------------------------------------------ - - public void writeLastRun(Run run) { - curator.set(lastRunPath(run.id().application(), run.id().type()), asJson(runSerializer.toSlime(run))); - } - - public void writeHistoricRuns(ApplicationId id, JobType type, Iterable<Run> runs) { - Path path = runsPath(id, type); - curator.set(path, asJson(runSerializer.toSlime(runs))); - } - - public Optional<Run> readLastRun(ApplicationId id, JobType type) { - return readSlime(lastRunPath(id, type)).map(runSerializer::runFromSlime); - } - - public NavigableMap<RunId, Run> readHistoricRuns(ApplicationId id, JobType type) { - Path path = runsPath(id, type); - return curator.getStat(path) - .map(stat -> cachedHistoricRuns.compute(path, (__, old) -> - old != null && old.getFirst() == stat.getVersion() - ? old - : new Pair<>(stat.getVersion(), runSerializer.runsFromSlime(readSlime(path).get()))).getSecond()) - .orElseGet(Collections::emptyNavigableMap); - } - - public void deleteRunData(ApplicationId id, JobType type) { - curator.delete(runsPath(id, type)); - curator.delete(lastRunPath(id, type)); - } - - public void deleteRunData(ApplicationId id) { - curator.delete(jobRoot.append(id.serializedForm())); - } - - public List<ApplicationId> applicationsWithJobs() { - return curator.getChildren(jobRoot).stream() - .map(ApplicationId::fromSerializedForm) - .toList(); - } - - - public Optional<byte[]> readLog(ApplicationId id, JobType type, long chunkId) { - return curator.getData(logPath(id, type, chunkId)); - } - - public void writeLog(ApplicationId id, JobType type, long chunkId, byte[] log) { - curator.set(logPath(id, type, chunkId), log); - } - - public void deleteLog(ApplicationId id, JobType type) { - curator.delete(runsPath(id, type).append("logs")); - } - - public Optional<Long> readLastLogEntryId(ApplicationId id, JobType type) { - return curator.getData(lastLogPath(id, type)) - .map(String::new).map(Long::parseLong); - } - - public void writeLastLogEntryId(ApplicationId id, JobType type, long lastId) { - curator.set(lastLogPath(id, type), Long.toString(lastId).getBytes()); - } - - public LongStream getLogChunkIds(ApplicationId id, JobType type) { - return curator.getChildren(runsPath(id, type).append("logs")).stream() - .mapToLong(Long::parseLong) - .sorted(); - } - - // -------------- Audit log ----------------------------------------------- - - public AuditLog readAuditLog() { - return readSlime(auditLogPath()).map(auditLogSerializer::fromSlime) - .orElse(AuditLog.empty); - } - - public void writeAuditLog(AuditLog log) { - curator.set(auditLogPath(), asJson(auditLogSerializer.toSlime(log))); - } - - - // -------------- Name service log ---------------------------------------- - - public NameServiceQueue readNameServiceQueue() { - return readSlime(nameServiceQueuePath()).map(nameServiceQueueSerializer::fromSlime) - .orElse(NameServiceQueue.EMPTY); - } - - public void writeNameServiceQueue(NameServiceQueue queue) { - curator.set(nameServiceQueuePath(), asJson(nameServiceQueueSerializer.toSlime(queue))); - } - - // -------------- Provisioning (called by internal code) ------------------ - - @SuppressWarnings("unused") - public Optional<byte[]> readProvisionState(String provisionId) { - return curator.getData(provisionStatePath(provisionId)); - } - - @SuppressWarnings("unused") - public void writeProvisionState(String provisionId, byte[] data) { - curator.set(provisionStatePath(provisionId), data); - } - - @SuppressWarnings("unused") - public List<String> readProvisionStateIds() { - return curator.getChildren(provisionStatePath()); - } - - // -------------- Routing policies ---------------------------------------- - - public void writeRoutingPolicies(ApplicationId application, List<RoutingPolicy> policies) { - for (var policy : policies) { - if (!policy.id().owner().equals(application)) { - throw new IllegalArgumentException(policy.id() + " does not belong to the application being written: " + - application.toShortString()); - } - } - curator.set(routingPolicyPath(application), asJson(routingPolicySerializer.toSlime(policies))); - } - - public Map<ApplicationId, List<RoutingPolicy>> readRoutingPolicies() { - return readRoutingPolicies((instance) -> true); - } - - public Map<ApplicationId, List<RoutingPolicy>> readRoutingPolicies(Predicate<ApplicationId> filter) { - return curator.getChildren(routingPoliciesRoot).stream() - .map(ApplicationId::fromSerializedForm) - .filter(filter) - .collect(Collectors.toUnmodifiableMap(Function.identity(), - this::readRoutingPolicies)); - } - - public List<RoutingPolicy> readRoutingPolicies(ApplicationId application) { - return readSlime(routingPolicyPath(application)).map(slime -> routingPolicySerializer.fromSlime(application, slime)) - .orElseGet(List::of); - } - - public void writeZoneRoutingPolicy(ZoneRoutingPolicy policy) { - curator.set(zoneRoutingPolicyPath(policy.zone()), asJson(zoneRoutingPolicySerializer.toSlime(policy))); - } - - public ZoneRoutingPolicy readZoneRoutingPolicy(ZoneId zone) { - return readSlime(zoneRoutingPolicyPath(zone)).map(data -> zoneRoutingPolicySerializer.fromSlime(zone, data)) - .orElseGet(() -> new ZoneRoutingPolicy(zone, RoutingStatus.DEFAULT)); - } - - public void writeDnsChallenge(DnsChallenge challenge) { - curator.set(dnsChallengePath(challenge.clusterId()), dnsChallengeSerializer.toJson(challenge)); - } - - public void deleteDnsChallenge(ClusterId id) { - curator.delete(dnsChallengePath(id)); - } - - public List<DnsChallenge> readDnsChallenges(DeploymentId id) { - return curator.getChildren(dnsChallengePath(id)).stream() - .map(cluster -> readDnsChallenge(new ClusterId(id, Id.from(cluster)))) - .toList(); - } - - private DnsChallenge readDnsChallenge(ClusterId clusterId) { - return curator.getData(dnsChallengePath(clusterId)) - .map(bytes -> dnsChallengeSerializer.fromJson(bytes, clusterId)) - .orElseThrow(() -> new IllegalArgumentException("no DNS challenge for " + clusterId)); - } - - private static Path dnsChallengePath(DeploymentId id) { - return dnsChallengesRoot.append(id.applicationId().serializedForm()) - .append(id.zoneId().value()); - } - - private static Path dnsChallengePath(ClusterId id) { - return dnsChallengePath(id.deploymentId()).append(id.clusterId().value()); - } - - // -------------- Application endpoint certificates ---------------------------- - - public void writeAssignedCertificate(AssignedCertificate certificate) { - try (NestedTransaction transaction = new NestedTransaction()) { - writeAssignedCertificate(certificate, transaction); - transaction.commit(); - } - } - - public void writeAssignedCertificate(AssignedCertificate certificate, NestedTransaction transaction) { - Path path = endpointCertificatePath(certificate.application(), certificate.instance()); - curator.create(path); - CuratorOperation operation = CuratorOperations.setData(path.getAbsolute(), - asJson(EndpointCertificateSerializer.toSlime(certificate.certificate()))); - transaction.add(CuratorTransaction.from(operation, curator)); - } - - public void removeAssignedCertificate(TenantAndApplicationId application, Optional<InstanceName> instanceName) { - curator.delete(endpointCertificatePath(application, instanceName)); - } - - public void removeAssignedCertificate(TenantAndApplicationId application, Optional<InstanceName> instanceName, NestedTransaction transaction) { - transaction.add(CuratorTransaction.from(CuratorOperations.delete(endpointCertificatePath(application, instanceName).getAbsolute()), curator)); - } - - // TODO(mpolden): Remove this. Caller should make an explicit decision to read certificate for a particular instance - public Optional<AssignedCertificate> readAssignedCertificate(ApplicationId applicationId) { - return readAssignedCertificate(TenantAndApplicationId.from(applicationId), Optional.of(applicationId.instance())); - } - - public Optional<AssignedCertificate> readAssignedCertificate(TenantAndApplicationId application, Optional<InstanceName> instance) { - return readSlime(endpointCertificatePath(application, instance)).map(Slime::get) - .map(EndpointCertificateSerializer::fromSlime) - .map(cert -> new AssignedCertificate(application, instance, cert, false)); - } - - public List<AssignedCertificate> readAssignedCertificates() { - List<AssignedCertificate> certificates = new ArrayList<>(); - for (String value : curator.getChildren(endpointCertificateRoot)) { - final TenantAndApplicationId application; - final Optional<InstanceName> instanceName; - if (value.split(":").length == 3) { - ApplicationId instance = ApplicationId.fromSerializedForm(value); - application = TenantAndApplicationId.from(instance); - instanceName = Optional.of(instance.instance()); - } else { - application = TenantAndApplicationId.fromSerialized(value); - instanceName = Optional.empty(); - } - Optional<AssignedCertificate> assigned = readAssignedCertificate(application, instanceName); - if (assigned.isEmpty()) continue; // Deleted while reading - certificates.add(assigned.get()); - } - return certificates; - } - - // -------------- Metering view refresh times ---------------------------- - - public void writeMeteringRefreshTime(long timestamp) { - curator.set(meteringRefreshPath(), Long.toString(timestamp).getBytes()); - } - - public long readMeteringRefreshTime() { - return curator.getData(meteringRefreshPath()) - .map(String::new).map(Long::parseLong) - .orElse(0L); - } - - // -------------- Archive buckets ----------------------------------------- - - public ArchiveBuckets readArchiveBuckets(ZoneId zoneId) { - return readSlime(archiveBucketsPath(zoneId)).map(ArchiveBucketsSerializer::fromSlime) - .orElse(ArchiveBuckets.EMPTY); - } - - public void writeArchiveBuckets(ZoneId zoneid, ArchiveBuckets archiveBuckets) { - curator.set(archiveBucketsPath(zoneid), asJson(ArchiveBucketsSerializer.toSlime(archiveBuckets))); - } - - // -------------- VCMRs --------------------------------------------------- - - public Optional<VespaChangeRequest> readChangeRequest(String changeRequestId) { - return readSlime(changeRequestPath(changeRequestId)).map(ChangeRequestSerializer::fromSlime); - } - - public List<VespaChangeRequest> readChangeRequests() { - return curator.getChildren(changeRequestsRoot) - .stream() - .map(this::readChangeRequest) - .flatMap(Optional::stream) - .toList(); - } - - public void writeChangeRequest(VespaChangeRequest changeRequest) { - curator.set(changeRequestPath(changeRequest.getId()), asJson(ChangeRequestSerializer.toSlime(changeRequest))); - } - - public void deleteChangeRequest(VespaChangeRequest changeRequest) { - curator.delete(changeRequestPath(changeRequest.getId())); - } - - // -------------- Notifications ------------------------------------------- - - public List<Notification> readNotifications(TenantName tenantName) { - return readSlime(notificationsPath(tenantName)) - .map(slime -> notificationsSerializer.fromSlime(tenantName, slime)).orElseGet(List::of); - } - - - public List<TenantName> listTenantsWithNotifications() { - return curator.getChildren(notificationsRoot).stream() - .map(TenantName::from) - .toList(); - } - - public void writeNotifications(TenantName tenantName, List<Notification> notifications) { - curator.set(notificationsPath(tenantName), asJson(notificationsSerializer.toSlime(notifications))); - } - - public void deleteNotifications(TenantName tenantName) { - curator.delete(notificationsPath(tenantName)); - } - - // -------------- Endpoint Support Access --------------------------------- - - public SupportAccess readSupportAccess(DeploymentId deploymentId) { - return readSlime(supportAccessPath(deploymentId)).map(SupportAccessSerializer::fromSlime).orElse(SupportAccess.DISALLOWED_NO_HISTORY); - } - - public void writeSupportAccess(DeploymentId deploymentId, SupportAccess supportAccess) { - curator.set(supportAccessPath(deploymentId), asJson(SupportAccessSerializer.toSlime(supportAccess))); - } - - // -------------- Job Retrigger entries ----------------------------------- - - public List<RetriggerEntry> readRetriggerEntries() { - return readSlime(deploymentRetriggerPath()).map(retriggerEntrySerializer::fromSlime).orElseGet(List::of); - } - - public void writeRetriggerEntries(List<RetriggerEntry> retriggerEntries) { - curator.set(deploymentRetriggerPath(), asJson(retriggerEntrySerializer.toSlime(retriggerEntries))); - } - - // -------------- Pending mail verification ------------------------------- - - public Optional<PendingMailVerification> getPendingMailVerification(String verificationCode) { - return readSlime(mailVerificationPath(verificationCode)).map(MailVerificationSerializer::fromSlime); - } - - public List<PendingMailVerification> listPendingMailVerifications() { - return curator.getChildren(mailVerificationRoot) - .stream() - .map(this::getPendingMailVerification) - .flatMap(Optional::stream) - .toList(); - } - - public void writePendingMailVerification(PendingMailVerification pendingMailVerification) { - curator.set(mailVerificationPath(pendingMailVerification.getVerificationCode()), asJson(MailVerificationSerializer.toSlime(pendingMailVerification))); - } - - public void deletePendingMailVerification(PendingMailVerification pendingMailVerification) { - curator.delete(mailVerificationPath(pendingMailVerification.getVerificationCode())); - } - - // -------------- Date plane tokens --------------------------------------- - - public void writeDataplaneTokens(TenantName tenantName, List<DataplaneTokenVersions> dataplaneTokenVersions) { - curator.set(dataplaneTokenPath(tenantName), asJson(DataplaneTokenSerializer.toSlime(dataplaneTokenVersions))); - } - - public List<DataplaneTokenVersions> readDataplaneTokens(TenantName tenantName) { - return readSlime(dataplaneTokenPath(tenantName)).map(DataplaneTokenSerializer::fromSlime).orElse(List.of()); - } - - // -------------- Endpoint certificate pool ------------------------------- - - public void writeUnassignedCertificate(UnassignedCertificate certificate) { - curator.set(certificatePoolPath(certificate.id()), asJson(unassignedCertificateSerializer.toSlime(certificate))); - } - - public Optional<UnassignedCertificate> readUnassignedCertificate(String id) { - return readSlime(certificatePoolPath(id)).map(unassignedCertificateSerializer::fromSlime); - } - - public void removeUnassignedCertificate(UnassignedCertificate certificate, NestedTransaction transaction) { - Path path = certificatePoolPath(certificate.id()); - CuratorTransaction curatorTransaction = CuratorTransaction.from(CuratorOperations.delete(path.getAbsolute()), curator); - transaction.add(curatorTransaction); - } - - public List<UnassignedCertificate> readUnassignedCertificates() { - return curator.getChildren(certificatePoolRoot).stream().flatMap(id -> readUnassignedCertificate(id).stream()).toList(); - } - - // -------------- Cloud trial notification -------------------------------- - - public void writeTrialNotifications(TrialNotifications tn) { - curator.set(trialNotificationsRoot, asJson(tn.toSlime())); - } - - public Optional<TrialNotifications> readTrialNotifications() { - return readSlime(trialNotificationsRoot).map(TrialNotifications::fromSlime); - } - - // -------------- Paths --------------------------------------------------- - - private static Path upgradesPerMinutePath() { - return root.append("upgrader").append("upgradesPerMinute"); - } - - private static Path confidenceOverridesPath() { - return root.append("upgrader").append("confidenceOverrides"); - } - - private static Path osVersionTargetsPath() { - return root.append("osUpgrader").append("targetVersion"); - } - - private static Path certifiedOsVersionsPath() { - return root.append("osUpgrader").append("certifiedVersion"); - } - - private static Path osVersionStatusPath() { - return root.append("osVersionStatus"); - } - - private static Path versionStatusPath() { - return root.append("versionStatus"); - } - - private static Path routingPolicyPath(ApplicationId application) { - return routingPoliciesRoot.append(application.serializedForm()); - } - - private static Path zoneRoutingPolicyPath(ZoneId zone) { return zoneRoutingPoliciesRoot.append(zone.value()); } - - private static Path nameServiceQueuePath() { - return root.append("nameServiceQueue"); - } - - private static Path auditLogPath() { - return root.append("auditLog"); - } - - private static Path provisionStatePath() { - return root.append("provisioning").append("states"); - } - - private static Path provisionStatePath(String provisionId) { - return provisionStatePath().append(provisionId); - } - - private static Path tenantPath(TenantName name) { - return tenantRoot.append(name.value()); - } - - private static Path applicationPath(TenantAndApplicationId id) { - return applicationRoot.append(id.serialized()); - } - - private static Path runsPath(ApplicationId id, JobType type) { - return jobRoot.append(id.serializedForm()).append(type.jobName()); - } - - private static Path lastRunPath(ApplicationId id, JobType type) { - return runsPath(id, type).append("last"); - } - - private static Path logPath(ApplicationId id, JobType type, long first) { - return runsPath(id, type).append("logs").append(Long.toString(first)); - } - - private static Path lastLogPath(ApplicationId id, JobType type) { - return runsPath(id, type).append("logs"); - } - - private static Path controllerPath(String hostname) { - return controllerRoot.append(hostname); - } - - private static Path endpointCertificatePath(TenantAndApplicationId application, Optional<InstanceName> instance) { - String id = instance.map(name -> application.instance(name).serializedForm()) - .orElseGet(application::serialized); - return endpointCertificateRoot.append(id); - } - - private static Path meteringRefreshPath() { - return root.append("meteringRefreshTime"); - } - - private static Path archiveBucketsPath(ZoneId zoneId) { - return archiveBucketsRoot.append(zoneId.value()); - } - - private static Path changeRequestPath(String id) { - return changeRequestsRoot.append(id); - } - - private static Path notificationsPath(TenantName tenantName) { - return notificationsRoot.append(tenantName.value()); - } - - private static Path supportAccessPath(DeploymentId deploymentId) { - return supportAccessRoot.append(deploymentId.dottedString()); - } - - private static Path deploymentRetriggerPath() { - return root.append("deploymentRetriggerQueue"); - } - - private static Path mailVerificationPath(String verificationCode) { - return mailVerificationRoot.append(verificationCode); - } - - private static Path dataplaneTokenPath(TenantName tenantName) { - return dataPlaneTokenRoot.append(tenantName.value()); - } - - private static Path certificatePoolPath(String id) { - return certificatePoolRoot.append(id); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DataplaneTokenSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DataplaneTokenSerializer.java deleted file mode 100644 index 6537bde467a..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DataplaneTokenSerializer.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneTokenVersions; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.FingerPrint; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId; - -import java.time.Instant; -import java.util.List; -import java.util.Optional; - -/** - * @author mortent - */ -public class DataplaneTokenSerializer { - - private static final String dataplaneTokenField = "dataplaneToken"; - private static final String idField = "id"; - private static final String tokenVersionsField = "tokenVersions"; - private static final String fingerPrintField = "fingerPrint"; - private static final String checkAccessHashField = "checkAccessHash"; - private static final String creationTimeField = "creationTime"; - private static final String authorField = "author"; - private static final String expirationField = "expiration"; - private static final String lastUpdatedField = "lastUpdated"; - - public static Slime toSlime(List<DataplaneTokenVersions> dataplaneTokenVersions) { - Slime slime = new Slime(); - Cursor cursor = slime.setObject(); - Cursor array = cursor.setArray(dataplaneTokenField); - dataplaneTokenVersions.forEach(tokenMetadata -> { - Cursor tokenCursor = array.addObject(); - tokenCursor.setString(idField, tokenMetadata.tokenId().value()); - tokenCursor.setLong(lastUpdatedField, tokenMetadata.lastUpdated().toEpochMilli()); - Cursor versionArray = tokenCursor.setArray(tokenVersionsField); - tokenMetadata.tokenVersions().forEach(version -> { - Cursor versionCursor = versionArray.addObject(); - versionCursor.setString(fingerPrintField, version.fingerPrint().value()); - versionCursor.setString(checkAccessHashField, version.checkAccessHash()); - versionCursor.setLong(creationTimeField, version.creationTime().toEpochMilli()); - versionCursor.setString(creationTimeField, version.creationTime().toString()); - versionCursor.setString(authorField, version.author()); - versionCursor.setString(expirationField, version.expiration().map(Instant::toString).orElse("<none>")); - }); - }); - return slime; - } - - public static List<DataplaneTokenVersions> fromSlime(Slime slime) { - Cursor cursor = slime.get(); - return SlimeUtils.entriesStream(cursor.field(dataplaneTokenField)) - .map(entry -> { - TokenId id = TokenId.of(entry.field(idField).asString()); - List<DataplaneTokenVersions.Version> versions = SlimeUtils.entriesStream(entry.field(tokenVersionsField)) - .map(versionCursor -> { - FingerPrint fingerPrint = FingerPrint.of(versionCursor.field(fingerPrintField).asString()); - String checkAccessHash = versionCursor.field(checkAccessHashField).asString(); - Instant creationTime = SlimeUtils.instant(versionCursor.field(creationTimeField)); - String author = versionCursor.field(authorField).asString(); - String expirationStr = versionCursor.field(expirationField).asString(); - Optional<Instant> expiration = expirationStr.equals("<none>") ? Optional.empty() - : (expirationStr.isBlank() - ? Optional.of(Instant.EPOCH) : Optional.of(Instant.parse(expirationStr))); - return new DataplaneTokenVersions.Version(fingerPrint, checkAccessHash, creationTime, expiration, author); - }) - .toList(); - return new DataplaneTokenVersions(id, versions, Instant.ofEpochMilli(entry.field(lastUpdatedField).asLong())); - }) - .toList(); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializer.java deleted file mode 100644 index 4991d03d7df..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/DnsChallengeSerializer.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; -import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.ChallengeState; - -import java.time.Instant; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * @author jonmv - */ -class DnsChallengeSerializer { - - private static final String nameField = "name"; - private static final String dataField = "data"; - private static final String serviceIdField = "serviceId"; - private static final String accountField = "account"; - private static final String createdAtField = "createdAt"; - private static final String stateField = "state"; - - DnsChallenge fromJson(byte[] json, ClusterId clusterId) { - Cursor object = SlimeUtils.jsonToSlime(json).get(); - return new DnsChallenge(RecordName.from(object.field(nameField).asString()), - RecordData.from(object.field(dataField).asString()), - clusterId, - object.field(serviceIdField).asString(), - SlimeUtils.optionalString(object.field(accountField)).map(CloudAccount::from), - Instant.ofEpochMilli(object.field(createdAtField).asLong()), - toState(object.field(stateField).asString())); - } - - byte[] toJson(DnsChallenge challenge) { - Slime slime = new Slime(); - Cursor object = slime.setObject(); - object.setString(nameField, challenge.name().name()); - object.setString(dataField, challenge.data().data()); - object.setString(serviceIdField, challenge.serviceId()); - challenge.account().ifPresent(account -> object.setString(accountField, account.value())); - object.setLong(createdAtField, challenge.createdAt().toEpochMilli()); - object.setString(stateField, toString(challenge.state())); - return uncheck(() -> SlimeUtils.toJsonBytes(slime)); - } - - private static ChallengeState toState(String value) { - return switch (value) { - case "pending" -> ChallengeState.pending; - case "ready" -> ChallengeState.ready; - case "running" -> ChallengeState.running; - case "done" -> ChallengeState.done; - default -> throw new IllegalArgumentException("invalid serialized state: " + value); - }; - } - - private static String toString(ChallengeState state) { - return switch (state) { - case pending -> "pending"; - case ready -> "ready"; - case running -> "running"; - case done -> "done"; - }; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateSerializer.java deleted file mode 100644 index b204e2fe328..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateSerializer.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.slime.Type; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; - -import java.util.Optional; -import java.util.stream.IntStream; - -/** - * Serializer for {@link EndpointCertificate}. - * - * @author andreer - */ -public class EndpointCertificateSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster, and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private final static String keyNameField = "keyName"; - private final static String certNameField = "certName"; - private final static String versionField = "version"; - private final static String lastRequestedField = "lastRequested"; - private final static String rootRequestIdField = "requestId"; - private final static String leafRequestIdField = "leafRequestId"; - private final static String requestedDnsSansField = "requestedDnsSans"; - private final static String issuerField = "issuer"; - private final static String expiryField = "expiry"; - private final static String lastRefreshedField = "lastRefreshed"; - private final static String generatedIdField = "randomizedId"; - - public static Slime toSlime(EndpointCertificate cert) { - Slime slime = new Slime(); - Cursor object = slime.setObject(); - toSlime(cert, object); - return slime; - } - - public static void toSlime(EndpointCertificate cert, Cursor object) { - object.setString(keyNameField, cert.keyName()); - object.setString(certNameField, cert.certName()); - object.setLong(versionField, cert.version()); - object.setLong(lastRequestedField, cert.lastRequested()); - object.setString(rootRequestIdField, cert.rootRequestId()); - cert.leafRequestId().ifPresent(leafRequestId -> object.setString(leafRequestIdField, leafRequestId)); - var cursor = object.setArray(requestedDnsSansField); - cert.requestedDnsSans().forEach(cursor::addString); - object.setString(issuerField, cert.issuer()); - cert.expiry().ifPresent(expiry -> object.setLong(expiryField, expiry)); - cert.lastRefreshed().ifPresent(refreshTime -> object.setLong(lastRefreshedField, refreshTime)); - cert.generatedId().ifPresent(id -> object.setString(generatedIdField, id)); - } - - public static EndpointCertificate fromSlime(Inspector inspector) { - if (inspector.type() != Type.OBJECT) - throw new IllegalArgumentException("Invalid format encountered for endpoint certificate"); - - return new EndpointCertificate( - inspector.field(keyNameField).asString(), - inspector.field(certNameField).asString(), - Math.toIntExact(inspector.field(versionField).asLong()), - inspector.field(lastRequestedField).asLong(), - inspector.field(rootRequestIdField).asString(), - SlimeUtils.optionalString(inspector.field(leafRequestIdField)), - IntStream.range(0, inspector.field(requestedDnsSansField).entries()) - .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).toList(), - inspector.field(issuerField).asString(), - inspector.field(expiryField).valid() ? - Optional.of(inspector.field(expiryField).asLong()) : - Optional.empty(), - inspector.field(lastRefreshedField).valid() ? - Optional.of(inspector.field(lastRefreshedField).asLong()) : - Optional.empty(), - inspector.field(generatedIdField).valid() ? - Optional.of(inspector.field(generatedIdField).asString()) : - Optional.empty()); - } - - public static EndpointCertificate fromJsonString(String zkData) { - return fromSlime(SlimeUtils.jsonToSlime(zkData).get()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobControlFlags.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobControlFlags.java deleted file mode 100644 index f699133ca53..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobControlFlags.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.concurrent.maintenance.JobControlState; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.ListFlag; -import com.yahoo.vespa.flags.PermanentFlags; - -import java.util.Set; - -/** - * An implementation of {@link JobControlState} that uses a feature flag to control maintenance jobs. - * - * @author mpolden - */ -public class JobControlFlags implements JobControlState { - - private final CuratorDb curator; - private final ListFlag<String> inactiveJobsFlag; - - public JobControlFlags(CuratorDb curator, FlagSource flagSource) { - this.curator = curator; - this.inactiveJobsFlag = PermanentFlags.INACTIVE_MAINTENANCE_JOBS.bindTo(flagSource); - } - - @Override - public Set<String> readInactiveJobs() { - return Set.copyOf(inactiveJobsFlag.value()); - } - - @Override - public Mutex lockMaintenanceJob(String job) { - return curator.lockMaintenanceJob(job); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java deleted file mode 100644 index 69fe9bb8fa1..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type; -import com.yahoo.vespa.hosted.controller.deployment.Step; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Serialisation of {@link LogEntry} objects. Not all fields are stored! - * - * @author jonmv - */ -class LogSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String idField = "id"; - private static final String typeField = "type"; - private static final String timestampField = "at"; - private static final String messageField = "message"; - - byte[] toJson(Map<Step, List<LogEntry>> log) { - try { - return SlimeUtils.toJsonBytes(toSlime(log)); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - Slime toSlime(Map<Step, List<LogEntry>> log) { - Slime root = new Slime(); - Cursor logObject = root.setObject(); - log.forEach((step, entries) -> { - Cursor recordsArray = logObject.setArray(RunSerializer.valueOf(step)); - entries.forEach(entry -> toSlime(entry, recordsArray.addObject())); - }); - return root; - } - - private void toSlime(LogEntry entry, Cursor entryObject) { - entryObject.setLong(idField, entry.id()); - entryObject.setLong(timestampField, entry.at().toEpochMilli()); - entryObject.setString(typeField, valueOf(entry.type())); - entryObject.setString(messageField, entry.message()); - } - - Map<Step, List<LogEntry>> fromJson(byte[] logJson, long after) { - return fromJson(Collections.singletonList(logJson), after); - } - - Map<Step, List<LogEntry>> fromJson(List<byte[]> logJsons, long after) { - return fromSlime(logJsons.stream() - .map(SlimeUtils::jsonToSlime) - .toList(), - after); - } - - Map<Step, List<LogEntry>> fromSlime(List<Slime> slimes, long after) { - Map<Step, List<LogEntry>> log = new HashMap<>(); - slimes.forEach(slime -> slime.get().traverse((ObjectTraverser) (stepName, entryArray) -> { - Step step = RunSerializer.stepOf(stepName); - List<LogEntry> entries = log.computeIfAbsent(step, __ -> new ArrayList<>()); - entryArray.traverse((ArrayTraverser) (__, entryObject) -> { - LogEntry entry = fromSlime(entryObject); - if (entry.id() > after) - entries.add(entry); - }); - })); - return log; - } - - private LogEntry fromSlime(Inspector entryObject) { - return new LogEntry(entryObject.field(idField).asLong(), - SlimeUtils.instant(entryObject.field(timestampField)), - typeOf(entryObject.field(typeField).asString()), - entryObject.field(messageField).asString()); - } - - static String valueOf(Type type) { - return switch (type) { - case debug -> "debug"; - case info -> "info"; - case warning -> "warning"; - case error -> "error"; - case html -> "html"; - }; - } - - static Type typeOf(String type) { - return switch (type) { - case "debug" -> Type.debug; - case "info" -> Type.info; - case "warning" -> Type.warning; - case "error" -> Type.error; - case "html" -> Type.html; - default -> throw new IllegalArgumentException("Unknown log entry type '" + type + "'!"); - }; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MailVerificationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MailVerificationSerializer.java deleted file mode 100644 index 44325853c15..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MailVerificationSerializer.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.tenant.PendingMailVerification; - -import java.time.Instant; - -/** - * @author olaa - */ -public class MailVerificationSerializer { - - private static final String tenantField = "tenant"; - private static final String emailField = "email"; - private static final String emailTypeField = "emailType"; - private static final String emailVerificationCodeField = "emailVerificationCode"; - private static final String emailVerificationDeadlineField = "emailVerificationDeadline"; - - public static Slime toSlime(PendingMailVerification pendingMailVerification) { - var slime = new Slime(); - var object = slime.setObject(); - toSlime(pendingMailVerification, object); - return slime; - } - - public static void toSlime(PendingMailVerification pendingMailVerification, Cursor object) { - object.setString(tenantField, pendingMailVerification.getTenantName().value()); - object.setString(emailVerificationCodeField, pendingMailVerification.getVerificationCode()); - object.setString(emailField, pendingMailVerification.getMailAddress()); - object.setLong(emailVerificationDeadlineField, pendingMailVerification.getVerificationDeadline().toEpochMilli()); - object.setString(emailTypeField, pendingMailVerification.getMailType().name()); - } - - public static PendingMailVerification fromSlime(Slime slime) { - return fromSlime(slime.get()); - } - - public static PendingMailVerification fromSlime(Inspector inspector) { - var tenant = TenantName.from(inspector.field(tenantField).asString()); - var address = inspector.field(emailField).asString(); - var verificationCode = inspector.field(emailVerificationCodeField).asString(); - var deadline = Instant.ofEpochMilli(inspector.field(emailVerificationDeadlineField).asLong()); - var type = PendingMailVerification.MailType.valueOf(inspector.field(emailTypeField).asString()); - return new PendingMailVerification(tenant, address, verificationCode, deadline, type); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java deleted file mode 100644 index 6ad77af08e2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.curator.mock.MockCurator; - -import java.time.Duration; - -/** - * A curator db backed by a mock curator. - * - * @author bratseth - */ -@SuppressWarnings("unused") // injected -public class MockCuratorDb extends CuratorDb { - - private final MockCurator curator; - - @Inject - public MockCuratorDb(ConfigserverConfig config) { - this("test-controller:2222"); - } - - public MockCuratorDb(SystemName system) { - this("test-controller:2222"); - } - - public MockCuratorDb(String zooKeeperEnsembleConnectionSpec) { - this(new MockCurator() { @Override public String zooKeeperEnsembleConnectionSpec() { return zooKeeperEnsembleConnectionSpec; } }); - } - - public MockCuratorDb(MockCurator curator) { - super(curator, Duration.ofMillis(100)); - this.curator = curator; - } - - public MockCurator curator() { return curator; } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java deleted file mode 100644 index 4192f19298f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - - -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.dns.CreateRecord; -import com.yahoo.vespa.hosted.controller.dns.CreateRecords; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; -import com.yahoo.vespa.hosted.controller.dns.NameServiceRequest; -import com.yahoo.vespa.hosted.controller.dns.RemoveRecords; - -import java.util.ArrayList; -import java.util.Optional; - -/** - * Serializer for {@link com.yahoo.vespa.hosted.controller.dns.NameServiceQueue}. - * - * @author mpolden - */ -public class NameServiceQueueSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String requestsField = "requests"; - private static final String requestType = "requestType"; - private static final String recordsField = "records"; - private static final String typeField = "type"; - private static final String nameField = "name"; - private static final String dataField = "data"; - private static final String ownerField = "owner"; - - public Slime toSlime(NameServiceQueue queue) { - var slime = new Slime(); - var root = slime.setObject(); - var array = root.setArray(requestsField); - - for (var request : queue.requests()) { - var object = array.addObject(); - - request.owner().ifPresent(owner -> object.setString(ownerField, owner.serialized())); - - if (request instanceof CreateRecords) toSlime(object, (CreateRecords) request); - else if (request instanceof CreateRecord) toSlime(object, (CreateRecord) request); - else if (request instanceof RemoveRecords) toSlime(object, (RemoveRecords) request); - else throw new IllegalArgumentException("No serialization defined for request of type " + - request.getClass().getName()); - } - - return slime; - } - - public NameServiceQueue fromSlime(Slime slime) { - var items = new ArrayList<NameServiceRequest>(); - var root = slime.get(); - root.field(requestsField).traverse((ArrayTraverser) (i, object) -> { - Optional<TenantAndApplicationId> owner = SlimeUtils.optionalString(object.field(ownerField)).map(TenantAndApplicationId::fromSerialized); - var request = Request.valueOf(object.field(requestType).asString()); - switch (request) { - case createRecords -> items.add(createRecordsFromSlime(object, owner)); - case createRecord -> items.add(createRecordFromSlime(object, owner)); - case removeRecords -> items.add(removeRecordsFromSlime(object, owner)); - default -> throw new IllegalArgumentException("No serialization defined for request " + request); - } - }); - return new NameServiceQueue(items); - } - - private void toSlime(Cursor object, CreateRecord createRecord) { - object.setString(requestType, Request.createRecord.name()); - toSlime(object, createRecord.record()); - } - - private void toSlime(Cursor object, CreateRecords createRecords) { - object.setString(requestType, Request.createRecords.name()); - var recordArray = object.setArray(recordsField); - createRecords.records().forEach(record -> toSlime(recordArray.addObject(), record)); - } - - private void toSlime(Cursor object, RemoveRecords removeRecords) { - object.setString(requestType, Request.removeRecords.name()); - object.setString(typeField, removeRecords.type().name()); - object.setString(nameField, removeRecords.name().asString()); - removeRecords.data().ifPresent(data -> object.setString(dataField, data.asString())); - } - - private void toSlime(Cursor object, Record record) { - object.setString(typeField, record.type().name()); - object.setString(nameField, record.name().asString()); - object.setString(dataField, record.data().asString()); - } - - private CreateRecords createRecordsFromSlime(Inspector object, Optional<TenantAndApplicationId> owner) { - var records = new ArrayList<Record>(); - object.field(recordsField).traverse((ArrayTraverser) (i, recordObject) -> records.add(recordFromSlime(recordObject))); - return new CreateRecords(owner, records); - } - - private CreateRecord createRecordFromSlime(Inspector object, Optional<TenantAndApplicationId> owner) { - return new CreateRecord(owner, recordFromSlime(object)); - } - - private RemoveRecords removeRecordsFromSlime(Inspector object, Optional<TenantAndApplicationId> owner) { - var type = Record.Type.valueOf(object.field(typeField).asString()); - var name = RecordName.from(object.field(nameField).asString()); - var data = SlimeUtils.optionalString(object.field(dataField)).map(RecordData::from); - return new RemoveRecords(owner, type, name, data); - } - - private Record recordFromSlime(Inspector object) { - return new Record(Record.Type.valueOf(object.field(typeField).asString()), - RecordName.from(object.field(nameField).asString()), - RecordData.from(object.field(dataField).asString())); - } - - private enum Request { - createRecord, - createRecords, - removeRecords, - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java deleted file mode 100644 index 1ac8aad74ba..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.versions.NodeVersion; - -import java.util.ArrayList; -import java.util.List; - -/** - * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.NodeVersion}. - * - * @author mpolden - */ -public class NodeVersionSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String hostnameField = "hostname"; - private static final String zoneField = "zone"; - private static final String wantedVersionField = "wantedVersion"; - private static final String suspendedAtField = "suspendedAt"; - - public void nodeVersionsToSlime(List<NodeVersion> nodeVersions, Cursor array) { - for (var nodeVersion : nodeVersions) { - var nodeVersionObject = array.addObject(); - nodeVersionObject.setString(hostnameField, nodeVersion.hostname().value()); - nodeVersionObject.setString(zoneField, nodeVersion.zone().value()); - nodeVersionObject.setString(wantedVersionField, nodeVersion.wantedVersion().toFullString()); - nodeVersion.suspendedAt().ifPresent(suspendedAt -> nodeVersionObject.setLong(suspendedAtField, - suspendedAt.toEpochMilli())); - } - } - - public List<NodeVersion> nodeVersionsFromSlime(Inspector array, Version version) { - List<NodeVersion> nodeVersions = new ArrayList<>(); - array.traverse((ArrayTraverser) (i, entry) -> { - var hostname = HostName.of(entry.field(hostnameField).asString()); - var zone = ZoneId.from(entry.field(zoneField).asString()); - var wantedVersion = Version.fromString(entry.field(wantedVersionField).asString()); - var suspendedAt = SlimeUtils.optionalInstant(entry.field(suspendedAtField)); - nodeVersions.add(new NodeVersion(hostname, zone, version, wantedVersion, suspendedAt)); - }); - return nodeVersions; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java deleted file mode 100644 index d5be4d22dc2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.notification.MailTemplating; -import com.yahoo.vespa.hosted.controller.notification.Notification; -import com.yahoo.vespa.hosted.controller.notification.NotificationSource; - -import java.util.List; -import java.util.Optional; - -/** - * (de)serializes notifications for a tenant - * - * @author freva - */ -public class NotificationsSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String notificationsFieldName = "notifications"; - private static final String atFieldName = "at"; - private static final String typeField = "type"; - private static final String levelField = "level"; - private static final String titleField = "title"; - private static final String messagesField = "messages"; - private static final String applicationField = "application"; - private static final String instanceField = "instance"; - private static final String zoneField = "zone"; - private static final String clusterIdField = "clusterId"; - private static final String jobTypeField = "jobId"; - private static final String runNumberField = "runNumber"; - - public Slime toSlime(List<Notification> notifications) { - Slime slime = new Slime(); - Cursor notificationsArray = slime.setObject().setArray(notificationsFieldName); - - for (Notification notification : notifications) { - Cursor notificationObject = notificationsArray.addObject(); - notificationObject.setLong(atFieldName, notification.at().toEpochMilli()); - notificationObject.setString(typeField, asString(notification.type())); - notificationObject.setString(levelField, asString(notification.level())); - notificationObject.setString(titleField, notification.title()); - Cursor messagesArray = notificationObject.setArray(messagesField); - notification.messages().forEach(messagesArray::addString); - - notification.source().application().ifPresent(application -> notificationObject.setString(applicationField, application.value())); - notification.source().instance().ifPresent(instance -> notificationObject.setString(instanceField, instance.value())); - notification.source().zoneId().ifPresent(zoneId -> notificationObject.setString(zoneField, zoneId.value())); - notification.source().clusterId().ifPresent(clusterId -> notificationObject.setString(clusterIdField, clusterId.value())); - notification.source().jobType().ifPresent(jobType -> notificationObject.setString(jobTypeField, jobType.serialized())); - notification.source().runNumber().ifPresent(runNumber -> notificationObject.setLong(runNumberField, runNumber)); - - notification.mailContent().ifPresent(mc -> { - notificationObject.setString("mail-template", mc.template().getId()); - mc.subject().ifPresent(s -> notificationObject.setString("mail-subject", s)); - var mailParamsCursor = notificationObject.setObject("mail-params"); - mc.values().forEach((key, value) -> { - if (value instanceof String str) { - mailParamsCursor.setString(key, str); - } else if (value instanceof List<?> l) { - var array = mailParamsCursor.setArray(key); - l.forEach(elem -> array.addString((String) elem)); - } else { - throw new ClassCastException("Unsupported param type: " + value.getClass()); - } - }); - }); - } - - return slime; - } - - public List<Notification> fromSlime(TenantName tenantName, Slime slime) { - return SlimeUtils.entriesStream(slime.get().field(notificationsFieldName)) - .filter(inspector -> { // TODO: remove in summer. - if (!inspector.field(jobTypeField).valid()) return true; - try { - JobType.ofSerialized(inspector.field(jobTypeField).asString()); - return true; - } catch (RuntimeException e) { - return false; - } - }) - .map(inspector -> fromInspector(tenantName, inspector)).toList(); - } - - private Notification fromInspector(TenantName tenantName, Inspector inspector) { - return new Notification( - SlimeUtils.instant(inspector.field(atFieldName)), - typeFrom(inspector.field(typeField)), - levelFrom(inspector.field(levelField)), - new NotificationSource( - tenantName, - SlimeUtils.optionalString(inspector.field(applicationField)).map(ApplicationName::from), - SlimeUtils.optionalString(inspector.field(instanceField)).map(InstanceName::from), - SlimeUtils.optionalString(inspector.field(zoneField)).map(ZoneId::from), - SlimeUtils.optionalString(inspector.field(clusterIdField)).map(ClusterSpec.Id::from), - SlimeUtils.optionalString(inspector.field(jobTypeField)).map(jobName -> JobType.ofSerialized(jobName)), - SlimeUtils.optionalLong(inspector.field(runNumberField))), - SlimeUtils.optionalString(inspector.field(titleField)).orElse(""), - SlimeUtils.entriesStream(inspector.field(messagesField)).map(Inspector::asString).toList(), - mailContentFrom(inspector)); - } - - private Optional<Notification.MailContent> mailContentFrom(final Inspector inspector) { - return SlimeUtils.optionalString(inspector.field("mail-template")).map(template -> { - var builder = Notification.MailContent.fromTemplate(MailTemplating.Template.fromId(template).orElseThrow()); - SlimeUtils.optionalString(inspector.field("mail-subject")).ifPresent(builder::subject); - inspector.field("mail-params").traverse((ObjectTraverser) (name, insp) -> { - switch (insp.type()) { - case STRING -> builder.with(name, insp.asString()); - case ARRAY -> builder.with(name, SlimeUtils.entriesStream(insp).map(Inspector::asString).toList()); - default -> throw new IllegalArgumentException("Unsupported param type: " + insp.type()); - } - }); - return builder.build(); - }); - } - - private static String asString(Notification.Type type) { - return switch (type) { - case applicationPackage -> "applicationPackage"; - case submission -> "submission"; - case testPackage -> "testPackage"; - case deployment -> "deployment"; - case feedBlock -> "feedBlock"; - case reindex -> "reindex"; - case account -> "account"; - }; - } - - private static Notification.Type typeFrom(Inspector field) { - return switch (field.asString()) { - case "applicationPackage" -> Notification.Type.applicationPackage; - case "submission" -> Notification.Type.submission; - case "testPackage" -> Notification.Type.testPackage; - case "deployment" -> Notification.Type.deployment; - case "feedBlock" -> Notification.Type.feedBlock; - case "reindex" -> Notification.Type.reindex; - case "account" -> Notification.Type.account; - default -> throw new IllegalArgumentException("Unknown serialized notification type value '" + field.asString() + "'"); - }; - } - - private static String asString(Notification.Level level) { - return switch (level) { - case info -> "info"; - case warning -> "warning"; - case error -> "error"; - }; - } - - private static Notification.Level levelFrom(Inspector field) { - return switch (field.asString()) { - case "info" -> Notification.Level.info; - case "warning" -> Notification.Level.warning; - case "error" -> Notification.Level.error; - default -> throw new IllegalArgumentException("Unknown serialized notification level value '" + field.asString() + "'"); - }; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java deleted file mode 100644 index 173ebf151aa..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.CloudName; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.versions.OsVersion; - -import java.util.Collections; -import java.util.Set; -import java.util.TreeSet; - -/** - * Serializer for an {@link OsVersion}. - * - * @author mpolden - */ -public class OsVersionSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String versionsField = "versions"; - private static final String versionField = "version"; - private static final String cloudField = "cloud"; - - public Slime toSlime(Set<OsVersion> osVersions) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor array = root.setArray(versionsField); - osVersions.forEach(osVersion -> toSlime(osVersion, array.addObject())); - return slime; - } - - public void toSlime(OsVersion osVersion, Cursor object) { - object.setString(versionField, osVersion.version().toFullString()); - object.setString(cloudField, osVersion.cloud().value()); - } - - public Set<OsVersion> fromSlime(Slime slime) { - Inspector array = slime.get().field(versionsField); - Set<OsVersion> osVersions = new TreeSet<>(); - array.traverse((ArrayTraverser) (i, inspector) -> osVersions.add(fromSlime(inspector))); - return Collections.unmodifiableSet(osVersions); - } - - public OsVersion fromSlime(Inspector object) { - return new OsVersion( - Version.fromString(object.field(versionField).asString()), - CloudName.from(object.field(cloudField).asString()) - ); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java deleted file mode 100644 index 40826079efd..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSortedMap; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.versions.NodeVersion; -import com.yahoo.vespa.hosted.controller.versions.OsVersion; -import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; - -import java.util.List; -import java.util.Objects; - -/** - * Serializer for {@link OsVersionStatus}. - * - * @author mpolden - */ -public class OsVersionStatusSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String versionsField = "versions"; - private static final String nodeVersionsField = "nodeVersions"; - - private final OsVersionSerializer osVersionSerializer; - private final NodeVersionSerializer nodeVersionSerializer; - - public OsVersionStatusSerializer(OsVersionSerializer osVersionSerializer, NodeVersionSerializer nodeVersionSerializer) { - this.osVersionSerializer = Objects.requireNonNull(osVersionSerializer, "osVersionSerializer must be non-null"); - this.nodeVersionSerializer = Objects.requireNonNull(nodeVersionSerializer, "nodeVersionSerializer must be non-null"); - } - - public Slime toSlime(OsVersionStatus status) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor versions = root.setArray(versionsField); - status.versions().forEach((version, nodes) -> { - Cursor object = versions.addObject(); - osVersionSerializer.toSlime(version, object); - nodeVersionSerializer.nodeVersionsToSlime(nodes, object.setArray(nodeVersionsField)); - }); - return slime; - } - - public OsVersionStatus fromSlime(Slime slime) { - return new OsVersionStatus(osVersionsFromSlime(slime.get().field(versionsField))); - } - - private ImmutableMap<OsVersion, List<NodeVersion>> osVersionsFromSlime(Inspector array) { - var versions = ImmutableSortedMap.<OsVersion, List<NodeVersion>>naturalOrder(); - array.traverse((ArrayTraverser) (i, object) -> { - OsVersion osVersion = osVersionSerializer.fromSlime(object); - versions.put(osVersion, nodeVersionSerializer.nodeVersionsFromSlime(object.field(nodeVersionsField), osVersion.version())); - }); - return versions.build(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java deleted file mode 100644 index 968cea33162..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.versions.OsVersion; -import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; - -import java.time.Instant; -import java.util.Collections; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; - -/** - * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.OsVersionTarget}. - * - * @author mpolden - */ -public class OsVersionTargetSerializer { - - private final OsVersionSerializer osVersionSerializer; - - private static final String versionsField = "versions"; - private static final String scheduledAtField = "scheduledAt"; - private static final String pinnedField = "pinned"; - private static final String downgradeField = "downgrade"; - - public OsVersionTargetSerializer(OsVersionSerializer osVersionSerializer) { - this.osVersionSerializer = osVersionSerializer; - } - - public Slime toSlime(SortedSet<OsVersionTarget> osVersionTargets) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor array = root.setArray(versionsField); - osVersionTargets.forEach(target -> toSlime(target, array.addObject())); - return slime; - } - - public Set<OsVersionTarget> fromSlime(Slime slime) { - Inspector array = slime.get().field(versionsField); - Set<OsVersionTarget> osVersionTargets = new TreeSet<>(); - array.traverse((ArrayTraverser) (i, inspector) -> { - OsVersion osVersion = osVersionSerializer.fromSlime(inspector); - Instant scheduledAt = SlimeUtils.instant(inspector.field(scheduledAtField)); - boolean pinned = inspector.field(pinnedField).asBool(); - boolean downgrade = inspector.field(downgradeField).asBool(); - osVersionTargets.add(new OsVersionTarget(osVersion, scheduledAt, pinned, downgrade)); - }); - return Collections.unmodifiableSet(osVersionTargets); - } - - private void toSlime(OsVersionTarget target, Cursor object) { - osVersionSerializer.toSlime(target.osVersion(), object); - object.setLong(scheduledAtField, target.scheduledAt().toEpochMilli()); - object.setBool(pinnedField, target.pinned()); - object.setBool(downgradeField, target.downgrade()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java deleted file mode 100644 index 5e3f6675955..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import ai.vespa.http.DomainName; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.zone.AuthMethod; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; -import com.yahoo.vespa.hosted.controller.routing.GeneratedEndpointList; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; -import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; - -/** - * Serializer and deserializer for a {@link RoutingPolicy}. - * - * @author mortent - * @author mpolden - */ -public class RoutingPolicySerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String routingPoliciesField = "routingPolicies"; - private static final String clusterField = "cluster"; - private static final String canonicalNameField = "canonicalName"; - private static final String ipAddressField = "ipAddress"; - private static final String zoneField = "zone"; - private static final String dnsZoneField = "dnsZone"; - private static final String instanceEndpointsField = "rotations"; - private static final String applicationEndpointsField = "applicationEndpoints"; - private static final String globalRoutingField = "globalRouting"; - private static final String agentField = "agent"; - private static final String changedAtField = "changedAt"; - private static final String statusField = "status"; - private static final String privateOnlyField = "private"; - private static final String generatedEndpointsField = "generatedEndpoints"; - private static final String clusterPartField = "clusterPart"; - private static final String applicationPartField = "applicationPart"; - private static final String authMethodField = "authMethod"; - private static final String endpointIdField = "endpointId"; - - public Slime toSlime(List<RoutingPolicy> routingPolicies) { - var slime = new Slime(); - var root = slime.setObject(); - var policyArray = root.setArray(routingPoliciesField); - routingPolicies.forEach(policy -> { - var policyObject = policyArray.addObject(); - policyObject.setString(clusterField, policy.id().cluster().value()); - policyObject.setString(zoneField, policy.id().zone().value()); - policy.canonicalName().map(DomainName::value).ifPresent(name -> policyObject.setString(canonicalNameField, name)); - policy.ipAddress().ifPresent(ipAddress -> policyObject.setString(ipAddressField, ipAddress)); - policy.dnsZone().ifPresent(dnsZone -> policyObject.setString(dnsZoneField, dnsZone)); - var instanceEndpointsArray = policyObject.setArray(instanceEndpointsField); - policy.instanceEndpoints().forEach(endpointId -> instanceEndpointsArray.addString(endpointId.id())); - var applicationEndpointsArray = policyObject.setArray(applicationEndpointsField); - policy.applicationEndpoints().forEach(endpointId -> applicationEndpointsArray.addString(endpointId.id())); - globalRoutingToSlime(policy.routingStatus(), policyObject.setObject(globalRoutingField)); - if ( ! policy.isPublic()) policyObject.setBool(privateOnlyField, true); - Cursor generatedEndpointsArray = policyObject.setArray(generatedEndpointsField); - policy.generatedEndpoints().forEach(generatedEndpoint -> { - Cursor generatedEndpointObject = generatedEndpointsArray.addObject(); - generatedEndpointObject.setString(clusterPartField, generatedEndpoint.clusterPart()); - generatedEndpointObject.setString(applicationPartField, generatedEndpoint.applicationPart()); - generatedEndpointObject.setString(authMethodField, authMethod(generatedEndpoint.authMethod())); - generatedEndpoint.endpoint().ifPresent(endpointId -> generatedEndpointObject.setString(endpointIdField, endpointId.id())); - }); - }); - return slime; - } - - public List<RoutingPolicy> fromSlime(ApplicationId owner, Slime slime) { - List<RoutingPolicy> policies = new ArrayList<>(); - var root = slime.get(); - var field = root.field(routingPoliciesField); - field.traverse((ArrayTraverser) (i, inspect) -> { - Set<EndpointId> instanceEndpoints = new LinkedHashSet<>(); - inspect.field(instanceEndpointsField).traverse((ArrayTraverser) (j, endpointId) -> instanceEndpoints.add(EndpointId.of(endpointId.asString()))); - Set<EndpointId> applicationEndpoints = new LinkedHashSet<>(); - inspect.field(applicationEndpointsField).traverse((ArrayTraverser) (idx, endpointId) -> applicationEndpoints.add(EndpointId.of(endpointId.asString()))); - RoutingPolicyId id = new RoutingPolicyId(owner, - ClusterSpec.Id.from(inspect.field(clusterField).asString()), - ZoneId.from(inspect.field(zoneField).asString())); - boolean isPublic = ! inspect.field(privateOnlyField).asBool(); - List<GeneratedEndpoint> generatedEndpoints = new ArrayList<>(); - Inspector generatedEndpointsArray = inspect.field(generatedEndpointsField); - if (generatedEndpointsArray.valid()) { - generatedEndpointsArray.traverse((ArrayTraverser) (idx, generatedEndpointObject) -> - generatedEndpoints.add(new GeneratedEndpoint(generatedEndpointObject.field(clusterPartField).asString(), - generatedEndpointObject.field(applicationPartField).asString(), - authMethodFromSlime(generatedEndpointObject.field(authMethodField)), - SlimeUtils.optionalString(generatedEndpointObject.field(endpointIdField)) - .map(EndpointId::of)))); - } - policies.add(new RoutingPolicy(id, - SlimeUtils.optionalString(inspect.field(canonicalNameField)).map(DomainName::of), - SlimeUtils.optionalString(inspect.field(ipAddressField)), - SlimeUtils.optionalString(inspect.field(dnsZoneField)), - instanceEndpoints, - applicationEndpoints, - routingStatusFromSlime(inspect.field(globalRoutingField)), - isPublic, - GeneratedEndpointList.copyOf(generatedEndpoints))); - }); - return Collections.unmodifiableList(policies); - } - - public void globalRoutingToSlime(RoutingStatus routingStatus, Cursor object) { - object.setString(statusField, routingStatus.value().name()); - object.setString(agentField, routingStatus.agent().name()); - object.setLong(changedAtField, routingStatus.changedAt().toEpochMilli()); - } - - public RoutingStatus routingStatusFromSlime(Inspector object) { - var status = RoutingStatus.Value.valueOf(object.field(statusField).asString()); - var agent = RoutingStatus.Agent.valueOf(object.field(agentField).asString()); - var changedAt = SlimeUtils.optionalInstant(object.field(changedAtField)).orElse(Instant.EPOCH); - return new RoutingStatus(status, agent, changedAt); - } - - private String authMethod(AuthMethod authMethod) { - return switch (authMethod) { - case token -> "token"; - case mtls -> "mtls"; - case none -> "none"; - }; - } - - private AuthMethod authMethodFromSlime(Inspector field) { - return switch (field.asString()) { - case "token" -> AuthMethod.token; - case "mtls" -> AuthMethod.mtls; - case "none" -> AuthMethod.none; - default -> throw new IllegalArgumentException("Unknown auth method '" + field.asString() + "'"); - }; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java deleted file mode 100644 index 1d28432039b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ /dev/null @@ -1,418 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.Run.Reason; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Step; -import com.yahoo.vespa.hosted.controller.deployment.Step.Status; -import com.yahoo.vespa.hosted.controller.deployment.StepInfo; -import com.yahoo.vespa.hosted.controller.deployment.Versions; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Collections; -import java.util.EnumMap; -import java.util.NavigableMap; -import java.util.Optional; -import java.util.TreeMap; - -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.cancelled; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCertificateTimeout; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.invalidApplication; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.quotaExceeded; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup; -import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.report; -import static com.yahoo.vespa.hosted.controller.deployment.Step.startStagingSetup; -import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; -import static java.util.Comparator.comparing; - -/** - * Serialises and deserialises {@link Run} objects for persistent storage. - * - * @author jonmv - */ -class RunSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String stepsField = "steps"; - private static final String stepDetailsField = "stepDetails"; - private static final String startTimeField = "startTime"; - private static final String applicationField = "id"; - private static final String jobTypeField = "type"; - private static final String numberField = "number"; - private static final String startField = "start"; - private static final String endField = "end"; - private static final String sleepingUntilField = "sleepingUntil"; - private static final String statusField = "status"; - private static final String versionsField = "versions"; - private static final String isRedeploymentField = "isRedeployment"; - private static final String platformVersionField = "platform"; - private static final String deployedDirectlyField = "deployedDirectly"; - private static final String buildField = "build"; - private static final String sourceField = "source"; - private static final String lastTestRecordField = "lastTestRecord"; - private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp"; - private static final String noNodesDownSinceField = "noNodesDownSince"; - private static final String convergenceSummaryField = "convergenceSummaryV2"; - private static final String testerCertificateField = "testerCertificate"; - private static final String isDryRunField = "isDryRun"; - private static final String cloudAccountField = "account"; - private static final String reasonField = "reason"; - private static final String dependentField = "dependent"; - private static final String changeField = "change"; - - Run runFromSlime(Slime slime) { - return runFromSlime(slime.get()); - } - - NavigableMap<RunId, Run> runsFromSlime(Slime slime) { - NavigableMap<RunId, Run> runs = new TreeMap<>(comparing(RunId::number)); - Inspector runArray = slime.get(); - runArray.traverse((ArrayTraverser) (__, runObject) -> { - Run run = runFromSlime(runObject); - runs.put(run.id(), run); - }); - return Collections.unmodifiableNavigableMap(runs); - } - - private Run runFromSlime(Inspector runObject) { - var steps = new EnumMap<Step, StepInfo>(Step.class); - Inspector detailsField = runObject.field(stepDetailsField); - runObject.field(stepsField).traverse((ObjectTraverser) (step, status) -> { - Step typedStep = stepOf(step); - - // For historical reasons are the step details stored in a separate JSON structure from the step statuses. - Inspector stepDetailsField = detailsField.field(step); - Inspector startTimeValue = stepDetailsField.field(startTimeField); - Optional<Instant> startTime = SlimeUtils.optionalInstant(startTimeValue); - - steps.put(typedStep, new StepInfo(typedStep, stepStatusOf(status.asString()), startTime)); - }); - RunId id = new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), - JobType.ofSerialized(runObject.field(jobTypeField).asString()), - runObject.field(numberField).asLong()); - return new Run(id, - steps, - versionsFromSlime(runObject.field(versionsField), id), - runObject.field(isRedeploymentField).asBool(), - SlimeUtils.instant(runObject.field(startField)), - SlimeUtils.optionalInstant(runObject.field(endField)), - SlimeUtils.optionalInstant(runObject.field(sleepingUntilField)), - runStatusOf(runObject.field(statusField).asString()), - runObject.field(lastTestRecordField).asLong(), - Instant.EPOCH.plus(runObject.field(lastVespaLogTimestampField).asLong(), ChronoUnit.MICROS), - SlimeUtils.optionalInstant(runObject.field(noNodesDownSinceField)), - convergenceSummaryFrom(runObject.field(convergenceSummaryField)), - SlimeUtils.optionalString(runObject.field(testerCertificateField)).map(X509CertificateUtils::fromPem), - runObject.field(isDryRunField).valid() && runObject.field(isDryRunField).asBool(), - SlimeUtils.optionalString(runObject.field(cloudAccountField)).map(CloudAccount::from), - reasonFrom(runObject)); - } - - private Versions versionsFromSlime(Inspector versionsObject, RunId id) { - Version targetPlatformVersion = Version.fromString(versionsObject.field(platformVersionField).asString()); - RevisionId targetRevision = revisionFrom(versionsObject, id); - - Optional<Version> sourcePlatformVersion = versionsObject.field(sourceField).valid() - ? Optional.of(Version.fromString(versionsObject.field(sourceField).field(platformVersionField).asString())) - : Optional.empty(); - Optional<RevisionId> sourceRevision = versionsObject.field(sourceField).valid() - ? Optional.of(revisionFrom(versionsObject.field(sourceField), id)) - : Optional.empty(); - - return new Versions(targetPlatformVersion, targetRevision, sourcePlatformVersion, sourceRevision); - } - - private RevisionId revisionFrom(Inspector versionObject, RunId id) { - long buildNumber = versionObject.field(buildField).asLong(); - boolean production = versionObject.field(deployedDirectlyField).valid() // TODO jonmv: remove after migration - && buildNumber > 0 - && ! versionObject.field(deployedDirectlyField).asBool(); - return production ? RevisionId.forProduction(buildNumber) : RevisionId.forDevelopment(buildNumber, id.job()); - } - - // Don't change this — introduce a separate array instead. - private Optional<ConvergenceSummary> convergenceSummaryFrom(Inspector summaryArray) { - if ( ! summaryArray.valid()) return Optional.empty(); - - if (summaryArray.entries() != 12 && summaryArray.entries() != 13) - throw new IllegalArgumentException("Convergence summary must have 13 entries"); - - return Optional.of(new ConvergenceSummary(summaryArray.entry(0).asLong(), - summaryArray.entry(1).asLong(), - summaryArray.entry(2).asLong(), - summaryArray.entry(3).asLong(), - summaryArray.entry(4).asLong(), - summaryArray.entry(5).asLong(), - summaryArray.entry(6).asLong(), - summaryArray.entry(7).asLong(), - summaryArray.entry(8).asLong(), - summaryArray.entry(9).asLong(), - summaryArray.entry(10).asLong(), - summaryArray.entry(11).asLong(), - summaryArray.entry(12).asLong())); - } - - Slime toSlime(Iterable<Run> runs) { - Slime slime = new Slime(); - Cursor runArray = slime.setArray(); - runs.forEach(run -> toSlime(run, runArray.addObject())); - return slime; - } - - Slime toSlime(Run run) { - Slime slime = new Slime(); - toSlime(run, slime.setObject()); - return slime; - } - - private void toSlime(Run run, Cursor runObject) { - runObject.setString(applicationField, run.id().application().serializedForm()); - runObject.setString(jobTypeField, run.id().type().serialized()); - runObject.setBool(isRedeploymentField, run.isRedeployment()); - runObject.setLong(numberField, run.id().number()); - runObject.setLong(startField, run.start().toEpochMilli()); - run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); - run.sleepUntil().ifPresent(end -> runObject.setLong(sleepingUntilField, end.toEpochMilli())); - runObject.setString(statusField, valueOf(run.status())); - runObject.setLong(lastTestRecordField, run.lastTestLogEntry()); - if (run.lastVespaLogTimestamp().isAfter(Instant.EPOCH)) runObject.setLong(lastVespaLogTimestampField, Instant.EPOCH.until(run.lastVespaLogTimestamp(), ChronoUnit.MICROS)); - run.noNodesDownSince().ifPresent(noNodesDownSince -> runObject.setLong(noNodesDownSinceField, noNodesDownSince.toEpochMilli())); - run.convergenceSummary().ifPresent(convergenceSummary -> toSlime(convergenceSummary, runObject.setArray(convergenceSummaryField))); - run.testerCertificate().ifPresent(certificate -> runObject.setString(testerCertificateField, X509CertificateUtils.toPem(certificate))); - - Cursor stepsObject = runObject.setObject(stepsField); - run.steps().forEach((step, statusInfo) -> stepsObject.setString(valueOf(step), valueOf(statusInfo.status()))); - - // For historical reasons are the step details stored in a different field from the step statuses. - Cursor stepDetailsObject = runObject.setObject(stepDetailsField); - run.steps().forEach((step, statusInfo) -> - statusInfo.startTime().ifPresent(startTime -> - stepDetailsObject.setObject(valueOf(step)).setLong(startTimeField, valueOf(startTime)))); - - Cursor versionsObject = runObject.setObject(versionsField); - toSlime(run.versions().targetPlatform(), run.versions().targetRevision(), versionsObject); - run.versions().sourcePlatform().ifPresent(sourcePlatformVersion -> { - toSlime(sourcePlatformVersion, - run.versions().sourceRevision() - .orElseThrow(() -> new IllegalArgumentException("Source versions must be both present or absent.")), - versionsObject.setObject(sourceField)); - }); - runObject.setBool(isDryRunField, run.isDryRun()); - run.cloudAccount().ifPresent(account -> runObject.setString(cloudAccountField, account.value())); - toSlime(run.reason(), runObject); - } - - private void toSlime(Version platformVersion, RevisionId revsion, Cursor versionsObject) { - versionsObject.setString(platformVersionField, platformVersion.toString()); - versionsObject.setLong(buildField, revsion.number()); - versionsObject.setBool(deployedDirectlyField, ! revsion.isProduction()); - } - - // Don't change this - introduce a separate array with new values if needed. - private void toSlime(ConvergenceSummary summary, Cursor summaryArray) { - summaryArray.addLong(summary.nodes()); - summaryArray.addLong(summary.down()); - summaryArray.addLong(summary.upgradingOs()); - summaryArray.addLong(summary.upgradingFirmware()); - summaryArray.addLong(summary.needPlatformUpgrade()); - summaryArray.addLong(summary.upgradingPlatform()); - summaryArray.addLong(summary.needReboot()); - summaryArray.addLong(summary.rebooting()); - summaryArray.addLong(summary.needRestart()); - summaryArray.addLong(summary.restarting()); - summaryArray.addLong(summary.services()); - summaryArray.addLong(summary.needNewConfig()); - summaryArray.addLong(summary.retiring()); - } - - static String valueOf(Step step) { - switch (step) { - case deployInitialReal : return "deployInitialReal"; - case installInitialReal : return "installInitialReal"; - case deployReal : return "deployReal"; - case installReal : return "installReal"; - case deactivateReal : return "deactivateReal"; - case deployTester : return "deployTester"; - case installTester : return "installTester"; - case deactivateTester : return "deactivateTester"; - case copyVespaLogs : return "copyVespaLogs"; - case startStagingSetup : return "startStagingSetup"; - case endStagingSetup : return "endStagingSetup"; - case startTests : return "startTests"; - case endTests : return "endTests"; - case report : return "report"; - - default: throw new AssertionError("No value defined for '" + step + "'!"); - } - } - - static Step stepOf(String step) { - switch (step) { - case "deployInitialReal" : return deployInitialReal; - case "installInitialReal" : return installInitialReal; - case "deployReal" : return deployReal; - case "installReal" : return installReal; - case "deactivateReal" : return deactivateReal; - case "deployTester" : return deployTester; - case "installTester" : return installTester; - case "deactivateTester" : return deactivateTester; - case "copyVespaLogs" : return copyVespaLogs; - case "startStagingSetup" : return startStagingSetup; - case "endStagingSetup" : return endStagingSetup; - case "startTests" : return startTests; - case "endTests" : return endTests; - case "report" : return report; - - default: throw new IllegalArgumentException("No step defined by '" + step + "'!"); - } - } - - static String valueOf(Status status) { - switch (status) { - case unfinished : return "unfinished"; - case failed : return "failed"; - case succeeded : return "succeeded"; - - default: throw new AssertionError("No value defined for '" + status + "'!"); - } - } - - static Status stepStatusOf(String status) { - switch (status) { - case "unfinished" : return unfinished; - case "failed" : return failed; - case "succeeded" : return succeeded; - - default: throw new IllegalArgumentException("No status defined by '" + status + "'!"); - } - } - - static Long valueOf(Instant instant) { - return instant.toEpochMilli(); - } - - static String valueOf(RunStatus status) { - return switch (status) { - case running -> "running"; - case nodeAllocationFailure -> "nodeAllocationFailure"; - case endpointCertificateTimeout -> "endpointCertificateTimeout"; - case deploymentFailed -> "deploymentFailed"; - case invalidApplication -> "invalidApplication"; - case installationFailed -> "installationFailed"; - case testFailure -> "testFailure"; - case noTests -> "noTests"; - case error -> "error"; - case success -> "success"; - case aborted -> "aborted"; - case cancelled -> "cancelled"; - case reset -> "reset"; - case quotaExceeded -> "quotaExceeded"; - }; - } - - static RunStatus runStatusOf(String status) { - return switch (status) { - case "running" -> running; - case "nodeAllocationFailure" -> nodeAllocationFailure; - case "endpointCertificateTimeout" -> endpointCertificateTimeout; - case "deploymentFailed" -> deploymentFailed; - case "invalidApplication" -> invalidApplication; - case "installationFailed" -> installationFailed; - case "noTests" -> noTests; - case "testFailure" -> testFailure; - case "error" -> error; - case "success" -> success; - case "aborted" -> aborted; - case "cancelled" -> cancelled; - case "reset" -> reset; - case "quotaExceeded" -> quotaExceeded; - default -> throw new IllegalArgumentException("No run status defined by '" + status + "'!"); - }; - } - - Reason reasonFrom(Inspector object) { - return new Reason(SlimeUtils.optionalString(object.field(reasonField)), - Optional.ofNullable(jobIdFrom(object.field(dependentField))), - Optional.ofNullable(toChange(object.field(changeField)))); - } - - void toSlime(Reason reason, Cursor object) { - reason.reason().ifPresent(value -> object.setString(reasonField, value)); - reason.dependent().ifPresent(dependent -> toSlime(dependent, object.setObject(dependentField))); - reason.change().ifPresent(change -> toSlime(change, object.setObject(changeField))); - } - - JobId jobIdFrom(Inspector object) { - if ( ! object.valid()) return null; - return new JobId(ApplicationId.fromSerializedForm(object.field(applicationField).asString()), - JobType.ofSerialized(object.field(jobTypeField).asString())); - } - - void toSlime(JobId jobId, Cursor object) { - object.setString(applicationField, jobId.application().serializedForm()); - object.setString(jobTypeField, jobId.type().serialized()); - } - - Change toChange(Inspector object) { - if ( ! object.valid()) return null; - Change change = Change.empty(); - if (object.field(platformVersionField).valid()) - change = change.with(Version.fromString(object.field(platformVersionField).asString())); - if (object.field(buildField).valid()) - change = change.with(RevisionId.forProduction(object.field(buildField).asLong())); - return change; - } - - void toSlime(Change change, Cursor object) { - change.platform().ifPresent(version -> object.setString(platformVersionField, version.toString())); - change.revision().ifPresent(revision -> object.setLong(buildField, revision.number())); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java deleted file mode 100644 index 33f4709cfdd..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/SupportAccessSerializer.java +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; -import com.yahoo.vespa.hosted.controller.support.access.SupportAccessChange; -import com.yahoo.vespa.hosted.controller.support.access.SupportAccessGrant; - -import java.time.Instant; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.stream.Collectors; - -/** - * (de)serializes support access status and history - * - * @author andreer - */ -public class SupportAccessSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String stateFieldName = "state"; - private static final String supportAccessFieldName = "supportAccess"; - private static final String untilFieldName = "until"; - private static final String byFieldName = "by"; - private static final String historyFieldName = "history"; - private static final String allowedStateName = "allowed"; - private static final String disallowedStateName = "disallowed"; - private static final String atFieldName = "at"; - private static final String grantFieldName = "grants"; - private static final String requestorFieldName = "requestor"; - private static final String notBeforeFieldName = "notBefore"; - private static final String notAfterFieldName = "notAfter"; - private static final String certificateFieldName = "certificate"; - - - public static Slime toSlime(SupportAccess supportAccess) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - serializeHistoricEvents(root, supportAccess.changeHistory(), List.of()); - serializeGrants(root, supportAccess.grantHistory(), true); - - return slime; - } - - public static Slime serializeCurrentState(SupportAccess supportAccess, Instant currentTime) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - Cursor status = root.setObject(stateFieldName); - SupportAccess.CurrentStatus currentState = supportAccess.currentStatus(currentTime); - status.setString(supportAccessFieldName, currentState.state().name()); - if (currentState.state() == SupportAccess.State.ALLOWED) { - status.setString(untilFieldName, serializeInstant(currentState.allowedUntil().orElseThrow())); - status.setString(byFieldName, currentState.allowedBy().orElseThrow()); - } - - List<SupportAccessGrant> inactiveGrants = supportAccess.grantHistory().stream() - .filter(grant -> currentTime.isAfter(grant.certificate().getNotAfter().toInstant())) - .toList(); - - serializeHistoricEvents(root, supportAccess.changeHistory(), inactiveGrants); - - // Active grants should show up in the grant section - List<SupportAccessGrant> activeGrants = supportAccess.grantHistory().stream() - .filter(grant -> currentTime.isBefore(grant.certificate().getNotAfter().toInstant())) - .toList(); - serializeGrants(root, activeGrants, false); - return slime; - } - - private static void serializeHistoricEvents(Cursor root, List<SupportAccessChange> changeEvents, List<SupportAccessGrant> historicGrants) { - Cursor historyRoot = root.setArray(historyFieldName); - for (SupportAccessChange change : changeEvents) { - Cursor historyObject = historyRoot.addObject(); - historyObject.setString(stateFieldName, change.accessAllowedUntil().isPresent() ? allowedStateName : disallowedStateName); - historyObject.setString(atFieldName, serializeInstant(change.changeTime())); - change.accessAllowedUntil().ifPresent(allowedUntil -> historyObject.setString(untilFieldName, serializeInstant(allowedUntil))); - historyObject.setString(byFieldName, change.madeBy()); - } - - for (SupportAccessGrant grant : historicGrants) { - Cursor historyObject = historyRoot.addObject(); - historyObject.setString(stateFieldName, "grant"); - historyObject.setString(atFieldName, serializeInstant(grant.certificate().getNotBefore().toInstant())); - historyObject.setString(untilFieldName, serializeInstant(grant.certificate().getNotAfter().toInstant())); - historyObject.setString(byFieldName, grant.requestor()); - } - } - - private static void serializeGrants(Cursor root, List<SupportAccessGrant> grants, boolean includeCertificates) { - Cursor grantsRoot = root.setArray(grantFieldName); - for (SupportAccessGrant grant : grants) { - Cursor grantObject = grantsRoot.addObject(); - grantObject.setString(requestorFieldName, grant.requestor()); - if (includeCertificates) { - grantObject.setString(certificateFieldName, X509CertificateUtils.toPem(grant.certificate())); - } - grantObject.setString(notBeforeFieldName, serializeInstant(grant.certificate().getNotBefore().toInstant())); - grantObject.setString(notAfterFieldName, serializeInstant(grant.certificate().getNotAfter().toInstant())); - } - - } - - private static String serializeInstant(Instant i) { - return DateTimeFormatter.ISO_INSTANT.format(i.truncatedTo(ChronoUnit.SECONDS)); - } - - public static SupportAccess fromSlime(Slime slime) { - List<SupportAccessGrant> grantHistory = SlimeUtils.entriesStream(slime.get().field(grantFieldName)) - .map(inspector -> - new SupportAccessGrant( - inspector.field(requestorFieldName).asString(), - X509CertificateUtils.fromPem(inspector.field(certificateFieldName).asString()) - )) - .toList(); - - List<SupportAccessChange> changeHistory = SlimeUtils.entriesStream(slime.get().field(historyFieldName)) - .map(inspector -> - new SupportAccessChange( - SlimeUtils.optionalString(inspector.field(untilFieldName)).map(Instant::parse), - Instant.parse(inspector.field(atFieldName).asString()), - inspector.field(byFieldName).asString()) - ) - .toList(); - - return new SupportAccess(changeHistory, grantHistory); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java deleted file mode 100644 index 961925cf620..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ /dev/null @@ -1,580 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.google.common.collect.BiMap; -import com.google.common.collect.ImmutableBiMap; -import com.yahoo.component.Version; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.TenantName; -import com.yahoo.security.KeyUtils; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.hosted.controller.api.identifiers.Property; -import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; -import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; -import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; -import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; -import com.yahoo.vespa.hosted.controller.tenant.BillingReference; -import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; -import com.yahoo.vespa.hosted.controller.tenant.Email; -import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; -import com.yahoo.vespa.hosted.controller.tenant.PurchaseOrder; -import com.yahoo.vespa.hosted.controller.tenant.TaxId; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.TenantAddress; -import com.yahoo.vespa.hosted.controller.tenant.TenantBilling; -import com.yahoo.vespa.hosted.controller.tenant.TenantContact; -import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; -import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; -import com.yahoo.vespa.hosted.controller.tenant.TermsOfServiceApproval; - -import java.net.URI; -import java.security.Principal; -import java.security.PublicKey; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * Slime serialization of {@link Tenant} sub-types. - * - * @author mpolden - */ -public class TenantSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String nameField = "name"; - private static final String typeField = "type"; - private static final String athenzDomainField = "athenzDomain"; - private static final String propertyField = "property"; - private static final String propertyIdField = "propertyId"; - private static final String creatorField = "creator"; - private static final String createdAtField = "createdAt"; - private static final String deletedAtField = "deletedAt"; - private static final String contactField = "contact"; - private static final String contactUrlField = "contactUrl"; - private static final String propertyUrlField = "propertyUrl"; - private static final String issueTrackerUrlField = "issueTrackerUrl"; - private static final String personsField = "persons"; - private static final String personField = "person"; - private static final String queueField = "queue"; - private static final String componentField = "component"; - private static final String billingInfoField = "billingInfo"; - private static final String customerIdField = "customerId"; - private static final String productCodeField = "productCode"; - private static final String pemDeveloperKeysField = "pemDeveloperKeys"; - private static final String tenantInfoField = "info"; - private static final String lastLoginInfoField = "lastLoginInfo"; - private static final String secretStoresField = "secretStores"; - private static final String archiveAccessRoleField = "archiveAccessRole"; - private static final String archiveAccessField = "archiveAccess"; - private static final String awsArchiveAccessRoleField = "awsArchiveAccessRole"; - private static final String gcpArchiveAccessMemberField = "gcpArchiveAccessMember"; - private static final String invalidateUserSessionsBeforeField = "invalidateUserSessionsBefore"; - private static final String tenantRolesLastMaintainedField = "tenantRolesLastMaintained"; - private static final String billingReferenceField = "billingReference"; - private static final String planIdField = "planId"; - private static final String cloudAccountsField = "cloudAccounts"; - private static final String accountField = "account"; - private static final String templateVersionField = "templateVersion"; - private static final String taxIdField = "taxId"; - private static final String taxIdCountryField = "country"; - private static final String taxIdTypeField = "type"; - private static final String taxIdCodeField = "code"; - private static final String purchaseOrderField = "purchaseOrder"; - private static final String invoiceEmailField = "invoiceEmail"; - private static final String tosApprovalField = "tosApproval"; - private static final String tosApprovalAtField = "at"; - private static final String tosApprovalByField = "by"; - - private static final String awsIdField = "awsId"; - private static final String roleField = "role"; - - public Slime toSlime(Tenant tenant) { - Slime slime = new Slime(); - Cursor tenantObject = slime.setObject(); - tenantObject.setString(nameField, tenant.name().value()); - tenantObject.setString(typeField, valueOf(tenant.type())); - tenantObject.setLong(createdAtField, tenant.createdAt().toEpochMilli()); - toSlime(tenant.lastLoginInfo(), tenantObject.setObject(lastLoginInfoField)); - tenantObject.setLong(tenantRolesLastMaintainedField, tenant.tenantRolesLastMaintained().toEpochMilli()); - cloudAccountsToSlime(tenant.cloudAccounts(), tenantObject.setArray(cloudAccountsField)); - - switch (tenant.type()) { - case athenz: toSlime((AthenzTenant) tenant, tenantObject); break; - case cloud: toSlime((CloudTenant) tenant, tenantObject); break; - case deleted: toSlime((DeletedTenant) tenant, tenantObject); break; - default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); - } - return slime; - } - - private void toSlime(AthenzTenant tenant, Cursor tenantObject) { - tenantObject.setString(athenzDomainField, tenant.domain().getName()); - tenantObject.setString(propertyField, tenant.property().id()); - tenant.propertyId().ifPresent(propertyId -> tenantObject.setString(propertyIdField, propertyId.id())); - tenant.contact().ifPresent(contact -> { - Cursor contactCursor = tenantObject.setObject(contactField); - writeContact(contact, contactCursor); - }); - } - - private void toSlime(CloudTenant tenant, Cursor root) { - // BillingInfo was never used and always just a static default value. To retire this - // field we continue to write the default value and stop reading it. - // TODO(ogronnesby, 2020-08-05): Remove when a version where we do not read the field has propagated. - var legacyBillingInfo = new BillingInfo("customer", "Vespa"); - tenant.creator().ifPresent(creator -> root.setString(creatorField, creator.getName())); - developerKeysToSlime(tenant.developerKeys(), root.setArray(pemDeveloperKeysField)); - toSlime(legacyBillingInfo, root.setObject(billingInfoField)); - toSlime(tenant.info(), root); - toSlime(tenant.tenantSecretStores(), root); - toSlime(tenant.archiveAccess(), root); - tenant.billingReference().ifPresent(b -> toSlime(b, root)); - tenant.invalidateUserSessionsBefore().ifPresent(instant -> root.setLong(invalidateUserSessionsBeforeField, instant.toEpochMilli())); - root.setString(planIdField, tenant.planId().value()); - } - - private void toSlime(ArchiveAccess archiveAccess, Cursor root) { - Cursor object = root.setObject(archiveAccessField); - archiveAccess.awsRole().ifPresent(role -> object.setString(awsArchiveAccessRoleField, role)); - archiveAccess.gcpMember().ifPresent(member -> object.setString(gcpArchiveAccessMemberField, member)); - } - - private void toSlime(DeletedTenant tenant, Cursor root) { - root.setLong(deletedAtField, tenant.deletedAt().toEpochMilli()); - } - - private void developerKeysToSlime(BiMap<PublicKey, ? extends Principal> keys, Cursor array) { - keys.forEach((key, user) -> { - Cursor object = array.addObject(); - object.setString("key", KeyUtils.toPem(key)); - object.setString("user", user.getName()); - }); - } - - private void toSlime(BillingInfo billingInfo, Cursor billingInfoObject) { - billingInfoObject.setString(customerIdField, billingInfo.customerId()); - billingInfoObject.setString(productCodeField, billingInfo.productCode()); - } - - private void toSlime(LastLoginInfo lastLoginInfo, Cursor lastLoginInfoObject) { - for (LastLoginInfo.UserLevel userLevel: LastLoginInfo.UserLevel.values()) { - lastLoginInfo.get(userLevel).ifPresent(lastLoginAt -> - lastLoginInfoObject.setLong(valueOf(userLevel), lastLoginAt.toEpochMilli())); - } - } - - private void cloudAccountsToSlime(List<CloudAccountInfo> cloudAccounts, Cursor cloudAccountsObject) { - cloudAccounts.forEach(cloudAccountInfo -> { - Cursor object = cloudAccountsObject.addObject(); - object.setString(accountField, cloudAccountInfo.cloudAccount().account()); - object.setString(templateVersionField, cloudAccountInfo.templateVersion().toFullString()); - }); - } - - public Tenant tenantFrom(Slime slime) { - Inspector tenantObject = slime.get(); - Tenant.Type type = typeOf(tenantObject.field(typeField).asString()); - - switch (type) { - case athenz: return athenzTenantFrom(tenantObject); - case cloud: return cloudTenantFrom(tenantObject); - case deleted: return deletedTenantFrom(tenantObject); - default: throw new IllegalArgumentException("Unexpected tenant type '" + type + "'."); - } - } - - private AthenzTenant athenzTenantFrom(Inspector tenantObject) { - TenantName name = TenantName.from(tenantObject.field(nameField).asString()); - AthenzDomain domain = new AthenzDomain(tenantObject.field(athenzDomainField).asString()); - Property property = new Property(tenantObject.field(propertyField).asString()); - Optional<PropertyId> propertyId = SlimeUtils.optionalString(tenantObject.field(propertyIdField)).map(PropertyId::new); - Optional<Contact> contact = contactFrom(tenantObject.field(contactField)); - Instant createdAt = SlimeUtils.instant(tenantObject.field(createdAtField)); - LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField)); - Instant tenantRolesLastMaintained = SlimeUtils.instant(tenantObject.field(tenantRolesLastMaintainedField)); - List<CloudAccountInfo> cloudAccountInfos = cloudAccountsFromSlime(tenantObject.field(cloudAccountsField)); - return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccountInfos); - } - - private CloudTenant cloudTenantFrom(Inspector tenantObject) { - TenantName name = TenantName.from(tenantObject.field(nameField).asString()); - Instant createdAt = SlimeUtils.instant(tenantObject.field(createdAtField)); - LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField)); - Optional<SimplePrincipal> creator = SlimeUtils.optionalString(tenantObject.field(creatorField)).map(SimplePrincipal::new); - BiMap<PublicKey, SimplePrincipal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField)); - TenantInfo info = tenantInfoFromSlime(tenantObject.field(tenantInfoField)); - List<TenantSecretStore> tenantSecretStores = secretStoresFromSlime(tenantObject.field(secretStoresField)); - ArchiveAccess archiveAccess = archiveAccessFromSlime(tenantObject); - Optional<Instant> invalidateUserSessionsBefore = SlimeUtils.optionalInstant(tenantObject.field(invalidateUserSessionsBeforeField)); - Instant tenantRolesLastMaintained = SlimeUtils.instant(tenantObject.field(tenantRolesLastMaintainedField)); - List<CloudAccountInfo> cloudAccountInfos = cloudAccountsFromSlime(tenantObject.field(cloudAccountsField)); - Optional<BillingReference> billingReference = billingReferenceFrom(tenantObject.field(billingReferenceField)); - PlanId planId = planId(tenantObject.field(planIdField)); - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, - archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, - cloudAccountInfos, billingReference, planId); - } - - private DeletedTenant deletedTenantFrom(Inspector tenantObject) { - TenantName name = TenantName.from(tenantObject.field(nameField).asString()); - Instant createdAt = SlimeUtils.instant(tenantObject.field(createdAtField)); - Instant deletedAt = SlimeUtils.instant(tenantObject.field(deletedAtField)); - return new DeletedTenant(name, createdAt, deletedAt); - } - - private BiMap<PublicKey, SimplePrincipal> developerKeysFromSlime(Inspector array) { - ImmutableBiMap.Builder<PublicKey, SimplePrincipal> keys = ImmutableBiMap.builder(); - array.traverse((ArrayTraverser) (__, keyObject) -> - keys.put(KeyUtils.fromPemEncodedPublicKey(keyObject.field("key").asString()), - new SimplePrincipal(keyObject.field("user").asString()))); - - return keys.build(); - } - - ArchiveAccess archiveAccessFromSlime(Inspector tenantObject) { - // TODO(enygaard, 2022-05-24): Remove when all tenants have been rewritten to use ArchiveAccess object - Optional<String> archiveAccessRole = SlimeUtils.optionalString(tenantObject.field(archiveAccessRoleField)); - if (archiveAccessRole.isPresent()) { - return new ArchiveAccess().withAWSRole(archiveAccessRole.get()); - } - Inspector object = tenantObject.field(archiveAccessField); - if (!object.valid()) { - return new ArchiveAccess(); - } - Optional<String> awsArchiveAccessRole = SlimeUtils.optionalString(object.field(awsArchiveAccessRoleField)); - Optional<String> gcpArchiveAccessMember = SlimeUtils.optionalString(object.field(gcpArchiveAccessMemberField)); - return new ArchiveAccess() - .withAWSRole(awsArchiveAccessRole) - .withGCPMember(gcpArchiveAccessMember); - } - - TenantInfo tenantInfoFromSlime(Inspector infoObject) { - if (!infoObject.valid()) return TenantInfo.empty(); - - return TenantInfo.empty() - .withName(infoObject.field("name").asString()) - .withEmail(infoObject.field("email").asString()) - .withWebsite(infoObject.field("website").asString()) - .withContact(TenantContact.from( - infoObject.field("contactName").asString(), - new Email(infoObject.field("contactEmail").asString(), asBoolOrTrue(infoObject.field("contactEmailVerified"))))) - .withAddress(tenantInfoAddressFromSlime(infoObject.field("address"))) - .withBilling(tenantInfoBillingContactFromSlime(infoObject.field("billingContact"))) - .withContacts(tenantContactsFrom(infoObject.field("contacts"))); - } - - private TenantAddress tenantInfoAddressFromSlime(Inspector addressObject) { - return TenantAddress.empty() - .withAddress(addressObject.field("addressLines").asString()) - .withCode(addressObject.field("postalCodeOrZip").asString()) - .withCity(addressObject.field("city").asString()) - .withRegion(addressObject.field("stateRegionProvince").asString()) - .withCountry(addressObject.field("country").asString()); - } - - private TenantBilling tenantInfoBillingContactFromSlime(Inspector billingObject) { - var taxIdInspector = billingObject.field(taxIdField); - var taxId = switch (taxIdInspector.type()) { - // TODO(bjorncs, 2023-11-02): Remove legacy tax id format - case STRING -> TaxId.legacy(taxIdInspector.asString()); - case OBJECT -> { - var taxIdCountry = taxIdInspector.field(taxIdCountryField).asString(); - var taxIdType = taxIdInspector.field(taxIdTypeField).asString(); - var taxIdCode = taxIdInspector.field(taxIdCodeField).asString(); - yield new TaxId(new TaxId.Country(taxIdCountry), new TaxId.Type(taxIdType), new TaxId.Code(taxIdCode)); - } - case NIX -> TaxId.empty(); - default -> throw new IllegalStateException(taxIdInspector.type().name()); - }; - var purchaseOrder = new PurchaseOrder(billingObject.field(purchaseOrderField).asString()); - var invoiceEmail = new Email(billingObject.field(invoiceEmailField).asString(), false); - var tosApprovalInspector = billingObject.field(tosApprovalField); - var tosApproval = switch (tosApprovalInspector.type()) { - case OBJECT -> new TermsOfServiceApproval(tosApprovalInspector.field(tosApprovalAtField).asString(), - tosApprovalInspector.field(tosApprovalByField).asString()); - case NIX -> TermsOfServiceApproval.empty(); - default -> throw new IllegalArgumentException(taxIdInspector.type().name()); - }; - - return TenantBilling.empty() - .withContact(TenantContact.from( - billingObject.field("name").asString(), - new Email(billingObject.field("email").asString(), billingObject.field("emailVerified").asBool()), - billingObject.field("phone").asString())) - .withAddress(tenantInfoAddressFromSlime(billingObject.field("address"))) - .withTaxId(taxId) - .withPurchaseOrder(purchaseOrder) - .withInvoiceEmail(invoiceEmail) - .withToSApproval(tosApproval); - } - - private List<TenantSecretStore> secretStoresFromSlime(Inspector secretStoresObject) { - if (!secretStoresObject.valid()) return List.of(); - - return SlimeUtils.entriesStream(secretStoresObject) - .map(inspector -> new TenantSecretStore( - inspector.field(nameField).asString(), - inspector.field(awsIdField).asString(), - inspector.field(roleField).asString())) - .toList(); - } - - private LastLoginInfo lastLoginInfoFromSlime(Inspector lastLoginInfoObject) { - Map<LastLoginInfo.UserLevel, Instant> lastLoginByUserLevel = new HashMap<>(); - lastLoginInfoObject.traverse((String name, Inspector value) -> - lastLoginByUserLevel.put(userLevelOf(name), SlimeUtils.instant(value))); - return new LastLoginInfo(lastLoginByUserLevel); - } - - private List<CloudAccountInfo> cloudAccountsFromSlime(Inspector cloudAccountsObject) { - return SlimeUtils.entriesStream(cloudAccountsObject) - .map(inspector -> new CloudAccountInfo( - CloudAccount.from(inspector.field(accountField).asString()), - Version.fromString(inspector.field(templateVersionField).asString()))) - .toList(); - } - - void toSlime(TenantInfo info, Cursor parentCursor) { - if (info.isEmpty()) return; - Cursor infoCursor = parentCursor.setObject("info"); - infoCursor.setString("name", info.name()); - infoCursor.setString("email", info.email()); - infoCursor.setString("website", info.website()); - infoCursor.setString("contactName", info.contact().name()); - infoCursor.setString("contactEmail", info.contact().email().getEmailAddress()); - infoCursor.setBool("contactEmailVerified", info.contact().email().isVerified()); - toSlime(info.address(), infoCursor); - toSlime(info.billingContact(), infoCursor); - toSlime(info.contacts(), infoCursor); - } - - private void toSlime(TenantAddress address, Cursor parentCursor) { - if (address.isEmpty()) return; - - Cursor addressCursor = parentCursor.setObject("address"); - addressCursor.setString("addressLines", address.address()); - addressCursor.setString("postalCodeOrZip", address.code()); - addressCursor.setString("city", address.city()); - addressCursor.setString("stateRegionProvince", address.region()); - addressCursor.setString("country", address.country()); - } - - private void toSlime(TenantBilling billingContact, Cursor parentCursor) { - if (billingContact.isEmpty()) return; - - Cursor billingCursor = parentCursor.setObject("billingContact"); - billingCursor.setString("name", billingContact.contact().name()); - billingCursor.setString("email", billingContact.contact().email().getEmailAddress()); - billingCursor.setBool("emailVerified", billingContact.contact().email().isVerified()); - billingCursor.setString("phone", billingContact.contact().phone()); - var taxIdCursor = billingCursor.setObject(taxIdField); - taxIdCursor.setString(taxIdCountryField, billingContact.getTaxId().country().value()); - taxIdCursor.setString(taxIdTypeField, billingContact.getTaxId().type().value()); - taxIdCursor.setString(taxIdCodeField, billingContact.getTaxId().code().value()); - billingCursor.setString(purchaseOrderField, billingContact.getPurchaseOrder().value()); - billingCursor.setString(invoiceEmailField, billingContact.getInvoiceEmail().getEmailAddress()); - toSlime(billingContact.address(), billingCursor); - if (!billingContact.getToSApproval().isEmpty()) { - var tosApprovalCursor = billingCursor.setObject(tosApprovalField); - tosApprovalCursor.setString(tosApprovalAtField, billingContact.getToSApproval().approvedAt().toString()); - tosApprovalCursor.setString(tosApprovalByField, billingContact.getToSApproval().approvedBy().get().getName()); - } - } - - private void toSlime(List<TenantSecretStore> tenantSecretStores, Cursor parentCursor) { - if (tenantSecretStores.isEmpty()) return; - - Cursor secretStoresCursor = parentCursor.setArray(secretStoresField); - tenantSecretStores.forEach(tenantSecretStore -> { - Cursor secretStoreCursor = secretStoresCursor.addObject(); - secretStoreCursor.setString(nameField, tenantSecretStore.getName()); - secretStoreCursor.setString(awsIdField, tenantSecretStore.getAwsId()); - secretStoreCursor.setString(roleField, tenantSecretStore.getRole()); - }); - } - - private void toSlime(TenantContacts contacts, Cursor parent) { - if (contacts.isEmpty()) return; - var cursor = parent.setArray("contacts"); - contacts.all().forEach(contact -> writeContact(contact, cursor.addObject())); - } - - private void toSlime(BillingReference reference, Cursor parent) { - var cursor = parent.setObject(billingReferenceField); - cursor.setString("reference", reference.reference()); - cursor.setLong("updated", reference.updated().toEpochMilli()); - } - - private Optional<BillingReference> billingReferenceFrom(Inspector object) { - if (! object.valid()) return Optional.empty(); - return Optional.of(new BillingReference( - object.field("reference").asString(), - SlimeUtils.instant(object.field("updated")))); - } - - private PlanId planId(Inspector object) { - if (! object.valid()) return PlanId.from("none"); - - return PlanId.from(object.asString()); - } - - private TenantContacts tenantContactsFrom(Inspector object) { - List<TenantContacts.Contact> contacts = SlimeUtils.entriesStream(object) - .map(this::readContact) - .toList(); - return new TenantContacts(contacts); - } - - private Optional<Contact> contactFrom(Inspector object) { - if ( ! object.valid()) return Optional.empty(); - - URI contactUrl = URI.create(object.field(contactUrlField).asString()); - URI propertyUrl = URI.create(object.field(propertyUrlField).asString()); - URI issueTrackerUrl = URI.create(object.field(issueTrackerUrlField).asString()); - List<List<String>> persons = personsFrom(object.field(personsField)); - String queue = object.field(queueField).asString(); - Optional<String> component = object.field(componentField).valid() ? Optional.of(object.field(componentField).asString()) : Optional.empty(); - return Optional.of(new Contact(contactUrl, - propertyUrl, - issueTrackerUrl, - persons, - queue, - component)); - } - - private void writeContact(Contact contact, Cursor contactCursor) { - contactCursor.setString(contactUrlField, contact.url().toString()); - contactCursor.setString(propertyUrlField, contact.propertyUrl().toString()); - contactCursor.setString(issueTrackerUrlField, contact.issueTrackerUrl().toString()); - Cursor personsArray = contactCursor.setArray(personsField); - contact.persons().forEach(personList -> { - Cursor personArray = personsArray.addArray(); - personList.forEach(person -> { - Cursor personObject = personArray.addObject(); - personObject.setString(personField, person); - }); - }); - contactCursor.setString(queueField, contact.queue()); - contact.component().ifPresent(component -> contactCursor.setString(componentField, component)); - } - - private List<List<String>> personsFrom(Inspector array) { - List<List<String>> personLists = new ArrayList<>(); - array.traverse((ArrayTraverser) (i, personArray) -> { - List<String> persons = new ArrayList<>(); - personArray.traverse((ArrayTraverser) (j, inspector) -> persons.add(inspector.field("person").asString())); - personLists.add(persons); - }); - return personLists; - } - - private void writeContact(TenantContacts.Contact contact, Cursor cursor) { - cursor.setString("type", contact.type().value()); - Cursor audiencesArray = cursor.setArray("audiences"); - contact.audiences().forEach(audience -> audiencesArray.addString(toAudience(audience))); - var data = cursor.setObject("data"); - switch (contact.type()) { - case EMAIL: - var email = (TenantContacts.EmailContact) contact; - data.setString("email", email.email().getEmailAddress()); - data.setBool("emailVerified", email.email().isVerified()); - return; - default: - throw new IllegalArgumentException("Serialization for contact type not implemented: " + contact.type()); - } - } - - private TenantContacts.Contact readContact(Inspector inspector) { - var type = TenantContacts.Type.from(inspector.field("type").asString()) - .orElseThrow(() -> new RuntimeException("Unknown type: " + inspector.field("type").asString())); - var audiences = SlimeUtils.entriesStream(inspector.field("audiences")) - .map(audience -> TenantSerializer.fromAudience(audience.asString())) - .toList(); - switch (type) { - case EMAIL: - var isVerified = asBoolOrTrue(inspector.field("data").field("emailVerified")); - return new TenantContacts.EmailContact(audiences, new Email(inspector.field("data").field("email").asString(), isVerified)); - default: - throw new IllegalArgumentException("Serialization for contact type not implemented: " + type); - } - - } - - private static Tenant.Type typeOf(String value) { - switch (value) { - case "athenz": return Tenant.Type.athenz; - case "cloud": return Tenant.Type.cloud; - case "deleted": return Tenant.Type.deleted; - default: throw new IllegalArgumentException("Unknown tenant type '" + value + "'."); - } - } - - private static String valueOf(Tenant.Type type) { - switch (type) { - case athenz: return "athenz"; - case cloud: return "cloud"; - case deleted: return "deleted"; - default: throw new IllegalArgumentException("Unexpected tenant type '" + type + "'."); - } - } - - private static LastLoginInfo.UserLevel userLevelOf(String value) { - switch (value) { - case "user": return LastLoginInfo.UserLevel.user; - case "developer": return LastLoginInfo.UserLevel.developer; - case "administrator": return LastLoginInfo.UserLevel.administrator; - default: throw new IllegalArgumentException("Unknown user level '" + value + "'."); - } - } - - private static String valueOf(LastLoginInfo.UserLevel userLevel) { - switch (userLevel) { - case user: return "user"; - case developer: return "developer"; - case administrator: return "administrator"; - default: throw new IllegalArgumentException("Unexpected user level '" + userLevel + "'."); - } - } - - private static TenantContacts.Audience fromAudience(String value) { - switch (value) { - case "tenant": return TenantContacts.Audience.TENANT; - case "notifications": return TenantContacts.Audience.NOTIFICATIONS; - default: throw new IllegalArgumentException("Unknown contact audience '" + value + "'."); - } - } - - private static String toAudience(TenantContacts.Audience audience) { - switch (audience) { - case TENANT: return "tenant"; - case NOTIFICATIONS: return "notifications"; - default: throw new IllegalArgumentException("Unexpected contact audience '" + audience + "'."); - } - } - - private boolean asBoolOrTrue(Inspector inspector) { - return !inspector.valid() || inspector.asBool(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java deleted file mode 100644 index 4ea4fe79e8f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.logging.Logger; - -/** - * @author bjorncs - */ -public record TrialNotifications(List<Status> tenants) { - private static final Logger log = Logger.getLogger(TrialNotifications.class.getName()); - - public TrialNotifications { tenants = List.copyOf(tenants); } - - public record Status(TenantName tenant, State state, Instant lastUpdate) {} - public enum State { SIGNED_UP, MID_CHECK_IN, EXPIRES_IMMEDIATELY, EXPIRED, UNKNOWN } - - public Slime toSlime() { - var slime = new Slime(); - var rootCursor = slime.setObject(); - var tenantsCursor = rootCursor.setArray("tenants"); - for (Status t : tenants) { - var tenantCursor = tenantsCursor.addObject(); - tenantCursor.setString("tenant", t.tenant().value()); - tenantCursor.setString("state", t.state().name()); - tenantCursor.setString("lastUpdate", t.lastUpdate().toString()); - } - log.fine(() -> "Generated json '%s' from '%s'".formatted(SlimeUtils.toJson(slime), this)); - return slime; - } - - public static TrialNotifications fromSlime(Slime slime) { - var rootCursor = slime.get(); - var tenantsCursor = rootCursor.field("tenants"); - var tenants = new ArrayList<Status>(); - for (int i = 0; i < tenantsCursor.entries(); i++) { - var tenantCursor = tenantsCursor.entry(i); - var name = TenantName.from(tenantCursor.field("tenant").asString()); - var stateStr = tenantCursor.field("state").asString(); - var state = Arrays.stream(State.values()) - .filter(s -> s.name().equals(stateStr)).findFirst().orElse(State.UNKNOWN); - var lastUpdate = Instant.parse(tenantCursor.field("lastUpdate").asString()); - tenants.add(new Status(name, state, lastUpdate)); - } - var tn = new TrialNotifications(tenants); - log.fine(() -> "Parsed '%s' from '%s'".formatted(tn, SlimeUtils.toJson(slime))); - return tn; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/UnassignedCertificateSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/UnassignedCertificateSerializer.java deleted file mode 100644 index 44f50800561..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/UnassignedCertificateSerializer.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; -import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; - -/** - * @author mpolden - */ -public class UnassignedCertificateSerializer { - - private static final String stateKey = "state"; - private static final String certificateKey = "certificate"; - - public Slime toSlime(UnassignedCertificate unassignedCertificate) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString(stateKey, unassignedCertificate.state().name()); - EndpointCertificateSerializer.toSlime(unassignedCertificate.certificate(), root.setObject(certificateKey)); - return slime; - } - - public UnassignedCertificate fromSlime(Slime slime) { - Cursor root = slime.get(); - UnassignedCertificate.State state = UnassignedCertificate.State.valueOf(root.field(stateKey).asString()); - EndpointCertificate certificate = EndpointCertificateSerializer.fromSlime(root.field(certificateKey)); - return new UnassignedCertificate(certificate, state); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java deleted file mode 100644 index e4de073e2c6..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.component.Version; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.versions.NodeVersion; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; - -/** - * Serializer for {@link VersionStatus}. - * - * @author mpolden - */ -public class VersionStatusSerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - // VersionStatus fields - private static final String versionsField = "versions"; - private static final String currentMajorField = "currentMajor"; - - // VespaVersion fields - private static final String releaseCommitField = "releaseCommit"; - private static final String committedAtField = "releasedAt"; - private static final String isControllerVersionField = "isCurrentControllerVersion"; - private static final String isSystemVersionField = "isCurrentSystemVersion"; - private static final String isReleasedField = "isReleased"; - private static final String deploymentStatisticsField = "deploymentStatistics"; - private static final String confidenceField = "confidence"; - - // NodeVersions fields - private static final String nodeVersionsField = "nodeVersions"; - - // DeploymentStatistics fields - private static final String versionField = "version"; - private static final String failingField = "failing"; - private static final String productionField = "production"; - private static final String deployingField = "deploying"; - - private final NodeVersionSerializer nodeVersionSerializer; - - public VersionStatusSerializer(NodeVersionSerializer nodeVersionSerializer) { - this.nodeVersionSerializer = Objects.requireNonNull(nodeVersionSerializer, "nodeVersionSerializer must be non-null"); - } - - public Slime toSlime(VersionStatus status) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - versionsToSlime(status.versions(), root.setArray(versionsField)); - root.setLong(currentMajorField, status.currentMajor()); - return slime; - } - - public VersionStatus fromSlime(Slime slime) { - Inspector root = slime.get(); - return new VersionStatus(vespaVersionsFromSlime(root.field(versionsField)), - (int) root.field(currentMajorField).asLong()); - } - - private void versionsToSlime(List<VespaVersion> versions, Cursor array) { - versions.forEach(version -> vespaVersionToSlime(version, array.addObject())); - } - - private void vespaVersionToSlime(VespaVersion version, Cursor object) { - object.setString(releaseCommitField, version.releaseCommit()); - object.setLong(committedAtField, version.committedAt().toEpochMilli()); - object.setBool(isControllerVersionField, version.isControllerVersion()); - object.setBool(isSystemVersionField, version.isSystemVersion()); - object.setBool(isReleasedField, version.isReleased()); - deploymentStatisticsToSlime(version.versionNumber(), object.setObject(deploymentStatisticsField)); - object.setString(confidenceField, version.confidence().name()); - nodeVersionsToSlime(version.nodeVersions(), object.setArray(nodeVersionsField)); - } - - private void nodeVersionsToSlime(List<NodeVersion> nodeVersions, Cursor array) { - nodeVersionSerializer.nodeVersionsToSlime(nodeVersions, array); - } - - private void deploymentStatisticsToSlime(Version version, Cursor object) { - object.setString(versionField, version.toString()); - // TODO jonmv: Remove the below. - object.setArray(failingField); - object.setArray(productionField); - object.setArray(deployingField); - } - - private List<VespaVersion> vespaVersionsFromSlime(Inspector array) { - List<VespaVersion> versions = new ArrayList<>(); - array.traverse((ArrayTraverser) (i, object) -> versions.add(vespaVersionFromSlime(object))); - return Collections.unmodifiableList(versions); - } - - private VespaVersion vespaVersionFromSlime(Inspector object) { - var version = Version.fromString(object.field(deploymentStatisticsField).field(versionField).asString()); - return new VespaVersion(version, - object.field(releaseCommitField).asString(), - SlimeUtils.instant(object.field(committedAtField)), - object.field(isControllerVersionField).asBool(), - object.field(isSystemVersionField).asBool(), - object.field(isReleasedField).asBool(), - nodeVersionSerializer.nodeVersionsFromSlime(object.field(nodeVersionsField), version), - VespaVersion.Confidence.valueOf(object.field(confidenceField).asString()) - ); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java deleted file mode 100644 index 97b0e340025..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; - -import java.util.Objects; - -/** - * Serializer for {@link ZoneRoutingPolicy}. - * - * @author mpolden - */ -public class ZoneRoutingPolicySerializer { - - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String GLOBAL_ROUTING_FIELD = "globalRouting"; - - private final RoutingPolicySerializer routingPolicySerializer; - - public ZoneRoutingPolicySerializer(RoutingPolicySerializer routingPolicySerializer) { - this.routingPolicySerializer = Objects.requireNonNull(routingPolicySerializer, "routingPolicySerializer must be non-null"); - } - - public ZoneRoutingPolicy fromSlime(ZoneId zone, Slime slime) { - var root = slime.get(); - return new ZoneRoutingPolicy(zone, routingPolicySerializer.routingStatusFromSlime(root.field(GLOBAL_ROUTING_FIELD))); - } - - public Slime toSlime(ZoneRoutingPolicy policy) { - var slime = new Slime(); - var root = slime.setObject(); - routingPolicySerializer.globalRoutingToSlime(policy.routingStatus(), root.setObject(GLOBAL_ROUTING_FIELD)); - return slime; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java deleted file mode 100644 index abb8ab08d89..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Persistence layer for the controller. - * - * @author bratseth - */ -@ExportPackage -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java deleted file mode 100644 index e623b7e440c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.proxy; - -import com.yahoo.container.jdisc.HttpResponse; - -/** - * Executes call against config servers and handles discovery requests. Rest URIs in the response are - * rewritten. - * - * @author Haakon Dybdahl - */ -public interface ConfigServerRestExecutor { - - HttpResponse handle(ProxyRequest request); - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java deleted file mode 100644 index bbed0554350..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.proxy; - -import ai.vespa.util.http.hc4.SslConnectionSocketFactory; -import com.yahoo.component.AbstractComponent; -import com.yahoo.component.annotation.Inject; -import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.text.Text; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; -import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.yolean.concurrent.Sleeper; -import org.apache.http.Header; -import org.apache.http.HttpResponse; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPatch; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.impl.DefaultConnectionReuseStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.protocol.HttpContext; -import org.apache.http.protocol.HttpCoreContext; -import org.apache.http.util.EntityUtils; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLSession; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.net.URI; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static com.yahoo.yolean.Exceptions.uncheck; - - -/** - * @author Haakon Dybdahl - * @author bjorncs - */ -@SuppressWarnings("unused") // Injected -public class ConfigServerRestExecutorImpl extends AbstractComponent implements ConfigServerRestExecutor { - - private static final Logger LOG = Logger.getLogger(ConfigServerRestExecutorImpl.class.getName()); - private static final Duration PROXY_REQUEST_TIMEOUT = Duration.ofSeconds(20); - private static final Duration PING_REQUEST_TIMEOUT = Duration.ofMillis(500); - private static final Duration SINGLE_TARGET_WAIT = Duration.ofSeconds(2); - private static final int SINGLE_TARGET_RETRIES = 3; - private static final Set<String> HEADERS_TO_COPY = Set.of("X-HTTP-Method-Override", "Content-Type"); - - private final CloseableHttpClient client; - private final Sleeper sleeper; - - @Inject - public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, ControllerIdentityProvider identityProvider) { - this(SslConnectionSocketFactory.of(identityProvider.getConfigServerSslSocketFactory(), new ControllerOrConfigserverHostnameVerifier(zoneRegistry)), - Sleeper.DEFAULT, // Specify - new ConnectionReuseStrategy(zoneRegistry)); - } - - ConfigServerRestExecutorImpl(SSLConnectionSocketFactory connectionSocketFactory, - Sleeper sleeper, ConnectionReuseStrategy connectionReuseStrategy) { - this.client = createHttpClient(connectionSocketFactory, connectionReuseStrategy); - this.sleeper = sleeper; - } - - @Override - public ProxyResponse handle(ProxyRequest request) { - List<URI> targets = new ArrayList<>(request.getTargets()); - - StringBuilder errorBuilder = new StringBuilder(); - boolean singleTarget = targets.size() == 1; - if (singleTarget) { - for (int i = 0; i < SINGLE_TARGET_RETRIES - 1; i++) { - targets.add(targets.get(0)); - } - } else if (queueFirstServerIfDown(targets)) { - errorBuilder.append("Change ordering due to failed ping."); - } - - for (URI url : targets) { - Optional<ProxyResponse> proxyResponse = proxy(request, url, errorBuilder); - if (proxyResponse.isPresent()) { - return proxyResponse.get(); - } - if (singleTarget) { - sleeper.sleep(SINGLE_TARGET_WAIT); - } - } - - throw new RuntimeException("Failed talking to config servers: " + errorBuilder); - } - - private Optional<ProxyResponse> proxy(ProxyRequest request, URI url, StringBuilder errorBuilder) { - HttpRequestBase requestBase = createHttpBaseRequest( - request.getMethod(), request.createConfigServerRequestUri(url), request.getData()); - // Empty list of headers to copy for now, add headers when needed, or rewrite logic. - copyHeaders(request.getHeaders(), requestBase); - - try (CloseableHttpResponse response = client.execute(requestBase)) { - String content = getContent(response); - int status = response.getStatusLine().getStatusCode(); - if (status / 100 == 5) { - errorBuilder.append("Talking to server ").append(url.getHost()) - .append(", got ").append(status).append(" ") - .append(content).append("\n"); - LOG.log(Level.FINE, () -> Text.format("Got response from %s with status code %d and content:\n %s", - url.getHost(), status, content)); - return Optional.empty(); - } - Header contentHeader = response.getLastHeader("Content-Type"); - String contentType; - if (contentHeader != null && contentHeader.getValue() != null && ! contentHeader.getValue().isEmpty()) { - contentType = contentHeader.getValue().replace("; charset=UTF-8",""); - } else { - contentType = "application/json"; - } - // Send response back - return Optional.of(new ProxyResponse(request, content, status, url, contentType)); - } catch (Exception e) { - errorBuilder.append("Talking to server ").append(url.getHost()) - .append(" got exception ").append(e.getMessage()) - .append("\n"); - LOG.log(Level.FINE, e, () -> "Got exception while sending request to " + url.getHost()); - return Optional.empty(); - } - } - - private static String getContent(CloseableHttpResponse response) { - return Optional.ofNullable(response.getEntity()) - .map(entity -> uncheck(() -> EntityUtils.toString(entity))) - .orElse(""); - } - - private static HttpRequestBase createHttpBaseRequest(Method method, URI url, InputStream data) { - switch (method) { - case GET: - return new HttpGet(url); - case POST: - HttpPost post = new HttpPost(url); - if (data != null) { - post.setEntity(new InputStreamEntity(data)); - } - return post; - case PUT: - HttpPut put = new HttpPut(url); - if (data != null) { - put.setEntity(new InputStreamEntity(data)); - } - return put; - case DELETE: - return new HttpDelete(url); - case PATCH: - HttpPatch patch = new HttpPatch(url); - if (data != null) { - patch.setEntity(new InputStreamEntity(data)); - } - return patch; - } - throw new IllegalArgumentException("Refusing to proxy " + method + " " + url + ": Unsupported method"); - } - - private static void copyHeaders(Map<String, List<String>> headers, HttpRequestBase toRequest) { - for (Map.Entry<String, List<String>> headerEntry : headers.entrySet()) { - if (HEADERS_TO_COPY.contains(headerEntry.getKey())) { - for (String value : headerEntry.getValue()) { - toRequest.addHeader(headerEntry.getKey(), value); - } - } - } - } - - /** - * During upgrade, one server can be down, this is normal. Therefore we do a quick ping on the first server, - * if it is not responding, we try the other servers first. False positive/negatives are not critical, - * but will increase latency to some extent. - */ - private boolean queueFirstServerIfDown(List<URI> allServers) { - if (allServers.size() < 2) { - return false; - } - URI uri = allServers.get(0); - HttpGet httpGet = new HttpGet(uri); - - RequestConfig config = RequestConfig.custom() - .setConnectTimeout((int) PING_REQUEST_TIMEOUT.toMillis()) - .setConnectionRequestTimeout((int) PING_REQUEST_TIMEOUT.toMillis()) - .setSocketTimeout((int) PING_REQUEST_TIMEOUT.toMillis()).build(); - httpGet.setConfig(config); - - try (CloseableHttpResponse response = client.execute(httpGet)) { - if (response.getStatusLine().getStatusCode() == 200) { - return false; - } - - } catch (IOException e) { - // We ignore this, if server is restarting this might happen. - } - // Some error happened, move this server to the back. The other servers should be running. - Collections.rotate(allServers, -1); - return true; - } - - @Override - public void deconstruct() { - try { - client.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static CloseableHttpClient createHttpClient(SSLConnectionSocketFactory connectionSocketFactory, - org.apache.http.ConnectionReuseStrategy connectionReuseStrategy) { - - RequestConfig config = RequestConfig.custom() - .setConnectTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) - .setConnectionRequestTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) - .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build(); - return HttpClientBuilder.create() - .setUserAgent("config-server-proxy-client") - .setSSLSocketFactory(connectionSocketFactory) - .setDefaultRequestConfig(config) - .setMaxConnPerRoute(10) - .setMaxConnTotal(500) - .setConnectionReuseStrategy(connectionReuseStrategy) - .setConnectionTimeToLive(1, TimeUnit.MINUTES) - .build(); - - } - - private static class ControllerOrConfigserverHostnameVerifier implements HostnameVerifier { - - private final HostnameVerifier configserverVerifier; - - ControllerOrConfigserverHostnameVerifier(ZoneRegistry registry) { - this.configserverVerifier = createConfigserverVerifier(registry); - } - - private static HostnameVerifier createConfigserverVerifier(ZoneRegistry registry) { - Set<AthenzIdentity> configserverIdentities = registry.zones().all().zones().stream() - .map(zone -> registry.getConfigServerHttpsIdentity(zone.getId())) - .collect(Collectors.toSet()); - return new AthenzIdentityVerifier(configserverIdentities); - } - - @Override - public boolean verify(String hostname, SSLSession session) { - return "localhost".equals(hostname) || configserverVerifier.verify(hostname, session); - } - } - - /** - * A connection reuse strategy which avoids reusing connections to VIPs. Since VIPs are TCP-level load balancers, - * a reconnect is needed to (potentially) switch real server. - */ - public static class ConnectionReuseStrategy extends DefaultConnectionReuseStrategy { - - private final Set<String> vips; - - public ConnectionReuseStrategy(ZoneRegistry zoneRegistry) { - this(zoneRegistry.zones().all().ids().stream() - .map(zoneRegistry::getConfigServerVipUri) - .map(URI::getHost) - .collect(Collectors.toUnmodifiableSet())); - } - - public ConnectionReuseStrategy(Set<String> vips) { - this.vips = Set.copyOf(vips); - } - - @Override - public boolean keepAlive(HttpResponse response, HttpContext context) { - HttpCoreContext coreContext = HttpCoreContext.adapt(context); - String host = coreContext.getTargetHost().getHostName(); - if (vips.contains(host)) { - return false; - } - return super.keepAlive(response, context); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java deleted file mode 100644 index 2a29e2b590d..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.proxy; - -import ai.vespa.http.HttpURL; -import ai.vespa.http.HttpURL.Path; -import ai.vespa.http.HttpURL.Query; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.text.Text; - -import java.io.InputStream; -import java.net.URI; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static com.yahoo.jdisc.http.HttpRequest.Method; - -/** - * Keeping information about the calls that are being proxied. - * A request is of form /zone/v2/[environment]/[region]/[config-server-path] - * - * @author Haakon Dybdahl - */ -public class ProxyRequest { - - private final Method method; - private final HttpURL requestUri; - private final Map<String, List<String>> headers; - private final InputStream requestData; - - private final List<URI> targets; - private final Path targetPath; - - ProxyRequest(Method method, URI uri, Map<String, List<String>> headers, InputStream body, List<URI> targets, Path path) { - this.requestUri = HttpURL.from(uri); - if ( requestUri.path().length() < path.length() - || ! requestUri.path().tail(path.length()).equals(path)) { - throw new IllegalArgumentException(Text.format("Request %s does not end with proxy %s", requestUri.path(), path)); - } - if (targets.isEmpty()) { - throw new IllegalArgumentException("targets must be non-empty"); - } - this.method = Objects.requireNonNull(method); - this.headers = Objects.requireNonNull(headers); - this.requestData = body; - this.targets = List.copyOf(targets); - this.targetPath = path; - } - - - public Method getMethod() { - return method; - } - - public Map<String, List<String>> getHeaders() { - return headers; - } - - public InputStream getData() { - return requestData; - } - - public List<URI> getTargets() { - return targets; - } - - public URI createConfigServerRequestUri(URI baseURI) { - return HttpURL.from(baseURI).withPath(targetPath).withQuery(requestUri.query()).asURI(); - } - - public URI getControllerPrefixUri() { - Path prefixPath = requestUri.path().cut(targetPath.length()).withTrailingSlash(); - return requestUri.withPath(prefixPath).withQuery(Query.empty()).asURI(); - } - - @Override - public String toString() { - return "[targets: " + targets + " request: " + targetPath + "]"; - } - - /** Create a proxy request that repeatedly tries a single target */ - public static ProxyRequest tryOne(URI target, Path path, HttpRequest request) { - return new ProxyRequest(request.getMethod(), request.getUri(), request.getJDiscRequest().headers(), - request.getData(), List.of(target), path); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java deleted file mode 100644 index caf2ff05814..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.proxy; - -import ai.vespa.http.HttpURL; -import ai.vespa.http.HttpURL.Path; -import com.yahoo.container.jdisc.HttpResponse; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; -import java.nio.charset.StandardCharsets; - -/** - * Response class that also rewrites URL from config server. - * - * @author Haakon Dybdahl - */ -public class ProxyResponse extends HttpResponse { - - private final String bodyResponseRewritten; - private final String contentType; - - public ProxyResponse( - ProxyRequest controllerRequest, - String bodyResponse, - int statusResponse, - URI configServer, - String contentType) { - super(statusResponse); - this.contentType = contentType; - - // Configserver always serves from 4443, therefore all responses will have port 4443 in them, - // but the request URI (loadbalancer/VIP) is not always 4443 - String configServerPrefix = HttpURL.from(configServer).withPort(4443).withPath(Path.empty()).asURI().toString(); - String controllerRequestPrefix = controllerRequest.getControllerPrefixUri().toString(); - bodyResponseRewritten = bodyResponse.replace(configServerPrefix, controllerRequestPrefix); - } - - @Override - public void render(OutputStream stream) throws IOException { - stream.write(bodyResponseRewritten.getBytes(StandardCharsets.UTF_8)); - } - - @Override - public String getContentType() { return contentType; } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java deleted file mode 100644 index 0acb064f52a..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author Haakon Dybdahl - */ -@ExportPackage -package com.yahoo.vespa.hosted.controller.proxy; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponses.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponses.java deleted file mode 100644 index 56844887caf..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponses.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.restapi.ErrorResponse; - -import java.util.UUID; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Helper class for creating error responses. - * - * @author mpolden - */ -public class ErrorResponses { - - private ErrorResponses() {} - - /** - * Returns a response for a failing request containing an unique request ID. Details of the error are logged through - * given logger. - */ - public static ErrorResponse logThrowing(HttpRequest request, Logger logger, Throwable t) { - String requestId = UUID.randomUUID().toString(); - logger.log(Level.SEVERE, "Unexpected error handling '" + request.getUri() + "' (request ID: " + - requestId + ")", t); - return ErrorResponse.internalServerError("Unexpected error occurred (request ID: " + requestId + ")"); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java deleted file mode 100644 index ebcc81ab756..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ /dev/null @@ -1,3479 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import ai.vespa.hosted.api.Signatures; -import ai.vespa.http.DomainName; -import ai.vespa.http.HttpURL; -import ai.vespa.http.HttpURL.Query; -import ai.vespa.validation.Validation; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableSet; -import com.yahoo.component.Version; -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.IntRange; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.container.handler.metrics.JsonResponse; -import com.yahoo.container.jdisc.EmptyResponse; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.io.IOUtils; -import com.yahoo.jdisc.http.filter.security.misc.User; -import com.yahoo.restapi.ByteArrayResponse; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.ResourceResponse; -import com.yahoo.restapi.RestApiException; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.security.KeyUtils; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.JsonParseException; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.text.Text; -import com.yahoo.vespa.athenz.api.OAuthCredentials; -import com.yahoo.vespa.athenz.client.zms.ZmsClientException; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.LockedTenant; -import com.yahoo.vespa.hosted.controller.NotExistsException; -import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.SearchNodeMetrics; -import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; -import com.yahoo.vespa.hosted.controller.api.integration.aws.TenantRoles; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult.LogEntry; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Load; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneToken; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.FingerPrint; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; -import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; -import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; -import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; -import com.yahoo.vespa.hosted.controller.application.AssignedRotation; -import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.EndpointList; -import com.yahoo.vespa.hosted.controller.application.QuotaUsage; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; -import com.yahoo.vespa.hosted.controller.deployment.JobStatus; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.Submission; -import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer; -import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; -import com.yahoo.vespa.hosted.controller.notification.Notification; -import com.yahoo.vespa.hosted.controller.notification.NotificationSource; -import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.vespa.hosted.controller.restapi.dataplanetoken.DataplaneTokenService; -import com.yahoo.vespa.hosted.controller.restapi.dataplanetoken.DataplaneTokenService.State; -import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; -import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; -import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; -import com.yahoo.vespa.hosted.controller.security.Credentials; -import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; -import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; -import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; -import com.yahoo.vespa.hosted.controller.tenant.Email; -import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; -import com.yahoo.vespa.hosted.controller.tenant.PendingMailVerification; -import com.yahoo.vespa.hosted.controller.tenant.PurchaseOrder; -import com.yahoo.vespa.hosted.controller.tenant.TaxId; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.TenantAddress; -import com.yahoo.vespa.hosted.controller.tenant.TenantBilling; -import com.yahoo.vespa.hosted.controller.tenant.TenantContact; -import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; -import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; -import com.yahoo.vespa.hosted.controller.tenant.TermsOfServiceApproval; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.yolean.Exceptions; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.security.DigestInputStream; -import java.security.Principal; -import java.security.PublicKey; -import java.time.DayOfWeek; -import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Base64; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Scanner; -import java.util.StringJoiner; -import java.util.function.BiFunction; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; -import static com.yahoo.jdisc.Response.Status.CONFLICT; -import static com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource.APPLICATION_TEST_ZIP; -import static com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource.APPLICATION_ZIP; -import static com.yahoo.yolean.Exceptions.uncheck; -import static java.util.Comparator.comparingInt; -import static java.util.Map.Entry.comparingByKey; -import static java.util.stream.Collectors.collectingAndThen; -import static java.util.stream.Collectors.joining; - -/** - * This implements the application/v4 API which is used to deploy and manage applications - * on hosted Vespa. - * - * @author bratseth - * @author mpolden - */ -@SuppressWarnings("unused") // created by injection -public class ApplicationApiHandler extends AuditLoggingRequestHandler { - - private static final ObjectMapper jsonMapper = new ObjectMapper(); - - private final Controller controller; - private final AccessControlRequests accessControlRequests; - private final TestConfigSerializer testConfigSerializer; - - @Inject - public ApplicationApiHandler(ThreadedHttpRequestHandler.Context parentCtx, - Controller controller, - AccessControlRequests accessControlRequests) { - super(parentCtx, controller.auditLogger()); - this.controller = controller; - this.accessControlRequests = accessControlRequests; - this.testConfigSerializer = new TestConfigSerializer(controller.system()); - } - - @Override - public Duration getTimeout() { - return Duration.ofMinutes(20); // deploys may take a long time; - } - - @Override - public HttpResponse auditAndHandle(HttpRequest request) { - try { - Path path = new Path(request.getUri()); - return switch (request.getMethod()) { - case GET: yield handleGET(path, request); - case PUT: yield handlePUT(path, request); - case POST: yield handlePOST(path, request); - case PATCH: yield handlePATCH(path, request); - case DELETE: yield handleDELETE(path, request); - case OPTIONS: yield handleOPTIONS(); - default: yield ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - }; - } - catch (RestApiException.Forbidden e) { - return ErrorResponse.forbidden(Exceptions.toMessageString(e)); - } - catch (RestApiException.Unauthorized e) { - return ErrorResponse.unauthorized(Exceptions.toMessageString(e)); - } - catch (NotExistsException e) { - return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); - } - catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } - catch (ConfigServerException e) { - return switch (e.code()) { - case NOT_FOUND -> ErrorResponse.notFoundError(Exceptions.toMessageString(e)); - case ACTIVATION_CONFLICT -> new ErrorResponse(CONFLICT, e.code().name(), Exceptions.toMessageString(e)); - case INTERNAL_SERVER_ERROR -> ErrorResponses.logThrowing(request, log, e); - default -> new ErrorResponse(BAD_REQUEST, e.code().name(), Exceptions.toMessageString(e)); - }; - } - catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse handleGET(Path path, HttpRequest request) { - if (path.matches("/application/v4/")) return root(request); - if (path.matches("/application/v4/search/{*}")) return search(path, request); - if (path.matches("/application/v4/notifications")) return notifications(request, Optional.ofNullable(request.getProperty("tenant")), true); - if (path.matches("/application/v4/tenant")) return tenants(request); - if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/access/request/operator")) return accessRequests(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/info")) return tenantInfo(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/info/profile")) return withCloudTenant(path.get("tenant"), this::tenantInfoProfile); - if (path.matches("/application/v4/tenant/{tenant}/info/billing")) return withCloudTenant(path.get("tenant"), this::tenantInfoBilling); - if (path.matches("/application/v4/tenant/{tenant}/info/contacts")) return withCloudTenant(path.get("tenant"), this::tenantInfoContacts); - if (path.matches("/application/v4/tenant/{tenant}/notifications")) return notifications(request, Optional.of(path.get("tenant")), false); - if (path.matches("/application/v4/tenant/{tenant}/secret-store/{name}/validate")) return validateSecretStore(path.get("tenant"), path.get("name"), request); - if (path.matches("/application/v4/tenant/{tenant}/token")) return listTokens(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), Optional.empty(), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/compile-version")) return compileVersion(path.get("tenant"), path.get("application"), request.getProperty("allowMajor")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deployment")) return JobControllerApiHandlerHelper.overviewResponse(controller, TenantAndApplicationId.from(path.get("tenant"), path.get("application")), request.getUri()); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/package")) return applicationPackage(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/diff/{number}")) return applicationPackageDiff(path.get("tenant"), path.get("application"), path.get("number")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), "default", request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance")) return applications(path.get("tenant"), Optional.of(path.get("application")), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return instance(path.get("tenant"), path.get("application"), path.get("instance"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller, new JobId(appIdFromPath(path), jobTypeFromPath(path)), Optional.ofNullable(request.getProperty("limit")), request.getUri()); // (((\(✘෴✘)/))) - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path)); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/diff/{number}")) return devApplicationPackageDiff(runIdFromPath(path)); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}/logs")) return JobControllerApiHandlerHelper.vespaLogsResponse(controller.jobController(), runIdFromPath(path), asLong(request.getProperty("from"), 0), request.getBooleanProperty("tester")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return getReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/state/v1/{*}")) return stateV1(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/orchestrator")) return orchestrator(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/content/{*}")) return content(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.getRest(), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/private-services")) return getPrivateServiceInfo(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/drop-documents")) return dropDocumentsStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/access/support")) return supportAccess(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/node/{node}/service-dump")) return getServiceDump(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("node"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/scaling")) return scaling(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - // TODO: Remove when not used anymore (migrated to ../metrics/searchnode) - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/metrics")) return searchNodeMetrics(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/metrics/searchnode")) return searchNodeMetrics(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("endpointId"))); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("endpointId"))); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse search(Path path, HttpRequest request) { - if (path.matches("/application/v4/search/deployment")) return searchDeploymentsByEndpoint(request); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse searchDeploymentsByEndpoint(HttpRequest request) { - String endpoint = request.getProperty("endpoint"); - if (endpoint == null) { - throw new IllegalArgumentException("Missing 'endpoint' query parameter"); - } - endpoint = endpoint.trim(); - if (endpoint.startsWith("https://") || endpoint.startsWith("http://")) { - // Trim scheme and port - endpoint = URI.create(endpoint).getHost(); - } - List<Application> applications = controller.applications().asList(); - record EndpointTarget(DeploymentId deployment, ClusterSpec.Id cluster) {} - List<EndpointTarget> targets = new ArrayList<>(); - out: - for (var app : applications) { - Optional<Endpoint> declaredEndpoint = controller.routing().readDeclaredEndpointsOf(app).dnsName(endpoint); - if (declaredEndpoint.isPresent()) { - for (var target : declaredEndpoint.get().targets()) { - targets.add(new EndpointTarget(target.deployment(), declaredEndpoint.get().cluster())); - } - break; - } else { - for (var instance : app.instances().values()) { - for (var deployment : instance.deployments().values()) { - DeploymentId id = new DeploymentId(instance.id(), deployment.zone()); - Optional<Endpoint> matchingEndpoint = controller.routing().readEndpointsOf(id).dnsName(endpoint); - if (matchingEndpoint.isPresent()) { - for (var target : matchingEndpoint.get().targets()) { - targets.add(new EndpointTarget(target.deployment(), matchingEndpoint.get().cluster())); - } - break out; - } - } - } - } - } - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor deploymentArray = root.setArray("deployments"); - for (var target : targets) { - toSlime(target.deployment, target.cluster, deploymentArray.addObject(), request); - } - return new SlimeJsonResponse(slime); - } - - - private HttpResponse handlePUT(Path path, HttpRequest request) { - if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/access/request/operator")) return requestSshAccess(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/access/approve/operator")) return approveAccessRequest(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/access/managed/operator")) return addManagedAccess(path.get("tenant")); - if (path.matches("/application/v4/tenant/{tenant}/info")) return updateTenantInfo(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/info/profile")) return withCloudTenant(path.get("tenant"), request, this::putTenantInfoProfile); - if (path.matches("/application/v4/tenant/{tenant}/info/billing")) return withCloudTenant(path.get("tenant"), request, this::putTenantInfoBilling); - if (path.matches("/application/v4/tenant/{tenant}/info/contacts")) return withCloudTenant(path.get("tenant"), request, this::putTenantInfoContacts); - if (path.matches("/application/v4/tenant/{tenant}/info/resend-mail-verification")) return withCloudTenant(path.get("tenant"), request, this::resendEmailVerification); - if (path.matches("/application/v4/tenant/{tenant}/archive-access/aws")) return allowAwsArchiveAccess(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/archive-access/gcp")) return allowGcpArchiveAccess(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/secret-store/{name}")) return addSecretStore(path.get("tenant"), path.get("name"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse handlePOST(Path path, HttpRequest request) { - if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/terms-of-service")) return approveTermsOfService(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/key")) return addDeveloperKey(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/token/{tokenid}")) return generateToken(path.get("tenant"), path.get("tokenid"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), "default", false, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform-pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application-pin")) return deployApplication(path.get("tenant"), path.get("application"), "default", true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", false, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return addDeployKey(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return createInstance(path.get("tenant"), path.get("application"), path.get("instance"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobtype}")) return jobDeploy(appIdFromPath(path), jobTypeFromPath(path), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), false, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/platform-pin")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/application-pin")) return deployApplication(path.get("tenant"), path.get("application"), path.get("instance"), true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), path.get("instance"), false, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit")) return submit(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return trigger(appIdFromPath(path), jobTypeFromPath(path), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return pause(appIdFromPath(path), jobTypeFromPath(path)); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deploySystemApplication(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/deploy")) return deploySystemApplication(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindex")) return reindex(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return enableReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspend")) return suspend(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/drop-documents")) return dropDocuments(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/access/support")) return allowSupportAccess(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/node/{node}/service-dump")) return requestServiceDump(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("node"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploySystemApplication(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploySystemApplication(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse handlePATCH(Path path, HttpRequest request) { - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return patchApplication(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return patchApplication(path.get("tenant"), path.get("application"), request); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse handleDELETE(Path path, HttpRequest request) { - if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/access/managed/operator")) return removeManagedAccess(path.get("tenant")); - if (path.matches("/application/v4/tenant/{tenant}/key")) return removeDeveloperKey(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/archive-access/aws")) return removeAwsArchiveAccess(path.get("tenant")); - if (path.matches("/application/v4/tenant/{tenant}/archive-access/gcp")) return removeGcpArchiveAccess(path.get("tenant")); - if (path.matches("/application/v4/tenant/{tenant}/secret-store/{name}")) return deleteSecretStore(path.get("tenant"), path.get("name"), request); - if (path.matches("/application/v4/tenant/{tenant}/terms-of-service")) return unapproveTermsOfService(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/token/{tokenid}")) return deleteToken(path.get("tenant"), path.get("tokenid"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deployment")) return removeAllProdDeployments(path.get("tenant"), path.get("application")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", "all"); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", path.get("choice")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return removeDeployKey(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit/{build}")) return cancelBuild(path.get("tenant"), path.get("application"), path.get("build")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return deleteInstance(path.get("tenant"), path.get("application"), path.get("instance"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), "all"); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("choice")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), request, appIdFromPath(path), jobTypeFromPath(path)); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return resume(appIdFromPath(path), jobTypeFromPath(path)); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return disableReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspend")) return suspend(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/access/support")) return disallowSupportAccess(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse handleOPTIONS() { - // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother - // spelling out the methods supported at each path, which we should - EmptyResponse response = new EmptyResponse(); - response.headers().put("Allow", "GET,PUT,POST,PATCH,DELETE,OPTIONS"); - return response; - } - - private HttpResponse recursiveRoot(HttpRequest request) { - Slime slime = new Slime(); - Cursor tenantArray = slime.setArray(); - List<Application> applications = controller.applications().asList(); - for (Tenant tenant : controller.tenants().asList(includeDeleted(request))) - toSlime(tenantArray.addObject(), - tenant, - applications.stream().filter(app -> app.id().tenant().equals(tenant.name())).toList(), - request); - return new SlimeJsonResponse(slime); - } - - private HttpResponse root(HttpRequest request) { - return recurseOverTenants(request) - ? recursiveRoot(request) - : new ResourceResponse(request, "tenant"); - } - - private HttpResponse tenants(HttpRequest request) { - Slime slime = new Slime(); - Cursor response = slime.setArray(); - for (Tenant tenant : controller.tenants().asList(includeDeleted(request))) - tenantInTenantsListToSlime(tenant, request.getUri(), response.addObject()); - return new SlimeJsonResponse(slime); - } - - private HttpResponse tenant(String tenantName, HttpRequest request) { - return controller.tenants().get(TenantName.from(tenantName), includeDeleted(request)) - .map(tenant -> tenant(tenant, request)) - .orElseGet(() -> ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist")); - } - - private HttpResponse tenant(Tenant tenant, HttpRequest request) { - Slime slime = new Slime(); - toSlime(slime.setObject(), tenant, controller.applications().asList(tenant.name()), request); - return new SlimeJsonResponse(slime); - } - - private HttpResponse accessRequests(String tenantName, HttpRequest request) { - var tenant = TenantName.from(tenantName); - if (controller.tenants().require(tenant).type() != Tenant.Type.cloud) - return ErrorResponse.badRequest("Can only see access requests for cloud tenants"); - - var accessControlService = controller.serviceRegistry().accessControlService(); - var slime = new Slime(); - var cursor = slime.setObject(); - try { - var accessRoleInformation = accessControlService.getAccessRoleInformation(tenant); - var managedAccess = accessControlService.getManagedAccess(tenant); - cursor.setBool("managedAccess", managedAccess); - accessRoleInformation.getPendingRequest() - .ifPresent(membershipRequest -> { - var requestCursor = cursor.setObject("pendingRequest"); - requestCursor.setString("requestTime", membershipRequest.getCreationTime()); - requestCursor.setString("reason", membershipRequest.getReason()); - }); - var auditLogCursor = cursor.setArray("auditLog"); - accessRoleInformation.getAuditLog() - .forEach(auditLogEntry -> { - var entryCursor = auditLogCursor.addObject(); - entryCursor.setString("created", auditLogEntry.getCreationTime()); - entryCursor.setString("approver", auditLogEntry.getApprover()); - entryCursor.setString("reason", auditLogEntry.getReason()); - entryCursor.setString("status", auditLogEntry.getAction()); - }); - } - catch (ZmsClientException e) { - if (e.getErrorCode() == 404) cursor.setBool("managedAccess", false); - } - return new SlimeJsonResponse(slime); - } - - private HttpResponse requestSshAccess(String tenantName, HttpRequest request) { - if (!isOperator(request)) { - return ErrorResponse.forbidden("Only operators are allowed to request ssh access"); - } - - if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) - return ErrorResponse.badRequest("Can only request access for cloud tenants"); - - controller.serviceRegistry().accessControlService().requestSshAccess(TenantName.from(tenantName)); - return new MessageResponse("OK"); - - } - - private HttpResponse approveAccessRequest(String tenantName, HttpRequest request) { - var tenant = TenantName.from(tenantName); - - if (controller.tenants().require(tenant).type() != Tenant.Type.cloud) - return ErrorResponse.badRequest("Can only see access requests for cloud tenants"); - - var inspector = toSlime(request.getData()).get(); - var expiry = inspector.field("expiry").valid() ? - Instant.ofEpochMilli(inspector.field("expiry").asLong()) : - Instant.now().plus(1, ChronoUnit.DAYS); - var approve = inspector.field("approve").asBool(); - - controller.serviceRegistry().accessControlService().decideSshAccess(tenant, expiry, OAuthCredentials.fromAuth0RequestContext(request.getJDiscRequest().context()), approve); - return new MessageResponse("OK"); - } - - private HttpResponse addManagedAccess(String tenantName) { - return setManagedAccess(tenantName, true); - } - - private HttpResponse removeManagedAccess(String tenantName) { - return setManagedAccess(tenantName, false); - } - - private HttpResponse setManagedAccess(String tenantName, boolean managedAccess) { - var tenant = TenantName.from(tenantName); - - if (controller.tenants().require(tenant).type() != Tenant.Type.cloud) - return ErrorResponse.badRequest("Can only set access privel for cloud tenants"); - - try { - controller.serviceRegistry().accessControlService().setManagedAccess(tenant, managedAccess); - var slime = new Slime(); - slime.setObject().setBool("managedAccess", managedAccess); - return new SlimeJsonResponse(slime); - } - catch (ZmsClientException e) { - if (e.getErrorCode() == 404) return ErrorResponse.conflict("Configuration not yet ready, please try again in a few minutes"); - throw e; - } - } - - private HttpResponse tenantInfo(String tenantName, HttpRequest request) { - return controller.tenants().get(TenantName.from(tenantName)) - .filter(tenant -> tenant.type() == Tenant.Type.cloud) - .map(tenant -> tenantInfo(((CloudTenant)tenant).info(), request)) - .orElseGet(() -> ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist or does not support this")); - } - - private HttpResponse withCloudTenant(String tenantName, Function<CloudTenant, SlimeJsonResponse> handler) { - return controller.tenants().get(TenantName.from(tenantName)) - .filter(tenant -> tenant.type() == Tenant.Type.cloud) - .map(tenant -> handler.apply((CloudTenant) tenant)) - .orElseGet(() -> ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist or does not support this")); - } - - private SlimeJsonResponse tenantInfo(TenantInfo info, HttpRequest request) { - Slime slime = new Slime(); - Cursor infoCursor = slime.setObject(); - if (!info.isEmpty()) { - infoCursor.setString("name", info.name()); - infoCursor.setString("email", info.email()); - infoCursor.setString("website", info.website()); - infoCursor.setString("contactName", info.contact().name()); - infoCursor.setString("contactEmail", info.contact().email().getEmailAddress()); - infoCursor.setBool("contactEmailVerified", info.contact().email().isVerified()); - toSlime(info.address(), infoCursor); - toSlime(info.billingContact(), infoCursor); - toSlime(info.contacts(), infoCursor); - } - - return new SlimeJsonResponse(slime); - } - - private SlimeJsonResponse tenantInfoProfile(CloudTenant cloudTenant) { - var slime = new Slime(); - var root = slime.setObject(); - var info = cloudTenant.info(); - - if (!info.isEmpty()) { - var contact = root.setObject("contact"); - contact.setString("name", info.contact().name()); - contact.setString("email", info.contact().email().getEmailAddress()); - contact.setBool("emailVerified", info.contact().email().isVerified()); - - var tenant = root.setObject("tenant"); - tenant.setString("company", info.name()); - tenant.setString("website", info.website()); - - toSlime(info.address(), root); // will create "address" on the parent - } - - return new SlimeJsonResponse(slime); - } - - private SlimeJsonResponse withCloudTenant(String tenantName, HttpRequest request, BiFunction<CloudTenant, Inspector, SlimeJsonResponse> handler) { - return controller.tenants().get(tenantName) - .map(tenant -> handler.apply((CloudTenant) tenant, toSlime(request.getData()).get())) - .orElseGet(() -> ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist or does not support this")); - } - - private SlimeJsonResponse putTenantInfoProfile(CloudTenant cloudTenant, Inspector inspector) { - var info = cloudTenant.info(); - - var mergedEmail = optional("email", inspector.field("contact")) - .filter(address -> !address.equals(info.contact().email().getEmailAddress())) - .map(address -> { - controller.mailVerifier().sendMailVerification(cloudTenant.name(), address, PendingMailVerification.MailType.TENANT_CONTACT); - return new Email(address, false); - }) - .orElse(info.contact().email()); - - var mergedContact = TenantContact.empty() - .withName(getString(inspector.field("contact").field("name"), info.contact().name())) - .withEmail(mergedEmail); - - var mergedAddress = updateTenantInfoAddress(inspector.field("address"), info.address()); - - var mergedInfo = info - .withName(getString(inspector.field("tenant").field("company"), info.name())) - .withWebsite(getString(inspector.field("tenant").field("website"), info.website())) - .withContact(mergedContact) - .withAddress(mergedAddress); - - validateMergedTenantInfo(mergedInfo); - - controller.tenants().lockOrThrow(cloudTenant.name(), LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withInfo(mergedInfo); - controller.tenants().store(lockedTenant); - }); - - return new MessageResponse("Tenant info updated"); - } - - private SlimeJsonResponse tenantInfoBilling(CloudTenant cloudTenant) { - var slime = new Slime(); - var root = slime.setObject(); - var info = cloudTenant.info(); - - if (!info.isEmpty()) { - var billingContact = info.billingContact(); - - var contact = root.setObject("contact"); - contact.setString("name", billingContact.contact().name()); - contact.setString("email", billingContact.contact().email().getEmailAddress()); - contact.setBool("emailVerified", billingContact.contact().email().isVerified()); - contact.setString("phone", billingContact.contact().phone()); - var taxIdCursor = root.setObject("taxId"); - taxIdCursor.setString("country", billingContact.getTaxId().country().value()); - taxIdCursor.setString("type", billingContact.getTaxId().type().value()); - taxIdCursor.setString("code", billingContact.getTaxId().code().value()); - root.setString("purchaseOrder", billingContact.getPurchaseOrder().value()); - root.setString("invoiceEmail", billingContact.getInvoiceEmail().getEmailAddress()); - var tosApprovalCursor = root.setObject("tosApproval"); - var tosApproval = billingContact.getToSApproval(); - tosApprovalCursor.setString("at", !tosApproval.isEmpty() ? tosApproval.approvedAt().toString() : ""); - tosApprovalCursor.setString("by", !tosApproval.isEmpty() ? tosApproval.approvedBy().get().getName() : ""); - - toSlime(billingContact.address(), root); // will create "address" on the parent - } - - return new SlimeJsonResponse(slime); - } - - private SlimeJsonResponse putTenantInfoBilling(CloudTenant cloudTenant, Inspector inspector) { - var info = cloudTenant.info(); - var billing = info.billingContact(); - var contact = billing.contact(); - var address = billing.address(); - - var mergedContact = updateBillingContact(inspector.field("contact"), cloudTenant.name(), contact); - var mergedAddress = updateTenantInfoAddress(inspector.field("address"), billing.address()); - var mergedTaxId = updateAndValidateTaxId(inspector.field("taxId"), billing.getTaxId()); - var mergedPurchaseOrder = optional("purchaseOrder", inspector).map(PurchaseOrder::new).orElse(billing.getPurchaseOrder()); - var mergedInvoiceEmail = optional("invoiceEmail", inspector).map(mail -> new Email(mail, false)).orElse(billing.getInvoiceEmail()); - - var mergedBilling = info.billingContact() - .withContact(mergedContact) - .withAddress(mergedAddress) - .withTaxId(mergedTaxId) - .withPurchaseOrder(mergedPurchaseOrder) - .withInvoiceEmail(mergedInvoiceEmail); - - var mergedInfo = info.withBilling(mergedBilling); - - // Store changes - controller.tenants().lockOrThrow(cloudTenant.name(), LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withInfo(mergedInfo); - controller.tenants().store(lockedTenant); - }); - - return new MessageResponse("Tenant info updated"); - } - - private SlimeJsonResponse tenantInfoContacts(CloudTenant cloudTenant) { - var slime = new Slime(); - var root = slime.setObject(); - toSlime(cloudTenant.info().contacts(), root); - return new SlimeJsonResponse(slime); - } - - private SlimeJsonResponse putTenantInfoContacts(CloudTenant cloudTenant, Inspector inspector) { - var mergedInfo = cloudTenant.info() - .withContacts(updateTenantInfoContacts(inspector.field("contacts"), cloudTenant.name(), cloudTenant.info().contacts())); - - // Store changes - controller.tenants().lockOrThrow(cloudTenant.name(), LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withInfo(mergedInfo); - controller.tenants().store(lockedTenant); - }); - - return new MessageResponse("Tenant info updated"); - } - - private void validateMergedTenantInfo(TenantInfo mergedInfo) { - // Assert that we have a valid tenant info - if (mergedInfo.contact().name().isBlank()) { - throw new IllegalArgumentException("'contactName' cannot be empty"); - } - if (mergedInfo.contact().email().getEmailAddress().isBlank()) { - throw new IllegalArgumentException("'contactEmail' cannot be empty"); - } - if (! mergedInfo.contact().email().getEmailAddress().contains("@")) { - // email address validation is notoriously hard - we should probably just try to send a - // verification email to this address. checking for @ is a simple best-effort. - throw new IllegalArgumentException("'contactEmail' needs to be an email address"); - } - if (! mergedInfo.website().isBlank()) { - try { - new URL(mergedInfo.website()); - } catch (MalformedURLException e) { - throw new IllegalArgumentException("'website' needs to be a valid address"); - } - } - if (! mergedInfo.billingContact().getInvoiceEmail().isBlank()) { - // TODO: Validate invoice email is set if collection method is INVOICE - if (! mergedInfo.billingContact().getInvoiceEmail().getEmailAddress().contains("@")) - throw new IllegalArgumentException("'Invoice email' needs to be an email address"); - } - } - - private void toSlime(TenantAddress address, Cursor parentCursor) { - if (address.isEmpty()) return; - - Cursor addressCursor = parentCursor.setObject("address"); - addressCursor.setString("addressLines", address.address()); - addressCursor.setString("postalCodeOrZip", address.code()); - addressCursor.setString("city", address.city()); - addressCursor.setString("stateRegionProvince", address.region()); - addressCursor.setString("country", address.country()); - } - - private void toSlime(TenantBilling billingContact, Cursor parentCursor) { - if (billingContact.isEmpty()) return; - - Cursor billingCursor = parentCursor.setObject("billingContact"); - billingCursor.setString("name", billingContact.contact().name()); - billingCursor.setString("email", billingContact.contact().email().getEmailAddress()); - billingCursor.setBool("emailVerified", billingContact.contact().email().isVerified()); - billingCursor.setString("phone", billingContact.contact().phone()); - var taxIdCursor = billingCursor.setObject("taxId"); - taxIdCursor.setString("country", billingContact.getTaxId().country().value()); - taxIdCursor.setString("type", billingContact.getTaxId().type().value()); - taxIdCursor.setString("code", billingContact.getTaxId().code().value()); - billingCursor.setString("purchaseOrder", billingContact.getPurchaseOrder().value()); - billingCursor.setString("invoiceEmail", billingContact.getInvoiceEmail().getEmailAddress()); - toSlime(billingContact.address(), billingCursor); - var tosApprovalCursor = billingCursor.setObject("tosApproval"); - var tosApproval = billingContact.getToSApproval(); - tosApprovalCursor.setString("at", !tosApproval.isEmpty() ? tosApproval.approvedAt().toString() : ""); - tosApprovalCursor.setString("by", !tosApproval.isEmpty() ? tosApproval.approvedBy().get().getName() : ""); - } - - private void toSlime(TenantContacts contacts, Cursor parentCursor) { - Cursor contactsCursor = parentCursor.setArray("contacts"); - contacts.all().forEach(contact -> { - Cursor contactCursor = contactsCursor.addObject(); - Cursor audiencesArray = contactCursor.setArray("audiences"); - contact.audiences().forEach(audience -> audiencesArray.addString(toAudience(audience))); - switch (contact.type()) { - case EMAIL: - var email = (TenantContacts.EmailContact) contact; - contactCursor.setString("email", email.email().getEmailAddress()); - contactCursor.setBool("emailVerified", email.email().isVerified()); - return; - default: - throw new IllegalArgumentException("Serialization for contact type not implemented: " + contact.type()); - } - }); - } - - private static TenantContacts.Audience fromAudience(String value) { - return switch (value) { - case "tenant": yield TenantContacts.Audience.TENANT; - case "notifications": yield TenantContacts.Audience.NOTIFICATIONS; - default: throw new IllegalArgumentException("Unknown contact audience '" + value + "'."); - }; - } - - private static String toAudience(TenantContacts.Audience audience) { - return switch (audience) { - case TENANT: yield "tenant"; - case NOTIFICATIONS: yield "notifications"; - }; - } - - - private HttpResponse updateTenantInfo(String tenantName, HttpRequest request) { - return controller.tenants().get(TenantName.from(tenantName)) - .filter(tenant -> tenant.type() == Tenant.Type.cloud) - .map(tenant -> updateTenantInfo(((CloudTenant)tenant), request)) - .orElseGet(() -> ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist or does not support this")); - } - - private String getString(Inspector field, String defaultVale) { - var string = field.valid() ? field.asString().trim() : defaultVale; - if (string.length() > 512) throw new IllegalArgumentException("Input value too long"); - return string; - } - - private SlimeJsonResponse updateTenantInfo(CloudTenant tenant, HttpRequest request) { - TenantInfo oldInfo = tenant.info(); - - // Merge info from request with the existing info - Inspector insp = toSlime(request.getData()).get(); - - var mergedEmail = optional("contactEmail", insp) - .filter(address -> !address.equals(oldInfo.contact().email().getEmailAddress())) - .map(address -> { - controller.mailVerifier().sendMailVerification(tenant.name(), address, PendingMailVerification.MailType.TENANT_CONTACT); - return new Email(address, false); - }) - .orElse(oldInfo.contact().email()); - - TenantContact mergedContact = TenantContact.empty() - .withName(getString(insp.field("contactName"), oldInfo.contact().name())) - .withEmail(mergedEmail); - - TenantInfo mergedInfo = TenantInfo.empty() - .withName(getString(insp.field("name"), oldInfo.name())) - .withEmail(getString(insp.field("email"), oldInfo.email())) - .withWebsite(getString(insp.field("website"), oldInfo.website())) - .withContact(mergedContact) - .withAddress(updateTenantInfoAddress(insp.field("address"), oldInfo.address())) - .withBilling(updateTenantInfoBillingContact(insp.field("billingContact"), tenant.name(), oldInfo.billingContact())) - .withContacts(updateTenantInfoContacts(insp.field("contacts"), tenant.name(), oldInfo.contacts())); - - validateMergedTenantInfo(mergedInfo); - - // Store changes - controller.tenants().lockOrThrow(tenant.name(), LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withInfo(mergedInfo); - controller.tenants().store(lockedTenant); - }); - - return new MessageResponse("Tenant info updated"); - } - - private TenantAddress updateTenantInfoAddress(Inspector insp, TenantAddress oldAddress) { - if (!insp.valid()) return oldAddress; - TenantAddress address = TenantAddress.empty() - .withCountry(getString(insp.field("country"), oldAddress.country())) - .withRegion(getString(insp.field("stateRegionProvince"), oldAddress.region())) - .withCity(getString(insp.field("city"), oldAddress.city())) - .withCode(getString(insp.field("postalCodeOrZip"), oldAddress.code())) - .withAddress(getString(insp.field("addressLines"), oldAddress.address())); - - List<String> fields = List.of(address.address(), - address.code(), - address.country(), - address.city(), - address.region()); - - if (fields.stream().allMatch(String::isBlank) || fields.stream().noneMatch(String::isBlank)) - return address; - - throw new IllegalArgumentException("All address fields must be set"); - } - - private TaxId updateAndValidateTaxId(Inspector insp, TaxId old) { - if (!insp.valid()) return old; - var taxId = new TaxId( - getString(insp.field("country"), old.country().value()), - getString(insp.field("type"), old.type().value()), - getString(insp.field("code"), old.code().value())); - controller.serviceRegistry().billingController().validateTaxId(taxId); - return taxId; - } - - private TenantContact updateBillingContact(Inspector insp, TenantName tenantName, TenantContact oldContact) { - if (!insp.valid()) return oldContact; - - var mergedEmail = optional("email", insp) - .filter(address -> !address.equals(oldContact.email().getEmailAddress())) - .map(address -> { - controller.mailVerifier().sendMailVerification(tenantName, address, PendingMailVerification.MailType.BILLING); - return new Email(address, false); - }) - .orElse(oldContact.email()); - - return TenantContact.empty() - .withName(getString(insp.field("name"), oldContact.name())) - .withEmail(mergedEmail) - .withPhone(getString(insp.field("phone"), oldContact.phone())); - } - - private TenantBilling updateTenantInfoBillingContact(Inspector insp, TenantName tenantName, TenantBilling oldContact) { - if (!insp.valid()) return oldContact; - - var purchaseOrder = optional("purchaseOrder", insp).map(PurchaseOrder::new).orElse(oldContact.getPurchaseOrder()); - var invoiceEmail = optional("invoiceEmail", insp).map(mail -> new Email(mail, false)).orElse(oldContact.getInvoiceEmail()); - return TenantBilling.empty() - .withContact(updateBillingContact(insp, tenantName, oldContact.contact())) - .withAddress(updateTenantInfoAddress(insp.field("address"), oldContact.address())) - .withTaxId(updateAndValidateTaxId(insp.field("taxId"), oldContact.getTaxId())) - .withPurchaseOrder(purchaseOrder) - .withInvoiceEmail(invoiceEmail); - } - - private TenantContacts updateTenantInfoContacts(Inspector insp, TenantName tenantName, TenantContacts oldContacts) { - if (!insp.valid()) return oldContacts; - - List<TenantContacts.EmailContact> contacts = SlimeUtils.entriesStream(insp).map(inspector -> { - String email = inspector.field("email").asString().trim(); - List<TenantContacts.Audience> audiences = SlimeUtils.entriesStream(inspector.field("audiences")) - .map(audience -> fromAudience(audience.asString())) - .toList(); - - // If contact exists, update audience. Otherwise, create new unverified contact - return oldContacts.ofType(TenantContacts.EmailContact.class) - .stream() - .filter(contact -> contact.email().getEmailAddress().equals(email)) - .findAny() - .map(emailContact -> new TenantContacts.EmailContact(audiences, emailContact.email())) - .orElseGet(() -> { - controller.mailVerifier().sendMailVerification(tenantName, email, PendingMailVerification.MailType.NOTIFICATIONS); - return new TenantContacts.EmailContact(audiences, new Email(email, false)); - }); - }).toList(); - - return new TenantContacts(contacts); - } - - private HttpResponse notifications(HttpRequest request, Optional<String> tenant, boolean includeTenantFieldInResponse) { - boolean productionOnly = showOnlyProductionInstances(request); - boolean excludeMessages = "true".equals(request.getProperty("excludeMessages")); - Slime slime = new Slime(); - Cursor notificationsArray = slime.setObject().setArray("notifications"); - - tenant.map(t -> Stream.of(TenantName.from(t))) - .orElseGet(() -> controller.notificationsDb().listTenantsWithNotifications().stream()) - .flatMap(tenantName -> controller.notificationsDb().listNotifications(NotificationSource.from(tenantName), productionOnly).stream()) - .filter(notification -> - propertyEquals(request, "application", ApplicationName::from, notification.source().application()) && - propertyEquals(request, "instance", InstanceName::from, notification.source().instance()) && - propertyEquals(request, "zone", ZoneId::from, notification.source().zoneId()) && - propertyEquals(request, "job", job -> JobType.fromJobName(job, controller.zoneRegistry()), notification.source().jobType()) && - propertyEquals(request, "type", Notification.Type::valueOf, Optional.of(notification.type())) && - propertyEquals(request, "level", Notification.Level::valueOf, Optional.of(notification.level()))) - .forEach(notification -> toSlime(notificationsArray.addObject(), notification, includeTenantFieldInResponse, excludeMessages)); - return new SlimeJsonResponse(slime); - } - - private HttpResponse listTokens(String tenant, HttpRequest request) { - Slime slime = new Slime(); - Cursor tokensArray = slime.setObject().setArray("tokens"); - controller.dataplaneTokenService().listTokensWithState(TenantName.from(tenant)).forEach((token, states) -> { - Cursor tokenObject = tokensArray.addObject(); - tokenObject.setString("id", token.tokenId().value()); - tokenObject.setLong("lastUpdatedMillis", token.lastUpdated().toEpochMilli()); - Cursor fingerprintsArray = tokenObject.setArray("versions"); - for (var tokenVersion : token.tokenVersions()) { - Cursor fingerprintObject = fingerprintsArray.addObject(); - fingerprintObject.setString("fingerprint", tokenVersion.fingerPrint().value()); - fingerprintObject.setString("created", tokenVersion.creationTime().toString()); - fingerprintObject.setString("author", tokenVersion.author()); - fingerprintObject.setString("expiration", tokenVersion.expiration().map(Instant::toString).orElse("none")); - String tokenState = tokenVersion.expiration().map(controller.clock().instant()::isAfter).orElse(false) - ? "expired" - : valueOf(states.get(tokenVersion.fingerPrint())); - fingerprintObject.setString("state", tokenState); - } - states.forEach((print, state) -> { - if (state != State.REVOKING) return; - Cursor fingerprintObject = fingerprintsArray.addObject(); - fingerprintObject.setString("fingerprint", print.value()); - fingerprintObject.setString("state", valueOf(state)); - }); - }); - return new SlimeJsonResponse(slime); - } - - private static String valueOf(DataplaneTokenService.State state) { - return switch (state) { - case UNUSED: yield "unused"; - case DEPLOYING: yield "deploying"; - case ACTIVE: yield "active"; - case REVOKING: yield "revoking"; - }; - } - - - private HttpResponse generateToken(String tenant, String tokenid, HttpRequest request) { - var expiration = resolveExpiration(request).orElse(null); - DataplaneToken token = controller.dataplaneTokenService().generateToken( - TenantName.from(tenant), TokenId.of(tokenid), expiration, request.getJDiscRequest().getUserPrincipal()); - Slime slime = new Slime(); - Cursor tokenObject = slime.setObject(); - tokenObject.setString("id", token.tokenId().value()); - tokenObject.setString("token", token.tokenValue()); - tokenObject.setString("fingerprint", token.fingerPrint().value()); - tokenObject.setString("expiration", token.expiration().map(Instant::toString).orElse("none")); - return new SlimeJsonResponse(slime); - } - - /** - * Specify 'expiration=none' for no expiration, no parameter or 'expiration=default' for default TTL. - * Use ISO-8601 format for timestamp or period, - * e.g 'expiration=PT1H' for 1 hour, 'expiration=2021-01-01T12:00:00Z' for a specific time. - */ - private Optional<Instant> resolveExpiration(HttpRequest r) { - var expirationParam = r.getProperty("expiration"); - var now = controller.clock().instant(); - if (expirationParam == null || expirationParam.equals("default")) - return Optional.of(now.plus(DataplaneTokenService.DEFAULT_TTL)); - if (expirationParam.equals("none")) return Optional.empty(); - return expirationParam.startsWith("P") - ? Optional.of(now.plus(Duration.parse(expirationParam))) - : Optional.of(Instant.parse(expirationParam)); - } - - private HttpResponse deleteToken(String tenant, String tokenid, HttpRequest request) { - String fingerprint = Optional.ofNullable(request.getProperty("fingerprint")).orElseThrow(() -> new IllegalArgumentException("Cannot delete token without fingerprint")); - controller.dataplaneTokenService().deleteToken(TenantName.from(tenant), TokenId.of(tokenid), FingerPrint.of(fingerprint)); - return new MessageResponse("Token version deleted"); - } - - private static <T> boolean propertyEquals(HttpRequest request, String property, Function<String, T> mapper, Optional<T> value) { - return Optional.ofNullable(request.getProperty(property)) - .map(propertyValue -> value.isPresent() && mapper.apply(propertyValue).equals(value.get())) - .orElse(true); - } - - private static void toSlime(Cursor cursor, Notification notification, boolean includeTenantFieldInResponse, boolean excludeMessages) { - cursor.setLong("at", notification.at().toEpochMilli()); - cursor.setString("level", notificationLevelAsString(notification.level())); - cursor.setString("type", notificationTypeAsString(notification.type())); - if (!excludeMessages) { - cursor.setString("title", notification.title()); - Cursor messagesArray = cursor.setArray("messages"); - notification.messages().forEach(messagesArray::addString); - } - - if (includeTenantFieldInResponse) cursor.setString("tenant", notification.source().tenant().value()); - notification.source().application().ifPresent(application -> cursor.setString("application", application.value())); - notification.source().instance().ifPresent(instance -> cursor.setString("instance", instance.value())); - notification.source().zoneId().ifPresent(zoneId -> { - cursor.setString("environment", zoneId.environment().value()); - cursor.setString("region", zoneId.region().value()); - }); - notification.source().clusterId().ifPresent(clusterId -> cursor.setString("clusterId", clusterId.value())); - notification.source().jobType().ifPresent(jobType -> cursor.setString("jobName", jobType.jobName())); - notification.source().runNumber().ifPresent(runNumber -> cursor.setLong("runNumber", runNumber)); - } - - private static String notificationTypeAsString(Notification.Type type) { - return switch (type) { - case submission, applicationPackage: yield "applicationPackage"; - case testPackage: yield "testPackage"; - case deployment: yield "deployment"; - case feedBlock: yield "feedBlock"; - case reindex: yield "reindex"; - case account: yield "account"; - }; - } - - private static String notificationLevelAsString(Notification.Level level) { - return switch (level) { - case info: yield "info"; - case warning: yield "warning"; - case error: yield "error"; - }; - } - - private HttpResponse applications(String tenantName, Optional<String> applicationName, HttpRequest request) { - TenantName tenant = TenantName.from(tenantName); - getTenantOrThrow(tenantName); - - List<Application> applications = applicationName.isEmpty() ? - controller.applications().asList(tenant) : - controller.applications().getApplication(TenantAndApplicationId.from(tenantName, applicationName.get())) - .map(List::of) - .orElseThrow(() -> new NotExistsException("Application '" + applicationName.get() + "' does not exist")); - - Slime slime = new Slime(); - Cursor applicationArray = slime.setArray(); - for (Application application : applications) { - Cursor applicationObject = applicationArray.addObject(); - applicationObject.setString("tenant", application.id().tenant().value()); - applicationObject.setString("application", application.id().application().value()); - applicationObject.setString("url", withPath("/application/v4" + - "/tenant/" + application.id().tenant().value() + - "/application/" + application.id().application().value(), - request.getUri()).toString()); - Cursor instanceArray = applicationObject.setArray("instances"); - for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet() - : application.instances().keySet()) { - Cursor instanceObject = instanceArray.addObject(); - instanceObject.setString("instance", instance.value()); - instanceObject.setString("url", withPath("/application/v4" + - "/tenant/" + application.id().tenant().value() + - "/application/" + application.id().application().value() + - "/instance/" + instance.value(), - request.getUri()).toString()); - } - } - return new SlimeJsonResponse(slime); - } - - private HttpResponse devApplicationPackage(ApplicationId id, JobType type) { - ZoneId zone = type.zone(); - RevisionId revision = controller.jobController().last(id, type).get().versions().targetRevision(); - return new ZipResponse(id.toFullString() + "." + zone.value() + ".zip", - controller.applications().applicationStore().stream(new DeploymentId(id, zone), revision)); - } - - private HttpResponse devApplicationPackageDiff(RunId runId) { - DeploymentId deploymentId = new DeploymentId(runId.application(), runId.job().type().zone()); - return controller.applications().applicationStore().getDevDiff(deploymentId, runId.number()) - .map(ByteArrayResponse::new) - .orElseThrow(() -> new NotExistsException("No application package diff found for " + runId)); - } - - private HttpResponse applicationPackage(String tenantName, String applicationName, HttpRequest request) { - TenantAndApplicationId tenantAndApplication = TenantAndApplicationId.from(tenantName, applicationName); - final long build; - String requestedBuild = request.getProperty("build"); - if (requestedBuild != null) { - if (requestedBuild.equals("latestDeployed")) { - build = controller.applications().requireApplication(tenantAndApplication).latestDeployedRevision() - .map(RevisionId::number) - .orElseThrow(() -> new NotExistsException("no application package has been deployed in production for " + tenantAndApplication)); - } else { - try { - build = Validation.requireAtLeast(Long.parseLong(request.getProperty("build")), "build number", 1L); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("invalid value for request parameter 'build'", e); - } - } - } else { - build = controller.applications().requireApplication(tenantAndApplication).revisions().last() - .map(version -> version.id().number()) - .orElseThrow(() -> new NotExistsException("no application package has been submitted for " + tenantAndApplication)); - } - RevisionId revision = RevisionId.forProduction(build); - boolean tests = request.getBooleanProperty("tests"); - String filename = tenantAndApplication + (tests ? "-tests" : "-build") + revision.number() + ".zip"; - InputStream applicationPackage = tests ? - controller.applications().applicationStore().streamTester(tenantAndApplication.tenant(), tenantAndApplication.application(), revision) : - controller.applications().applicationStore().stream(new DeploymentId(tenantAndApplication.defaultInstance(), ZoneId.defaultId()), revision); - return new ZipResponse(filename, applicationPackage); - } - - private HttpResponse applicationPackageDiff(String tenant, String application, String number) { - TenantAndApplicationId tenantAndApplication = TenantAndApplicationId.from(tenant, application); - return controller.applications().applicationStore().getDiff(tenantAndApplication.tenant(), tenantAndApplication.application(), Long.parseLong(number)) - .map(ByteArrayResponse::new) - .orElseThrow(() -> new NotExistsException("No application package diff found for '" + tenantAndApplication + "' with build number " + number)); - } - - private HttpResponse application(String tenantName, String applicationName, HttpRequest request) { - Slime slime = new Slime(); - toSlime(slime.setObject(), getApplication(tenantName, applicationName), request); - return new SlimeJsonResponse(slime); - } - - private HttpResponse compileVersion(String tenantName, String applicationName, String allowMajorParam) { - Slime slime = new Slime(); - OptionalInt allowMajor = OptionalInt.empty(); - if (allowMajorParam != null) { - try { - allowMajor = OptionalInt.of(Integer.parseInt(allowMajorParam)); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Invalid major version '" + allowMajorParam + "'", e); - } - } - Version compileVersion = controller.applications().compileVersion(TenantAndApplicationId.from(tenantName, applicationName), allowMajor); - slime.setObject().setString("compileVersion", compileVersion.toFullString()); - return new SlimeJsonResponse(slime); - } - - private HttpResponse instance(String tenantName, String applicationName, String instanceName, HttpRequest request) { - Slime slime = new Slime(); - toSlime(slime.setObject(), getInstance(tenantName, applicationName, instanceName), - controller.jobController().deploymentStatus(getApplication(tenantName, applicationName)), request); - return new SlimeJsonResponse(slime); - } - - private HttpResponse approveTermsOfService(String tenant, HttpRequest req) { - if (controller.tenants().require(TenantName.from(tenant)).type() != Tenant.Type.cloud) - throw new IllegalArgumentException("Tenant '" + tenant + "' is not a cloud tenant"); - var approvedBy = SimplePrincipal.of(req.getJDiscRequest().getUserPrincipal()); - var approvedAt = controller.clock().instant(); - - controller.tenants().lockOrThrow(TenantName.from(tenant), LockedTenant.Cloud.class, t -> { - var updatedTenant = t.withInfo(t.get().info().withBilling(t.get().info().billingContact().withToSApproval( - new TermsOfServiceApproval(approvedAt, approvedBy)))); - controller.tenants().store(updatedTenant); - }); - return new MessageResponse("Terms of service approved by %s".formatted(approvedBy.getName())); - } - - private HttpResponse unapproveTermsOfService(String tenant, HttpRequest req) { - if (controller.tenants().require(TenantName.from(tenant)).type() != Tenant.Type.cloud) - throw new IllegalArgumentException("Tenant '" + tenant + "' is not a cloud tenant"); - controller.tenants().lockOrThrow(TenantName.from(tenant), LockedTenant.Cloud.class, t -> { - var updatedTenant = t.withInfo(t.get().info().withBilling(t.get().info().billingContact().withToSApproval( - TermsOfServiceApproval.empty()))); - controller.tenants().store(updatedTenant); - }); - return new MessageResponse("Terms of service approval removed"); - } - - private HttpResponse addDeveloperKey(String tenantName, HttpRequest request) { - if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) - throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); - - Principal user = request.getJDiscRequest().getUserPrincipal(); - String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString(); - PublicKey developerKey = KeyUtils.fromPemEncodedPublicKey(pemDeveloperKey); - Slime root = new Slime(); - controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> { - tenant = tenant.withDeveloperKey(developerKey, user); - toSlime(root.setObject().setArray("keys"), tenant.get().developerKeys()); - controller.tenants().store(tenant); - }); - return new SlimeJsonResponse(root); - } - - private HttpResponse validateSecretStore(String tenantName, String secretStoreName, HttpRequest request) { - var awsRegion = request.getProperty("aws-region"); - var parameterName = request.getProperty("parameter-name"); - var applicationId = ApplicationId.fromFullString(request.getProperty("application-id")); - if (!applicationId.tenant().equals(TenantName.from(tenantName))) - return ErrorResponse.badRequest("Invalid application id"); - var zoneId = requireZone(ZoneId.from(request.getProperty("zone"))); - var deploymentId = new DeploymentId(applicationId, zoneId); - - var tenant = controller.tenants().require(applicationId.tenant(), CloudTenant.class); - - var tenantSecretStore = tenant.tenantSecretStores() - .stream() - .filter(secretStore -> secretStore.getName().equals(secretStoreName)) - .findFirst(); - - if (tenantSecretStore.isEmpty()) - return ErrorResponse.notFoundError("No secret store '" + secretStoreName + "' configured for tenant '" + tenantName + "'"); - - var response = controller.serviceRegistry().configServer().validateSecretStore(deploymentId, tenantSecretStore.get(), awsRegion, parameterName); - try { - var responseRoot = new Slime(); - var responseCursor = responseRoot.setObject(); - responseCursor.setString("target", deploymentId.toString()); - var responseResultCursor = responseCursor.setObject("result"); - var responseSlime = SlimeUtils.jsonToSlime(response); - SlimeUtils.copyObject(responseSlime.get(), responseResultCursor); - return new SlimeJsonResponse(responseRoot); - } catch (JsonParseException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse removeDeveloperKey(String tenantName, HttpRequest request) { - if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) - throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); - - String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString(); - PublicKey developerKey = KeyUtils.fromPemEncodedPublicKey(pemDeveloperKey); - Principal user = controller.tenants().require(TenantName.from(tenantName), CloudTenant.class).developerKeys().get(developerKey); - Slime root = new Slime(); - controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> { - tenant = tenant.withoutDeveloperKey(developerKey); - toSlime(root.setObject().setArray("keys"), tenant.get().developerKeys()); - controller.tenants().store(tenant); - }); - return new SlimeJsonResponse(root); - } - - private void toSlime(Cursor keysArray, Map<PublicKey, ? extends Principal> keys) { - keys.forEach((key, principal) -> { - Cursor keyObject = keysArray.addObject(); - keyObject.setString("key", KeyUtils.toPem(key)); - keyObject.setString("user", principal.getName()); - }); - } - - private HttpResponse addDeployKey(String tenantName, String applicationName, HttpRequest request) { - String pemDeployKey = toSlime(request.getData()).get().field("key").asString(); - PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey); - Slime root = new Slime(); - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> { - application = application.withDeployKey(deployKey); - application.get().deployKeys().stream() - .map(KeyUtils::toPem) - .forEach(root.setObject().setArray("keys")::addString); - controller.applications().store(application); - }); - return new SlimeJsonResponse(root); - } - - private HttpResponse removeDeployKey(String tenantName, String applicationName, HttpRequest request) { - String pemDeployKey = toSlime(request.getData()).get().field("key").asString(); - PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey); - Slime root = new Slime(); - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> { - application = application.withoutDeployKey(deployKey); - application.get().deployKeys().stream() - .map(KeyUtils::toPem) - .forEach(root.setObject().setArray("keys")::addString); - controller.applications().store(application); - }); - return new SlimeJsonResponse(root); - } - - private HttpResponse addSecretStore(String tenantName, String name, HttpRequest request) { - if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) - throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); - - var data = toSlime(request.getData()).get(); - var awsId = mandatory("awsId", data).asString(); - var externalId = mandatory("externalId", data).asString(); - var role = mandatory("role", data).asString(); - - var tenant = controller.tenants().require(TenantName.from(tenantName), CloudTenant.class); - var tenantSecretStore = new TenantSecretStore(name, awsId, role); - - if (!tenantSecretStore.isValid()) { - return ErrorResponse.badRequest("Secret store " + tenantSecretStore + " is invalid"); - } - if (tenant.tenantSecretStores().contains(tenantSecretStore)) { - return ErrorResponse.badRequest("Secret store " + tenantSecretStore + " is already configured"); - } - - controller.serviceRegistry().roleService().createTenantPolicy(TenantName.from(tenantName), name, awsId, role); - controller.serviceRegistry().tenantSecretService().addSecretStore(tenant.name(), tenantSecretStore, externalId); - // Store changes - controller.tenants().lockOrThrow(tenant.name(), LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withSecretStore(tenantSecretStore); - controller.tenants().store(lockedTenant); - }); - - tenant = controller.tenants().require(TenantName.from(tenantName), CloudTenant.class); - var slime = new Slime(); - toSlime(slime.setObject(), tenant.tenantSecretStores()); - return new SlimeJsonResponse(slime); - } - - private HttpResponse deleteSecretStore(String tenantName, String name, HttpRequest request) { - var tenant = controller.tenants().require(TenantName.from(tenantName), CloudTenant.class); - - var optionalSecretStore = tenant.tenantSecretStores().stream() - .filter(secretStore -> secretStore.getName().equals(name)) - .findFirst(); - - if (optionalSecretStore.isEmpty()) - return ErrorResponse.notFoundError("Could not delete secret store '" + name + "': Secret store not found"); - - var tenantSecretStore = optionalSecretStore.get(); - controller.serviceRegistry().tenantSecretService().deleteSecretStore(tenant.name(), tenantSecretStore); - controller.serviceRegistry().roleService().deleteTenantPolicy(tenant.name(), tenantSecretStore.getName(), tenantSecretStore.getRole()); - controller.tenants().lockOrThrow(tenant.name(), LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withoutSecretStore(tenantSecretStore); - controller.tenants().store(lockedTenant); - }); - - tenant = controller.tenants().require(TenantName.from(tenantName), CloudTenant.class); - var slime = new Slime(); - toSlime(slime.setObject(), tenant.tenantSecretStores()); - return new SlimeJsonResponse(slime); - } - - private HttpResponse allowAwsArchiveAccess(String tenantName, HttpRequest request) { - if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) - throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); - - var data = toSlime(request.getData()).get(); - var role = mandatory("role", data).asString(); - - if (role.isBlank()) { - return ErrorResponse.badRequest("AWS archive access role can't be whitespace only"); - } - - controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, lockedTenant -> { - var access = lockedTenant.get().archiveAccess(); - lockedTenant = lockedTenant.withArchiveAccess(access.withAWSRole(role)); - controller.tenants().store(lockedTenant); - }); - - return new MessageResponse("AWS archive access role set to '" + role + "' for tenant " + tenantName + "."); - } - - private HttpResponse removeAwsArchiveAccess(String tenantName) { - if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) - throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); - - controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, lockedTenant -> { - var access = lockedTenant.get().archiveAccess(); - lockedTenant = lockedTenant.withArchiveAccess(access.removeAWSRole()); - controller.tenants().store(lockedTenant); - }); - - return new MessageResponse("AWS archive access role removed for tenant " + tenantName + "."); - } - - private HttpResponse allowGcpArchiveAccess(String tenantName, HttpRequest request) { - if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) - throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); - - var data = toSlime(request.getData()).get(); - var member = mandatory("member", data).asString(); - - if (member.isBlank()) { - return ErrorResponse.badRequest("GCP archive access role can't be whitespace only"); - } - - controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, lockedTenant -> { - var access = lockedTenant.get().archiveAccess(); - lockedTenant = lockedTenant.withArchiveAccess(access.withGCPMember(member)); - controller.tenants().store(lockedTenant); - }); - - return new MessageResponse("GCP archive access member set to '" + member + "' for tenant " + tenantName + "."); - } - - private HttpResponse removeGcpArchiveAccess(String tenantName) { - if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) - throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); - - controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, lockedTenant -> { - var access = lockedTenant.get().archiveAccess(); - lockedTenant = lockedTenant.withArchiveAccess(access.removeGCPMember()); - controller.tenants().store(lockedTenant); - }); - - return new MessageResponse("GCP archive access member removed for tenant " + tenantName + "."); - } - - private HttpResponse patchApplication(String tenantName, String applicationName, HttpRequest request) { - Inspector requestObject = toSlime(request.getData()).get(); - StringJoiner messageBuilder = new StringJoiner("\n").setEmptyValue("No applicable changes."); - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> { - Inspector majorVersionField = requestObject.field("majorVersion"); - if (majorVersionField.valid()) { - Integer majorVersion = majorVersionField.asLong() == 0 ? null : (int) majorVersionField.asLong(); - application = application.withMajorVersion(majorVersion); - messageBuilder.add("Set major version to " + (majorVersion == null ? "empty" : majorVersion)); - } - - // TODO jonmv: Remove when clients are updated. - Inspector pemDeployKeyField = requestObject.field("pemDeployKey"); - if (pemDeployKeyField.valid()) { - String pemDeployKey = pemDeployKeyField.asString(); - PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey); - application = application.withDeployKey(deployKey); - messageBuilder.add("Added deploy key " + pemDeployKey); - } - - controller.applications().store(application); - }); - return new MessageResponse(messageBuilder.toString()); - } - - private Application getApplication(String tenantName, String applicationName) { - TenantAndApplicationId applicationId = TenantAndApplicationId.from(tenantName, applicationName); - return controller.applications().getApplication(applicationId) - .orElseThrow(() -> new NotExistsException(applicationId + " not found")); - } - - private Instance getInstance(String tenantName, String applicationName, String instanceName) { - ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); - return controller.applications().getInstance(applicationId) - .orElseThrow(() -> new NotExistsException(applicationId + " not found")); - } - - private HttpResponse nodes(String tenantName, String applicationName, String instanceName, String environment, String region) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = requireZone(environment, region); - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(zone, NodeFilter.all().applications(id)); - - Slime slime = new Slime(); - Cursor nodesArray = slime.setObject().setArray("nodes"); - for (Node node : nodes) { - Cursor nodeObject = nodesArray.addObject(); - nodeObject.setString("hostname", node.hostname().value()); - nodeObject.setString("state", valueOf(node.state())); - node.reservedTo().ifPresent(tenant -> nodeObject.setString("reservedTo", tenant.value())); - nodeObject.setString("orchestration", valueOf(node.serviceState())); - nodeObject.setString("version", node.currentVersion().toString()); - node.flavor().ifPresent(flavor -> nodeObject.setString("flavor", flavor)); - toSlime(node.resources(), nodeObject); - nodeObject.setString("clusterId", node.clusterId()); - nodeObject.setString("clusterType", valueOf(node.clusterType())); - nodeObject.setBool("down", node.down()); - nodeObject.setBool("retired", node.retired() || node.wantToRetire()); - nodeObject.setBool("restarting", node.wantedRestartGeneration() > node.restartGeneration()); - nodeObject.setBool("rebooting", node.wantedRebootGeneration() > node.rebootGeneration()); - nodeObject.setString("group", node.group()); - nodeObject.setLong("index", node.index()); - } - return new SlimeJsonResponse(slime); - } - - private HttpResponse clusters(String tenantName, String applicationName, String instanceName, String environment, String region) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = requireZone(environment, region); - com.yahoo.vespa.hosted.controller.api.integration.configserver.Application application = controller.serviceRegistry().configServer().nodeRepository().getApplication(zone, id); - - Slime slime = new Slime(); - Cursor clustersObject = slime.setObject().setObject("clusters"); - for (Cluster cluster : application.clusters().values()) { - Cursor clusterObject = clustersObject.setObject(cluster.id().value()); - clusterObject.setString("type", cluster.type().name()); - toSlime(cluster.min(), clusterObject.setObject("min")); - toSlime(cluster.max(), clusterObject.setObject("max")); - if ( ! cluster.groupSize().isEmpty()) - toSlime(cluster.groupSize(), clusterObject.setObject("groupSize")); - toSlime(cluster.current(), clusterObject.setObject("current")); - toSlime(cluster.target(), clusterObject.setObject("target")); - toSlime(cluster.suggested(), clusterObject.setObject("suggested")); - scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); - clusterObject.setLong("scalingDuration", cluster.scalingDuration().toMillis()); - } - return new SlimeJsonResponse(slime); - } - - private static String valueOf(Node.State state) { - return switch (state) { - case failed: yield "failed"; - case parked: yield "parked"; - case dirty: yield "dirty"; - case ready: yield "ready"; - case active: yield "active"; - case inactive: yield "inactive"; - case reserved: yield "reserved"; - case provisioned: yield "provisioned"; - case breakfixed: yield "breakfixed"; - case deprovisioned: yield "deprovisioned"; - default: throw new IllegalArgumentException("Unexpected node state '" + state + "'."); - }; - } - - static String valueOf(Node.ServiceState state) { - switch (state) { - case expectedUp: return "expectedUp"; - case allowedDown: return "allowedDown"; - case permanentlyDown: return "permanentlyDown"; - case unorchestrated: return "unorchestrated"; - case unknown: break; - } - - return "unknown"; - } - - private static String valueOf(Node.ClusterType type) { - return switch (type) { - case admin: yield "admin"; - case content: yield "content"; - case container: yield "container"; - case combined: yield "combined"; - case unknown: throw new IllegalArgumentException("Unexpected node cluster type '" + type + "'."); - }; - } - - private static String valueOf(NodeResources.DiskSpeed diskSpeed) { - return switch (diskSpeed) { - case fast : yield "fast"; - case slow : yield "slow"; - case any : yield "any"; - }; - } - - private static String valueOf(NodeResources.StorageType storageType) { - return switch (storageType) { - case remote : yield "remote"; - case local : yield "local"; - case any : yield "any"; - }; - } - - private static String valueOf(NodeResources.Architecture architecture) { - return switch (architecture) { - case x86_64 : yield "x86_64"; - case arm64 : yield "arm64"; - case any : yield "any"; - }; - } - - private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region, Map<String, String> queryParameters) { - ApplicationId application = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = requireZone(environment, region); - DeploymentId deployment = new DeploymentId(application, zone); - InputStream logStream = controller.serviceRegistry().configServer().getLogs(deployment, queryParameters); - return new HttpResponse(200) { - @Override - public void render(OutputStream outputStream) throws IOException { - try (logStream) { - logStream.transferTo(outputStream); - } - } - @Override - public long maxPendingBytes() { - return 1 << 26; - } - }; - } - - private HttpResponse supportAccess(String tenantName, String applicationName, String instanceName, String environment, String region, Map<String, String> queryParameters) { - DeploymentId deployment = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - SupportAccess supportAccess = controller.supportAccess().forDeployment(deployment); - return new SlimeJsonResponse(SupportAccessSerializer.serializeCurrentState(supportAccess, controller.clock().instant())); - } - - // TODO support access: only let tenants (not operators!) allow access - // TODO support access: configurable period of access? - private HttpResponse allowSupportAccess(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - DeploymentId deployment = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - Principal principal = requireUserPrincipal(request); - Instant now = controller.clock().instant(); - SupportAccess allowed = controller.supportAccess().allow(deployment, now.plus(7, ChronoUnit.DAYS), principal.getName()); - return new SlimeJsonResponse(SupportAccessSerializer.serializeCurrentState(allowed, now)); - } - - private HttpResponse disallowSupportAccess(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - DeploymentId deployment = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - Principal principal = requireUserPrincipal(request); - SupportAccess disallowed = controller.supportAccess().disallow(deployment, principal.getName()); - controller.applications().deploymentTrigger().reTriggerOrAddToQueue(deployment, "re-triggered to disallow support access, by " + request.getJDiscRequest().getUserPrincipal().getName()); - return new SlimeJsonResponse(SupportAccessSerializer.serializeCurrentState(disallowed, controller.clock().instant())); - } - - private HttpResponse searchNodeMetrics(String tenantName, String applicationName, String instanceName, String environment, String region) { - ApplicationId application = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = requireZone(environment, region); - DeploymentId deployment = new DeploymentId(application, zone); - List<SearchNodeMetrics> searchNodeMetrics = controller.serviceRegistry().configServer().getSearchNodeMetrics(deployment); - return buildResponseFromSearchNodeMetrics(searchNodeMetrics); - } - - private HttpResponse scaling(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - var from = Optional.ofNullable(request.getProperty("from")) - .map(Long::valueOf) - .map(Instant::ofEpochSecond) - .orElse(Instant.EPOCH); - var until = Optional.ofNullable(request.getProperty("until")) - .map(Long::valueOf) - .map(Instant::ofEpochSecond) - .orElse(Instant.now(controller.clock())); - - var application = ApplicationId.from(tenantName, applicationName, instanceName); - var zone = requireZone(environment, region); - var deployment = new DeploymentId(application, zone); - var events = controller.serviceRegistry().resourceDatabase().scalingEvents(from, until, deployment); - var slime = new Slime(); - var root = slime.setObject(); - for (var entry : events.entrySet()) { - var serviceRoot = root.setArray(entry.getKey().clusterId().value()); - scalingEventsToSlime(entry.getValue(), serviceRoot); - } - return new SlimeJsonResponse(slime); - } - - private JsonResponse buildResponseFromSearchNodeMetrics(List<SearchNodeMetrics> searchnodeMetrics) { - try { - var jsonObject = jsonMapper.createObjectNode(); - var jsonArray = jsonMapper.createArrayNode(); - for (SearchNodeMetrics metrics : searchnodeMetrics) { - jsonArray.add(metrics.toJson()); - } - jsonObject.set("metrics", jsonArray); - return new JsonResponse(200, jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonObject)); - } catch (JsonProcessingException e) { - log.log(Level.WARNING, "Unable to build JsonResponse with Proton data: " + e.getMessage(), e); - return new JsonResponse(500, ""); - } - } - - - - private HttpResponse trigger(ApplicationId id, JobType type, HttpRequest request) { - Inspector requestObject = toSlime(request.getData()).get(); - boolean requireTests = ! requestObject.field("skipTests").asBool(); - boolean reTrigger = requestObject.field("reTrigger").asBool(); - boolean upgradeRevision = ! requestObject.field("skipRevision").asBool(); - boolean upgradePlatform = ! requestObject.field("skipUpgrade").asBool(); - String triggered = reTrigger - ? controller.applications().deploymentTrigger() - .reTrigger(id, type, "re-triggered by " + request.getJDiscRequest().getUserPrincipal().getName()).type().jobName() - : controller.applications().deploymentTrigger() - .forceTrigger(id, type, "triggered by " + request.getJDiscRequest().getUserPrincipal().getName(), requireTests, upgradeRevision, upgradePlatform) - .stream().map(job -> job.type().jobName()).collect(joining(", ")); - String suppressedUpgrades = ( ! upgradeRevision || ! upgradePlatform ? ", without " : "") + - (upgradeRevision ? "" : "revision") + - ( ! upgradeRevision && ! upgradePlatform ? " and " : "") + - (upgradePlatform ? "" : "platform") + - ( ! upgradeRevision || ! upgradePlatform ? " upgrade" : ""); - return new MessageResponse(triggered.isEmpty() ? "Job " + type.jobName() + " for " + id + " not triggered" - : "Triggered " + triggered + " for " + id + suppressedUpgrades); - } - - private HttpResponse pause(ApplicationId id, JobType type) { - Instant until = controller.clock().instant().plus(DeploymentTrigger.maxPause); - controller.applications().deploymentTrigger().pauseJob(id, type, until); - return new MessageResponse(type.jobName() + " for " + id + " paused for " + DeploymentTrigger.maxPause); - } - - private HttpResponse resume(ApplicationId id, JobType type) { - controller.applications().deploymentTrigger().resumeJob(id, type); - return new MessageResponse(type.jobName() + " for " + id + " resumed"); - } - - private SlimeJsonResponse resendEmailVerification(CloudTenant tenant, Inspector inspector) { - var mail = mandatory("mail", inspector).asString(); - var type = mandatory("mailType", inspector).asString(); - - var mailType = switch (type) { - case "contact" -> PendingMailVerification.MailType.TENANT_CONTACT; - case "notifications" -> PendingMailVerification.MailType.NOTIFICATIONS; - case "billing" -> PendingMailVerification.MailType.BILLING; - default -> throw new IllegalArgumentException("Unknown mail type " + type); - }; - - var pendingVerification = controller.mailVerifier().resendMailVerification(tenant.name(), mail, mailType); - return pendingVerification.isPresent() ? new MessageResponse("Re-sent verification mail to " + mail) : - ErrorResponse.notFoundError("No pending mail verification found for " + mail); - } - - private void toSlime(Cursor object, Application application, HttpRequest request) { - object.setString("tenant", application.id().tenant().value()); - object.setString("application", application.id().application().value()); - object.setString("deployments", withPath("/application/v4" + - "/tenant/" + application.id().tenant().value() + - "/application/" + application.id().application().value() + - "/job/", - request.getUri()).toString()); - - DeploymentStatus status = controller.jobController().deploymentStatus(application); - application.revisions().last().ifPresent(version -> JobControllerApiHandlerHelper.toSlime(object.setObject("latestVersion"), version)); - - application.projectId().ifPresent(id -> object.setLong("projectId", id)); - - // TODO jonmv: Remove this when users are updated. - application.instances().values().stream().findFirst().ifPresent(instance -> { - // Currently deploying change - if ( ! instance.change().isEmpty()) - toSlime(object.setObject("deploying"), instance.change(), application); - - // Outstanding change - if ( ! status.outstandingChange(instance.name()).isEmpty()) - toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name()), application); - }); - - application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion)); - - Cursor instancesArray = object.setArray("instances"); - for (Instance instance : showOnlyProductionInstances(request) ? application.productionInstances().values() - : application.instances().values()) - toSlime(instancesArray.addObject(), status, instance, application.deploymentSpec(), request); - - application.deployKeys().stream().map(KeyUtils::toPem).forEach(object.setArray("pemDeployKeys")::addString); - - // Metrics - Cursor metricsObject = object.setObject("metrics"); - metricsObject.setDouble("queryServiceQuality", application.metrics().queryServiceQuality()); - metricsObject.setDouble("writeServiceQuality", application.metrics().writeServiceQuality()); - - // Activity - Cursor activity = object.setObject("activity"); - application.activity().lastQueried().ifPresent(instant -> activity.setLong("lastQueried", instant.toEpochMilli())); - application.activity().lastWritten().ifPresent(instant -> activity.setLong("lastWritten", instant.toEpochMilli())); - application.activity().lastQueriesPerSecond().ifPresent(value -> activity.setDouble("lastQueriesPerSecond", value)); - application.activity().lastWritesPerSecond().ifPresent(value -> activity.setDouble("lastWritesPerSecond", value)); - - application.ownershipIssueId().ifPresent(issueId -> object.setString("ownershipIssueId", issueId.value())); - application.issueOwner().ifPresent(owner -> object.setString("owner", owner.value())); - application.deploymentIssueId().ifPresent(issueId -> object.setString("deploymentIssueId", issueId.value())); - } - - // TODO: Eliminate duplicated code in this and toSlime(Cursor, Instance, DeploymentStatus, HttpRequest) - private void toSlime(Cursor object, DeploymentStatus status, Instance instance, DeploymentSpec deploymentSpec, HttpRequest request) { - object.setString("instance", instance.name().value()); - - if (deploymentSpec.instance(instance.name()).isPresent()) { - // Jobs ordered according to deployment spec - Collection<JobStatus> jobStatus = status.instanceJobs(instance.name()).values(); - - if ( ! instance.change().isEmpty()) - toSlime(object.setObject("deploying"), instance.change(), status.application()); - - // Outstanding change - if ( ! status.outstandingChange(instance.name()).isEmpty()) - toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name()), status.application()); - - // Change blockers - Cursor changeBlockers = object.setArray("changeBlockers"); - deploymentSpec.instance(instance.name()).ifPresent(spec -> spec.changeBlocker().forEach(changeBlocker -> { - Cursor changeBlockerObject = changeBlockers.addObject(); - changeBlockerObject.setBool("versions", changeBlocker.blocksVersions()); - changeBlockerObject.setBool("revisions", changeBlocker.blocksRevisions()); - changeBlockerObject.setString("timeZone", changeBlocker.window().zone().getId()); - Cursor days = changeBlockerObject.setArray("days"); - changeBlocker.window().days().stream().map(DayOfWeek::getValue).forEach(days::addLong); - Cursor hours = changeBlockerObject.setArray("hours"); - changeBlocker.window().hours().forEach(hours::addLong); - })); - } - - // Rotation ID - addRotationId(object, instance); - - // Deployments sorted according to deployment spec - List<Deployment> deployments = deploymentSpec.instance(instance.name()) - .map(spec -> sortedDeployments(instance.deployments().values(), spec)) - .orElse(List.copyOf(instance.deployments().values())); - - Cursor deploymentsArray = object.setArray("deployments"); - for (Deployment deployment : deployments) { - Cursor deploymentObject = deploymentsArray.addObject(); - - // Rotation status for this deployment - if (deployment.zone().environment() == Environment.prod && ! instance.rotations().isEmpty()) - toSlime(instance.rotations(), instance.rotationStatus(), deployment, deploymentObject); - - if (recurseOverDeployments(request)) // List full deployment information when recursive. - toSlime(deploymentObject, new DeploymentId(instance.id(), deployment.zone()), deployment, request); - else { - deploymentObject.setString("environment", deployment.zone().environment().value()); - deploymentObject.setString("region", deployment.zone().region().value()); - addAvailabilityZone(deploymentObject, deployment.zone()); - deploymentObject.setString("url", withPath(request.getUri().getPath() + - "/instance/" + instance.name().value() + - "/environment/" + deployment.zone().environment().value() + - "/region/" + deployment.zone().region().value(), - request.getUri()).toString()); - } - } - } - - // TODO(mpolden): Remove once MultiRegionTest stops expecting this field - private void addRotationId(Cursor object, Instance instance) { - // Legacy field. Identifies the first assigned rotation, if any. - instance.rotations().stream() - .map(AssignedRotation::rotationId) - .findFirst() - .ifPresent(rotation -> object.setString("rotationId", rotation.asString())); - } - - private void toSlime(Cursor object, Instance instance, DeploymentStatus status, HttpRequest request) { - Application application = status.application(); - object.setString("tenant", instance.id().tenant().value()); - object.setString("application", instance.id().application().value()); - object.setString("instance", instance.id().instance().value()); - object.setString("deployments", withPath("/application/v4" + - "/tenant/" + instance.id().tenant().value() + - "/application/" + instance.id().application().value() + - "/instance/" + instance.id().instance().value() + "/job/", - request.getUri()).toString()); - - application.revisions().last().ifPresent(version -> { - version.sourceUrl().ifPresent(url -> object.setString("sourceUrl", url)); - version.commit().ifPresent(commit -> object.setString("commit", commit)); - }); - - application.projectId().ifPresent(id -> object.setLong("projectId", id)); - - if (application.deploymentSpec().instance(instance.name()).isPresent()) { - // Jobs ordered according to deployment spec - Collection<JobStatus> jobStatus = status.instanceJobs(instance.name()).values(); - - if ( ! instance.change().isEmpty()) - toSlime(object.setObject("deploying"), instance.change(), application); - - // Outstanding change - if ( ! status.outstandingChange(instance.name()).isEmpty()) - toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name()), application); - - // Change blockers - Cursor changeBlockers = object.setArray("changeBlockers"); - application.deploymentSpec().instance(instance.name()).ifPresent(spec -> spec.changeBlocker().forEach(changeBlocker -> { - Cursor changeBlockerObject = changeBlockers.addObject(); - changeBlockerObject.setBool("versions", changeBlocker.blocksVersions()); - changeBlockerObject.setBool("revisions", changeBlocker.blocksRevisions()); - changeBlockerObject.setString("timeZone", changeBlocker.window().zone().getId()); - Cursor days = changeBlockerObject.setArray("days"); - changeBlocker.window().days().stream().map(DayOfWeek::getValue).forEach(days::addLong); - Cursor hours = changeBlockerObject.setArray("hours"); - changeBlocker.window().hours().forEach(hours::addLong); - })); - } - - application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion)); - - // Rotation ID - addRotationId(object, instance); - - // Deployments sorted according to deployment spec - List<Deployment> deployments = application.deploymentSpec().instance(instance.name()) - .map(spec -> sortedDeployments(instance.deployments().values(), spec)) - .orElse(List.copyOf(instance.deployments().values())); - Cursor instancesArray = object.setArray("instances"); - for (Deployment deployment : deployments) { - Cursor deploymentObject = instancesArray.addObject(); - - // Rotation status for this deployment - if (deployment.zone().environment() == Environment.prod) { - // 0 rotations: No fields written - // 1 rotation : Write legacy field and endpointStatus field - // >1 rotation : Write only endpointStatus field - if (instance.rotations().size() == 1) { - // TODO(mpolden): Stop writing this field once clients stop expecting it - toSlime(instance.rotationStatus().of(instance.rotations().get(0).rotationId(), deployment), - deploymentObject); - } - if ( ! recurseOverDeployments(request) && ! instance.rotations().isEmpty()) { // TODO jonmv: clean up when clients have converged. - toSlime(instance.rotations(), instance.rotationStatus(), deployment, deploymentObject); - } - - } - - if (recurseOverDeployments(request)) // List full deployment information when recursive. - toSlime(deploymentObject, new DeploymentId(instance.id(), deployment.zone()), deployment, request); - else { - deploymentObject.setString("environment", deployment.zone().environment().value()); - deploymentObject.setString("region", deployment.zone().region().value()); - deploymentObject.setString("instance", instance.id().instance().value()); // pointless - addAvailabilityZone(deploymentObject, deployment.zone()); - deploymentObject.setString("url", withPath(request.getUri().getPath() + - "/environment/" + deployment.zone().environment().value() + - "/region/" + deployment.zone().region().value(), - request.getUri()).toString()); - } - } - // Add dummy values for not-yet-existent prod deployments, and running dev/perf deployments. - Stream.concat(status.jobSteps().keySet().stream() - .filter(job -> job.application().instance().equals(instance.name())) - .filter(job -> job.type().isProduction() && job.type().isDeployment()), - controller.jobController().active(instance.id()).stream() - .map(run -> run.id().job()) - .filter(job -> job.type().environment().isManuallyDeployed())) - .map(job -> job.type().zone()) - .filter(zone -> ! instance.deployments().containsKey(zone)) - .forEach(zone -> { - Cursor deploymentObject = instancesArray.addObject(); - deploymentObject.setString("environment", zone.environment().value()); - deploymentObject.setString("region", zone.region().value()); - }); - - - // TODO jonmv: Remove when clients are updated - application.deployKeys().stream().findFirst().ifPresent(key -> object.setString("pemDeployKey", KeyUtils.toPem(key))); - - application.deployKeys().stream().map(KeyUtils::toPem).forEach(object.setArray("pemDeployKeys")::addString); - - // Metrics - Cursor metricsObject = object.setObject("metrics"); - metricsObject.setDouble("queryServiceQuality", application.metrics().queryServiceQuality()); - metricsObject.setDouble("writeServiceQuality", application.metrics().writeServiceQuality()); - - // Activity - Cursor activity = object.setObject("activity"); - application.activity().lastQueried().ifPresent(instant -> activity.setLong("lastQueried", instant.toEpochMilli())); - application.activity().lastWritten().ifPresent(instant -> activity.setLong("lastWritten", instant.toEpochMilli())); - application.activity().lastQueriesPerSecond().ifPresent(value -> activity.setDouble("lastQueriesPerSecond", value)); - application.activity().lastWritesPerSecond().ifPresent(value -> activity.setDouble("lastWritesPerSecond", value)); - - application.ownershipIssueId().ifPresent(issueId -> object.setString("ownershipIssueId", issueId.value())); - application.issueOwner().ifPresent(owner -> object.setString("owner", owner.value())); - application.deploymentIssueId().ifPresent(issueId -> object.setString("deploymentIssueId", issueId.value())); - } - - private HttpResponse deployment(String tenantName, String applicationName, String instanceName, String environment, - String region, HttpRequest request) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - Instance instance = controller.applications().getInstance(id) - .orElseThrow(() -> new NotExistsException(id + " not found")); - - DeploymentId deploymentId = new DeploymentId(instance.id(), - requireZone(environment, region)); - - Deployment deployment = instance.deployments().get(deploymentId.zoneId()); - if (deployment == null) - throw new NotExistsException(instance + " is not deployed in " + deploymentId.zoneId()); - - Slime slime = new Slime(); - toSlime(slime.setObject(), deploymentId, deployment, request); - return new SlimeJsonResponse(slime); - } - - private void toSlime(Cursor object, Change change, Application application) { - change.platform().ifPresent(version -> object.setString("version", version.toString())); - change.revision().ifPresent(revision -> JobControllerApiHandlerHelper.toSlime(object.setObject("revision"), application.revisions().get(revision))); - } - - private void toSlime(Endpoint endpoint, Cursor object) { - if (endpoint.scope().multiDeployment()) { - object.setString("id", endpoint.name()); - } - object.setString("cluster", endpoint.cluster().value()); - object.setBool("tls", endpoint.tls()); - object.setString("url", endpoint.url().toString()); - object.setString("scope", endpointScopeString(endpoint.scope())); - object.setString("routingMethod", routingMethodString(endpoint.routingMethod())); - object.setBool("legacy", endpoint.legacy()); - switch (endpoint.authMethod()) { - case mtls -> object.setString("authMethod", "mtls"); - case token -> object.setString("authMethod", "token"); - case none -> object.setString("authMethod", "none"); - } - } - - private void toSlime(Cursor response, DeploymentId deploymentId, Deployment deployment, HttpRequest request) { - response.setString("tenant", deploymentId.applicationId().tenant().value()); - response.setString("application", deploymentId.applicationId().application().value()); - response.setString("instance", deploymentId.applicationId().instance().value()); // pointless - response.setString("environment", deploymentId.zoneId().environment().value()); - response.setString("region", deploymentId.zoneId().region().value()); - addAvailabilityZone(response, deployment.zone()); - var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); - boolean includeAllEndpoints = request.getBooleanProperty("includeAllEndpoints"); - boolean includeWeightedEndpoints = includeAllEndpoints || request.getBooleanProperty("includeWeightedEndpoints"); - boolean includeLegacyEndpoints = includeAllEndpoints || request.getBooleanProperty("includeLegacyEndpoints"); - var endpointArray = response.setArray("endpoints"); - for (var endpoint : endpointsOf(deploymentId, application, includeLegacyEndpoints, includeWeightedEndpoints)) { - toSlime(endpoint, endpointArray.addObject()); - } - response.setString("clusters", withPath(toPath(deploymentId) + "/clusters", request.getUri()).toString()); - response.setString("nodes", withPathAndQuery("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/", "recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); - response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); - response.setString("version", deployment.version().toFullString()); - response.setString("revision", application.revisions().get(deployment.revision()).stringId()); // TODO jonmv or freva: ƪ(`▿▿▿▿´ƪ) - response.setLong("build", deployment.revision().number()); - Instant lastDeploymentStart = controller.jobController().lastDeploymentStart(deploymentId.applicationId(), deployment); - response.setLong("deployTimeEpochMs", lastDeploymentStart.toEpochMilli()); - controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zoneId()) - .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", lastDeploymentStart.plus(deploymentTimeToLive).toEpochMilli())); - - application.projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i))); - - if (controller.zoneRegistry().isExternal(deployment.cloudAccount())) { - Cursor enclave = response.setObject("enclave"); - enclave.setString("cloudAccount", deployment.cloudAccount().value()); - controller.zoneRegistry().cloudAccountAthenzDomain(deployment.cloudAccount()).ifPresent(domain -> enclave.setString("athensDomain", domain.value())); - } - - var instance = application.instances().get(deploymentId.applicationId().instance()); - if (instance != null) { - if (!instance.rotations().isEmpty() && deployment.zone().environment() == Environment.prod) - toSlime(instance.rotations(), instance.rotationStatus(), deployment, response); - - if (!deployment.zone().environment().isManuallyDeployed()) { - DeploymentStatus status = controller.jobController().deploymentStatus(application); - JobId jobId = new JobId(instance.id(), JobType.deploymentTo(deployment.zone())); - Optional.ofNullable(status.jobSteps().get(jobId)) - .ifPresent(stepStatus -> { - JobControllerApiHandlerHelper.toSlime(response.setObject("applicationVersion"), application.revisions().get(deployment.revision())); - if ( ! status.jobsToRun().containsKey(stepStatus.job().get())) - response.setString("status", "complete"); - else if ( ! stepStatus.readiness(instance.change()).okAt(controller.clock().instant())) - response.setString("status", "pending"); - else - response.setString("status", "running"); - }); - } else { - var deploymentRun = controller.jobController().last(deploymentId.applicationId(), JobType.deploymentTo(deploymentId.zoneId())); - deploymentRun.ifPresent(run -> { - response.setString("status", run.hasEnded() ? "complete" : "running"); - }); - } - } - - response.setDouble("quota", deployment.quota().rate()); - deployment.cost().ifPresent(cost -> response.setDouble("cost", cost)); - - (controller.zoneRegistry().isExclave(deployment.cloudAccount()) ? - controller.archiveBucketDb().archiveUriFor(deploymentId.zoneId(), deployment.cloudAccount(), false) : - controller.archiveBucketDb().archiveUriFor(deploymentId.zoneId(), deploymentId.applicationId().tenant(), false)) - .ifPresent(archiveUri -> response.setString("archiveUri", archiveUri.toString())); - - Cursor activity = response.setObject("activity"); - deployment.activity().lastQueried().ifPresent(instant -> activity.setLong("lastQueried", - instant.toEpochMilli())); - deployment.activity().lastWritten().ifPresent(instant -> activity.setLong("lastWritten", - instant.toEpochMilli())); - deployment.activity().lastQueriesPerSecond().ifPresent(value -> activity.setDouble("lastQueriesPerSecond", value)); - deployment.activity().lastWritesPerSecond().ifPresent(value -> activity.setDouble("lastWritesPerSecond", value)); - - // Metrics - DeploymentMetrics metrics = deployment.metrics(); - Cursor metricsObject = response.setObject("metrics"); - metricsObject.setDouble("queriesPerSecond", metrics.queriesPerSecond()); - metricsObject.setDouble("writesPerSecond", metrics.writesPerSecond()); - metricsObject.setDouble("documentCount", metrics.documentCount()); - metricsObject.setDouble("queryLatencyMillis", metrics.queryLatencyMillis()); - metricsObject.setDouble("writeLatencyMillis", metrics.writeLatencyMillis()); - metrics.instant().ifPresent(instant -> metricsObject.setLong("lastUpdated", instant.toEpochMilli())); - } - - private EndpointList endpointsOf(DeploymentId deploymentId, Application application, boolean includeLegacy, boolean includeWeighted) { - EndpointList zoneEndpoints = controller.routing().readEndpointsOf(deploymentId).direct(); - EndpointList declaredEndpoints = controller.routing().readDeclaredEndpointsOf(application).targets(deploymentId); - EndpointList endpoints = zoneEndpoints.and(declaredEndpoints); - if (!includeLegacy) { - endpoints = endpoints.not().legacy(); - } - if (!includeWeighted) { - endpoints = endpoints.not().scope(Endpoint.Scope.weighted); - } - return endpoints; - } - - private void toSlime(RotationState state, Cursor object) { - Cursor bcpStatus = object.setObject("bcpStatus"); - bcpStatus.setString("rotationStatus", rotationStateString(state)); - } - - private void toSlime(List<AssignedRotation> rotations, RotationStatus status, Deployment deployment, Cursor object) { - var array = object.setArray("endpointStatus"); - for (var rotation : rotations) { - var statusObject = array.addObject(); - var targets = status.of(rotation.rotationId()); - statusObject.setString("endpointId", rotation.endpointId().id()); - statusObject.setString("rotationId", rotation.rotationId().asString()); - statusObject.setString("clusterId", rotation.clusterId().value()); - statusObject.setString("status", rotationStateString(status.of(rotation.rotationId(), deployment))); - statusObject.setLong("lastUpdated", targets.lastUpdated().toEpochMilli()); - } - } - - private URI monitoringSystemUri(DeploymentId deploymentId) { - return controller.zoneRegistry().getMonitoringSystemUri(deploymentId); - } - - private HttpResponse setGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region, boolean inService, HttpRequest request) { - Instance instance = controller.applications().requireInstance(ApplicationId.from(tenantName, applicationName, instanceName)); - ZoneId zone = requireZone(environment, region); - Deployment deployment = instance.deployments().get(zone); - if (deployment == null) { - throw new NotExistsException(instance + " has no deployment in " + zone); - } - DeploymentId deploymentId = new DeploymentId(instance.id(), zone); - RoutingStatus.Agent agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; - RoutingStatus.Value status = inService ? RoutingStatus.Value.in : RoutingStatus.Value.out; - controller.routing().of(deploymentId).setRoutingStatus(status, agent); - return new MessageResponse(Text.format("Successfully set %s in %s %s service", - instance.id().toShortString(), zone, inService ? "in" : "out of")); - } - - private String serviceTypeIn(DeploymentId id) { - CloudName cloud = controller.zoneRegistry().zones().all().get(id.zoneId()).get().getCloudName(); - if (CloudName.AWS.equals(cloud)) return "aws-private-link"; - if (CloudName.GCP.equals(cloud)) return "gcp-service-connect"; - return "unknown"; - } - - private HttpResponse getPrivateServiceInfo(String tenantName, String applicationName, String instanceName, String environment, String region) { - DeploymentId id = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - ZoneId.from(environment, region)); - List<LoadBalancer> lbs = controller.serviceRegistry().configServer().getLoadBalancers(id.applicationId(), id.zoneId()); - Slime slime = new Slime(); - Cursor lbArray = slime.setObject().setArray("privateServices"); - for (LoadBalancer lb : lbs) { - Cursor serviceObject = lbArray.addObject(); - serviceObject.setString("cluster", lb.cluster().value()); - lb.service().ifPresent(service -> { - serviceObject.setString("serviceId", service.id()); // Really the "serviceName", but this is what the user needs >_< - serviceObject.setString("type", serviceTypeIn(id)); - Cursor urnsArray = serviceObject.setArray("allowedUrns"); - for (AllowedUrn urn : service.allowedUrns()) { - Cursor urnObject = urnsArray.addObject(); - urnObject.setString("type", switch (urn.type()) { - case awsPrivateLink -> "aws-private-link"; - case gcpServiceConnect -> "gcp-service-connect"; - }); - urnObject.setString("urn", urn.urn()); - } - Cursor endpointsArray = serviceObject.setArray("endpoints"); - controller.serviceRegistry().vpcEndpointService() - .getConnections(new ClusterId(id, lb.cluster()), lb.cloudAccount()) - .forEach(endpoint -> { - Cursor endpointObject = endpointsArray.addObject(); - endpointObject.setString("endpointId", endpoint.endpointId()); - endpointObject.setString("state", endpoint.stateValue().name()); - endpointObject.setString("detail", endpoint.stateString()); - }); - }); - } - return new SlimeJsonResponse(slime); - } - - private HttpResponse dropDocumentsStatus(String tenant, String application, String instance, String environment, String region, Optional<ClusterSpec.Id> clusterId) { - ZoneId zone = ZoneId.from(environment, region); - if (!zone.environment().isManuallyDeployed()) - throw new IllegalArgumentException("Drop documents status is only available for manually deployed environments"); - - ApplicationId applicationId = ApplicationId.from(tenant, application, instance); - NodeFilter filters = NodeFilter.all() - .states(Node.State.active) - .applications(applicationId) - .clusterTypes(Node.ClusterType.content, Node.ClusterType.combined); - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(zone, clusterId.map(filters::clusterIds).orElse(filters)); - if (nodes.isEmpty()) { - throw new NotExistsException("No content nodes found for %s%s in %s".formatted( - applicationId.toFullString(), clusterId.map(id -> " cluster " + id).orElse(""), zone)); - } - - Instant readiedAt = null; - int numNoReport = 0, numInitial = 0, numDropped = 0, numReadied = 0, numStarted = 0; - for (Node node : nodes) { - Inspector report = Optional.ofNullable(node.reports().get("dropDocuments")) - .map(json -> SlimeUtils.jsonToSlime(json).get()).orElse(null); - if (report == null) numNoReport++; - else if (report.field("startedAt").valid()) { - numStarted++; - readiedAt = SlimeUtils.instant(report.field("readiedAt")); - } else if (report.field("readiedAt").valid()) numReadied++; - else if (report.field("droppedAt").valid()) numDropped++; - else numInitial++; - } - - if (numInitial + numDropped > 0 && numNoReport + numReadied + numStarted > 0) - return ErrorResponse.conflict("Last dropping of documents may have failed to clear all documents due " + - "to concurrent topology changes, consider retrying"); - - Slime slime = new Slime(); - Cursor root = slime.setObject(); - if (numStarted + numNoReport == nodes.size()) { - if (readiedAt != null) root.setLong("lastDropped", readiedAt.toEpochMilli()); - } else { - Cursor progress = root.setObject("progress"); - progress.setLong("total", nodes.size()); - progress.setLong("dropped", numDropped + numReadied + numStarted); - progress.setLong("started", numStarted + numNoReport); - } - - return new SlimeJsonResponse(slime); - } - - private HttpResponse dropDocuments(String tenant, String application, String instance, String environment, String region, Optional<ClusterSpec.Id> clusterId) { - ZoneId zone = ZoneId.from(environment, region); - if (!zone.environment().isManuallyDeployed()) - throw new IllegalArgumentException("Drop documents status is only available for manually deployed environments"); - - ApplicationId applicationId = ApplicationId.from(tenant, application, instance); - controller.serviceRegistry().configServer().nodeRepository().dropDocuments(zone, applicationId, clusterId); - return new MessageResponse("Triggered drop documents for " + applicationId.toFullString() + - clusterId.map(id -> " and cluster " + id).orElse("") + " in " + zone); - } - - private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - requireZone(environment, region)); - Slime slime = new Slime(); - Cursor array = slime.setObject().setArray("globalrotationoverride"); - Optional<Endpoint> primaryEndpoint = controller.routing().readDeclaredEndpointsOf(deploymentId.applicationId()) - .requiresRotation() - .first(); - if (primaryEndpoint.isPresent()) { - DeploymentRoutingContext context = controller.routing().of(deploymentId); - RoutingStatus status = context.routingStatus(); - array.addString(primaryEndpoint.get().upstreamName(deploymentId)); - Cursor statusObject = array.addObject(); - statusObject.setString("status", status.value().name()); - statusObject.setString("reason", ""); - statusObject.setString("agent", status.agent().name()); - statusObject.setLong("timestamp", status.changedAt().getEpochSecond()); - } - return new SlimeJsonResponse(slime); - } - - private HttpResponse rotationStatus(String tenantName, String applicationName, String instanceName, String environment, String region, Optional<String> endpointId) { - ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); - Instance instance = controller.applications().requireInstance(applicationId); - ZoneId zone = requireZone(environment, region); - RotationId rotation = findRotationId(instance, endpointId); - Deployment deployment = instance.deployments().get(zone); - if (deployment == null) { - throw new NotExistsException(instance + " has no deployment in " + zone); - } - - Slime slime = new Slime(); - Cursor response = slime.setObject(); - toSlime(instance.rotationStatus().of(rotation, deployment), response); - return new SlimeJsonResponse(slime); - } - - private HttpResponse deploying(String tenantName, String applicationName, String instanceName, HttpRequest request) { - Instance instance = controller.applications().requireInstance(ApplicationId.from(tenantName, applicationName, instanceName)); - Slime slime = new Slime(); - Cursor root = slime.setObject(); - if ( ! instance.change().isEmpty()) { - instance.change().platform().ifPresent(version -> root.setString("platform", version.toString())); - instance.change().revision().ifPresent(revision -> root.setString("application", revision.toString())); - root.setBool("pinned", instance.change().isPlatformPinned()); - root.setBool("platform-pinned", instance.change().isPlatformPinned()); - root.setBool("application-pinned", instance.change().isRevisionPinned()); - } - return new SlimeJsonResponse(slime); - } - - private HttpResponse suspended(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - requireZone(environment, region)); - boolean suspended = controller.applications().isSuspended(deploymentId); - Slime slime = new Slime(); - Cursor response = slime.setObject(); - response.setBool("suspended", suspended); - return new SlimeJsonResponse(slime); - } - - private HttpResponse status(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host, HttpURL.Path restPath, HttpRequest request) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - return controller.serviceRegistry().configServer().getServiceNodePage(deploymentId, - serviceName, - DomainName.of(host), - HttpURL.Path.parse("/status").append(restPath), - Query.empty().add(request.getJDiscRequest().parameters())); - } - - private HttpResponse orchestrator(String tenantName, String applicationName, String instanceName, String environment, String region) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - return controller.serviceRegistry().configServer().getServiceNodes(deploymentId); - } - - private HttpResponse stateV1(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host, HttpURL.Path rest, HttpRequest request) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - Query query = Query.empty().add(request.getJDiscRequest().parameters()); - query = query.set("forwarded-url", HttpURL.from(request.getUri()).withQuery(Query.empty()).asURI().toString()); - return controller.serviceRegistry().configServer().getServiceNodePage( - deploymentId, serviceName, DomainName.of(host), HttpURL.Path.parse("/state/v1").append(rest), query); - } - - private HttpResponse content(String tenantName, String applicationName, String instanceName, String environment, String region, HttpURL.Path restPath, HttpRequest request) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - return controller.serviceRegistry().configServer().getApplicationPackageContent(deploymentId, restPath, request.getUri()); - } - - private HttpResponse updateTenant(String tenantName, HttpRequest request) { - getTenantOrThrow(tenantName); - TenantName tenant = TenantName.from(tenantName); - Inspector requestObject = toSlime(request.getData()).get(); - controller.tenants().update(accessControlRequests.specification(tenant, requestObject), - accessControlRequests.credentials(tenant, requestObject, request.getJDiscRequest())); - return tenant(controller.tenants().require(TenantName.from(tenantName)), request); - } - - private HttpResponse createTenant(String tenantName, HttpRequest request) { - TenantName tenant = TenantName.from(tenantName); - Inspector requestObject = toSlime(request.getData()).get(); - controller.tenants().create(accessControlRequests.specification(tenant, requestObject), - accessControlRequests.credentials(tenant, requestObject, request.getJDiscRequest())); - if (controller.system().isPublic()) { - User user = getAttribute(request, User.ATTRIBUTE_NAME, User.class); - TenantInfo info = controller.tenants().require(tenant, CloudTenant.class) - .info() - .withContact(TenantContact.from(user.name(), new Email(user.email(), true))); - // Store changes - controller.tenants().lockOrThrow(tenant, LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withInfo(info); - controller.tenants().store(lockedTenant); - }); - } - return tenant(controller.tenants().require(TenantName.from(tenantName)), request); - } - - private HttpResponse createApplication(String tenantName, String applicationName, HttpRequest request) { - Inspector requestObject = toSlime(request.getData()).get(); - TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); - Credentials credentials = accessControlRequests.credentials(id.tenant(), requestObject, request.getJDiscRequest()); - Application application = controller.applications().createApplication(id, credentials); - Slime slime = new Slime(); - toSlime(id, slime.setObject(), request); - return new SlimeJsonResponse(slime); - } - - // TODO jonmv: Remove when clients are updated. - private HttpResponse createInstance(String tenantName, String applicationName, String instanceName, HttpRequest request) { - TenantAndApplicationId applicationId = TenantAndApplicationId.from(tenantName, applicationName); - if (controller.applications().getApplication(applicationId).isEmpty()) - createApplication(tenantName, applicationName, request); - - controller.applications().createInstance(applicationId.instance(instanceName)); - - Slime slime = new Slime(); - toSlime(applicationId.instance(instanceName), slime.setObject(), request); - return new SlimeJsonResponse(slime); - } - - /** Trigger deployment of the given Vespa version if a valid one is given, e.g., "7.8.9". */ - private HttpResponse deployPlatform(String tenantName, String applicationName, String instanceName, boolean pin, HttpRequest request) { - String versionString = readToString(request.getData()); - ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - StringBuilder response = new StringBuilder(); - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - Version version = Version.fromString(versionString); - VersionStatus versionStatus = controller.readVersionStatus(); - if (version.equals(Version.emptyVersion)) - version = controller.systemVersion(versionStatus); - if ( ! versionStatus.isActive(version) && ! isOperator(request)) - throw new IllegalArgumentException("Cannot trigger deployment of version '" + version + "': " + - "Version is not active in this system. " + - "Active versions: " + versionStatus.versions() - .stream() - .map(VespaVersion::versionNumber) - .map(Version::toString) - .collect(joining(", "))); - Change change = Change.of(version); - if (pin) - change = change.withPlatformPin(); - - controller.applications().deploymentTrigger().forceChange(id, change, isOperator(request)); - response.append("Triggered ").append(change).append(" for ").append(id); - }); - return new MessageResponse(response.toString()); - } - - /** Trigger deployment to the last known application package for the given application. */ - private HttpResponse deployApplication(String tenantName, String applicationName, String instanceName, boolean pin, HttpRequest request) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - Inspector buildField = toSlime(request.getData()).get().field("build"); - long build = buildField.valid() ? buildField.asLong() : -1; - - StringBuilder response = new StringBuilder(); - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - RevisionId revision = build == -1 ? application.get().revisions().last().get().id() - : getRevision(application.get(), build); - Change change = Change.of(revision); - if (pin) - change = change.withRevisionPin(); - controller.applications().deploymentTrigger().forceChange(id, change, isOperator(request)); - response.append("Triggered ").append(change).append(" for ").append(id); - }); - return new MessageResponse(response.toString()); - } - - private RevisionId getRevision(Application application, long build) { - return application.revisions().withPackage().stream() - .map(ApplicationVersion::id) - .filter(version -> version.number() == build) - .findFirst() - .filter(version -> controller.applications().applicationStore().hasBuild(application.id().tenant(), - application.id().application(), - build)) - .orElseThrow(() -> new IllegalArgumentException("Build number '" + build + "' was not found")); - } - - private HttpResponse cancelBuild(String tenantName, String applicationName, String build){ - TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); - RevisionId revision = RevisionId.forProduction(Long.parseLong(build)); - controller.applications().lockApplicationOrThrow(id, application -> { - controller.applications().store(application.withRevisions(revisions -> revisions.with(revisions.get(revision).skipped()))); - for (Instance instance : application.get().instances().values()) - if (instance.change().revision().equals(Optional.of(revision))) - controller.applications().deploymentTrigger().cancelChange(instance.id(), ChangesToCancel.APPLICATION); - }); - return new MessageResponse("Marked build '" + build + "' as non-deployable"); - } - - /** Cancel ongoing change for given application, e.g., everything with {"cancel":"all"} */ - private HttpResponse cancelDeploy(String tenantName, String applicationName, String instanceName, String choice) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - StringBuilder response = new StringBuilder(); - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - Change change = application.get().require(id.instance()).change(); - if (change.isEmpty()) { - response.append("No deployment in progress for ").append(id).append(" at this time"); - return; - } - - ChangesToCancel cancel = ChangesToCancel.valueOf(choice.replaceAll("-", "_").toUpperCase()); - controller.applications().deploymentTrigger().cancelChange(id, cancel); - response.append("Changed deployment from '").append(change).append("' to '").append(controller.applications().requireInstance(id).change()).append("' for ").append(id); - }); - - return new MessageResponse(response.toString()); - } - - /** Schedule reindexing of an application, or a subset of clusters, possibly on a subset of documents. */ - private HttpResponse reindex(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = requireZone(environment, region); - List<String> clusterNames = Optional.ofNullable(request.getProperty("clusterId")).stream() - .flatMap(clusters -> Stream.of(clusters.split(","))) - .filter(cluster -> ! cluster.isBlank()) - .toList(); - List<String> documentTypes = Optional.ofNullable(request.getProperty("documentType")).stream() - .flatMap(types -> Stream.of(types.split(","))) - .filter(type -> ! type.isBlank()) - .toList(); - - Double speed = request.hasProperty("speed") ? Double.parseDouble(request.getProperty("speed")) : null; - boolean indexedOnly = request.getBooleanProperty("indexedOnly"); - controller.applications().reindex(id, zone, clusterNames, documentTypes, indexedOnly, speed, "reindexing triggered by " + requireUserPrincipal(request).getName()); - return new MessageResponse("Requested reindexing of " + id + " in " + zone + - (clusterNames.isEmpty() ? "" : ", on clusters " + String.join(", ", clusterNames)) + - (documentTypes.isEmpty() ? "" : ", for types " + String.join(", ", documentTypes)) + - (indexedOnly ? ", for indexed types" : "") + - (speed != null ? ", with speed " + speed : "")); - } - - /** Gets reindexing status of an application in a zone. */ - private HttpResponse getReindexing(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = requireZone(environment, region); - ApplicationReindexing reindexing = controller.applications().applicationReindexing(id, zone); - - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - root.setBool("enabled", reindexing.enabled()); - - Cursor clustersArray = root.setArray("clusters"); - reindexing.clusters().entrySet().stream().sorted(comparingByKey()) - .forEach(cluster -> { - Cursor clusterObject = clustersArray.addObject(); - clusterObject.setString("name", cluster.getKey()); - - Cursor pendingArray = clusterObject.setArray("pending"); - cluster.getValue().pending().entrySet().stream().sorted(comparingByKey()) - .forEach(pending -> { - Cursor pendingObject = pendingArray.addObject(); - pendingObject.setString("type", pending.getKey()); - pendingObject.setLong("requiredGeneration", pending.getValue()); - }); - - Cursor readyArray = clusterObject.setArray("ready"); - cluster.getValue().ready().entrySet().stream().sorted(comparingByKey()) - .forEach(ready -> { - Cursor readyObject = readyArray.addObject(); - readyObject.setString("type", ready.getKey()); - setStatus(readyObject, ready.getValue()); - }); - }); - return new SlimeJsonResponse(slime); - } - - void setStatus(Cursor statusObject, ApplicationReindexing.Status status) { - status.readyAt().ifPresent(readyAt -> statusObject.setLong("readyAtMillis", readyAt.toEpochMilli())); - status.startedAt().ifPresent(startedAt -> statusObject.setLong("startedAtMillis", startedAt.toEpochMilli())); - status.endedAt().ifPresent(endedAt -> statusObject.setLong("endedAtMillis", endedAt.toEpochMilli())); - status.state().map(ApplicationApiHandler::toString).ifPresent(state -> statusObject.setString("state", state)); - status.message().ifPresent(message -> statusObject.setString("message", message)); - status.progress().ifPresent(progress -> statusObject.setDouble("progress", progress)); - status.speed().ifPresent(speed -> statusObject.setDouble("speed", speed)); - status.cause().ifPresent(cause -> statusObject.setString("cause", cause)); - } - - private static String toString(ApplicationReindexing.State state) { - return switch (state) { - case PENDING: yield "pending"; - case RUNNING: yield "running"; - case FAILED: yield "failed"; - case SUCCESSFUL: yield "successful"; - }; - } - - /** Enables reindexing of an application in a zone. */ - private HttpResponse enableReindexing(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = requireZone(environment, region); - controller.applications().enableReindexing(id, zone); - return new MessageResponse("Enabled reindexing of " + id + " in " + zone); - } - - /** Disables reindexing of an application in a zone. */ - private HttpResponse disableReindexing(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = requireZone(environment, region); - controller.applications().disableReindexing(id, zone); - return new MessageResponse("Disabled reindexing of " + id + " in " + zone); - } - - /** Schedule restart of deployment, or specific host in a deployment */ - private HttpResponse restart(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - requireZone(environment, region)); - RestartFilter restartFilter = new RestartFilter() - .withHostName(Optional.ofNullable(request.getProperty("hostname")).map(HostName::of)) - .withClusterType(Optional.ofNullable(request.getProperty("clusterType")).map(ClusterSpec.Type::from)) - .withClusterId(Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)); - - controller.applications().restart(deploymentId, restartFilter); - return new MessageResponse("Requested restart of " + deploymentId); - } - - /** Set suspension status of the given deployment. */ - private HttpResponse suspend(String tenantName, String applicationName, String instanceName, String environment, String region, boolean suspend) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - requireZone(environment, region)); - controller.applications().setSuspension(deploymentId, suspend); - return new MessageResponse((suspend ? "Suspended" : "Resumed") + " orchestration of " + deploymentId); - } - - private HttpResponse jobDeploy(ApplicationId id, JobType type, HttpRequest request) { - if ( ! type.environment().isManuallyDeployed() && ! (isOperator(request) || controller.system().isCd())) - throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments."); - - controller.applications().verifyPlan(id.tenant()); - - Map<String, byte[]> dataParts = parseDataParts(request); - if ( ! dataParts.containsKey("applicationZip")) - throw new IllegalArgumentException("Missing required form part 'applicationZip'"); - - ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(APPLICATION_ZIP)); - controller.applications().verifyApplicationIdentityConfiguration(id.tenant(), - Optional.of(new DeploymentId(id, type.zone())), - applicationPackage, - Optional.of(requireUserPrincipal(request))); - - Optional<Version> version = Optional.ofNullable(dataParts.get("deployOptions")) - .map(json -> SlimeUtils.jsonToSlime(json).get()) - .flatMap(options -> optional("vespaVersion", options)) - .map(Version::fromString); - - ensureApplicationExists(TenantAndApplicationId.from(id), request); - - boolean dryRun = Optional.ofNullable(dataParts.get("deployOptions")) - .map(json -> SlimeUtils.jsonToSlime(json).get()) - .flatMap(options -> optional("dryRun", options)) - .map(Boolean::valueOf) - .orElse(false); - - controller.jobController().deploy(id, type, version, applicationPackage, dryRun, isOperator(request)); - RunId runId = controller.jobController().last(id, type).get().id(); - Slime slime = new Slime(); - Cursor rootObject = slime.setObject(); - rootObject.setString("message", "Deployment started in " + runId + - ". This may take about 15 minutes the first time."); - rootObject.setLong("run", runId.number()); - return new SlimeJsonResponse(slime); - } - - private HttpResponse deploySystemApplication(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = requireZone(environment, region); - - // Get deployOptions - Map<String, byte[]> dataParts = parseDataParts(request); - if ( ! dataParts.containsKey("deployOptions")) - return ErrorResponse.badRequest("Missing required form part 'deployOptions'"); - Inspector deployOptions = SlimeUtils.jsonToSlime(dataParts.get("deployOptions")).get(); - - // Resolve system application - Optional<SystemApplication> systemApplication = SystemApplication.matching(applicationId); - if (systemApplication.isEmpty() || ! systemApplication.get().hasApplicationPackage()) { - return ErrorResponse.badRequest("Deployment of " + applicationId + " is not supported through this API"); - } - - // Make it explicit that version is not yet supported here - String vespaVersion = deployOptions.field("vespaVersion").asString(); - if ( ! vespaVersion.isEmpty()) { - return ErrorResponse.badRequest("Specifying version for " + applicationId + " is not permitted"); - } - - // To avoid second guessing the orchestrated upgrades of system applications we don't allow - // deploying these during a system upgrade, i.e., when a new Vespa version is being rolled out - VersionStatus versionStatus = controller.readVersionStatus(); - if (versionStatus.isUpgrading()) { - throw new IllegalArgumentException("Deployment of system applications during a system upgrade is not allowed"); - } - Optional<VespaVersion> systemVersion = versionStatus.systemVersion(); - if (systemVersion.isEmpty()) { - throw new IllegalArgumentException("Deployment of system applications is not permitted until system version is determined"); - } - DeploymentResult result = controller.applications() - .deploySystemApplicationPackage(systemApplication.get(), zone, systemVersion.get().versionNumber()); - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("message", "Deployed " + systemApplication.get() + " in " + zone + " on " + systemVersion.get().versionNumber()); - - Cursor logArray = root.setArray("prepareMessages"); - for (LogEntry logMessage : result.log()) { - Cursor logObject = logArray.addObject(); - logObject.setLong("time", logMessage.epochMillis()); - logObject.setString("level", logMessage.level().getName()); - logObject.setString("message", logMessage.message()); - } - - return new SlimeJsonResponse(slime); - } - - private HttpResponse deleteTenant(String tenantName, HttpRequest request) { - boolean forget = request.getBooleanProperty("forget"); - if (forget && ! isOperator(request)) - return ErrorResponse.forbidden("Only operators can forget a tenant"); - - controller.tenants().delete(TenantName.from(tenantName), - Optional.of(accessControlRequests.credentials(TenantName.from(tenantName), - toSlime(request.getData()).get(), - request.getJDiscRequest())), - forget); - - return new MessageResponse("Deleted tenant " + tenantName); - } - - private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) { - TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); - Credentials credentials = accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest()); - controller.applications().deleteApplication(id, credentials); - return new MessageResponse("Deleted application " + id); - } - - private HttpResponse deleteInstance(String tenantName, String applicationName, String instanceName, HttpRequest request) { - TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); - controller.applications().deleteInstance(id.instance(instanceName)); - if (controller.applications().requireApplication(id).instances().isEmpty()) { - Credentials credentials = accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest()); - controller.applications().deleteApplication(id, credentials); - } - return new MessageResponse("Deleted instance " + id.instance(instanceName).toFullString()); - } - - private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - DeploymentId id = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - requireZone(environment, region)); - // Attempt to deactivate application even if the deployment is not known by the controller - controller.applications().deactivate(id.applicationId(), id.zoneId()); - controller.jobController().last(id.applicationId(), JobType.deploymentTo(id.zoneId())) - .filter(run -> ! run.hasEnded()) - .ifPresent(last -> controller.jobController().abort(last.id(), "deployment deactivated by " + request.getJDiscRequest().getUserPrincipal().getName(), true)); - return new MessageResponse("Deactivated " + id); - } - - /** Returns test config for indicated job, with production deployments of the default instance if the given is not in deployment spec. */ - private HttpResponse testConfig(ApplicationId id, JobType type) { - Application application = controller.applications().requireApplication(TenantAndApplicationId.from(id)); - ApplicationId prodInstanceId = application.deploymentSpec().instance(id.instance()).isPresent() - ? id : TenantAndApplicationId.from(id).defaultInstance(); - HashSet<DeploymentId> deployments = controller.applications() - .getInstance(prodInstanceId).stream() - .flatMap(instance -> instance.productionDeployments().keySet().stream()) - .map(zone -> new DeploymentId(prodInstanceId, zone)) - .collect(Collectors.toCollection(HashSet::new)); - - - // If a production job is specified, the production deployment of the orchestrated instance is the relevant one, - // as user instances should not exist in prod. - ApplicationId toTest = type.isProduction() ? prodInstanceId : id; - if ( ! type.isProduction()) - deployments.add(new DeploymentId(toTest, type.zone())); - - Deployment deployment = application.require(toTest.instance()).deployments().get(type.zone()); - if (deployment == null) - throw new NotExistsException(toTest + " is not deployed in " + type.zone()); - - return new SlimeJsonResponse(testConfigSerializer.configSlime(id, - type, - false, - deployment.version(), - deployment.revision(), - deployment.at(), - controller.routing().readStepRunnerEndpointsOf(deployments), - controller.applications().reachableContentClustersByZone(deployments))); - } - - private HttpResponse requestServiceDump(String tenant, String application, String instance, String environment, - String region, String hostname, HttpRequest request) { - NodeRepository nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - ZoneId zone = requireZone(environment, region); - - // Check that no other service dump is in progress - Slime report = getReport(nodeRepository, zone, tenant, application, instance, hostname).orElse(null); - if (report != null) { - Cursor cursor = report.get(); - // Note: same behaviour for both value '0' and missing value. - boolean force = request.getBooleanProperty("force"); - if (!force && cursor.field("failedAt").asLong() == 0 && cursor.field("completedAt").asLong() == 0) { - throw new IllegalArgumentException("Service dump already in progress for " + cursor.field("configId").asString()); - } - } - Slime requestPayload; - try { - requestPayload = SlimeUtils.jsonToSlimeOrThrow(request.getData().readAllBytes()); - } catch (Exception e) { - throw new IllegalArgumentException("Missing or invalid JSON in request content", e); - } - Cursor requestPayloadCursor = requestPayload.get(); - String configId = requestPayloadCursor.field("configId").asString(); - long expiresAt = requestPayloadCursor.field("expiresAt").asLong(); - if (configId.isEmpty()) { - throw new IllegalArgumentException("Missing configId"); - } - Cursor artifactsCursor = requestPayloadCursor.field("artifacts"); - int artifactEntries = artifactsCursor.entries(); - if (artifactEntries == 0) { - throw new IllegalArgumentException("Missing or empty 'artifacts'"); - } - - Slime dumpRequest = new Slime(); - Cursor dumpRequestCursor = dumpRequest.setObject(); - dumpRequestCursor.setLong("createdMillis", controller.clock().millis()); - dumpRequestCursor.setString("configId", configId); - Cursor dumpRequestArtifactsCursor = dumpRequestCursor.setArray("artifacts"); - for (int i = 0; i < artifactEntries; i++) { - dumpRequestArtifactsCursor.addString(artifactsCursor.entry(i).asString()); - } - if (expiresAt > 0) { - dumpRequestCursor.setLong("expiresAt", expiresAt); - } - Cursor dumpOptionsCursor = requestPayloadCursor.field("dumpOptions"); - if (dumpOptionsCursor.children() > 0) { - SlimeUtils.copyObject(dumpOptionsCursor, dumpRequestCursor.setObject("dumpOptions")); - } - var reportsUpdate = Map.of("serviceDump", new String(uncheck(() -> SlimeUtils.toJsonBytes(dumpRequest)))); - nodeRepository.updateReports(zone, hostname, reportsUpdate); - boolean wait = request.getBooleanProperty("wait"); - if (!wait) return new MessageResponse("Request created"); - return waitForServiceDumpResult(nodeRepository, zone, tenant, application, instance, hostname); - } - - private HttpResponse getServiceDump(String tenant, String application, String instance, String environment, - String region, String hostname, HttpRequest request) { - NodeRepository nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - ZoneId zone = requireZone(environment, region); - Slime report = getReport(nodeRepository, zone, tenant, application, instance, hostname) - .orElseThrow(() -> new NotExistsException("No service dump for node " + hostname)); - return new SlimeJsonResponse(report); - } - - private HttpResponse waitForServiceDumpResult(NodeRepository nodeRepository, ZoneId zone, String tenant, - String application, String instance, String hostname) { - int pollInterval = 2; - Slime report; - while (true) { - report = getReport(nodeRepository, zone, tenant, application, instance, hostname).get(); - Cursor cursor = report.get(); - if (cursor.field("completedAt").asLong() > 0 || cursor.field("failedAt").asLong() > 0) { - break; - } - final Slime copyForLambda = report; - log.fine(() -> uncheck(() -> new String(SlimeUtils.toJsonBytes(copyForLambda)))); - log.fine("Sleeping " + pollInterval + " seconds before checking report status again"); - controller.sleeper().sleep(Duration.ofSeconds(pollInterval)); - } - return new SlimeJsonResponse(report); - } - - private Optional<Slime> getReport(NodeRepository nodeRepository, ZoneId zone, String tenant, - String application, String instance, String hostname) { - Node node; - try { - node = nodeRepository.getNode(zone, hostname); - } catch (IllegalArgumentException e) { - throw new NotExistsException(hostname); - } - ApplicationId app = ApplicationId.from(tenant, application, instance); - ApplicationId owner = node.owner().orElseThrow(() -> new IllegalArgumentException("Node has no owner")); - if (!app.equals(owner)) { - throw new IllegalArgumentException("Node is not owned by " + app.toFullString()); - } - String json = node.reports().get("serviceDump"); - if (json == null) return Optional.empty(); - return Optional.of(SlimeUtils.jsonToSlimeOrThrow(json)); - } - - private static SourceRevision toSourceRevision(Inspector object) { - if (!object.field("repository").valid() || - !object.field("branch").valid() || - !object.field("commit").valid()) { - throw new IllegalArgumentException("Must specify \"repository\", \"branch\", and \"commit\"."); - } - return new SourceRevision(object.field("repository").asString(), - object.field("branch").asString(), - object.field("commit").asString()); - } - - private Tenant getTenantOrThrow(String tenantName) { - return controller.tenants().get(tenantName) - .orElseThrow(() -> new NotExistsException(new TenantId(tenantName))); - } - - private void toSlime(Cursor object, Tenant tenant, List<Application> applications, HttpRequest request) { - object.setString("tenant", tenant.name().value()); - object.setString("type", tenantType(tenant)); - switch (tenant.type()) { - case athenz: - AthenzTenant athenzTenant = (AthenzTenant) tenant; - object.setString("athensDomain", athenzTenant.domain().getName()); - object.setString("property", athenzTenant.property().id()); - athenzTenant.propertyId().ifPresent(id -> object.setString("propertyId", id.toString())); - athenzTenant.contact().ifPresent(c -> { - object.setString("propertyUrl", c.propertyUrl().toString()); - object.setString("contactsUrl", c.url().toString()); - object.setString("issueCreationUrl", c.issueTrackerUrl().toString()); - Cursor contactsArray = object.setArray("contacts"); - c.persons().forEach(persons -> { - Cursor personArray = contactsArray.addArray(); - persons.forEach(personArray::addString); - }); - }); - break; - case cloud: { - CloudTenant cloudTenant = (CloudTenant) tenant; - - cloudTenant.creator().ifPresent(creator -> object.setString("creator", creator.getName())); - Cursor pemDeveloperKeysArray = object.setArray("pemDeveloperKeys"); - cloudTenant.developerKeys().forEach((key, user) -> { - Cursor keyObject = pemDeveloperKeysArray.addObject(); - keyObject.setString("key", KeyUtils.toPem(key)); - keyObject.setString("user", user.getName()); - }); - - // TODO: remove this once console is updated - toSlime(object, cloudTenant.tenantSecretStores()); - - toSlime(object.setObject("integrations").setObject("aws"), - controller.serviceRegistry().roleService().getTenantRole(tenant.name()), - cloudTenant.tenantSecretStores()); - - try { - var usedQuota = applications.stream() - .map(Application::quotaUsage) - .reduce(QuotaUsage.none, QuotaUsage::add); - - toSlime(object.setObject("quota"), usedQuota); - } catch (Exception e) { - log.warning(String.format("Failed to get quota for tenant %s: %s", tenant.name(), Exceptions.toMessageString(e))); - } - - toSlime(cloudTenant.archiveAccess(), object.setObject("archiveAccess")); - - break; - } - case deleted: break; - default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); - } - // TODO jonmv: This should list applications, not instances. - Cursor applicationArray = object.setArray("applications"); - for (Application application : applications) { - DeploymentStatus status = null; - Collection<Instance> instances = showOnlyProductionInstances(request) ? application.productionInstances().values() - : application.instances().values(); - - if (instances.isEmpty() && !showOnlyActiveInstances(request)) - toSlime(application.id(), applicationArray.addObject(), request); - - for (Instance instance : instances) { - if (showOnlyActiveInstances(request) && instance.deployments().isEmpty()) - continue; - if (recurseOverApplications(request)) { - if (status == null) status = controller.jobController().deploymentStatus(application); - toSlime(applicationArray.addObject(), instance, status, request); - } else { - toSlime(instance.id(), applicationArray.addObject(), request); - } - } - } - tenantMetaDataToSlime(tenant, applications, object.setObject("metaData")); - - if (!tenant.cloudAccounts().isEmpty()) { - Cursor cloudAccounts = object.setArray("cloudAccounts"); - tenant.cloudAccounts().forEach(accountInfo -> { - Cursor accountObject = cloudAccounts.addObject(); - accountObject.setString("cloudAccount", accountInfo.cloudAccount().value()); - accountObject.setString("templateVersion", accountInfo.templateVersion().toFullString()); - }); - } - } - - private void toSlime(ArchiveAccess archiveAccess, Cursor object) { - archiveAccess.awsRole().ifPresent(role -> object.setString("awsRole", role)); - archiveAccess.gcpMember().ifPresent(member -> object.setString("gcpMember", member)); - } - - private void toSlime(Cursor object, QuotaUsage usage) { - object.setDouble("budgetUsed", usage.rate()); - } - - private void toSlime(ClusterResources resources, Cursor object) { - object.setLong("nodes", resources.nodes()); - object.setLong("groups", resources.groups()); - toSlime(resources.nodeResources(), object.setObject("nodeResources")); - - double cost = ResourceMeterMaintainer.cost(resources, controller.serviceRegistry().zoneRegistry().system()); - object.setDouble("cost", cost); - } - - private void toSlime(IntRange range, Cursor object) { - range.from().ifPresent(from -> object.setLong("from", from)); - range.to().ifPresent(to -> object.setLong("to", to)); - } - - private void toSlime(Cluster.Autoscaling autoscaling, Cursor autoscalingObject) { - autoscalingObject.setString("status", autoscaling.status()); - autoscalingObject.setString("description", autoscaling.description()); - autoscaling.resources().ifPresent(resources -> toSlime(resources, autoscalingObject.setObject("resources"))); - autoscalingObject.setLong("at", autoscaling.at().toEpochMilli()); - toSlime(autoscaling.peak(), autoscalingObject.setObject("peak")); - toSlime(autoscaling.ideal(), autoscalingObject.setObject("ideal")); - } - - private void toSlime(Load load, Cursor loadObject) { - loadObject.setDouble("cpu", load.cpu()); - loadObject.setDouble("memory", load.memory()); - loadObject.setDouble("disk", load.disk()); - } - - private void scalingEventsToSlime(List<Cluster.ScalingEvent> scalingEvents, Cursor scalingEventsArray) { - for (Cluster.ScalingEvent scalingEvent : scalingEvents) { - Cursor scalingEventObject = scalingEventsArray.addObject(); - toSlime(scalingEvent.from(), scalingEventObject.setObject("from")); - toSlime(scalingEvent.to(), scalingEventObject.setObject("to")); - scalingEventObject.setLong("at", scalingEvent.at().toEpochMilli()); - scalingEvent.completion().ifPresent(completion -> scalingEventObject.setLong("completion", completion.toEpochMilli())); - } - } - - private void toSlime(NodeResources resources, Cursor object) { - object.setDouble("vcpu", resources.vcpu()); - object.setDouble("memoryGb", resources.memoryGb()); - object.setDouble("diskGb", resources.diskGb()); - object.setDouble("bandwidthGbps", resources.bandwidthGbps()); - object.setString("diskSpeed", valueOf(resources.diskSpeed())); - object.setString("storageType", valueOf(resources.storageType())); - object.setString("architecture", valueOf(resources.architecture())); - object.setLong("gpuCount", resources.gpuResources().count()); - object.setDouble("gpuMemoryGb", resources.gpuResources().memoryGb()); - } - - // A tenant has different content when in a list ... antipattern, but not solvable before application/v5 - private void tenantInTenantsListToSlime(Tenant tenant, URI requestURI, Cursor object) { - object.setString("tenant", tenant.name().value()); - Cursor metaData = object.setObject("metaData"); - metaData.setString("type", tenantType(tenant)); - switch (tenant.type()) { - case athenz: - AthenzTenant athenzTenant = (AthenzTenant) tenant; - metaData.setString("athensDomain", athenzTenant.domain().getName()); - metaData.setString("property", athenzTenant.property().id()); - break; - case cloud: break; - case deleted: break; - default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); - } - object.setString("url", withPath("/application/v4/tenant/" + tenant.name().value(), requestURI).toString()); - } - - private void tenantMetaDataToSlime(Tenant tenant, List<Application> applications, Cursor object) { - Optional<Instant> lastDev = applications.stream() - .flatMap(application -> application.instances().values().stream()) - .flatMap(instance -> instance.deployments().values().stream() - .filter(deployment -> deployment.zone().environment() == Environment.dev) - .map(deployment -> controller.jobController().lastDeploymentStart(instance.id(), deployment))) - .max(Comparator.naturalOrder()) - .or(() -> applications.stream() - .flatMap(application -> application.instances().values().stream()) - .flatMap(instance -> JobType.allIn(controller.zoneRegistry()).stream() - .filter(job -> job.environment() == Environment.dev) - .flatMap(jobType -> controller.jobController().last(instance.id(), jobType).stream())) - .map(Run::start) - .max(Comparator.naturalOrder())); - Optional<Instant> lastSubmission = applications.stream() - .flatMap(app -> app.revisions().last().flatMap(ApplicationVersion::buildTime).stream()) - .max(Comparator.naturalOrder()); - object.setLong("createdAtMillis", tenant.createdAt().toEpochMilli()); - if (tenant.type() == Tenant.Type.deleted) - object.setLong("deletedAtMillis", ((DeletedTenant) tenant).deletedAt().toEpochMilli()); - lastDev.ifPresent(instant -> object.setLong("lastDeploymentToDevMillis", instant.toEpochMilli())); - lastSubmission.ifPresent(instant -> object.setLong("lastSubmissionToProdMillis", instant.toEpochMilli())); - - tenant.lastLoginInfo().get(LastLoginInfo.UserLevel.user) - .ifPresent(instant -> object.setLong("lastLoginByUserMillis", instant.toEpochMilli())); - tenant.lastLoginInfo().get(LastLoginInfo.UserLevel.developer) - .ifPresent(instant -> object.setLong("lastLoginByDeveloperMillis", instant.toEpochMilli())); - tenant.lastLoginInfo().get(LastLoginInfo.UserLevel.administrator) - .ifPresent(instant -> object.setLong("lastLoginByAdministratorMillis", instant.toEpochMilli())); - } - - /** Returns a copy of the given URI with the host and port from the given URI, the path set to the given path and the query set to given query*/ - private URI withPathAndQuery(String newPath, String newQuery, URI uri) { - try { - return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), newPath, newQuery, null); - } - catch (URISyntaxException e) { - throw new RuntimeException("Will not happen", e); - } - } - - /** Returns a copy of the given URI with the host and port from the given URI and the path set to the given path */ - private URI withPath(String newPath, URI uri) { - return withPathAndQuery(newPath, null, uri); - } - - private String toPath(DeploymentId id) { - return path("/application", "v4", - "tenant", id.applicationId().tenant(), - "application", id.applicationId().application(), - "instance", id.applicationId().instance(), - "environment", id.zoneId().environment(), - "region", id.zoneId().region()); - } - - private long asLong(String valueOrNull, long defaultWhenNull) { - if (valueOrNull == null) return defaultWhenNull; - try { - return Long.parseLong(valueOrNull); - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("Expected an integer but got '" + valueOrNull + "'"); - } - } - - private Slime toSlime(InputStream jsonStream) { - try { - byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000); - return SlimeUtils.jsonToSlime(jsonBytes); - } catch (IOException e) { - throw new RuntimeException(); - } - } - - private static Principal requireUserPrincipal(HttpRequest request) { - Principal principal = request.getJDiscRequest().getUserPrincipal(); - if (principal == null) throw new IllegalArgumentException("Expected a user principal"); - return principal; - } - - private Inspector mandatory(String key, Inspector object) { - if ( ! object.field(key).valid()) - throw new IllegalArgumentException("'" + key + "' is missing"); - return object.field(key); - } - - private Optional<String> optional(String key, Inspector object) { - return SlimeUtils.optionalString(object.field(key)); - } - - private static String path(Object... elements) { - return Joiner.on("/").join(elements); - } - - private void toSlime(TenantAndApplicationId id, Cursor object, HttpRequest request) { - object.setString("tenant", id.tenant().value()); - object.setString("application", id.application().value()); - object.setString("url", withPath("/application/v4" + - "/tenant/" + id.tenant().value() + - "/application/" + id.application().value(), - request.getUri()).toString()); - } - - private void toSlime(ApplicationId id, Cursor object, HttpRequest request) { - object.setString("tenant", id.tenant().value()); - object.setString("application", id.application().value()); - object.setString("instance", id.instance().value()); - object.setString("url", withPath("/application/v4" + - "/tenant/" + id.tenant().value() + - "/application/" + id.application().value() + - "/instance/" + id.instance().value(), - request.getUri()).toString()); - } - - private void toSlime(DeploymentId id, ClusterSpec.Id cluster, Cursor object, HttpRequest request) { - object.setString("tenant", id.applicationId().tenant().value()); - object.setString("application", id.applicationId().application().value()); - object.setString("instance", id.applicationId().instance().value()); - object.setString("environment", id.zoneId().environment().value()); - object.setString("region", id.zoneId().region().value()); - object.setString("cluster", cluster.value()); - object.setString("url", withPath("/application/v4" + - "/tenant/" + id.applicationId().tenant().value() + - "/application/" + id.applicationId().application().value() + - "/instance/" + id.applicationId().instance().value() + - "/environment/" + id.zoneId().environment().value() + - "/region/" + id.zoneId().region().value(), - request.getUri()).toString()); - } - - private void stringsToSlime(List<String> strings, Cursor array) { - for (String string : strings) - array.addString(string); - } - - private void toSlime(Cursor object, List<TenantSecretStore> tenantSecretStores) { - Cursor secretStore = object.setArray("secretStores"); - tenantSecretStores.forEach(store -> { - toSlime(secretStore.addObject(), store); - }); - } - - private void toSlime(Cursor object, TenantRoles tenantRoles, List<TenantSecretStore> tenantSecretStores) { - object.setString("tenantRole", tenantRoles.containerRole()); - var stores = object.setArray("accounts"); - tenantSecretStores.forEach(secretStore -> { - toSlime(stores.addObject(), secretStore); - }); - } - - private void toSlime(Cursor object, TenantSecretStore secretStore) { - object.setString("name", secretStore.getName()); - object.setString("awsId", secretStore.getAwsId()); - object.setString("role", secretStore.getRole()); - } - - private String readToString(InputStream stream) { - Scanner scanner = new Scanner(stream).useDelimiter("\\A"); - if ( ! scanner.hasNext()) return null; - return scanner.next(); - } - - private static boolean recurseOverTenants(HttpRequest request) { - return recurseOverApplications(request) || "tenant".equals(request.getProperty("recursive")); - } - - private static boolean recurseOverApplications(HttpRequest request) { - return recurseOverDeployments(request) || "application".equals(request.getProperty("recursive")); - } - - private static boolean recurseOverDeployments(HttpRequest request) { - return ImmutableSet.of("all", "true", "deployment").contains(request.getProperty("recursive")); - } - - private static boolean showOnlyProductionInstances(HttpRequest request) { - return "true".equals(request.getProperty("production")); - } - - private static boolean showOnlyActiveInstances(HttpRequest request) { - return "true".equals(request.getProperty("activeInstances")); - } - - private static boolean includeDeleted(HttpRequest request) { - return "true".equals(request.getProperty("includeDeleted")); - } - - private static String tenantType(Tenant tenant) { - return switch (tenant.type()) { - case athenz: yield "ATHENS"; - case cloud: yield "CLOUD"; - case deleted: yield "DELETED"; - }; - } - - private static ApplicationId appIdFromPath(Path path) { - return ApplicationId.from(path.get("tenant"), path.get("application"), path.get("instance")); - } - - private JobType jobTypeFromPath(Path path) { - return JobType.fromJobName(path.get("jobtype"), controller.zoneRegistry()); - } - - private RunId runIdFromPath(Path path) { - long number = Long.parseLong(path.get("number")); - return new RunId(appIdFromPath(path), jobTypeFromPath(path), number); - } - - private HttpResponse submit(String tenant, String application, HttpRequest request) { - TenantName tenantName = TenantName.from(tenant); - controller.applications().verifyPlan(tenantName); - - Map<String, byte[]> dataParts = parseDataParts(request); - Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get(); - long projectId = submitOptions.field("projectId").asLong(); // Absence of this means it's not a prod app :/ - projectId = projectId == 0 ? 1 : projectId; - Optional<String> repository = optional("repository", submitOptions); - Optional<String> branch = optional("branch", submitOptions); - Optional<String> commit = optional("commit", submitOptions); - Optional<SourceRevision> sourceRevision = repository.isPresent() && branch.isPresent() && commit.isPresent() - ? Optional.of(new SourceRevision(repository.get(), branch.get(), commit.get())) - : Optional.empty(); - Optional<String> sourceUrl = optional("sourceUrl", submitOptions); - Optional<String> authorEmail = optional("authorEmail", submitOptions); - Optional<String> description = optional("description", submitOptions); - int risk = (int) submitOptions.field("risk").asLong(); - - sourceUrl.map(URI::create).ifPresent(url -> { - if (url.getHost() == null || url.getScheme() == null) - throw new IllegalArgumentException("Source URL must include scheme and host"); - }); - - ApplicationPackage applicationPackage = - new ApplicationPackage(dataParts.get(APPLICATION_ZIP), true, controller.system().isPublic()); - byte[] testPackage = dataParts.getOrDefault(APPLICATION_TEST_ZIP, new byte[0]); - Submission submission = new Submission(applicationPackage, testPackage, sourceUrl, sourceRevision, authorEmail, description, controller.clock().instant(), risk); - - controller.applications().verifyApplicationIdentityConfiguration(tenantName, - Optional.empty(), - applicationPackage, - Optional.of(requireUserPrincipal(request))); - - TenantAndApplicationId id = TenantAndApplicationId.from(tenant, application); - ensureApplicationExists(id, request); - return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), id, submission, projectId); - } - - private HttpResponse removeAllProdDeployments(String tenant, String application) { - JobControllerApiHandlerHelper.submitResponse(controller.jobController(), - TenantAndApplicationId.from(tenant, application), - new Submission(ApplicationPackage.deploymentRemoval(), new byte[0], Optional.empty(), - Optional.empty(), Optional.empty(), Optional.empty(), controller.clock().instant(), 0), - 0); - return new MessageResponse("All deployments removed"); - } - - private void addAvailabilityZone(Cursor object, ZoneId zoneId) { - ZoneApi zone = controller.zoneRegistry().get(zoneId); - if (!zone.getCloudName().equals(CloudName.AWS)) return; - object.setString("availabilityZone", zone.getCloudNativeAvailabilityZone()); - } - - private ZoneId requireZone(String environment, String region) { - return requireZone(ZoneId.from(environment, region)); - } - - private ZoneId requireZone(ZoneId zone) { - // TODO(mpolden): Find a way to not hardcode this. Some APIs allow this "virtual" zone, e.g. /logs - if (zone.environment() == Environment.prod && zone.region().value().equals("controller")) { - return zone; - } - if (!controller.zoneRegistry().hasZone(zone)) { - throw new IllegalArgumentException("Zone " + zone + " does not exist in this system"); - } - return zone; - } - - private static Map<String, byte[]> parseDataParts(HttpRequest request) { - String contentHash = request.getHeader("X-Content-Hash"); - if (contentHash == null) - return new MultipartParser().parse(request); - - DigestInputStream digester = Signatures.sha256Digester(request.getData()); - var dataParts = new MultipartParser().parse(request.getHeader("Content-Type"), digester, request.getUri()); - if ( ! Arrays.equals(digester.getMessageDigest().digest(), Base64.getDecoder().decode(contentHash))) - throw new IllegalArgumentException("Value of X-Content-Hash header does not match computed content hash"); - - return dataParts; - } - - private static RotationId findRotationId(Instance instance, Optional<String> endpointId) { - if (instance.rotations().isEmpty()) { - throw new NotExistsException("global rotation does not exist for " + instance); - } - if (endpointId.isPresent()) { - return instance.rotations().stream() - .filter(r -> r.endpointId().id().equals(endpointId.get())) - .map(AssignedRotation::rotationId) - .findFirst() - .orElseThrow(() -> new NotExistsException("endpoint " + endpointId.get() + - " does not exist for " + instance)); - } else if (instance.rotations().size() > 1) { - throw new IllegalArgumentException(instance + " has multiple rotations. Query parameter 'endpointId' must be given"); - } - return instance.rotations().get(0).rotationId(); - } - - private static String rotationStateString(RotationState state) { - return switch (state) { - case in: yield "IN"; - case out: yield "OUT"; - case unknown: yield "UNKNOWN"; - }; - } - - private static String endpointScopeString(Endpoint.Scope scope) { - return switch (scope) { - case weighted: yield "weighted"; - case application: yield "application"; - case global: yield "global"; - case zone: yield "zone"; - }; - } - - private static String routingMethodString(RoutingMethod method) { - return switch (method) { - case exclusive: yield "exclusive"; - case sharedLayer4: yield "sharedLayer4"; - }; - } - - private static <T> T getAttribute(HttpRequest request, String attributeName, Class<T> cls) { - return Optional.ofNullable(request.getJDiscRequest().context().get(attributeName)) - .filter(cls::isInstance) - .map(cls::cast) - .orElseThrow(() -> new IllegalArgumentException("Attribute '" + attributeName + "' was not set on request")); - } - - /** Returns whether given request is by an operator */ - private static boolean isOperator(HttpRequest request) { - var securityContext = getAttribute(request, SecurityContext.ATTRIBUTE_NAME, SecurityContext.class); - return securityContext.roles().stream() - .map(Role::definition) - .anyMatch(definition -> definition == RoleDefinition.hostedOperator); - } - - private void ensureApplicationExists(TenantAndApplicationId id, HttpRequest request) { - if (controller.applications().getApplication(id).isEmpty()) { - if (controller.system().isPublic() || hasOktaContext(request)) { - log.fine("Application does not exist in public, creating: " + id); - var credentials = accessControlRequests.credentials(id.tenant(), null /* not used on public */ , request.getJDiscRequest()); - controller.applications().createApplication(id, credentials); - } else { - log.fine("Application does not exist in hosted, failing: " + id); - throw new IllegalArgumentException("Application does not exist. Create application in Console first."); - } - } - } - - private boolean hasOktaContext(HttpRequest request) { - try { - OAuthCredentials.fromOktaRequestContext(request.getJDiscRequest().context()); - return true; - } catch (IllegalArgumentException e) { - return false; - } - } - - private List<Deployment> sortedDeployments(Collection<Deployment> deployments, DeploymentInstanceSpec spec) { - List<ZoneId> productionZones = spec.zones().stream() - .filter(z -> z.region().isPresent()) - .map(z -> ZoneId.from(z.environment(), z.region().get())) - .toList(); - return deployments.stream() - .sorted(comparingInt(deployment -> productionZones.indexOf(deployment.zone()))) - .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); - } - -} - diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java deleted file mode 100644 index 3bf2f070f97..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.container.jdisc.HttpResponse; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; - -/** - * @author freva - */ -public class HtmlResponse extends HttpResponse { - - private final String content; - - public HtmlResponse(String content) { - super(200); - this.content = content; - } - - @Override - public void render(OutputStream stream) throws IOException { - stream.write(content.getBytes(StandardCharsets.UTF_8)); - } - - @Override - public String getContentType() { return "text/html"; } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java deleted file mode 100644 index 18221d82e44..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ /dev/null @@ -1,533 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.DeploymentSpec.ChangeBlocker; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.NotExistsException; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness; -import com.yahoo.vespa.hosted.controller.deployment.JobController; -import com.yahoo.vespa.hosted.controller.deployment.JobStatus; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.Run.Reason; -import com.yahoo.vespa.hosted.controller.deployment.RunLog; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Step; -import com.yahoo.vespa.hosted.controller.deployment.Submission; -import com.yahoo.vespa.hosted.controller.deployment.Versions; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.time.Instant; -import java.time.format.TextStyle; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Optional; -import java.util.stream.Stream; - -import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.canary; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; -import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.broken; -import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.normal; -import static java.util.Comparator.reverseOrder; - -/** - * Implements the REST API for the job controller delegated from the Application API. - * - * @see JobController - * @see ApplicationApiHandler - * - * @author smorgrav - * @author jonmv - */ -class JobControllerApiHandlerHelper { - - /** - * @return Response with all job types that have recorded runs for the application _and_ the status for the last run of that type - */ - static HttpResponse jobTypeResponse(Controller controller, ApplicationId id, URI baseUriForJobs) { - Slime slime = new Slime(); - Cursor responseObject = slime.setObject(); - - Cursor jobsArray = responseObject.setArray("deployment"); - JobType.allIn(controller.zoneRegistry()).stream() - .filter(type -> type.environment().isManuallyDeployed()) - .map(devType -> new JobId(id, devType)) - .forEach(job -> { - Collection<Run> runs = controller.jobController().runs(job).descendingMap().values(); - if (runs.isEmpty()) - return; - - Cursor jobObject = jobsArray.addObject(); - jobObject.setString("jobName", job.type().jobName()); - toSlime(jobObject.setArray("runs"), runs, controller.applications().requireApplication(TenantAndApplicationId.from(id)), 10, baseUriForJobs); - }); - - return new SlimeJsonResponse(slime); - } - - /** Returns a response with the runs for the given job type. */ - static HttpResponse runResponse(Controller controller, JobId id, Optional<String> limitStr, URI baseUriForJobType) { - Slime slime = new Slime(); - Cursor cursor = slime.setObject(); - Application application = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())); - NavigableMap<RunId, Run> runs = controller.jobController().runs(id).descendingMap(); - - int limit = limitStr.map(Integer::parseInt).orElse(Integer.MAX_VALUE); - toSlime(cursor.setArray("runs"), runs.values(), application, limit, baseUriForJobType); - Optional.ofNullable(runs.lastEntry()) - .map(entry -> new DeploymentId(id.application(), entry.getValue().id().job().type().zone())) // Urgh, must use a job with actual zone. - .flatMap(deployment -> controller.applications().decideCloudAccountOf(deployment, application.deploymentSpec())) - .ifPresent(cloudAccount -> cursor.setObject("enclave").setString("cloudAccount", cloudAccount.value())); - - return new SlimeJsonResponse(slime); - } - - /** - * @return Response with logs from a single run - */ - static HttpResponse runDetailsResponse(JobController jobController, RunId runId, String after) { - Slime slime = new Slime(); - Cursor detailsObject = slime.setObject(); - - Run run = jobController.run(runId); - detailsObject.setBool("active", ! run.hasEnded()); - detailsObject.setString("status", nameOf(run.status())); - run.reason().reason().ifPresent(reason -> detailsObject.setString("reason", reason)); - run.reason().dependent().ifPresent(dependent -> { - Cursor dependentObject = detailsObject.setObject("dependent"); - dependentObject.setString("instance", dependent.application().instance().value()); - dependentObject.setString("region", dependent.type().zone().region().value()); - run.reason().change().flatMap(Change::platform).ifPresent(platform -> dependentObject.setString("platform", platform.toFullString())); - run.reason().change().flatMap(Change::revision).ifPresent(revision -> dependentObject.setLong("build", revision.number())); - }); - try { - jobController.updateTestLog(runId); - jobController.updateVespaLog(runId); - } - catch (RuntimeException ignored) { } // Return response when this fails, which it does when, e.g., logserver is booting. - - RunLog runLog = (after == null ? jobController.details(runId) : jobController.details(runId, Long.parseLong(after))) - .orElseThrow(() -> new NotExistsException(Text.format( - "No run details exist for application: %s, job type: %s, number: %d", - runId.application().toShortString(), runId.type().jobName(), runId.number()))); - - Cursor logObject = detailsObject.setObject("log"); - for (Step step : Step.values()) { - if ( ! runLog.get(step).isEmpty()) - toSlime(logObject.setArray(step.name()), runLog.get(step)); - } - runLog.lastId().ifPresent(id -> detailsObject.setLong("lastId", id)); - - Cursor stepsObject = detailsObject.setObject("steps"); - run.steps().forEach((step, info) -> { - Cursor stepCursor = stepsObject.setObject(step.name()); - stepCursor.setString("status", info.status().name()); - info.startTime().ifPresent(startTime -> stepCursor.setLong("startMillis", startTime.toEpochMilli())); - run.convergenceSummary().ifPresent(summary -> { - // If initial installation never succeeded, but is part of the job, summary concerns it. - // If initial succeeded, or is not part of this job, summary concerns upgrade installation. - if ( step == installInitialReal && info.status() != succeeded - || step == installReal && run.stepStatus(installInitialReal).map(status -> status == succeeded).orElse(true)) - toSlime(stepCursor.setObject("convergence"), summary); - }); - }); - - // If a test report is available, include it in the response. - Optional<String> testReport = jobController.getTestReports(runId); - testReport.map(SlimeUtils::jsonToSlime) - .map(Slime::get) - .ifPresent(reportArrayCursor -> SlimeUtils.copyArray(reportArrayCursor, detailsObject.setArray("testReports"))); - - boolean logsStored = run.stepStatus(copyVespaLogs).map(succeeded::equals).orElse(false); - if (run.hasStep(copyVespaLogs) && ! runId.type().isProduction() && JobController.deploymentCompletedAt(run, false).isPresent()) - detailsObject.setBool("vespaLogsActive", ! logsStored); - - if (runId.type().isTest() && JobController.deploymentCompletedAt(run, true).isPresent()) - detailsObject.setBool("testerLogsActive", ! logsStored); - - return new SlimeJsonResponse(slime); - } - - /** Proxies a Vespa log request for a run to S3 once logs have been copied, or to logserver before this. */ - static HttpResponse vespaLogsResponse(JobController jobController, RunId runId, long fromMillis, boolean tester) { - return new HttpResponse(200) { - @Override public void render(OutputStream out) throws IOException { - try (InputStream logs = jobController.getVespaLogs(runId, fromMillis, tester)) { - logs.transferTo(out); - } - } - }; - } - - private static void toSlime(Cursor summaryObject, ConvergenceSummary summary) { - summaryObject.setLong("nodes", summary.nodes()); - summaryObject.setLong("down", summary.down()); - summaryObject.setLong("needPlatformUpgrade", summary.needPlatformUpgrade()); - summaryObject.setLong("upgrading", summary.upgradingPlatform()); - summaryObject.setLong("needReboot", summary.needReboot()); - summaryObject.setLong("rebooting", summary.rebooting()); - summaryObject.setLong("needRestart", summary.needRestart()); - summaryObject.setLong("restarting", summary.restarting()); - summaryObject.setLong("upgradingOs", summary.upgradingOs()); - summaryObject.setLong("upgradingFirmware", summary.upgradingFirmware()); - summaryObject.setLong("services", summary.services()); - summaryObject.setLong("needNewConfig", summary.needNewConfig()); - summaryObject.setLong("retiring", summary.retiring()); - } - - private static void toSlime(Cursor entryArray, List<LogEntry> entries) { - entries.forEach(entry -> toSlime(entryArray.addObject(), entry)); - } - - private static void toSlime(Cursor entryObject, LogEntry entry) { - entryObject.setLong("at", entry.at().toEpochMilli()); - entryObject.setString("type", entry.type().name()); - entryObject.setString("message", entry.message()); - } - - /** - * Unpack payload and submit to job controller. Defaults instance to 'default' and renders the - * application version on success. - * - * @return Response with the new application version - */ - static HttpResponse submitResponse(JobController jobController, TenantAndApplicationId id, Submission submission, long projectId) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - ApplicationVersion submitted = jobController.submit(id, submission, projectId); - String skipped = submitted.shouldSkip() - ? "; only applying deployment spec changes, as this build is otherwise equal to the previous" - : ""; - root.setString("message", "application " + submitted + skipped); - root.setLong("build", submitted.buildNumber()); - return new SlimeJsonResponse(slime); - } - - /** Aborts any job of the given type. */ - static HttpResponse abortJobResponse(JobController jobs, HttpRequest request, ApplicationId id, JobType type) { - Slime slime = new Slime(); - Cursor responseObject = slime.setObject(); - Optional<Run> run = jobs.last(id, type).flatMap(last -> jobs.active(last.id())); - if (run.isPresent()) { - jobs.abort(run.get().id(), "aborted by " + request.getJDiscRequest().getUserPrincipal().getName(), true); - responseObject.setString("message", "Aborting " + run.get().id()); - } - else - responseObject.setString("message", "Nothing to abort."); - return new SlimeJsonResponse(slime); - } - - private static String nameOf(RunStatus status) { - return switch (status) { - case reset, running -> "running"; - case cancelled, aborted -> "aborted"; - case error -> "error"; - case testFailure -> "testFailure"; - case noTests -> "noTests"; - case endpointCertificateTimeout -> "endpointCertificateTimeout"; - case nodeAllocationFailure -> "nodeAllocationFailure"; - case installationFailed -> "installationFailed"; - case invalidApplication, deploymentFailed -> "deploymentFailed"; - case success -> "success"; - case quotaExceeded -> "quotaExceeded"; - }; - } - - /** - * Returns response with all job types that have recorded runs for the application - * _and_ the status for the last run of that type - */ - static HttpResponse overviewResponse(Controller controller, TenantAndApplicationId id, URI baseUriForDeployments) { - Application application = controller.applications().requireApplication(id); - DeploymentStatus status = controller.jobController().deploymentStatus(application); - - Slime slime = new Slime(); - Cursor responseObject = slime.setObject(); - responseObject.setString("tenant", id.tenant().value()); - responseObject.setString("application", id.application().value()); - application.projectId().ifPresent(projectId -> responseObject.setLong("projectId", projectId)); - - Map<JobId, List<DeploymentStatus.Job>> jobsToRun = status.jobsToRun(); - Cursor stepsArray = responseObject.setArray("steps"); - VersionStatus versionStatus = controller.readVersionStatus(); - for (DeploymentStatus.StepStatus stepStatus : status.allSteps()) { - Change change = status.application().require(stepStatus.instance()).change(); - Cursor stepObject = stepsArray.addObject(); - stepObject.setString("type", stepStatus.type().name()); - stepStatus.dependencies().stream() - .map(status.allSteps()::indexOf) - .forEach(stepObject.setArray("dependencies")::addLong); - stepObject.setBool("declared", stepStatus.isDeclared()); - stepObject.setString("instance", stepStatus.instance().value()); - - // TODO: recursively search dependents for what is the relevant partial change when this is a delay step ... - Instant now = controller.clock().instant(); - Readiness readiness = stepStatus.pausedUntil().okAt(now) - ? stepStatus.job().map(jobsToRun::get).map(job -> job.get(0).readiness()) - .orElse(stepStatus.readiness(change)) - : stepStatus.pausedUntil(); - if (readiness.ok()) { - // TODO jonmv: remove after UI changes. - stepObject.setLong("readyAt", readiness.at().toEpochMilli()); - - if ( ! readiness.okAt(now)) stepObject.setLong("delayedUntil", readiness.at().toEpochMilli()); - } - - // TODO jonmv: remove after UI changes. - if (readiness.cause() == DelayCause.coolingDown) stepObject.setLong("coolingDownUntil", readiness.at().toEpochMilli()); - if (readiness.cause() == DelayCause.paused) stepObject.setLong("pausedUntil", readiness.at().toEpochMilli()); - - Readiness platformReadiness = stepStatus.blockedUntil(Change.of(controller.systemVersion(versionStatus))); // Dummy version — just anything with a platform. - if ( ! platformReadiness.okAt(now)) - stepObject.setLong("platformBlockedUntil", platformReadiness.at().toEpochMilli()); - Readiness applicationReadiness = stepStatus.blockedUntil(Change.of(RevisionId.forProduction(1))); // Dummy version — just anything with an application. - if ( ! applicationReadiness.okAt(now)) - stepObject.setLong("applicationBlockedUntil", applicationReadiness.at().toEpochMilli()); - - if (stepStatus.type() == DeploymentStatus.StepType.delay) - stepStatus.completedAt(change).ifPresent(completed -> stepObject.setLong("completedAt", completed.toEpochMilli())); - - if (stepStatus.type() == DeploymentStatus.StepType.instance) { - Cursor deployingObject = stepObject.setObject("deploying"); - if ( ! change.isEmpty()) { - change.platform().ifPresent(version -> deployingObject.setString("platform", version.toFullString())); - change.revision().ifPresent(revision -> toSlime(deployingObject.setObject("application"), application.revisions().get(revision))); - if (change.isPlatformPinned()) deployingObject.setBool("pinned", true); - if (change.isPlatformPinned()) deployingObject.setBool("platformPinned", true); - if (change.isRevisionPinned()) deployingObject.setBool("revisionPinned", true); - } - - Cursor latestVersionsObject = stepObject.setObject("latestVersions"); - List<ChangeBlocker> blockers = application.deploymentSpec().requireInstance(stepStatus.instance()).changeBlocker(); - var deployments = application.require(stepStatus.instance()).productionDeployments().values(); - List<VespaVersion> availablePlatforms = availablePlatforms(versionStatus.versions(), - application.deploymentSpec().requireInstance(stepStatus.instance()).upgradePolicy()); - if ( ! availablePlatforms.isEmpty()) { - Cursor latestPlatformObject = latestVersionsObject.setObject("platform"); - VespaVersion latestPlatform = availablePlatforms.get(0); - latestPlatformObject.setString("platform", latestPlatform.versionNumber().toFullString()); - latestPlatformObject.setLong("at", latestPlatform.committedAt().toEpochMilli()); - latestPlatformObject.setBool("upgrade", change.platform().map(latestPlatform.versionNumber()::isAfter).orElse(true) && deployments.isEmpty() - || deployments.stream().anyMatch(deployment -> deployment.version().isBefore(latestPlatform.versionNumber()))); - - Cursor availableArray = latestPlatformObject.setArray("available"); - boolean isUpgrade = true; - for (VespaVersion available : availablePlatforms) { - if ( deployments.stream().anyMatch(deployment -> deployment.version().isAfter(available.versionNumber())) - || deployments.stream().noneMatch(deployment -> deployment.version().isBefore(available.versionNumber())) && ! deployments.isEmpty() - || status.hasCompleted(stepStatus.instance(), Change.of(available.versionNumber())) - || change.platform().map(available.versionNumber()::compareTo).orElse(1) < 0) - isUpgrade = false; - - Cursor platformObject = availableArray.addObject(); - platformObject.setString("platform", available.versionNumber().toFullString()); - platformObject.setBool("upgrade", isUpgrade || change.platform().map(available.versionNumber()::equals).orElse(false)); - } - toSlime(latestPlatformObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksVersions)); - } - List<ApplicationVersion> availableApplications = new ArrayList<>(application.revisions().deployable(false)); - if ( ! availableApplications.isEmpty()) { - var latestApplication = availableApplications.get(0); - Cursor latestApplicationObject = latestVersionsObject.setObject("application"); - toSlime(latestApplicationObject.setObject("application"), latestApplication); - latestApplicationObject.setLong("at", latestApplication.buildTime().orElse(Instant.EPOCH).toEpochMilli()); - latestApplicationObject.setBool("upgrade", change.revision().map(latestApplication.id()::compareTo).orElse(1) > 0 && deployments.isEmpty() - || deployments.stream().anyMatch(deployment -> deployment.revision().compareTo(latestApplication.id()) < 0)); - - Cursor availableArray = latestApplicationObject.setArray("available"); - for (ApplicationVersion available : availableApplications) - toSlime(availableArray.addObject().setObject("application"), available); - - toSlime(latestApplicationObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksRevisions)); - } - } - - boolean showDelayCause = true; - if (stepStatus.job().isPresent()) { - JobId job = stepStatus.job().get(); - stepObject.setString("jobName", job.type().jobName()); - URI baseUriForJob = baseUriForDeployments.resolve(baseUriForDeployments.getPath() + - "/../instance/" + job.application().instance().value() + - "/job/" + job.type().jobName()).normalize(); - stepObject.setString("url", baseUriForJob.toString()); - stepObject.setString("environment", job.type().environment().value()); - if ( ! job.type().environment().isTest()) { - stepObject.setString("region", job.type().zone().value()); - } - - if (job.type().isProduction() && job.type().isDeployment()) { - status.deploymentFor(job).ifPresent(deployment -> { - stepObject.setString("currentPlatform", deployment.version().toFullString()); - toSlime(stepObject.setObject("currentApplication"), application.revisions().get(deployment.revision())); - }); - } - - JobStatus jobStatus = status.jobs().get(job).get(); - Cursor toRunArray = stepObject.setArray("toRun"); - showDelayCause = readiness.cause() == DelayCause.paused; - for (DeploymentStatus.Job jobToRun : jobsToRun.getOrDefault(job, List.of())) { - boolean running = jobStatus.lastTriggered() - .map(run -> jobStatus.isRunning() - && jobToRun.versions().targetsMatch(run.versions()) - && (job.type().isProduction() || jobToRun.versions().sourcesMatchIfPresent(run.versions()))) - .orElse(false); - if (running) - continue; // Run will be contained in the "runs" array. - - showDelayCause = true; - Cursor runObject = toRunArray.addObject(); - toSlime(runObject, jobToRun.versions(), jobToRun.reason(), application); - } - - if ( ! jobStatus.runs().isEmpty()) - controller.applications().decideCloudAccountOf(new DeploymentId(job.application(), - jobStatus.runs().lastEntry().getValue().id().job().type().zone()), // Urgh, must use a job with actual zone. - status.application().deploymentSpec()) - .ifPresent(cloudAccount -> stepObject.setObject("enclave").setString("cloudAccount", cloudAccount.value())); - - - toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), application, 10, baseUriForJob); - } - stepObject.setString("delayCause", - ! showDelayCause - ? (String) null - : switch (readiness.cause()) { - case none -> null; - case invalidPackage -> "invalidPackage"; - case paused -> "paused"; - case coolingDown -> "coolingDown"; - case changeBlocked -> "changeBlocked"; - case blocked -> "blocked"; - case running -> "running"; - case notReady -> "notReady"; - case unverified -> "unverified"; - }); - } - - Cursor buildsArray = responseObject.setArray("builds"); - application.revisions().withPackage().stream().sorted(reverseOrder()).forEach(version -> toRichSlime(buildsArray.addObject(), version)); - - return new SlimeJsonResponse(slime); - } - - static void toRichSlime(Cursor versionObject, ApplicationVersion version) { - toSlime(versionObject, version); - version.description().ifPresent(description -> versionObject.setString("description", description)); - if (version.risk() != 0) versionObject.setLong("risk", version.risk()); - versionObject.setBool("deployable", version.isDeployable()); - version.submittedAt().ifPresent(submittedAt -> versionObject.setLong("submittedAt", submittedAt.toEpochMilli())); - } - - static void toSlime(Cursor versionObject, ApplicationVersion version) { - versionObject.setLong("build", version.buildNumber()); - version.compileVersion().ifPresent(platform -> versionObject.setString("compileVersion", platform.toFullString())); - version.sourceUrl().ifPresent(url -> versionObject.setString("sourceUrl", url)); - version.commit().ifPresent(commit -> versionObject.setString("commit", commit)); - } - - private static void toSlime(Cursor versionsObject, Versions versions, Application application) { - versionsObject.setString("targetPlatform", versions.targetPlatform().toFullString()); - toSlime(versionsObject.setObject("targetApplication"), application.revisions().get(versions.targetRevision())); - versions.sourcePlatform().ifPresent(platform -> versionsObject.setString("sourcePlatform", platform.toFullString())); - versions.sourceRevision().ifPresent(revision -> toSlime(versionsObject.setObject("sourceApplication"), application.revisions().get(revision))); - } - - private static void toSlime(Cursor blockersArray, Stream<ChangeBlocker> blockers) { - blockers.forEach(blocker -> { - Cursor blockerObject = blockersArray.addObject(); - blocker.window().days().stream() - .map(day -> day.getDisplayName(TextStyle.SHORT, Locale.ENGLISH)) - .forEach(blockerObject.setArray("days")::addString); - blocker.window().hours() - .forEach(blockerObject.setArray("hours")::addLong); - blockerObject.setString("zone", blocker.window().zone().toString()); - }); - } - - private static List<VespaVersion> availablePlatforms(List<VespaVersion> versions, DeploymentSpec.UpgradePolicy policy) { - int i; - for (i = versions.size(); i-- > 0; ) - if (versions.get(i).isSystemVersion()) - break; - - if (i < 0) - return List.of(); - - List<VespaVersion> candidates = new ArrayList<>(); - VespaVersion.Confidence required = policy == canary ? broken : normal; - for (int j = i; j >= 0; j--) - if (versions.get(j).confidence().equalOrHigherThan(required)) - candidates.add(versions.get(j)); - - if (candidates.isEmpty()) - candidates.add(versions.get(i)); - - return candidates; - } - - private static void toSlime(Cursor runObject, Versions versions, Reason reason, Application application) { - reason.reason().ifPresent(because -> runObject.setString("reason", because)); - reason.dependent().ifPresent(dependent -> { - Cursor dependentObject = runObject.setObject("dependent"); - dependentObject.setString("instance", dependent.application().instance().value()); - dependentObject.setString("region", dependent.type().zone().region().value()); - reason.change().flatMap(Change::platform).ifPresent(platform -> dependentObject.setString("platform", platform.toFullString())); - reason.change().flatMap(Change::revision).ifPresent(revision -> dependentObject.setLong("build", revision.number())); - }); - toSlime(runObject.setObject("versions"), versions, application); - } - - private static void toSlime(Cursor runsArray, Collection<Run> runs, Application application, int limit, URI baseUriForJob) { - runs.stream().limit(limit).forEach(run -> { - Cursor runObject = runsArray.addObject(); - runObject.setLong("id", run.id().number()); - runObject.setString("url", baseUriForJob.resolve(baseUriForJob.getPath() + "/run/" + run.id().number()).toString()); - runObject.setLong("start", run.start().toEpochMilli()); - run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli())); - runObject.setString("status", nameOf(run.status())); - toSlime(runObject, run.versions(), run.reason(), application); - run.cloudAccount().filter(account -> ! account.isUnspecified()) - .ifPresent(cloudAccount -> runObject.setObject("enclave").setString("cloudAccount", cloudAccount.value())); - Cursor runStepsArray = runObject.setArray("steps"); - run.steps().forEach((step, info) -> { - Cursor runStepObject = runStepsArray.addObject(); - runStepObject.setString("name", step.name()); - runStepObject.setString("status", info.status().name()); - }); - }); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java deleted file mode 100644 index 35eb495a564..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; -import org.apache.commons.fileupload.MultipartStream; -import org.apache.commons.fileupload.ParameterParser; -import org.apache.commons.fileupload.util.Streams; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.util.HashMap; -import java.util.Map; - -/** - * Provides reading a multipart/form-data request type into a map of bytes for each part, - * indexed by the parts (form field) name. - * - * @author bratseth - */ -public class MultipartParser { - - private final long maxDataLength; - - public MultipartParser() { - this(2 * (long) Math.pow(1024, 3)); // 2 GB - } - - MultipartParser(long maxDataLength) { - this.maxDataLength = maxDataLength; - } - - /** - * Parses the given multipart request and returns all the parts indexed by their name. - * - * @throws IllegalArgumentException if this request is not a well-formed request with Content-Type multipart/form-data - */ - public Map<String, byte[]> parse(HttpRequest request) { - return parse(request.getHeader("Content-Type"), request.getData(), request.getUri()); - } - - /** - * Parses the given data stream for the given uri using the provided content-type header to determine boundaries. - * - * @throws IllegalArgumentException if this is not a well-formed request with Content-Type multipart/form-data - */ - public Map<String, byte[]> parse(String contentTypeHeader, InputStream data, URI uri) { - try { - LimitedOutputStream output = new LimitedOutputStream(maxDataLength); - ParameterParser parameterParser = new ParameterParser(); - Map<String, String> contentType = parameterParser.parse(contentTypeHeader, ';'); - if (contentType.containsKey("application/zip")) { - Streams.copy(data, output, false); - return Map.of(EnvironmentResource.APPLICATION_ZIP, output.toByteArray()); - } - if ( ! contentType.containsKey("multipart/form-data")) - throw new IllegalArgumentException("Expected a multipart or application/zip message, but got Content-Type: " + contentTypeHeader); - String boundary = contentType.get("boundary"); - if (boundary == null) - throw new IllegalArgumentException("Missing boundary property in Content-Type header"); - MultipartStream multipartStream = new MultipartStream(data, boundary.getBytes(), 1 << 20, null); - boolean nextPart = multipartStream.skipPreamble(); - Map<String, byte[]> parts = new HashMap<>(); - while (nextPart) { - String[] headers = multipartStream.readHeaders().split("\r\n"); - String contentDispositionContent = findContentDispositionHeader(headers); - if (contentDispositionContent == null) - throw new IllegalArgumentException("Missing Content-Disposition header in a multipart body part"); - Map<String, String> contentDisposition = parameterParser.parse(contentDispositionContent, ';'); - multipartStream.readBodyData(output); - parts.put(contentDisposition.get("name"), output.toByteArray()); - output.reset(); - nextPart = multipartStream.readBoundary(); - } - return parts; - } - catch (MultipartStream.MalformedStreamException e) { - throw new IllegalArgumentException("Malformed multipart/form-data request", e); - } - catch (IOException e) { - throw new IllegalArgumentException("IO error reading multipart request " + uri, e); - } - } - - private String findContentDispositionHeader(String[] headers) { - String contentDisposition = "Content-Disposition:"; - for (String header : headers) { - if (header.length() < contentDisposition.length()) continue; - if ( ! header.substring(0, contentDisposition.length()).equalsIgnoreCase(contentDisposition)) continue; - return header.substring(contentDisposition.length() + 1); - } - return null; - } - - /** A {@link java.io.ByteArrayOutputStream} that limits the number of bytes written to it */ - private static class LimitedOutputStream extends ByteArrayOutputStream { - - private long remaining; - - /** Create a new OutputStream that can fit up to len bytes */ - private LimitedOutputStream(long len) { - this.remaining = len; - } - - @Override - public synchronized void write(int b) { - requireCapacity(1); - super.write(b); - remaining--; - } - - @Override - public synchronized void write(byte[] b, int off, int len) { - requireCapacity(len); - super.write(b, off, len); - remaining -= len; - } - - private void requireCapacity(int len) { - if (len > remaining) throw new IllegalArgumentException("Too many bytes to write"); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ZipResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ZipResponse.java deleted file mode 100644 index 73f9db7165c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ZipResponse.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.container.jdisc.HttpResponse; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -/** - * A HTTP response containing a named ZIP file. - * - * @author mpolden - */ -public class ZipResponse extends HttpResponse { - - private final InputStream zipContent; - - public ZipResponse(String filename, InputStream zipContent) { - super(200); - this.zipContent = zipContent; - this.headers().add("Content-Disposition", "attachment; filename=\"" + filename + "\""); - } - - @Override - public String getContentType() { - return "application/zip"; - } - - @Override - public void render(OutputStream outputStream) throws IOException { - try (zipContent) { - zipContent.transferTo(outputStream); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java deleted file mode 100644 index 2ff0c1ab05c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.athenz; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.SystemName; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.ResourceResponse; -import com.yahoo.restapi.RestApi; -import com.yahoo.restapi.RestApiRequestHandler; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.Property; -import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; -import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; -import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; - -import java.util.Map; -import java.util.logging.Logger; - -import static com.yahoo.restapi.RestApi.route; - -/** - * This API proxies requests to an Athenz server. - * - * @author jonmv - */ -@SuppressWarnings("unused") // Handler -public class AthenzApiHandler extends RestApiRequestHandler<AthenzApiHandler> { - - private final static Logger log = Logger.getLogger(AthenzApiHandler.class.getName()); - - private final AthenzFacade athenz; - private final AthenzDomain sandboxDomain; - private final EntityService properties; - - @Inject - public AthenzApiHandler(Context parentCtx, AthenzFacade athenz, Controller controller) { - super(parentCtx, AthenzApiHandler::createRestApi); - this.athenz = athenz; - this.sandboxDomain = new AthenzDomain(sandboxDomainIn(controller.system())); - this.properties = controller.serviceRegistry().entityService(); - } - - private static RestApi createRestApi(AthenzApiHandler self) { - return RestApi.builder() - .addRoute(route("/athenz/v1") - .get(self::root)) - .addRoute(route("/athenz/v1/domains") - .get(self::domainList)) - .addRoute(route("/athenz/v1/properties") - .get(self::properties)) - .addRoute(route("/athenz/v1/user") - .post(self::signup)) - .build(); - } - - private HttpResponse root(RestApi.RequestContext ctx) { - return new ResourceResponse(ctx.request(), "domains", "properties"); - } - - private Slime properties(RestApi.RequestContext ctx) { - Slime slime = new Slime(); - Cursor response = slime.setObject(); - Cursor array = response.setArray("properties"); - for (Map.Entry<PropertyId, Property> entry : properties.listProperties().entrySet()) { - Cursor propertyObject = array.addObject(); - propertyObject.setString("propertyid", entry.getKey().id()); - propertyObject.setString("property", entry.getValue().id()); - } - return slime; - } - - private Slime domainList(RestApi.RequestContext ctx) { - Slime slime = new Slime(); - Cursor array = slime.setObject().setArray("data"); - for (AthenzDomain athenzDomain : athenz.getDomainList(ctx.queryParameters().getString("prefix").orElse(null))) - array.addString(athenzDomain.getName()); - - return slime; - } - - private String signup(RestApi.RequestContext ctx) { - AthenzUser user = athenzUser(ctx); - athenz.addTenantAdmin(sandboxDomain, user); - return "User '" + user.getName() + "' added to admin role of '" + sandboxDomain.getName() + "'"; - } - - private static AthenzUser athenzUser(RestApi.RequestContext ctx) { - return ctx.userPrincipal() - .filter(AthenzPrincipal.class::isInstance) - .map(AthenzPrincipal.class::cast) - .map(AthenzPrincipal::getIdentity) - .filter(AthenzUser.class::isInstance) - .map(AthenzUser.class::cast) - .orElseThrow(() -> new IllegalArgumentException("No Athenz user principal on request")); - } - - static String sandboxDomainIn(SystemName system) { - switch (system) { - case main: return "vespa.vespa.tenants.sandbox"; - case cd: return "vespa.vespa.cd.tenants.sandbox"; - default: throw new IllegalArgumentException("No sandbox domain in system '" + system + "'"); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java deleted file mode 100644 index bdd89abfa4c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ /dev/null @@ -1,683 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.billing; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.RestApi; -import com.yahoo.restapi.RestApiException; -import com.yahoo.restapi.RestApiRequestHandler; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.slime.Type; -import com.yahoo.vespa.hosted.controller.ApplicationController; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.TenantController; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Bill; -import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; -import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporter; -import com.yahoo.vespa.hosted.controller.api.integration.billing.CollectionMethod; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; -import com.yahoo.vespa.hosted.controller.api.integration.billing.StatusHistory; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.vespa.hosted.controller.tenant.BillingReference; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.math.BigDecimal; -import java.time.Clock; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.logging.Logger; - -/** - * @author ogronnesby - */ -public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandlerV2> { - - private static final Logger log = Logger.getLogger(BillingApiHandlerV2.class.getName()); - - private static final String[] CSV_INVOICE_HEADER = new String[]{ "ID", "Tenant", "From", "To", "CpuHours", "MemoryHours", "DiskHours", "Cpu", "Memory", "Disk", "Additional" }; - - private final ApplicationController applications; - private final TenantController tenants; - private final BillingController billing; - private final BillingReporter billingReporter; - private final PlanRegistry planRegistry; - private final Clock clock; - - public BillingApiHandlerV2(ThreadedHttpRequestHandler.Context context, Controller controller) { - super(context, BillingApiHandlerV2::createRestApi); - this.applications = controller.applications(); - this.tenants = controller.tenants(); - this.billing = controller.serviceRegistry().billingController(); - this.planRegistry = controller.serviceRegistry().planRegistry(); - this.clock = controller.serviceRegistry().clock(); - this.billingReporter = controller.serviceRegistry().billingReporter(); - } - - private static RestApi createRestApi(BillingApiHandlerV2 self) { - return RestApi.builder() - /* - * This is the API that is tenant agnostic - */ - .addRoute(RestApi.route("/billing/v2/countries") - .get(self::acceptedCountries)) - - /* - * This is the API that is available to tenants to view their status - */ - .addRoute(RestApi.route("/billing/v2/tenant/{tenant}") - .get(self::tenant) - .patch(Slime.class, self::patchTenant)) - .addRoute(RestApi.route("/billing/v2/tenant/{tenant}/usage") - .get(self::tenantUsage)) - .addRoute(RestApi.route("/billing/v2/tenant/{tenant}/bill") - .get(self::tenantInvoiceList)) - .addRoute(RestApi.route("/billing/v2/tenant/{tenant}/bill/{invoice}") - .get(self::tenantInvoice)) - /* - * This is the API that is created for accountant role in Vespa Cloud - */ - .addRoute(RestApi.route("/billing/v2/accountant") - .get(self::accountant)) - .addRoute(RestApi.route("/billing/v2/accountant/preview") - .get(self::accountantPreview)) - .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}") - .get(self::accountantTenant)) - .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/preview") - .get(self::previewBill) - .post(Slime.class, self::createBill)) - .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/items") - .get(self::additionalItems) - .post(Slime.class, self::newAdditionalItem)) - .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/item/{item}") - .delete(self::deleteAdditionalItem)) - .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/plan") - .get(self::accountantTenantPlan) - .post(Slime.class, self::setAccountantTenantPlan)) - .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/collection") - .get(self::accountantTenantCollection) - .post(Slime.class, self::setAccountantTenantCollection)) - .addRoute(RestApi.route("/billing/v2/accountant/bill/{invoice}/summary") - .get(self::accountantInvoiceSummary)) - .addRoute(RestApi.route("/billing/v2/accountant/bill/{invoice}/export") - .put(Slime.class, self::putAccountantInvoiceExport)) - .addRoute(RestApi.route("/billing/v2/accountant/plans") - .get(self::plans)) - .addExceptionMapper(RuntimeException.class, (c, e) -> ErrorResponses.logThrowing(c.request(), log, e)) - .build(); - } - - // ---------- AUX API ------------- - - private SlimeJsonResponse acceptedCountries(RestApi.RequestContext ctx) { - var response = new Slime(); - var countries = response.setObject().setArray("countries"); - billing.getAcceptedCountries().countries().forEach(country -> { - var countryCursor = countries.addObject(); - countryCursor.setString("code", country.code()); - countryCursor.setString("name", country.displayName()); - countryCursor.setBool("taxIdMandatory", country.taxIdMandatory()); - var taxTypesCursors = countryCursor.setArray("taxTypes"); - country.taxTypes().forEach(taxType -> { - var taxTypeCursor = taxTypesCursors.addObject(); - taxTypeCursor.setString("id", taxType.id()); - taxTypeCursor.setString("description", taxType.description()); - taxTypeCursor.setString("pattern", taxType.pattern()); - taxTypeCursor.setString("example", taxType.example()); - }); - }); - return new SlimeJsonResponse(response, /*compact*/false); - } - - // ---------- TENANT API ---------- - - private Slime tenant(RestApi.RequestContext requestContext) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.require(tenantName, CloudTenant.class); - - var plan = planFor(tenant.name()); - var collectionMethod = billing.getCollectionMethod(tenant.name()); - - var response = new Slime(); - var cursor = response.setObject(); - cursor.setString("tenant", tenant.name().value()); - - toSlime(cursor.setObject("plan"), plan); - cursor.setString("collection", collectionMethod.name()); - return response; - } - - private Slime patchTenant(RestApi.RequestContext requestContext, Slime body) { - var security = requestContext.attributes().get(SecurityContext.ATTRIBUTE_NAME) - .map(SecurityContext.class::cast) - .orElseThrow(() -> new RestApiException.Forbidden("Must be logged in")); - - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.require(tenantName, CloudTenant.class); - - var newPlan = body.get().field("plan"); - var newCollection = body.get().field("collection"); - - if (newPlan.valid() && newPlan.type() == Type.STRING) { - var planId = PlanId.from(newPlan.asString()); - var hasDeployments = tenantHasDeployments(tenant.name()); - var result = billing.setPlan(tenant.name(), planId, hasDeployments, false); - if (! result.isSuccess()) { - throw new RestApiException.Forbidden(result.getErrorMessage().get()); - } - } - - if (newCollection.valid() && newCollection.type() == Type.STRING) { - if (security.roles().contains(Role.hostedAccountant())) { - var collection = CollectionMethod.valueOf(newCollection.asString()); - billing.setCollectionMethod(tenant.name(), collection); - } else { - throw new RestApiException.Forbidden("Only accountant can change billing method"); - } - } - - var response = new Slime(); - var cursor = response.setObject(); - cursor.setString("tenant", tenant.name().value()); - toSlime(cursor.setObject("plan"), planFor(tenant.name())); - cursor.setString("collection", billing.getCollectionMethod(tenant.name()).name()); - return response; - } - - private Slime tenantInvoiceList(RestApi.RequestContext requestContext) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.require(tenantName, CloudTenant.class); - - var slime = new Slime(); - invoicesSummaryToSlime(slime.setObject().setArray("invoices"), billing.getBillsForTenant(tenant.name())); - return slime; - } - - private HttpResponse tenantInvoice(RestApi.RequestContext requestContext) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.require(tenantName, CloudTenant.class); - var invoiceId = requestContext.pathParameters().getStringOrThrow("invoice"); - var format = requestContext.queryParameters().getString("format").orElse("json"); - - var invoice = billing.getBillsForTenant(tenant.name()).stream() - .filter(inv -> inv.id().value().equals(invoiceId)) - .findAny() - .orElseThrow(RestApiException.NotFound::new); - - if (format.equals("json")) { - var slime = new Slime(); - toSlime(slime.setObject(), invoice); - return new SlimeJsonResponse(slime); - } - - if (format.equals("csv")) { - var csv = toCsv(invoice); - return new CsvResponse(CSV_INVOICE_HEADER, csv); - } - - throw new RestApiException.BadRequest("Unknown format: " + format); - } - - private boolean tenantHasDeployments(TenantName tenant) { - return applications.asList(tenant).stream() - .flatMap(app -> app.instances().values().stream()) - .mapToLong(instance -> instance.deployments().size()) - .sum() > 0; - } - - private Slime tenantUsage(RestApi.RequestContext requestContext) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.require(tenantName, CloudTenant.class); - var untilAt = untilParameter(requestContext); - var usage = billing.createUncommittedBill(tenant.name(), untilAt); - var slime = new Slime(); - usageToSlime(slime.setObject(), usage); - return slime; - } - - // --------- ACCOUNTANT API ---------- - - private Slime accountant(RestApi.RequestContext requestContext) { - var response = new Slime(); - var tenantsResponse = response.setObject().setArray("tenants"); - - tenants.asList().stream().sorted(Comparator.comparing(Tenant::name)).forEach(tenant -> { - var tenantResponse = tenantsResponse.addObject(); - tenantResponse.setString("tenant", tenant.name().value()); - toSlime(tenantResponse.setObject("plan"), planFor(tenant.name())); - toSlime(tenantResponse.setObject("quota"), billing.getQuota(tenant.name())); - tenantResponse.setString("collection", billing.getCollectionMethod(tenant.name()).name()); - tenantResponse.setString("lastBill", LocalDateTime.ofInstant(Instant.EPOCH, ZoneOffset.UTC).format(DateTimeFormatter.ISO_DATE)); - tenantResponse.setString("unbilled", "0.00"); - }); - - return response; - } - - private Slime accountantPreview(RestApi.RequestContext requestContext) { - var untilAt = untilParameter(requestContext); - var usagePerTenant = billing.createUncommittedBills(untilAt); - - var response = new Slime(); - var tenantsResponse = response.setObject().setArray("tenants"); - - usagePerTenant.entrySet().stream().sorted(Comparator.comparing(x -> x.getValue().sum())).forEachOrdered(x -> { - var tenant = x.getKey(); - var usage = x.getValue(); - var tenantResponse = tenantsResponse.addObject(); - tenantResponse.setString("tenant", tenant.value()); - toSlime(tenantResponse.setObject("plan"), planFor(tenant)); - toSlime(tenantResponse.setObject("quota"), billing.getQuota(tenant)); - tenantResponse.setString("collection", billing.getCollectionMethod(tenant).name()); - tenantResponse.setString("lastBill", usage.getStartDate().format(DateTimeFormatter.ISO_DATE)); - tenantResponse.setString("unbilled", usage.sum().toPlainString()); - }); - - return response; - } - - private Slime previewBill(RestApi.RequestContext requestContext) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.require(tenantName, CloudTenant.class); - var untilAt = untilParameter(requestContext); - - var usage = billing.createUncommittedBill(tenant.name(), untilAt); - - var slime = new Slime(); - toSlime(slime.setObject(), usage); - return slime; - } - - private HttpResponse createBill(RestApi.RequestContext requestContext, Slime slime) { - var body = slime.get(); - var security = requestContext.attributes().get(SecurityContext.ATTRIBUTE_NAME) - .map(SecurityContext.class::cast) - .orElseThrow(() -> new RestApiException.Forbidden("Must be logged in")); - - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.require(tenantName, CloudTenant.class); - - var startAt = LocalDate.parse(getInspectorFieldOrThrow(body, "from")).atStartOfDay(ZoneOffset.UTC); - var endAt = LocalDate.parse(getInspectorFieldOrThrow(body, "to")).plusDays(1).atStartOfDay(ZoneOffset.UTC); - - var invoiceId = billing.createBillForPeriod(tenant.name(), startAt, endAt, security.principal().getName()); - - // TODO: Make a redirect to the bill itself - return new MessageResponse("Created bill " + invoiceId.value()); - } - - private HttpResponse plans(RestApi.RequestContext ctx) { - var slime = new Slime(); - var root = slime.setObject(); - var plans = root.setArray("plans"); - for (var plan : planRegistry.all()) { - var p = plans.addObject(); - p.setString("id", plan.id().value()); - p.setString("name", plan.displayName()); - } - return new SlimeJsonResponse(slime); - } - - private HttpResponse putAccountantInvoiceExport(RestApi.RequestContext ctx, Slime slime) { - var billId = Bill.Id.of(ctx.pathParameters().getStringOrThrow("invoice")); - - // TODO: try to find a way to retrieve the cloud tenant from BillingControllerImpl - var bill = billing.getBill(billId); - var cloudTenant = tenants.require(bill.tenant(), CloudTenant.class); - - var exportMethod = slime.get().field("method").asString(); - var result = billingReporter.exportBill(bill, exportMethod, cloudTenant); - - var responseSlime = new Slime(); - responseSlime.setObject().setString("invoiceId", result); - return new SlimeJsonResponse(responseSlime); - } - - private MessageResponse deleteAdditionalItem(RestApi.RequestContext requestContext) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.get(tenantName).orElseThrow(() -> new RestApiException.NotFound("No such tenant: " + tenantName)); - - var itemId = requestContext.pathParameters().getStringOrThrow("item"); - - var items = billing.getUnusedLineItems(tenant.name()); - var candidate = items.stream().filter(item -> item.id().equals(itemId)).findAny(); - - if (candidate.isEmpty()) { - throw new RestApiException.NotFound("Could not find item with ID " + itemId); - } - - billing.deleteLineItem(itemId);; - - return new MessageResponse("Successfully deleted line item " + itemId); - } - - private MessageResponse newAdditionalItem(RestApi.RequestContext requestContext, Slime body) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.get(tenantName).orElseThrow(() -> new RestApiException.NotFound("No such tenant: " + tenantName)); - - var inspector = body.get(); - - var billId = SlimeUtils.optionalString(inspector.field("billId")).map(Bill.Id::of); - - billing.addLineItem( - tenant.name(), - getInspectorFieldOrThrow(inspector, "description"), - new BigDecimal(getInspectorFieldOrThrow(inspector, "amount")), - billId, - requestContext.userPrincipalOrThrow().getName()); - - return new MessageResponse("Added line item for tenant " + tenantName); - } - - private Slime additionalItems(RestApi.RequestContext requestContext) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.get(tenantName).orElseThrow(() -> new RestApiException.NotFound("No such tenant: " + tenantName)); - - var slime = new Slime(); - var items = slime.setObject().setArray("items"); - - billing.getUnusedLineItems(tenant.name()).forEach(item -> { - var itemCursor = items.addObject(); - toSlime(itemCursor, item); - }); - - return slime; - } - - private MessageResponse setAccountantTenantPlan(RestApi.RequestContext requestContext, Slime body) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.require(tenantName, CloudTenant.class); - - var planId = PlanId.from(getInspectorFieldOrThrow(body.get(), "id")); - var response = billing.setPlan(tenant.name(), planId, false, true); - - if (response.isSuccess()) { - return new MessageResponse("Plan: " + planId.value()); - } else { - throw new RestApiException.BadRequest("Could not change plan: " + response.getErrorMessage().get()); - } - } - - private Slime accountantTenantPlan(RestApi.RequestContext requestContext) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.require(tenantName, CloudTenant.class); - - var planId = billing.getPlan(tenant.name()); - var plan = planRegistry.plan(planId); - - if (plan.isEmpty()) { - throw new RestApiException.BadRequest("Plan with ID '" + planId.value() + "' does not exist"); - } - - var slime = new Slime(); - var root = slime.setObject(); - root.setString("id", plan.get().id().value()); - root.setString("name", plan.get().displayName()); - - return slime; - } - - private Slime accountantInvoiceSummary(RestApi.RequestContext requestContext) { - var billId = requestContext.pathParameters().getString("invoice").map(Bill.Id::of).orElseThrow(RestApiException.NotFound::new); - var requestParam = requestContext.queryParameters().getString("keys").stream() - .flatMap(s -> Arrays.stream(s.split(","))) - .map(Bill.ItemKeyType::valueOf) - .toList(); - - var requestKeys = Bill.ItemRequest.of(requestParam); - var bill = billing.getBill(billId); - var response = bill.summarizeBy(requestKeys); - - var slime = new Slime(); - toSlime(slime.setObject(), bill, response, bill.lineItems().stream().filter(Bill.LineItem::isAdditional).toList()); - return slime; - } - - private MessageResponse setAccountantTenantCollection(RestApi.RequestContext requestContext, Slime body) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.require(tenantName, CloudTenant.class); - - var collection = CollectionMethod.valueOf(getInspectorFieldOrThrow(body.get(), "collection")); - var result = billing.setCollectionMethod(tenant.name(), collection); - - if (result.isSuccess()) { - return new MessageResponse("Collection: " + collection.name()); - } else { - throw new RestApiException.BadRequest("Could not change collection method: " + result.getErrorMessage()); - } - } - - private Slime accountantTenantCollection(RestApi.RequestContext requestContext) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.require(tenantName, CloudTenant.class); - - var collection = billing.getCollectionMethod(tenant.name()); - - var slime = new Slime(); - var root = slime.setObject(); - root.setString("collection", collection.name()); - - return slime; - } - - private Slime accountantTenant(RestApi.RequestContext requestContext) { - var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); - var tenant = tenants.require(tenantName, CloudTenant.class); - - var slime = new Slime(); - var root = slime.setObject(); - - var planId = billing.getPlan(tenant.name()); - var plan = planRegistry.plan(planId); - - var collection = billing.getCollectionMethod(tenant.name()); - - toSlime(root, tenant, planId, plan, collection); - - return slime; - } - - // --------- INVOICE RENDERING ---------- - - private void invoicesSummaryToSlime(Cursor slime, List<Bill> bills) { - bills.forEach(invoice -> invoiceSummaryToSlime(slime.addObject(), invoice)); - } - - private void invoiceSummaryToSlime(Cursor slime, Bill bill) { - slime.setString("id", bill.id().value()); - slime.setString("from", bill.getStartDate().format(DateTimeFormatter.ISO_LOCAL_DATE)); - slime.setString("to", bill.getEndDate().format(DateTimeFormatter.ISO_LOCAL_DATE)); - slime.setString("total", bill.sum().toString()); - slime.setString("status", bill.status().value()); - } - - private void usageToSlime(Cursor slime, Bill bill) { - slime.setString("from", bill.getStartDate().format(DateTimeFormatter.ISO_LOCAL_DATE)); - slime.setString("to", bill.getEndTime().format(DateTimeFormatter.ISO_LOCAL_DATE)); - slime.setString("total", bill.sum().toString()); - toSlime(slime.setArray("items"), bill.lineItems()); - } - - private void toSlime(Cursor slime, Bill bill) { - slime.setString("id", bill.id().value()); - slime.setString("from", bill.getStartDate().format(DateTimeFormatter.ISO_LOCAL_DATE)); - slime.setString("to", bill.getEndDate().format(DateTimeFormatter.ISO_LOCAL_DATE)); - slime.setString("total", bill.sum().toString()); - slime.setString("status", bill.status().value()); - toSlime(slime.setArray("statusHistory"), bill.statusHistory()); - toSlime(slime.setArray("items"), bill.lineItems()); - } - - private void toSlime(Cursor slime, StatusHistory history) { - history.getHistory().forEach((key, value) -> { - var c = slime.addObject(); - c.setString("at", key.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - c.setString("status", value.value()); - }); - } - - private void toSlime(Cursor slime, List<Bill.LineItem> items) { - items.forEach(item -> toSlime(slime.addObject(), item)); - } - - private void toSlime(Cursor slime, Bill.LineItem item) { - slime.setString("id", item.id()); - slime.setString("description", item.description()); - slime.setString("amount",item.amount().toString()); - toSlime(slime.setObject("plan"), planRegistry.plan(item.plan()).orElseThrow(() -> new RuntimeException("No such plan: '" + item.plan() + "'"))); - item.getArchitecture().ifPresent(arch -> slime.setString("architecture", arch.name())); - slime.setLong("majorVersion", item.getMajorVersion()); - if (! item.getCloudAccount().isUnspecified()) - slime.setString("cloudAccount", item.getCloudAccount().value()); - - item.applicationId().ifPresent(appId -> { - slime.setString("application", appId.application().value()); - slime.setString("instance", appId.instance().value()); - }); - - item.zoneId().ifPresent(z -> slime.setString("zone", z.value())); - - toSlime(slime.setObject("cpu"), item.getCpuHours(), item.getCpuCost()); - toSlime(slime.setObject("memory"), item.getMemoryHours(), item.getMemoryCost()); - toSlime(slime.setObject("disk"), item.getDiskHours(), item.getDiskCost()); - toSlime(slime.setObject("gpu"), item.getGpuHours(), item.getGpuCost()); - } - - private void toSlime(Cursor slime, Optional<BigDecimal> hours, Optional<BigDecimal> cost) { - hours.ifPresent(h -> slime.setString("hours", h.toString())); - cost.ifPresent(c -> slime.setString("cost", c.toString())); - } - - private void toSlime(Cursor slime, CloudTenant tenant, PlanId planId, Optional<Plan> plan, CollectionMethod method) { - slime.setString("tenant", tenant.name().value()); - toSlime(slime.setObject("plan"), planId, plan); - toSlime(slime.setObject("billing"), tenant.billingReference()); - slime.setString("collection", method.name()); - } - - private void toSlime(Cursor slime, PlanId planId, Optional<Plan> plan) { - slime.setString("id", planId.value()); - if (plan.isPresent()) { - slime.setString("name", plan.get().displayName()); - slime.setBool("billed", plan.get().isBilled()); - slime.setBool("supported", plan.get().isSupported()); - } else { - slime.setString("name", "UNKNOWN"); - slime.setBool("billed", false); - slime.setBool("supported", false); - } - } - - private void toSlime(Cursor slime, Optional<BillingReference> billingReference) { - if (billingReference.isPresent()) { - slime.setString("id", billingReference.get().reference()); - slime.setLong("lastUpdated", billingReference.get().updated().toEpochMilli()); - } - } - - private void toSlime(Cursor slime, Bill bill, Map<Bill.ItemKey, Bill.ItemSummary> summaries, List<Bill.LineItem> additional) { - slime.setString("id", bill.id().value()); - var summaryCursor = slime.setArray("summary"); - summaries.forEach((key, summary) -> { - toSlime(summaryCursor.addObject(), key, summary); - }); - var additionalCursor = slime.setArray("additional"); - additional.forEach(item -> { - additionalSummaryToSlime(additionalCursor, item); - }); - } - - private void additionalSummaryToSlime(Cursor slime, Bill.LineItem item) { - slime.setString("description", item.description()); - slime.setString("amount", item.amount().toPlainString()); - } - - private void toSlime(Cursor slime, Bill.ItemKey key, Bill.ItemSummary summary) { - toSlime(slime.setObject("key"), key); - toSlime(slime.setObject("summary"), summary); - } - - private void toSlime(Cursor slime, Bill.ItemKey key) { - key.keys().forEach((keyType, keyValue) -> { - if (keyValue == null) slime.setNix(keyType.name()); - else slime.setString(keyType.name(), keyValue.toString()); - }); - } - - private void toSlime(Cursor slime, Bill.ItemSummary summary) { - var cpu = slime.setObject("cpu"); - cpu.setString("cost", summary.cpuCost().toPlainString()); - cpu.setString("hours", summary.cpuUsage().toPlainString()); - - var ram = slime.setObject("memory"); - ram.setString("cost", summary.ramCost().toPlainString()); - ram.setString("hours", summary.ramUsage().toPlainString()); - - var disk = slime.setObject("disk"); - disk.setString("cost", summary.diskCost().toPlainString()); - disk.setString("hours", summary.diskUsage().toPlainString()); - - var gpu = slime.setObject("gpu"); - gpu.setString("cost", summary.gpuCost().toPlainString()); - gpu.setString("hours", summary.gpuUsage().toPlainString()); - } - - private List<Object[]> toCsv(Bill bill) { - return List.<Object[]>of(new Object[]{ - bill.id().value(), bill.tenant().value(), - bill.getStartDate().format(DateTimeFormatter.ISO_DATE), - bill.getEndDate().format(DateTimeFormatter.ISO_DATE), - bill.sumCpuHours(), bill.sumMemoryHours(), bill.sumDiskHours(), - bill.sumCpuCost(), bill.sumMemoryCost(), bill.sumDiskCost(), - bill.sumAdditionalCost() - }); - } - - // ---------- END INVOICE RENDERING ---------- - - private LocalDate untilParameter(RestApi.RequestContext ctx) { - return ctx.queryParameters().getString("until") - .map(LocalDate::parse) - .orElseGet(() -> LocalDate.now(clock)); - } - - private static String getInspectorFieldOrThrow(Inspector inspector, String field) { - if (!inspector.field(field).valid()) - throw new RestApiException.BadRequest("Field " + field + " cannot be null"); - return inspector.field(field).asString(); - } - - private void toSlime(Cursor cursor, Plan plan) { - cursor.setString("id", plan.id().value()); - cursor.setString("name", plan.displayName()); - } - - private void toSlime(Cursor cursor, Quota quota) { - cursor.setDouble("budget", quota.budget().map(BigDecimal::doubleValue).orElse(-1.0)); - } - - private Plan planFor(TenantName tenant) { - var planId = billing.getPlan(tenant); - return planRegistry.plan(planId) - .orElseThrow(() -> new RuntimeException("No such plan: '" + planId + "'")); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/CsvResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/CsvResponse.java deleted file mode 100644 index cf45bfb67f0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/CsvResponse.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.billing; - -import com.yahoo.container.jdisc.HttpResponse; -import org.apache.commons.csv.CSVFormat; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.util.List; - -/** - * @author ogronnesby - */ -class CsvResponse extends HttpResponse { - - private final String[] header; - private final List<Object[]> rows; - - CsvResponse(String[] header, List<Object[]> rows) { - super(200); - this.header = header; - this.rows = rows; - } - - @Override - public void render(OutputStream outputStream) throws IOException { - var writer = new OutputStreamWriter(outputStream); - var printer = CSVFormat.DEFAULT.withRecordSeparator('\n').withHeader(this.header).print(writer); - for (var row : this.rows) printer.printRecord(row); - printer.flush(); - } - - @Override - public String getContentType() { - return "text/csv; encoding=utf-8"; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java deleted file mode 100644 index b38bb73a98a..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.certificate; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.RestApiException; -import com.yahoo.restapi.StringResponse; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.StringFlag; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateRequest; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.persistence.EndpointCertificateSerializer; -import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; - -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Executor; -import java.util.stream.Collectors; - -import static com.yahoo.jdisc.http.HttpRequest.Method.GET; -import static com.yahoo.jdisc.http.HttpRequest.Method.POST; - -/** - * List all certificate requests for a system, with their requested DNS names. - * Used for debugging, and verifying basic functionality of Cameo client in CD. - * - * @author andreer - */ - -public class EndpointCertificatesHandler extends ThreadedHttpRequestHandler { - - private final EndpointCertificateProvider endpointCertificateProvider; - private final CuratorDb curator; - private final BooleanFlag useAlternateCertProvider; - private final StringFlag endpointCertificateAlgo; - private final Controller controller; - - public EndpointCertificatesHandler(Executor executor, ServiceRegistry serviceRegistry, CuratorDb curator, Controller controller) { - super(executor); - this.endpointCertificateProvider = serviceRegistry.endpointCertificateProvider(); - this.curator = curator; - this.controller = controller; - this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); - this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); - } - - public HttpResponse handle(HttpRequest request) { - if (request.getMethod().equals(GET)) return listEndpointCertificates(); - if (request.getMethod().equals(POST)) return reRequestEndpointCertificateFor(request.getProperty("application"), request.getProperty("ignoreExistingMetadata") != null); - throw new RestApiException.MethodNotAllowed(request); - } - - public HttpResponse listEndpointCertificates() { - List<EndpointCertificateRequest> request = endpointCertificateProvider.listCertificates(); - - String requestsWithNames = request.stream() - .map(r -> r.requestId() + " : " + - String.join(", ", r.dnsNames().stream() - .map(EndpointCertificateRequest.DnsNameStatus::dnsName) - .collect(Collectors.joining(", ")))) - .collect(Collectors.joining("\n")); - - return new StringResponse(requestsWithNames); - } - - public StringResponse reRequestEndpointCertificateFor(String instanceId, boolean ignoreExisting) { - ApplicationId applicationId = ApplicationId.fromFullString(instanceId); - if (controller.routing().endpointConfig(applicationId) == EndpointConfig.generated) { - throw new IllegalArgumentException("Cannot re-request certificate. " + instanceId + " is assigned certificate from a pool"); - } - try (var lock = curator.lock(TenantAndApplicationId.from(applicationId))) { - AssignedCertificate assignedCertificate = curator.readAssignedCertificate(TenantAndApplicationId.from(applicationId), Optional.of(applicationId.instance())) - .orElseThrow(() -> new RestApiException.NotFound("No certificate found for application " + applicationId.serializedForm())); - - String algo = this.endpointCertificateAlgo.with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value(); - boolean useAlternativeProvider = useAlternateCertProvider.with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value(); - String keyPrefix = applicationId.toFullString(); - - EndpointCertificate cert = endpointCertificateProvider.requestCaSignedCertificate( - keyPrefix, assignedCertificate.certificate().requestedDnsSans(), - ignoreExisting ? - Optional.empty() : - Optional.of(assignedCertificate.certificate()), - algo, useAlternativeProvider); - - curator.writeAssignedCertificate(assignedCertificate.with(cert)); - - return new StringResponse(EndpointCertificateSerializer.toSlime(cert).toString()); - } - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java deleted file mode 100644 index f3b28691262..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.changemanagement; - -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.RestApiException; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; -import com.yahoo.vespa.hosted.controller.maintenance.ChangeManagementAssessor; -import com.yahoo.vespa.hosted.controller.persistence.ChangeRequestSerializer; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.yolean.Exceptions; - -import java.io.IOException; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { - - private final ChangeManagementAssessor assessor; - private final Controller controller; - private static final DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneOffset.UTC); - - public ChangeManagementApiHandler(ThreadedHttpRequestHandler.Context ctx, Controller controller) { - super(ctx, controller.auditLogger()); - this.assessor = new ChangeManagementAssessor(controller.serviceRegistry().configServer().nodeRepository()); - this.controller = controller; - } - - @Override - public HttpResponse auditAndHandle(HttpRequest request) { - try { - return switch (request.getMethod()) { - case GET -> get(request); - case POST -> post(request); - case PATCH -> patch(request); - case DELETE -> delete(request); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); - }; - } catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/changemanagement/v1/assessment/{changeRequestId}")) return changeRequestAssessment(path.get("changeRequestId")); - if (path.matches("/changemanagement/v1/vcmr")) return getVCMRs(); - if (path.matches("/changemanagement/v1/vcmr/{vcmrId}")) return getVCMR(path.get("vcmrId")); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse post(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/changemanagement/v1/assessment")) return doAssessment(request); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse patch(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/changemanagement/v1/vcmr/{vcmrId}")) return patchVCMR(request, path.get("vcmrId")); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse delete(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/changemanagement/v1/vcmr/{vcmrId}")) return deleteVCMR(path.get("vcmrId")); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private Inspector inspectorOrThrow(HttpRequest request) { - try { - return SlimeUtils.jsonToSlime(request.getData().readAllBytes()).get(); - } catch (IOException e) { - throw new RestApiException.BadRequest("Failed to parse request body"); - } - } - - private static Inspector getInspectorFieldOrThrow(Inspector inspector, String field) { - if (!inspector.field(field).valid()) - throw new RestApiException.BadRequest("Field " + field + " cannot be null"); - return inspector.field(field); - } - - private HttpResponse changeRequestAssessment(String changeRequestId) { - var optionalChangeRequest = controller.curator().readChangeRequests() - .stream() - .filter(request -> changeRequestId.equals(request.getChangeRequestSource().id())) - .findFirst(); - - if (optionalChangeRequest.isEmpty()) - return ErrorResponse.notFoundError("Could not find any upcoming change requests with id " + changeRequestId); - - var changeRequest = optionalChangeRequest.get(); - - return doAssessment(changeRequest.getImpactedHosts()); - } - - // The structure here should be - // - // { - // hosts: string[] - // switches: string[] - // switchInSequence: boolean - // } - // - // Only hosts is supported right now - private HttpResponse doAssessment(HttpRequest request) { - - Inspector inspector = inspectorOrThrow(request); - - // For now; mandatory fields - Inspector hostArray = inspector.field("hosts"); - Inspector switchArray = inspector.field("switches"); - - - // The impacted hostnames - List<String> hostNames = new ArrayList<>(); - if (hostArray.valid()) { - hostArray.traverse((ArrayTraverser) (i, host) -> hostNames.add(host.asString())); - } - - if (switchArray.valid()) { - List<String> switchNames = new ArrayList<>(); - switchArray.traverse((ArrayTraverser) (i, switchName) -> switchNames.add(switchName.asString())); - hostNames.addAll(hostsOnSwitch(switchNames)); - } - - if (hostNames.isEmpty()) - return ErrorResponse.badRequest("No prod hosts in provided host/switch list"); - - return doAssessment(hostNames); - } - - private HttpResponse doAssessment(List<String> hostNames) { - var zone = affectedZone(hostNames); - if (zone.isEmpty()) - return ErrorResponse.notFoundError("Could not infer prod zone from host list: " + hostNames); - - ChangeManagementAssessor.Assessment assessments = assessor.assessment(hostNames, zone.get()); - - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - // This is the main structure that might be part of something bigger later - Cursor assessmentCursor = root.setObject("assessment"); - - // Updated gives clue to if the assessment is old - assessmentCursor.setString("updated", formatter.format(controller.clock().instant())); - - // Assessment on the cluster level - Cursor clustersCursor = assessmentCursor.setArray("clusters"); - - assessments.getClusterAssessments().forEach(assessment -> { - Cursor oneCluster = clustersCursor.addObject(); - oneCluster.setString("app", assessment.app); - oneCluster.setString("zone", assessment.zone); - oneCluster.setString("cluster", assessment.cluster); - oneCluster.setLong("clusterSize", assessment.clusterSize); - oneCluster.setLong("clusterImpact", assessment.clusterImpact); - oneCluster.setLong("groupsTotal", assessment.groupsTotal); - oneCluster.setLong("groupsImpact", assessment.groupsImpact); - oneCluster.setString("upgradePolicy", assessment.upgradePolicy); - oneCluster.setString("suggestedAction", assessment.suggestedAction); - oneCluster.setString("impact", assessment.impact); - }); - - Cursor hostsCursor = assessmentCursor.setArray("hosts"); - assessments.getHostAssessments().forEach(assessment -> { - Cursor hostObject = hostsCursor.addObject(); - hostObject.setString("hostname", assessment.hostName); - hostObject.setString("switchName", assessment.switchName); - hostObject.setLong("numberOfChildren", assessment.numberOfChildren); - hostObject.setLong("numberOfProblematicChildren", assessment.numberOfProblematicChildren); - }); - - return new SlimeJsonResponse(slime); - } - - private HttpResponse getVCMRs() { - var changeRequests = controller.curator().readChangeRequests(); - var slime = new Slime(); - var cursor = slime.setObject().setArray("vcmrs"); - changeRequests.forEach(changeRequest -> { - var changeCursor = cursor.addObject(); - ChangeRequestSerializer.writeChangeRequest(changeCursor, changeRequest); - }); - return new SlimeJsonResponse(slime); - } - - private HttpResponse getVCMR(String vcmrId) { - var changeRequest = controller.curator().readChangeRequest(vcmrId); - - if (changeRequest.isEmpty()) { - return ErrorResponse.notFoundError("No VCMR with id: " + vcmrId); - } - - var slime = new Slime(); - var cursor = slime.setObject(); - - ChangeRequestSerializer.writeChangeRequest(cursor, changeRequest.get()); - return new SlimeJsonResponse(slime); - } - - private HttpResponse patchVCMR(HttpRequest request, String vcmrId) { - var optionalChangeRequest = controller.curator().readChangeRequest(vcmrId); - - if (optionalChangeRequest.isEmpty()) { - return ErrorResponse.notFoundError("No VCMR with id: " + vcmrId); - } - - var changeRequest = optionalChangeRequest.get(); - var inspector = inspectorOrThrow(request); - - if (inspector.field("approval").valid()) { - var approval = ChangeRequest.Approval.valueOf(inspector.field("approval").asString()); - changeRequest = changeRequest.withApproval(approval); - } - - if (inspector.field("actionPlan").valid()) { - var actionPlan = ChangeRequestSerializer.readHostActionPlan(inspector.field("actionPlan")); - changeRequest = changeRequest.withActionPlan(actionPlan); - } - - if (inspector.field("status").valid()) { - var status = VespaChangeRequest.Status.valueOf(inspector.field("status").asString()); - changeRequest = changeRequest.withStatus(status); - } - - try (var lock = controller.curator().lockChangeRequests()) { - controller.curator().writeChangeRequest(changeRequest); - } - - var slime = new Slime(); - var cursor = slime.setObject(); - ChangeRequestSerializer.writeChangeRequest(cursor, changeRequest); - return new SlimeJsonResponse(slime); - } - - private HttpResponse deleteVCMR(String vcmrId) { - var changeRequest = controller.curator().readChangeRequest(vcmrId); - - if (changeRequest.isEmpty()) { - return ErrorResponse.notFoundError("No VCMR with id: " + vcmrId); - } - - try (var lock = controller.curator().lockChangeRequests()) { - controller.curator().deleteChangeRequest(changeRequest.get()); - } - - var slime = new Slime(); - var cursor = slime.setObject(); - ChangeRequestSerializer.writeChangeRequest(cursor, changeRequest.get()); - return new SlimeJsonResponse(slime); - } - - private Optional<ZoneId> affectedZone(List<String> hosts) { - NodeFilter affectedHosts = NodeFilter.all().hostnames(hosts.stream() - .map(HostName::of) - .collect(Collectors.toSet())); - for (var zone : getProdZones()) { - var affectedHostsInZone = controller.serviceRegistry().configServer().nodeRepository().list(zone, affectedHosts); - if (!affectedHostsInZone.isEmpty()) - return Optional.of(zone); - } - - return Optional.empty(); - } - - private List<String> hostsOnSwitch(List<String> switches) { - return getProdZones().stream() - .flatMap(zone -> controller.serviceRegistry().configServer().nodeRepository().list(zone, NodeFilter.all()).stream()) - .filter(node -> node.switchHostname().map(switches::contains).orElse(false)) - .map(node -> node.hostname().value()) - .toList(); - } - - private List<ZoneId> getProdZones() { - return controller.zoneRegistry() - .zones() - .reachable() - .in(Environment.prod) - .ids(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java deleted file mode 100644 index b1e44756802..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.configserver; - -import ai.vespa.http.HttpURL; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.config.provision.zone.ZoneList; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; -import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; -import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.yolean.Exceptions; - -import java.net.URI; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static ai.vespa.http.HttpURL.Path.parse; - -/** - * REST API for proxying operator APIs to config servers in a given zone. - * - * @author freva - */ -@SuppressWarnings("unused") -public class ConfigServerApiHandler extends AuditLoggingRequestHandler { - - private static final URI CONTROLLER_URI = URI.create("https://localhost:4443/"); - private static final List<HttpURL.Path> WHITELISTED_APIS = List.of(parse("/flags/v1/"), - parse("/nodes/v2/"), - parse("/orchestrator/v1/"), - parse("/state/v1/")); - - private final ZoneRegistry zoneRegistry; - private final ConfigServerRestExecutor proxy; - private final ZoneId controllerZone; - - public ConfigServerApiHandler(Context parentCtx, ServiceRegistry serviceRegistry, - ConfigServerRestExecutor proxy, Controller controller) { - super(parentCtx, controller.auditLogger()); - this.zoneRegistry = serviceRegistry.zoneRegistry(); - this.controllerZone = zoneRegistry.systemZone().getVirtualId(); - this.proxy = proxy; - } - - @Override - public HttpResponse auditAndHandle(HttpRequest request) { - try { - return switch (request.getMethod()) { - case GET -> get(request); - case POST, PUT, DELETE, PATCH -> proxy(request); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); - }; - } catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/configserver/v1")) { - return root(request); - } - return proxy(request); - } - - private HttpResponse proxy(HttpRequest request) { - Path path = new Path(request.getUri()); - if ( ! path.matches("/configserver/v1/{environment}/{region}/{*}")) { - return ErrorResponse.notFoundError("Nothing at " + path); - } - - ZoneId zoneId = ZoneId.from(path.get("environment"), path.get("region")); - if ( ! zoneRegistry.hasZone(zoneId) && ! controllerZone.equals(zoneId)) { - throw new IllegalArgumentException("No such zone: " + zoneId.value()); - } - - if (path.getRest().length() < 2 || ! WHITELISTED_APIS.contains(path.getRest().head(2).withTrailingSlash())) { - return ErrorResponse.forbidden("Cannot access " + path.getRest() + - " through /configserver/v1, following APIs are permitted: " + WHITELISTED_APIS.stream() - .map(p -> "/" + String.join("/", p.segments()) + "/") - .collect(Collectors.joining(", "))); - } - - return proxy.handle(ProxyRequest.tryOne(getEndpoint(zoneId), path.getRest(), request)); - } - - private HttpResponse root(HttpRequest request) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - ZoneList zoneList = zoneRegistry.zones().reachable(); - - Cursor zones = root.setArray("zones"); - Stream.concat(Stream.of(controllerZone), zoneRegistry.zones().reachable().ids().stream()) - .forEach(zone -> { - Cursor object = zones.addObject(); - object.setString("environment", zone.environment().value()); - object.setString("region", zone.region().value()); - object.setString("uri", request.getUri().resolve( - "/configserver/v1/" + zone.environment().value() + "/" + zone.region().value()).toString()); - }); - return new SlimeJsonResponse(slime); - } - - private HttpResponse notFound(Path path) { - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private URI getEndpoint(ZoneId zoneId) { - return controllerZone.equals(zoneId) ? CONTROLLER_URI : zoneRegistry.getConfigServerVipUri(zoneId); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/package-info.java deleted file mode 100644 index 91dde82e233..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author freva - */ -package com.yahoo.vespa.hosted.controller.restapi.configserver; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AccessRequestResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AccessRequestResponse.java deleted file mode 100644 index 4863b91b3eb..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AccessRequestResponse.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.controller.restapi.controller; - -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzUser; - -import java.util.Collection; - -public class AccessRequestResponse extends SlimeJsonResponse { - - public AccessRequestResponse(Collection<AthenzUser> members) { - super(toSlime(members)); - } - - private static Slime toSlime(Collection<AthenzUser> members) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor array = root.setArray("members"); - members.stream() - .map(AthenzIdentity::getFullName) - .forEach(array::addString); - return slime; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java deleted file mode 100644 index 859281dbe18..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.controller; - -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; - -/** - * @author mpolden - */ -public class AuditLogResponse extends SlimeJsonResponse { - - public AuditLogResponse(AuditLog log) { - super(toSlime(log)); - } - - private static Slime toSlime(AuditLog log) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor entryArray = root.setArray("entries"); - log.entries().forEach(entry -> { - Cursor entryObject = entryArray.addObject(); - entryObject.setString("time", entry.at().toString()); - entryObject.setString("client", entry.client().name()); - entryObject.setString("user", entry.principal()); - entryObject.setString("method", entry.method().name()); - entryObject.setString("resource", entry.resource()); - entry.data().ifPresent(data -> entryObject.setString("data", data)); - }); - return slime; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java deleted file mode 100644 index b9ba4f691fc..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.controller; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.ResourceResponse; -import com.yahoo.restapi.RestApiException; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.text.Text; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; -import com.yahoo.vespa.hosted.controller.config.CoreDumpTokenResealingConfig; -import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; -import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; -import com.yahoo.yolean.Exceptions; - -import java.io.InputStream; -import java.security.Principal; -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.util.Optional; -import java.util.Scanner; - -import static com.yahoo.vespa.hosted.controller.restapi.controller.RequestUtils.requireField; -import static com.yahoo.vespa.hosted.controller.restapi.controller.RequestUtils.toJsonBytes; - -/** - * This implements the controller/v1 API which provides operators with information about, - * and control over the Controller. - * - * @author bratseth - */ -@SuppressWarnings("unused") // Created by injection -public class ControllerApiHandler extends AuditLoggingRequestHandler { - - private final ControllerMaintenance maintenance; - private final Controller controller; - private final SecretStore secretStore; - private final CoreDumpTokenResealingConfig tokenResealingConfig; - - public ControllerApiHandler(ThreadedHttpRequestHandler.Context parentCtx, - Controller controller, - ControllerMaintenance maintenance, - SecretStore secretStore, - CoreDumpTokenResealingConfig tokenResealingConfig) { - super(parentCtx, controller.auditLogger()); - this.controller = controller; - this.maintenance = maintenance; - this.secretStore = secretStore; - this.tokenResealingConfig = tokenResealingConfig; - } - - @Override - public HttpResponse auditAndHandle(HttpRequest request) { - try { - return switch (request.getMethod()) { - case GET -> get(request); - case POST -> post(request); - case DELETE -> delete(request); - case PATCH -> patch(request); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - }; - } - catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } - catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/controller/v1/")) return root(request); - if (path.matches("/controller/v1/auditlog/")) return new AuditLogResponse(controller.auditLogger().readLog()); - if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(controller.jobControl()); - if (path.matches("/controller/v1/stats")) return new StatsResponse(controller); - if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader()); - if (path.matches("/controller/v1/metering/tenant/{tenant}/month/{month}")) return new MeteringResponse(controller.serviceRegistry().resourceDatabase(), path.get("tenant"), path.get("month")); - return notFound(path); - } - - private HttpResponse post(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/controller/v1/jobs/upgrader/confidence/{version}")) return overrideConfidence(request, path.get("version")); - if (path.matches("/controller/v1/access/requests/{user}")) return approveMembership(request, path.get("user")); - if (path.matches("/controller/v1/access/grants/{user}")) return grantAccess(request, path.get("user")); - if (path.matches("/controller/v1/access/cores/reseal")) return DecryptionTokenResealer.handleResealRequest(request, tokenResealingConfig.resealingPrivateKeyName(), secretStore); - return notFound(path); - } - - private HttpResponse approveMembership(HttpRequest request, String user) { - AthenzUser athenzUser = AthenzUser.fromUserId(user); - byte[] jsonBytes = toJsonBytes(request.getData()); - Inspector inspector = SlimeUtils.jsonToSlime(jsonBytes).get(); - ApplicationId applicationId = requireField(inspector, "applicationId", ApplicationId::fromSerializedForm); - ZoneId zone = requireField(inspector, "zone", ZoneId::from); - if(controller.supportAccess().allowDataplaneMembership(athenzUser, new DeploymentId(applicationId, zone))) { - return new AccessRequestResponse(controller.serviceRegistry().accessControlService().listMembers()); - } else { - return new MessageResponse(400, "Unable to approve membership request"); - } - } - - private HttpResponse grantAccess(HttpRequest request, String user) { - Principal principal = requireUserPrincipal(request); - Instant now = controller.clock().instant(); - - byte[] jsonBytes = toJsonBytes(request.getData()); - Inspector requestObject = SlimeUtils.jsonToSlime(jsonBytes).get(); - X509Certificate certificate = requireField(requestObject, "certificate", X509CertificateUtils::fromPem); - ApplicationId applicationId = requireField(requestObject, "applicationId", ApplicationId::fromSerializedForm); - ZoneId zone = requireField(requestObject, "zone", ZoneId::from); - DeploymentId deployment = new DeploymentId(applicationId, zone); - - // Register grant - SupportAccess supportAccess = controller.supportAccess().registerGrant(deployment, principal.getName(), certificate); - - // Trigger deployment to include operator cert - Optional<JobId> jobId = controller.applications().deploymentTrigger().reTriggerOrAddToQueue(deployment, "re-triggered to grant access, by " + request.getJDiscRequest().getUserPrincipal().getName()); - return new MessageResponse( - jobId.map(id -> Text.format("Operator %s granted access and job %s triggered", principal.getName(), id.type().jobName())) - .orElseGet(() -> Text.format("Operator %s granted access and job trigger queued", principal.getName()))); - } - - private HttpResponse delete(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/controller/v1/jobs/upgrader/confidence/{version}")) return removeConfidenceOverride(path.get("version")); - return notFound(path); - } - - private HttpResponse patch(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/controller/v1/jobs/upgrader")) return configureUpgrader(request); - return notFound(path); - } - - private HttpResponse notFound(Path path) { return ErrorResponse.notFoundError("Nothing at " + path); } - - private HttpResponse root(HttpRequest request) { - return new ResourceResponse(request, "auditlog", "maintenance", "stats", "jobs/upgrader", "metering/tenant"); - } - - private HttpResponse configureUpgrader(HttpRequest request) { - String upgradesPerMinuteField = "upgradesPerMinute"; - - byte[] jsonBytes = toJsonBytes(request.getData()); - Inspector inspect = SlimeUtils.jsonToSlime(jsonBytes).get(); - Upgrader upgrader = maintenance.upgrader(); - - if (inspect.field(upgradesPerMinuteField).valid()) { - upgrader.setUpgradesPerMinute(inspect.field(upgradesPerMinuteField).asDouble()); - } else { - return ErrorResponse.badRequest("No such modifiable field(s)"); - } - - return new UpgraderResponse(maintenance.upgrader()); - } - - private HttpResponse removeConfidenceOverride(String version) { - maintenance.upgrader().removeConfidenceOverride(Version.fromString(version)); - return new UpgraderResponse(maintenance.upgrader()); - } - - private HttpResponse overrideConfidence(HttpRequest request, String version) { - Confidence confidence = Confidence.valueOf(asString(request.getData()).trim()); - maintenance.upgrader().overrideConfidence(Version.fromString(version), confidence); - return new UpgraderResponse(maintenance.upgrader()); - } - - private static String asString(InputStream in) { - Scanner scanner = new Scanner(in).useDelimiter("\\A"); - if (scanner.hasNext()) { - return scanner.next(); - } - return ""; - } - - private static Principal requireUserPrincipal(HttpRequest request) { - Principal principal = request.getJDiscRequest().getUserPrincipal(); - if (principal == null) throw new RestApiException.InternalServerError("Expected a user principal"); - return principal; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/DecryptionTokenResealer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/DecryptionTokenResealer.java deleted file mode 100644 index f2e51b51752..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/DecryptionTokenResealer.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.controller; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.security.KeyId; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SharedKeyResealingSession; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.SlimeUtils; - -import java.util.Optional; - -import static com.yahoo.vespa.hosted.controller.restapi.controller.RequestUtils.requireField; -import static com.yahoo.vespa.hosted.controller.restapi.controller.RequestUtils.toJsonBytes; - -/** - * @author vekterli - */ -class DecryptionTokenResealer { - - private static int checkKeyNameAndExtractVersion(KeyId tokenKeyId, String expectedKeyName) { - String keyStr = tokenKeyId.asString(); - int versionSepIdx = keyStr.lastIndexOf('.'); - if (versionSepIdx == -1) { - throw new IllegalArgumentException("Key ID is not of the form 'name.version'"); - } - String keyName = keyStr.substring(0, versionSepIdx); - if (!expectedKeyName.equals(keyName)) { - throw new IllegalArgumentException("Token is not generated for the expected key"); - } - int keyVersion; - try { - keyVersion = Integer.parseInt(keyStr.substring(versionSepIdx + 1)); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Key version is not a valid integer"); - } - if (keyVersion < 0) { - throw new IllegalArgumentException("Key version is out of range"); - } - return keyVersion; - } - - /** - * Extracts a resealing requests from an <strong>already authenticated</strong> HTTP request - * and re-seals it towards the requested public key, using the provided private key name to - * decrypt the token contained in the request. - * - * @param request a request with a JSON payload that contains a resealing request. - * @param privateKeyName The key name used to look up the decryption secret. - * The token must have a matching key name, or the request will be rejected. - * @param secretStore SecretStore instance that holds the private key. The request will fail otherwise. - * @return a response with a JSON payload containing a resealing response (any failure will throw). - */ - static HttpResponse handleResealRequest(HttpRequest request, String privateKeyName, SecretStore secretStore) { - if (privateKeyName.isEmpty()) { - throw new IllegalArgumentException("Private key ID is not set"); - } - byte[] jsonBytes = toJsonBytes(request.getData()); - var inspector = SlimeUtils.jsonToSlime(jsonBytes).get(); - var resealRequest = requireField(inspector, "resealRequest", SharedKeyResealingSession.ResealingRequest::fromSerializedString); - int keyVersion = checkKeyNameAndExtractVersion(resealRequest.sealedKey().keyId(), privateKeyName); - - var b58EncodedPrivateKey = secretStore.getSecret(privateKeyName, keyVersion); - if (b58EncodedPrivateKey == null) { - throw new IllegalArgumentException("Unknown key ID or version"); - } - var privateKey = KeyUtils.fromBase58EncodedX25519PrivateKey(b58EncodedPrivateKey); - var resealResponse = SharedKeyResealingSession.reseal(resealRequest, (keyId) -> Optional.of(privateKey)); - return new ResealedTokenResponse(resealResponse); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java deleted file mode 100644 index 0d15d9b2971..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.controller; - -import com.yahoo.concurrent.maintenance.JobControl; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; - -import java.util.TreeSet; - -/** - * A response containing maintenance job status - * - * @author bratseth - */ -public class JobsResponse extends SlimeJsonResponse { - - public JobsResponse(JobControl jobControl) { - super(toSlime(jobControl)); - } - - private static Slime toSlime(JobControl jobControl) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - Cursor jobArray = root.setArray("jobs"); - for (String jobName : jobControl.jobs()) - jobArray.addObject().setString("name", jobName); - - Cursor inactiveArray = root.setArray("inactive"); - for (String jobName : new TreeSet<>(jobControl.inactiveJobs())) - inactiveArray.addString(jobName); - - return slime; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java deleted file mode 100644 index 5a8c4847ce6..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.controller; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; - -import java.time.YearMonth; -import java.util.List; - -/** - * @author olaa - */ -public class MeteringResponse extends SlimeJsonResponse { - - public MeteringResponse(ResourceDatabaseClient resourceClient, String tenantName, String month) { - super(toSlime(resourceClient, tenantName, month)); - } - - private static Slime toSlime(ResourceDatabaseClient resourceClient, String tenantName, String month) { - Slime slime = new Slime(); - Cursor root = slime.setArray(); - List<ResourceSnapshot> snapshots = resourceClient.getRawSnapshotHistoryForTenant(TenantName.from(tenantName), YearMonth.parse(month)); - snapshots.forEach(snapshot -> { - Cursor object = root.addObject(); - object.setString("applicationId", snapshot.getApplicationId().toFullString()); - object.setLong("timestamp", snapshot.getTimestamp().toEpochMilli()); - object.setString("zoneId", snapshot.getZoneId().value()); - object.setDouble("cpu", snapshot.resources().vcpu()); - object.setDouble("memory", snapshot.resources().memoryGb()); - object.setDouble("disk", snapshot.resources().diskGb()); - object.setString("architecture", snapshot.resources().architecture().name()); - object.setLong("version", snapshot.getMajorVersion()); - object.setDouble("gpuMemoryGb", snapshot.resources().gpuResources().memoryGb()); - object.setLong("gpuCount", snapshot.resources().gpuResources().count()); - }); - return slime; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/RequestUtils.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/RequestUtils.java deleted file mode 100644 index 746f1d8ce2e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/RequestUtils.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.controller; - -import com.yahoo.io.IOUtils; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.SlimeUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.util.function.Function; - -class RequestUtils { - - static <T> T requireField(Inspector inspector, String field, Function<String, T> mapper) { - return SlimeUtils.optionalString(inspector.field(field)) - .map(mapper::apply) - .orElseThrow(() -> new IllegalArgumentException("Expected field \"" + field + "\" in request")); - } - - static byte[] toJsonBytes(InputStream jsonStream) { - try { - return IOUtils.readBytes(jsonStream, 1000 * 1000); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ResealedTokenResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ResealedTokenResponse.java deleted file mode 100644 index 2aab64a7c30..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ResealedTokenResponse.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.controller; - -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.security.SharedKeyResealingSession; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; - -/** - * A response that contains a decryption token "resealing response". - * - * @author vekterli - */ -public class ResealedTokenResponse extends SlimeJsonResponse { - - public ResealedTokenResponse(SharedKeyResealingSession.ResealingResponse response) { - super(toSlime(response)); - } - - private static Slime toSlime(SharedKeyResealingSession.ResealingResponse response) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("resealResponse", response.toSerializedString()); - return slime; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java deleted file mode 100644 index ab12187c069..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.controller; - -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationStats; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Load; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepoStats; - -/** - * A response containing statistics about this controller and its zones. - * - * @author bratseth - */ -public class StatsResponse extends SlimeJsonResponse { - - public StatsResponse(Controller controller) { - super(toSlime(controller)); - } - - private static Slime toSlime(Controller controller) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor zonesArray = root.setArray("zones"); - for (ZoneId zone : controller.zoneRegistry().zones().reachable().ids()) { - NodeRepoStats stats = controller.serviceRegistry().configServer().nodeRepository().getStats(zone); - if (stats.applicationStats().isEmpty()) continue; // skip empty zones - Cursor zoneObject = zonesArray.addObject(); - zoneObject.setString("id", zone.toString()); - zoneObject.setDouble("totalCost", stats.totalCost()); - zoneObject.setDouble("totalAllocatedCost", stats.totalAllocatedCost()); - toSlime(stats.load(), zoneObject.setObject("load")); - toSlime(stats.activeLoad(), zoneObject.setObject("activeLoad")); - Cursor applicationsArray = zoneObject.setArray("applications"); - for (var applicationStats : stats.applicationStats()) - toSlime(applicationStats, applicationsArray.addObject()); - } - return slime; - } - - private static void toSlime(ApplicationStats stats, Cursor applicationObject) { - applicationObject.setString("id", stats.id().toFullString()); - toSlime(stats.load(), applicationObject.setObject("load")); - applicationObject.setDouble("cost", stats.cost()); - applicationObject.setDouble("unutilizedCost", stats.unutilizedCost()); - } - - private static void toSlime(Load load, Cursor loadObject) { - loadObject.setDouble("cpu", load.cpu()); - loadObject.setDouble("memory", load.memory()); - loadObject.setDouble("disk", load.disk()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java deleted file mode 100644 index e8ba1177c67..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.controller; - -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; - -/** - * @author mpolden - */ -public class UpgraderResponse extends SlimeJsonResponse { - - public UpgraderResponse(Upgrader upgrader) { - super(toSlime(upgrader)); - } - - private static Slime toSlime(Upgrader upgrader) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setDouble("upgradesPerMinute", upgrader.upgradesPerMinute()); - - Cursor array = root.setArray("confidenceOverrides"); - upgrader.confidenceOverrides().forEach((version, confidence) -> { - Cursor object = array.addObject(); - object.setString(version.toFullString(), confidence.name()); - }); - - return slime; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/WellKnownApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/WellKnownApiHandler.java deleted file mode 100644 index 63f600aaa50..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/WellKnownApiHandler.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.controller; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.StringResponse; -import com.yahoo.vespa.hosted.controller.config.WellKnownFolderConfig; -import com.yahoo.yolean.Exceptions; - - -/** - * Responsible for serving contents from the RFC 8615 well-known directory - * @author olaa - */ -public class WellKnownApiHandler extends ThreadedHttpRequestHandler { - - private final String securityTxt; - - public WellKnownApiHandler(Context context, WellKnownFolderConfig wellKnownFolderConfig) { - super(context); - this.securityTxt = wellKnownFolderConfig.securityTxt(); - } - - @Override - public HttpResponse handle(HttpRequest request) { - return switch (request.getMethod()) { - case GET -> get(request); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - }; - } - - private HttpResponse get(HttpRequest request) { - try { - Path path = new Path(request.getUri()); - if (path.matches("/.well-known/security.txt")) return securityTxt(); - return ErrorResponse.notFoundError("Nothing at " + path); - } - catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } - } - - private HttpResponse securityTxt() { - return new StringResponse(securityTxt); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/dataplanetoken/DataplaneTokenService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/dataplanetoken/DataplaneTokenService.java deleted file mode 100644 index 834133e7eb5..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/dataplanetoken/DataplaneTokenService.java +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.dataplanetoken; - -import com.yahoo.concurrent.DaemonThreadFactory; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.security.token.Token; -import com.yahoo.security.token.TokenCheckHash; -import com.yahoo.security.token.TokenDomain; -import com.yahoo.security.token.TokenGenerator; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneToken; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneTokenVersions; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneTokenVersions.Version; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.FingerPrint; -import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.security.Principal; -import java.time.Duration; -import java.time.Instant; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Phaser; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Comparator.comparing; -import static java.util.Comparator.naturalOrder; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.toMap; - -/** - * Service to list, generate and delete data plane tokens - * - * @author mortent - */ -public class DataplaneTokenService { - - private static final String TOKEN_PREFIX = "vespa_cloud_"; - private static final int TOKEN_BYTES = 32; - private static final int CHECK_HASH_BYTES = 32; - public static final Duration DEFAULT_TTL = Duration.ofDays(30); - - private final ExecutorService executor = Executors.newCachedThreadPool(new DaemonThreadFactory("dataplane-token-service-")); - private final Controller controller; - - public DataplaneTokenService(Controller controller) { - this.controller = controller; - } - - /** - * List valid tokens for a tenant - */ - public List<DataplaneTokenVersions> listTokens(TenantName tenantName) { - return controller.curator().readDataplaneTokens(tenantName); - } - - public enum State { UNUSED, DEPLOYING, ACTIVE, REVOKING } - - /** List all known tokens for a tenant, with the state of each token version (both current and deactivating). */ - public Map<DataplaneTokenVersions, Map<FingerPrint, State>> listTokensWithState(TenantName tenantName) { - List<DataplaneTokenVersions> currentTokens = listTokens(tenantName); - Set<TokenId> usedTokens = new HashSet<>(); - Map<HostName, Map<TokenId, List<FingerPrint>>> activeTokens = listActiveTokens(tenantName, usedTokens); - Map<TokenId, Map<FingerPrint, Boolean>> activeFingerprints = computeStates(activeTokens); - Map<DataplaneTokenVersions, Map<FingerPrint, State>> tokens = new TreeMap<>(comparing(DataplaneTokenVersions::tokenId)); - for (DataplaneTokenVersions token : currentTokens) { - Map<FingerPrint, State> states = new TreeMap<>(); - // Current tokens are active iff. they are active everywhere. - for (Version version : token.tokenVersions()) { - // If the token was not seen anywhere, it is deploying or unused. - // Otherwise, it is active iff. it is active everywhere. - Boolean isActive = activeFingerprints.getOrDefault(token.tokenId(), Map.of()).get(version.fingerPrint()); - states.put(version.fingerPrint(), - isActive == null ? usedTokens.contains(token.tokenId()) ? State.DEPLOYING : State.UNUSED - : isActive ? State.ACTIVE : State.DEPLOYING); - } - // Active, non-current token versions are deactivating. - for (FingerPrint print : activeFingerprints.getOrDefault(token.tokenId(), Map.of()).keySet()) { - states.putIfAbsent(print, State.REVOKING); - } - tokens.put(token, states); - } - // Active, non-current tokens are also deactivating. - activeFingerprints.forEach((id, prints) -> { - if (currentTokens.stream().noneMatch(token -> token.tokenId().equals(id))) { - Map<FingerPrint, State> states = new TreeMap<>(); - for (FingerPrint print : prints.keySet()) states.put(print, State.REVOKING); - tokens.put(new DataplaneTokenVersions(id, List.of(), Instant.EPOCH), states); - } - }); - return tokens; - } - - private Map<HostName, Map<TokenId, List<FingerPrint>>> listActiveTokens(TenantName tenantName, Set<TokenId> usedTokens) { - Map<HostName, Map<TokenId, List<FingerPrint>>> tokens = new ConcurrentHashMap<>(); - Phaser phaser = new Phaser(1); - for (Application application : controller.applications().asList(tenantName)) { - for (Instance instance : application.instances().values()) { - instance.deployments().forEach((zone, deployment) -> { - DeploymentId id = new DeploymentId(instance.id(), zone); - usedTokens.addAll(deployment.dataPlaneTokens().keySet()); - phaser.register(); - executor.execute(() -> { - try { tokens.putAll(controller.serviceRegistry().configServer().activeTokenFingerprints(id)); } - finally { phaser.arrive(); } - }); - }); - } - } - phaser.arriveAndAwaitAdvance(); - return tokens; - } - - /** Computes whether each print is active on all hosts where its token is present. */ - private Map<TokenId, Map<FingerPrint, Boolean>> computeStates(Map<HostName, Map<TokenId, List<FingerPrint>>> activeTokens) { - Map<TokenId, Map<FingerPrint, Boolean>> states = new HashMap<>(); - for (Map<TokenId, List<FingerPrint>> token : activeTokens.values()) { - token.forEach((id, prints) -> { - states.merge(id, - prints.stream().collect(toMap(print -> print, __ -> true)), - (a, b) -> new HashMap<>() {{ // true iff. present in both, false iff. present in one. - a.forEach((p, s) -> put(p, s && b.getOrDefault(p, false))); - b.forEach((p, s) -> putIfAbsent(p, false)); - }}); - }); - } - return states; - } - - /** Triggers redeployment of all applications which reference a token which has changed. */ - public void triggerTokenChangeDeployments() { - controller.applications().asList().stream() - .collect(groupingBy(application -> application.id().tenant())) - .forEach((tenant, applications) -> { - List<DataplaneTokenVersions> currentTokens = listTokens(tenant); - for (Application application : applications) { - for (Instance instance : application.instances().values()) { - instance.deployments().forEach((zone, deployment) -> { - if (zone.environment().isTest()) return; - if (deployment.dataPlaneTokens().isEmpty()) return; - boolean needsRetrigger = false; - // If a token has a newer change than the deployed token data, we need to re-trigger. - for (DataplaneTokenVersions token : currentTokens) - needsRetrigger |= deployment.dataPlaneTokens().getOrDefault(token.tokenId(), Instant.MAX).isBefore(token.lastUpdated()); - - // If a token is no longer current, but was deployed with at least one version, we need to re-trigger. - for (var entry : deployment.dataPlaneTokens().entrySet()) - needsRetrigger |= ! Instant.EPOCH.equals(entry.getValue()) - && currentTokens.stream().noneMatch(token -> token.tokenId().equals(entry.getKey())); - - if (needsRetrigger && controller.jobController().last(instance.id(), JobType.deploymentTo(zone)).map(Run::hasEnded).orElse(true)) - controller.applications().deploymentTrigger().reTrigger(instance.id(), - JobType.deploymentTo(zone), - "Data plane tokens changed"); - }); - } - } - }); - } - - /** - * Generates a token using tenant name as the check access context. - * Persists the token fingerprint and check access hash, but not the token value - * - * @param tenantName name of the tenant to connect the token to - * @param tokenId The user generated name/id of the token - * @param expiration Token expiration - * @param principal The principal making the request - * @return a DataplaneToken containing the secret generated token - */ - public DataplaneToken generateToken(TenantName tenantName, TokenId tokenId, Instant expiration, Principal principal) { - TokenDomain tokenDomain = TokenDomain.of("Vespa Cloud tenant data plane:%s".formatted(tenantName.value())); - Token token = TokenGenerator.generateToken(tokenDomain, TOKEN_PREFIX, TOKEN_BYTES); - TokenCheckHash checkHash = TokenCheckHash.of(token, CHECK_HASH_BYTES); - Instant now = controller.clock().instant(); - DataplaneTokenVersions.Version newTokenVersion = new DataplaneTokenVersions.Version( - FingerPrint.of(token.fingerprint().toDelimitedHexString()), - checkHash.toHexString(), - now, - Optional.ofNullable(expiration), - principal.getName()); - - CuratorDb curator = controller.curator(); - try (Mutex lock = curator.lock(tenantName)) { - List<DataplaneTokenVersions> dataplaneTokenVersions = curator.readDataplaneTokens(tenantName); - Optional<DataplaneTokenVersions> existingToken = dataplaneTokenVersions.stream().filter(t -> Objects.equals(t.tokenId(), tokenId)).findFirst(); - if (existingToken.isPresent()) { - List<DataplaneTokenVersions.Version> versions = existingToken.get().tokenVersions(); - versions = Stream.concat( - versions.stream(), - Stream.of(newTokenVersion)) - .toList(); - dataplaneTokenVersions = Stream.concat( - dataplaneTokenVersions.stream().filter(t -> !Objects.equals(t.tokenId(), tokenId)), - Stream.of(new DataplaneTokenVersions(tokenId, versions, now))) - .toList(); - } else { - DataplaneTokenVersions newToken = new DataplaneTokenVersions(tokenId, List.of(newTokenVersion), now); - dataplaneTokenVersions = Stream.concat(dataplaneTokenVersions.stream(), Stream.of(newToken)).toList(); - } - curator.writeDataplaneTokens(tenantName, dataplaneTokenVersions); - } - - // Return the data plane token including the secret token. - return new DataplaneToken(tokenId, FingerPrint.of(token.fingerprint().toDelimitedHexString()), - token.secretTokenString(), Optional.ofNullable(expiration)); - } - - /** - * Deletes the token version identitfied by tokenId and tokenFingerPrint - * @throws IllegalArgumentException if the version could not be found - */ - public void deleteToken(TenantName tenantName, TokenId tokenId, FingerPrint tokenFingerprint) { - CuratorDb curator = controller.curator(); - try (Mutex lock = curator.lock(tenantName)) { - List<DataplaneTokenVersions> dataplaneTokenVersions = curator.readDataplaneTokens(tenantName); - Optional<DataplaneTokenVersions> existingToken = dataplaneTokenVersions.stream().filter(t -> Objects.equals(t.tokenId(), tokenId)).findFirst(); - if (existingToken.isPresent()) { - List<DataplaneTokenVersions.Version> versions = existingToken.get().tokenVersions(); - versions = versions.stream().filter(v -> !Objects.equals(v.fingerPrint(), tokenFingerprint)).toList(); - if (versions.isEmpty()) { - dataplaneTokenVersions = dataplaneTokenVersions.stream().filter(t -> !Objects.equals(t.tokenId(), tokenId)).toList(); - } else { - Optional<Version> existingVersion = existingToken.get().tokenVersions().stream().filter(v -> v.fingerPrint().equals(tokenFingerprint)).findAny(); - if (existingVersion.isPresent()) { - Instant now = controller.clock().instant(); - // If we removed an expired token, we keep the old lastUpdated timestamp. - Instant lastUpdated = existingVersion.get().expiration().map(now::isAfter).orElse(false) ? existingToken.get().lastUpdated() : now; - dataplaneTokenVersions = Stream.concat(dataplaneTokenVersions.stream().filter(t -> !Objects.equals(t.tokenId(), tokenId)), - Stream.of(new DataplaneTokenVersions(tokenId, versions, lastUpdated))).toList(); - } else { - throw new IllegalArgumentException("Fingerprint does not exist: " + tokenFingerprint); - } - } - curator.writeDataplaneTokens(tenantName, dataplaneTokenVersions); - } else { - throw new IllegalArgumentException("Token does not exist: " + tokenId); - } - } - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java deleted file mode 100644 index 839dbf76faa..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.deployment; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jdisc.EmptyResponse; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.JobStatus; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.yolean.Exceptions; - -import java.io.IOException; -import java.io.OutputStream; -import java.time.Instant; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.logging.Logger; - -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * This API serves redirects to a badge server. - * - * @author jonmv - */ -@SuppressWarnings("unused") // Handler -public class BadgeApiHandler extends ThreadedHttpRequestHandler { - - private final static Logger log = Logger.getLogger(BadgeApiHandler.class.getName()); - - private final Controller controller; - private final Map<Key, Value> badgeCache = new ConcurrentHashMap<>(); - - public BadgeApiHandler(Context parentCtx, Controller controller) { - super(parentCtx); - this.controller = controller; - } - - @Override - public HttpResponse handle(HttpRequest request) { - Method method = request.getMethod(); - try { - return switch (method) { - case OPTIONS -> new SvgHttpResponse("") {{ - headers().add("Allow", "GET, HEAD, OPTIONS"); - headers().add("Access-Control-Allow-Origin", "*"); - headers().add("Access-Control-Allow-Methods", "GET, HEAD, OPTIONS"); - }}; - case HEAD, GET -> get(request); - default -> ErrorResponse.methodNotAllowed("Method '" + method + "' is unsupported"); - }; - } catch (IllegalArgumentException|IllegalStateException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/badge/v1/{tenant}/{application}/{instance}")) return overviewBadge(path.get("tenant"), path.get("application"), path.get("instance")); - if (path.matches("/badge/v1/{tenant}/{application}/{instance}/{jobName}")) return historyBadge(path.get("tenant"), path.get("application"), path.get("instance"), path.get("jobName"), request.getProperty("historyLength")); - - return ErrorResponse.notFoundError(Text.format("No '%s' handler at '%s'", request.getMethod(), - request.getUri().getPath())); - } - - /** Returns a URI which points to an overview badge for the given application. */ - private HttpResponse overviewBadge(String tenant, String application, String instance) { - ApplicationId id = ApplicationId.from(tenant, application, instance); - return cachedResponse(new Key(id, null, 0), - controller.clock().instant(), - () -> { - DeploymentStatus status = controller.jobController().deploymentStatus(controller.applications().requireApplication(TenantAndApplicationId.from(id))); - Predicate<JobStatus> isDeclaredJob = job -> status.jobSteps().get(job.id()) != null && status.jobSteps().get(job.id()).isDeclared(); - return Badges.overviewBadge(id, status.jobs().instance(id.instance()).matching(isDeclaredJob)); - }); - } - - /** Returns a URI which points to a history badge for the given application and job type. */ - private HttpResponse historyBadge(String tenant, String application, String instance, String jobName, String historyLength) { - ApplicationId id = ApplicationId.from(tenant, application, instance); - JobType type = JobType.fromJobName(jobName, controller.zoneRegistry()); - int length = historyLength == null ? 5 : Math.min(32, Math.max(0, Integer.parseInt(historyLength))); - return cachedResponse(new Key(id, type, length), - controller.clock().instant(), - () -> Badges.historyBadge(id, - controller.jobController().jobStatus(new JobId(id, type)), - length) - ); - } - - private HttpResponse cachedResponse(Key key, Instant now, Supplier<String> badge) { - return new SvgHttpResponse(badgeCache.compute(key, (__, value) -> { - return value != null && value.expiry.isAfter(now) ? value : new Value(badge.get(), now); - }).badgeSvg); - } - - private static class SvgHttpResponse extends HttpResponse { - private final String svg; - SvgHttpResponse(String svg) { super(200); this.svg = svg; } - @Override public void render(OutputStream outputStream) throws IOException { - outputStream.write(svg.getBytes(UTF_8)); - } - @Override public String getContentType() { - return "image/svg+xml"; - } - } - - - private static class Key { - - private final ApplicationId id; - private final JobType type; - private final int historyLength; - - private Key(ApplicationId id, JobType type, int historyLength) { - this.id = id; - this.type = type; - this.historyLength = historyLength; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Key key = (Key) o; - return historyLength == key.historyLength && id.equals(key.id) && Objects.equals(type, key.type); - } - - @Override - public int hashCode() { - return Objects.hash(id, type, historyLength); - } - - } - - private static class Value { - - private final String badgeSvg; - private final Instant expiry; - - private Value(String badgeSvg, Instant created) { - this.badgeSvg = badgeSvg; - this.expiry = created.plusSeconds(60); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java deleted file mode 100644 index 41b5c833ec8..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java +++ /dev/null @@ -1,310 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.deployment; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.deployment.JobList; -import com.yahoo.vespa.hosted.controller.deployment.JobStatus; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; - -public class Badges { - - // https://chrishewett.com/blog/calculating-text-width-programmatically/ thank you! - private static final String characterWidths = "[[\" \",35.156],[\"!\",39.355],[\"\\\"\",45.898],[\"#\",81.836],[\"$\",63.574],[\"%\",107.617],[\"&\",72.656],[\"'\",26.855],[\"(\",45.41],[\")\",45.41],[\"*\",63.574],[\"+\",81.836],[\",\",36.377],[\"-\",45.41],[\".\",36.377],[\"/\",45.41],[\"0\",63.574],[\"1\",63.574],[\"2\",63.574],[\"3\",63.574],[\"4\",63.574],[\"5\",63.574],[\"6\",63.574],[\"7\",63.574],[\"8\",63.574],[\"9\",63.574],[\":\",45.41],[\";\",45.41],[\"<\",81.836],[\"=\",81.836],[\">\",81.836],[\"?\",54.541],[\"@\",100],[\"A\",68.359],[\"B\",68.555],[\"C\",69.824],[\"D\",77.051],[\"E\",63.232],[\"F\",57.471],[\"G\",77.539],[\"H\",75.146],[\"I\",42.09],[\"J\",45.459],[\"K\",69.287],[\"L\",55.664],[\"M\",84.277],[\"N\",74.805],[\"O\",78.711],[\"P\",60.303],[\"Q\",78.711],[\"R\",69.531],[\"S\",68.359],[\"T\",61.621],[\"U\",73.193],[\"V\",68.359],[\"W\",98.877],[\"X\",68.506],[\"Y\",61.523],[\"Z\",68.506],[\"[\",45.41],[\"\\\\\",45.41],[\"]\",45.41],[\"^\",81.836],[\"_\",63.574],[\"`\",63.574],[\"a\",60.059],[\"b\",62.305],[\"c\",52.1],[\"d\",62.305],[\"e\",59.57],[\"f\",35.156],[\"g\",62.305],[\"h\",63.281],[\"i\",27.441],[\"j\",34.424],[\"k\",59.18],[\"l\",27.441],[\"m\",97.266],[\"n\",63.281],[\"o\",60.693],[\"p\",62.305],[\"q\",62.305],[\"r\",42.676],[\"s\",52.1],[\"t\",39.404],[\"u\",63.281],[\"v\",59.18],[\"w\",81.836],[\"x\",59.18],[\"y\",59.18],[\"z\",52.539],[\"{\",63.477],[\"|\",45.41],[\"}\",63.477],[\"~\",81.836],[\"_median\",63.281]]"; - private static final double[] widths = new double[128]; // 0-94 hold widths for corresponding chars (+32); 95 holds the fallback width. - - static { - SlimeUtils.jsonToSlimeOrThrow(characterWidths).get() - .traverse((ArrayTraverser) (i, pair) -> { - if (i < 95) - assert Arrays.equals(new byte[]{(byte) (i + 32)}, pair.entry(0).asUtf8()) : i + ": " + pair.entry(0).asString(); - else - assert "_median".equals(pair.entry(0).asString()); - - widths[i] = pair.entry(1).asDouble(); - }); - } - - /** Character pixel width of a 100px size Verdana font rendering of the given code point, for code points in the range [32, 126]. */ - public static double widthOf(int codePoint) { - return 32 <= codePoint && codePoint <= 126 ? widths[codePoint - 32] : widths[95]; - } - - /** Computes an approximate pixel width of the given size Verdana font rendering of the given string, ignoring kerning. */ - public static double widthOf(String text, int size) { - return text.codePoints().mapToDouble(Badges::widthOf).sum() * (size - 0.5) / 100; - } - - /** Computes an approximate pixel width of a 11px size Verdana font rendering of the given string, ignoring kerning. */ - public static double widthOf(String text) { - return widthOf(text, 11); - } - - static String colorOf(Run run, Optional<RunStatus> previous) { - return switch (run.status()) { - case running -> switch (previous.orElse(RunStatus.success)) { - case success -> "url(#run-on-success)"; - case cancelled, aborted, noTests -> "url(#run-on-warning)"; - default -> "url(#run-on-failure)"; - }; - case success -> success; - case cancelled, aborted, noTests -> warning; - default -> failure; - }; - } - - static String nameOf(JobType type) { - return type.isTest() ? type.isProduction() ? "test" - : type.jobName() - : type.jobName().replace("production-", ""); - } - - static final double xPad = 6; - static final double logoSize = 16; - static final String dark = "#404040"; - static final String success = "#00f844"; - static final String running = "#ab83ff"; - static final String failure = "#bf103c"; - static final String warning = "#bd890b"; - - static void addText(List<String> texts, String text, double x, double width) { - addText(texts, text, x, width, 11); - } - - static void addText(List<String> texts, String text, double x, double width, int size) { - texts.add(" <text font-size='" + size + "' x='" + (x + 0.5) + "' y='" + (15) + "' fill='#000' fill-opacity='.4' textLength='" + width + "'>" + text + "</text>\n"); - texts.add(" <text font-size='" + size + "' x='" + x + "' y='" + (14) + "' fill='#fff' textLength='" + width + "'>" + text + "</text>\n"); - } - - static void addShade(List<String> sections, double x, double width) { - sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (width + 6) + "' height='20' fill='url(#shade)'/>\n"); - } - - static void addShadow(List<String> sections, double x) { - sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + 8 + "' height='20' fill='url(#shadow)'/>\n"); - } - - static String historyBadge(ApplicationId id, JobStatus status, int length) { - List<String> sections = new ArrayList<>(); - List<String> texts = new ArrayList<>(); - - double x = 0; - String text = id.toFullString(); - double textWidth = widthOf(text); - double dx = xPad + logoSize + xPad + textWidth + xPad; - - addShade(sections, x, dx); - sections.add(" <rect width='" + dx + "' height='20' fill='" + dark + "'/>\n"); - addText(texts, text, x + (xPad + logoSize + dx) / 2, textWidth); - x += dx; - - if (status.lastTriggered().isEmpty()) - return badge(sections, texts, x); - - Run lastTriggered = status.lastTriggered().get(); - List<Run> runs = status.runs().descendingMap().values().stream() - .filter(Run::hasEnded) - .skip(1) - .limit(length) - .toList(); - - text = lastTriggered.id().type().jobName(); - textWidth = widthOf(text); - dx = xPad + textWidth + xPad; - addShade(sections, x, dx); - sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(lastTriggered, status.lastStatus()) + "'/>\n"); - addShadow(sections, x + dx); - addText(texts, text, x + dx / 2, textWidth); - x += dx; - - dx = xPad * (192.0 / (32 + runs.size())); // Broader sections with shorter history. - for (Run run : runs) { - addShade(sections, x, dx); - sections.add(" <rect x='" + (x - 6) + "' rx='3' width='" + (dx + 6) + "' height='20' fill='" + colorOf(run, Optional.empty()) + "'/>\n"); - addShadow(sections, x + dx); - dx *= Math.pow(0.3, 1.0 / (runs.size() + 8)); // Gradually narrowing sections with age. - x += dx; - } - Collections.reverse(sections); - - return badge(sections, texts, x); - } - - static String overviewBadge(ApplicationId id, JobList jobs) { - // Put production tests right after their deployments, for a more compact rendering. - List<Run> runs = new ArrayList<>(jobs.lastTriggered().asList()); - boolean anyTest = false; - for (int i = 0; i < runs.size(); i++) { - Run run = runs.get(i); - if (run.id().type().isProduction() && run.id().type().isTest()) { - anyTest = true; - int j = i; - while ( ! runs.get(j - 1).id().type().zone().equals(run.id().type().zone())) - runs.set(j, runs.get(--j)); - runs.set(j, run); - } - } - - List<String> sections = new ArrayList<>(); - List<String> texts = new ArrayList<>(); - - double x = 0; - String text = id.toFullString(); - double textWidth = widthOf(text); - double dx = xPad + logoSize + xPad + textWidth + xPad; - double tdx = xPad + widthOf("test"); - - addShade(sections, 0, dx); - sections.add(" <rect width='" + dx + "' height='20' fill='" + dark + "'/>\n"); - addText(texts, text, x + (xPad + logoSize + dx) / 2, textWidth); - x += dx; - - for (int i = 0; i < runs.size(); i++) { - Run run = runs.get(i); - Run test = i + 1 < runs.size() ? runs.get(i + 1) : null; - if (test == null || ! test.id().type().isTest() || ! test.id().type().isProduction()) - test = null; - - boolean isTest = run.id().type().isTest() && run.id().type().isProduction(); - text = nameOf(run.id().type()); - textWidth = widthOf(text, isTest ? 9 : 11); - dx = xPad + textWidth + (isTest ? 0 : xPad); - Optional<RunStatus> previous = jobs.get(run.id().job()).flatMap(JobStatus::lastStatus); - - addText(texts, text, x + (dx - (isTest ? xPad : 0)) / 2, textWidth, isTest ? 9 : 11); - - // Add "deploy" when appropriate - if ( ! run.id().type().isTest() && anyTest) { - String deploy = "deploy"; - textWidth = widthOf(deploy, 9); - addText(texts, deploy, x + dx + textWidth / 2, textWidth, 9); - dx += textWidth + xPad; - } - - // Add shade across zone section. - if ( ! (isTest)) - addShade(sections, x, dx + (test != null ? tdx : 0)); - - // Add colored section for job ... - if (test == null) - sections.add(" <rect x='" + (x - 16) + "' rx='3' width='" + (dx + 16) + "' height='20' fill='" + colorOf(run, previous) + "'/>\n"); - // ... with a slant if a test is next. - else - sections.add(" <polygon points='" + (x - 6) + " 0 " + (x - 6) + " 20 " + (x + dx - 7) + " 20 " + (x + dx + 1) + " 0' fill='" + colorOf(run, previous) + "'/>\n"); - - // Cast a shadow onto the next zone ... - if (test == null) - addShadow(sections, x + dx); - - x += dx; - } - Collections.reverse(sections); - - return badge(sections, texts, x); - } - - static String badge(List<String> sections, List<String> texts, double width) { - return "<svg xmlns='http://www.w3.org/2000/svg' width='" + width + "' height='20' role='img' aria-label='Deployment Status'>\n" + - " <title>Deployment Status</title>\n" + - // Lighting to give the badge a 3d look--dispersion at the top, shadow at the bottom. - " <linearGradient id='light' x2='0' y2='100%'>\n" + - " <stop offset='0' stop-color='#fff' stop-opacity='.5'/>\n" + - " <stop offset='.1' stop-color='#fff' stop-opacity='.15'/>\n" + - " <stop offset='.9' stop-color='#000' stop-opacity='.15'/>\n" + - " <stop offset='1' stop-color='#000' stop-opacity='.5'/>\n" + - " </linearGradient>\n" + - // Dispersed light at the left of the badge. - " <linearGradient id='left-light' x2='100%' y2='0'>\n" + - " <stop offset='0' stop-color='#fff' stop-opacity='.3'/>\n" + - " <stop offset='.5' stop-color='#fff' stop-opacity='.1'/>\n" + - " <stop offset='1' stop-color='#fff' stop-opacity='.0'/>\n" + - " </linearGradient>\n" + - // Shadow at the right of the badge. - " <linearGradient id='right-shadow' x2='100%' y2='0'>\n" + - " <stop offset='0' stop-color='#000' stop-opacity='.0'/>\n" + - " <stop offset='.5' stop-color='#000' stop-opacity='.1'/>\n" + - " <stop offset='1' stop-color='#000' stop-opacity='.3'/>\n" + - " </linearGradient>\n" + - // Shadow to highlight the border between sections, without using a heavy separator. - " <linearGradient id='shadow' x2='100%' y2='0'>\n" + - " <stop offset='0' stop-color='#222' stop-opacity='.3'/>\n" + - " <stop offset='.625' stop-color='#555' stop-opacity='.3'/>\n" + - " <stop offset='.9' stop-color='#555' stop-opacity='.05'/>\n" + - " <stop offset='1' stop-color='#555' stop-opacity='.0'/>\n" + - " </linearGradient>\n" + - // Weak shade across each panel to highlight borders further. - " <linearGradient id='shade' x2='100%' y2='0'>\n" + - " <stop offset='0' stop-color='#000' stop-opacity='.20'/>\n" + - " <stop offset='0.05' stop-color='#000' stop-opacity='.10'/>\n" + - " <stop offset='1' stop-color='#000' stop-opacity='.0'/>\n" + - " </linearGradient>\n" + - // Running color sloshing back and forth on top of the failure color. - " <linearGradient id='run-on-failure' x1='40%' x2='80%' y2='0%'>\n" + - " <stop offset='0' stop-color='" + running + "' />\n" + - " <stop offset='1' stop-color='" + failure + "' />\n" + - " <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />\n" + - " <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />\n" + - " </linearGradient>\n" + - // Running color sloshing back and forth on top of the warning color. - " <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'>\n" + - " <stop offset='0' stop-color='" + running + "' />\n" + - " <stop offset='1' stop-color='" + warning + "' />\n" + - " <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />\n" + - " <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />\n" + - " </linearGradient>\n" + - // Running color sloshing back and forth on top of the success color. - " <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'>\n" + - " <stop offset='0' stop-color='" + running + "' />\n" + - " <stop offset='1' stop-color='" + success + "' />\n" + - " <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />\n" + - " <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />\n" + - " </linearGradient>\n" + - // Clipping to give the badge rounded corners. - " <clipPath id='rounded'>\n" + - " <rect width='" + width + "' height='20' rx='3' fill='#fff'/>\n" + - " </clipPath>\n" + - // Badge section backgrounds with status colors and shades for distinction. - " <g clip-path='url(#rounded)'>\n" + - String.join("", sections) + - " <rect width='" + 2 + "' height='20' fill='url(#left-light)'/>\n" + - " <rect x='" + (width - 2) + "' width='" + 2 + "' height='20' fill='url(#right-shadow)'/>\n" + - " <rect width='" + width + "' height='20' fill='url(#light)'/>\n" + - " </g>\n" + - " <g fill='#fff' text-anchor='middle' font-family='Verdana,Geneva,DejaVu Sans,sans-serif' text-rendering='geometricPrecision' font-size='11'>\n" + - // The vespa.ai logo (with a slightly coloured shadow)! - " <svg x='" + (xPad + 0.5) + "' y='" + ((20 - logoSize) / 2 + 1) + "' width='" + logoSize + "' height='" + logoSize + "' viewBox='0 0 150 150'>\n" + - " <polygon fill='#402a14' fill-opacity='0.5' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/>\n" + - " <polygon fill='#402a14' fill-opacity='0.5' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/>\n" + - " <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/>\n" + - " <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/>\n" + - " </svg>\n" + - " <svg x='" + xPad + "' y='" + ((20 - logoSize) / 2) + "' width='" + logoSize + "' height='" + logoSize + "' viewBox='0 0 150 150'>\n" + - " <linearGradient id='yellow-shaded' x1='91.17' y1='44.83' x2='136.24' y2='73.4' gradientUnits='userSpaceOnUse'>\n" + - " <stop offset='0.01' stop-color='#c6783e'/>\n" + - " <stop offset='0.54' stop-color='#ff9750'/>\n" + - " </linearGradient>\n" + - " <linearGradient id='blue-shaded' x1='60.71' y1='104.56' x2='-15.54' y2='63' gradientUnits='userSpaceOnUse'>\n" + - " <stop offset='0' stop-color='#005a8e'/>\n" + - " <stop offset='0.54' stop-color='#1a7db6'/>\n" + - " </linearGradient>\n" + - " <polygon fill='#ff9d4b' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/>\n" + - " <polygon fill='url(#yellow-shaded)' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/>\n" + - " <polygon fill='#1a7db6' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/>\n" + - " <polygon fill='url(#blue-shaded)' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/>\n" + - " </svg>\n" + - // Application ID and job names. - String.join("", texts) + - " </g>\n" + - "</svg>\n"; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/CliApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/CliApiHandler.java deleted file mode 100644 index 150acd297c2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/CliApiHandler.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.deployment; - -import com.yahoo.component.Version; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.yolean.Exceptions; - -/** - * This handler implements the /cli/v1/ API. The API allows Vespa CLI to retrieve information about the system, without - * authorization. One example of such information is the minimum Vespa CLI version supported by our APIs. - * - * @author mpolden - */ -public class CliApiHandler extends ThreadedHttpRequestHandler { - - /** - * The minimum version of Vespa CLI which is considered compatible with our APIs. If a version of Vespa CLI below - * this version tries to use our APIs, Vespa CLI will print a warning instructing the user to upgrade. - */ - private static final Version MIN_CLI_VERSION = Version.fromString("7.547.18"); - - public CliApiHandler(Context context) { - super(context); - } - - @Override - public HttpResponse handle(HttpRequest request) { - try { - return switch (request.getMethod()) { - case GET -> get(request); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - }; - } - catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } - catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/cli/v1/")) return root(); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse root() { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("minVersion", MIN_CLI_VERSION.toFullString()); - return new SlimeJsonResponse(slime); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java deleted file mode 100644 index edfa4d01d78..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.deployment; - -import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.container.jdisc.EmptyResponse; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.restapi.UriBuilder; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Versions; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.yolean.Exceptions; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toUnmodifiableMap; - -/** - * This implements the deployment/v1 API which provides information about the status of Vespa platform and - * application deployments. - * - * @author bratseth - */ -@SuppressWarnings("unused") // Injected -public class DeploymentApiHandler extends ThreadedHttpRequestHandler { - - private final Controller controller; - - public DeploymentApiHandler(ThreadedHttpRequestHandler.Context parentCtx, Controller controller) { - super(parentCtx); - this.controller = controller; - } - - @Override - public HttpResponse handle(HttpRequest request) { - try { - return switch (request.getMethod()) { - case GET -> handleGET(request); - case OPTIONS -> handleOPTIONS(); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - }; - } - catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } - catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse handleGET(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/deployment/v1/")) return root(request); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse handleOPTIONS() { - // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother - // spelling out the methods supported at each path, which we should - EmptyResponse response = new EmptyResponse(); - response.headers().put("Allow", "GET,OPTIONS"); - return response; - } - - private HttpResponse root(HttpRequest request) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor platformArray = root.setArray("versions"); - var versionStatus = controller.readVersionStatus(); - ApplicationList applications = ApplicationList.from(controller.applications().asList()).withJobs(); - var deploymentStatuses = controller.jobController().deploymentStatuses(applications, versionStatus); - Map<Version, DeploymentStatistics> deploymentStatistics = DeploymentStatistics.compute(versionStatus.versions().stream().map(VespaVersion::versionNumber).toList(), - deploymentStatuses) - .stream().collect(toMap(DeploymentStatistics::version, identity())); - for (VespaVersion version : versionStatus.versions()) { - Cursor versionObject = platformArray.addObject(); - versionObject.setString("version", version.versionNumber().toString()); - versionObject.setString("confidence", version.confidence().name()); - versionObject.setString("commit", version.releaseCommit()); - versionObject.setLong("date", version.committedAt().toEpochMilli()); - versionObject.setBool("controllerVersion", version.isControllerVersion()); - versionObject.setBool("systemVersion", version.isSystemVersion()); - - Cursor configServerArray = versionObject.setArray("configServers"); - for (var nodeVersion : version.nodeVersions()) { - Cursor configServerObject = configServerArray.addObject(); - configServerObject.setString("hostname", nodeVersion.hostname().value()); - } - - DeploymentStatistics statistics = deploymentStatistics.get(version.versionNumber()); - Cursor failingArray = versionObject.setArray("failingApplications"); - for (Run run : statistics.failingUpgrades()) { - Cursor applicationObject = failingArray.addObject(); - toSlime(applicationObject, run.id().application(), request); - applicationObject.setString("failing", run.id().type().jobName()); - applicationObject.setString("status", nameOf(run.status())); - } - - var statusByInstance = deploymentStatuses.asList().stream() - .flatMap(status -> status.instanceJobs().keySet().stream() - .map(instance -> Map.entry(instance, status))) - .collect(toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); - var jobsByInstance = statusByInstance.entrySet().stream() - .collect(toUnmodifiableMap(Map.Entry::getKey, - entry -> entry.getValue().instanceJobs().get(entry.getKey()))); - Cursor productionArray = versionObject.setArray("productionApplications"); - statistics.productionSuccesses().stream() - .collect(groupingBy(run -> run.id().application(), TreeMap::new, toList())) - .forEach((id, runs) -> { - Cursor applicationObject = productionArray.addObject(); - toSlime(applicationObject, id, request); - applicationObject.setLong("productionJobs", jobsByInstance.get(id).production().size()); - applicationObject.setLong("productionSuccesses", runs.size()); - }); - - Cursor runningArray = versionObject.setArray("deployingApplications"); - for (Run run : statistics.runningUpgrade()) { - Cursor applicationObject = runningArray.addObject(); - toSlime(applicationObject, run.id().application(), request); - applicationObject.setString("running", run.id().type().jobName()); - } - - Cursor instancesArray = versionObject.setArray("applications"); - Stream.of(statistics.failingUpgrades().stream().map(run -> new RunInfo(run, true)), - statistics.otherFailing().stream().map(run -> new RunInfo(run, false)), - statistics.runningUpgrade().stream().map(run -> new RunInfo(run, true)), - statistics.otherRunning().stream().map(run -> new RunInfo(run, false)), - statistics.productionSuccesses().stream().map(run -> new RunInfo(run, true))) - .flatMap(identity()) - .collect(Collectors.groupingBy(run -> run.run.id().application(), - LinkedHashMap::new, // Put apps with failing and running jobs first. - groupingBy(run -> run.run.id().type(), - LinkedHashMap::new, - toList()))) - .forEach((instance, runs) -> { - var status = statusByInstance.get(instance); - var jobsToRun = status.jobsToRun(); - Cursor instanceObject = instancesArray.addObject(); - instanceObject.setString("tenant", instance.tenant().value()); - instanceObject.setString("application", instance.application().value()); - instanceObject.setString("instance", instance.instance().value()); - instanceObject.setBool("upgrading", status.application().require(instance.instance()).change().platform().equals(Optional.of(statistics.version()))); - instanceObject.setBool("pinned", status.application().require(instance.instance()).change().isPlatformPinned()); - instanceObject.setBool("platformPinned", status.application().require(instance.instance()).change().isPlatformPinned()); - instanceObject.setBool("revisionPinned", status.application().require(instance.instance()).change().isRevisionPinned()); - DeploymentStatus.StepStatus stepStatus = status.instanceSteps().get(instance.instance()); - if (stepStatus != null) { // Instance may not have any steps, i.e. an empty deployment spec has been submitted - Readiness platformReadiness = stepStatus.blockedUntil(Change.of(statistics.version())); - if (platformReadiness.cause() == DelayCause.changeBlocked) - instanceObject.setLong("blockedUntil", platformReadiness.at().toEpochMilli()); - } - instanceObject.setString("upgradePolicy", toString(status.application().deploymentSpec().instance(instance.instance()) - .map(DeploymentInstanceSpec::upgradePolicy) - .orElse(DeploymentSpec.UpgradePolicy.defaultPolicy))); - status.application().revisions().last().flatMap(ApplicationVersion::compileVersion) - .ifPresent(compiled -> instanceObject.setString("compileVersion", compiled.toFullString())); - Cursor jobsArray = instanceObject.setArray("jobs"); - status.jobSteps().forEach((job, jobStatus) -> { - if ( ! job.application().equals(instance)) return; - Cursor jobObject = jobsArray.addObject(); - jobObject.setString("name", job.type().jobName()); - if (jobsToRun.containsKey(job)) { - Readiness readiness = jobsToRun.get(job).get(0).readiness(); - switch (readiness.cause()) { - case paused -> jobObject.setLong("pausedUntil", readiness.at().toEpochMilli()); - case coolingDown -> jobObject.setLong("coolingDownUntil", readiness.at().toEpochMilli()); - } - List<Versions> versionsOnThisPlatform = jobsToRun.get(job).stream() - .map(DeploymentStatus.Job::versions) - .filter(versions -> versions.targetPlatform().equals(statistics.version())) - .toList(); - if ( ! versionsOnThisPlatform.isEmpty()) - jobObject.setString("pending", versionsOnThisPlatform.stream() - .allMatch(versions -> versions.sourcePlatform() - .map(statistics.version()::equals) - .orElse(true)) - ? "application" : "platform"); - } - }); - Cursor allRunsObject = instanceObject.setObject("allRuns"); - Cursor upgradeRunsObject = instanceObject.setObject("upgradeRuns"); - runs.forEach((type, rs) -> { - Cursor runObject = allRunsObject.setObject(type.jobName()); - Cursor upgradeObject = upgradeRunsObject.setObject(type.jobName()); - CloudAccount cloudAccount = controller.applications().decideCloudAccountOf(new DeploymentId(instance, type.zone()), - status.application().deploymentSpec()) - .orElse(null); - for (RunInfo run : rs) { - toSlime(runObject, run.run, cloudAccount); - if (run.upgrade) - toSlime(upgradeObject, run.run, cloudAccount); - } - }); - }); - } - JobType.allIn(controller.zoneRegistry()).stream() - .filter(job -> ! job.environment().isManuallyDeployed()) - .map(JobType::jobName).forEach(root.setArray("jobs")::addString); - return new SlimeJsonResponse(slime); - } - - private void toSlime(Cursor jobObject, Run run, CloudAccount cloudAccount) { - String key = run.hasFailed() ? "failing" : run.hasEnded() ? "success" : "running"; - Cursor runObject = jobObject.setObject(key); - runObject.setLong("number", run.id().number()); - runObject.setLong("start", run.start().toEpochMilli()); - run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli())); - runObject.setString("status", nameOf(run.status())); - if (cloudAccount != null) runObject.setObject("enclave").setString("cloudAccount", cloudAccount.value()); - } - - private void toSlime(Cursor object, ApplicationId id, HttpRequest request) { - object.setString("tenant", id.tenant().value()); - object.setString("application", id.application().value()); - object.setString("instance", id.instance().value()); - object.setString("url", new UriBuilder(request.getUri()).withPath("/application/v4/tenant/" + - id.tenant().value() + - "/application/" + - id.application().value()).toString()); - object.setString("upgradePolicy", toString(controller.applications().requireApplication(TenantAndApplicationId.from(id)) - .deploymentSpec().instance(id.instance()).map(DeploymentInstanceSpec::upgradePolicy) - .orElse(DeploymentSpec.UpgradePolicy.defaultPolicy))); - } - - private static String toString(DeploymentSpec.UpgradePolicy upgradePolicy) { - if (upgradePolicy == DeploymentSpec.UpgradePolicy.defaultPolicy) { - return "default"; - } - return upgradePolicy.name(); - } - - public static String nameOf(RunStatus status) { - return switch (status) { - case reset, running -> "running"; - case cancelled, aborted -> "aborted"; - case error -> "error"; - case testFailure -> "testFailure"; - case noTests -> "noTests"; - case endpointCertificateTimeout -> "endpointCertificateTimeout"; - case nodeAllocationFailure -> "nodeAllocationFailure"; - case installationFailed -> "installationFailed"; - case invalidApplication, deploymentFailed -> "deploymentFailed"; - case success -> "success"; - case quotaExceeded -> "quotaExceeded"; - }; - } - - private static class RunInfo { - final Run run; - final boolean upgrade; - - RunInfo(Run run, boolean upgrade) { - this.run = run; - this.upgrade = upgrade; - } - - @Override - public String toString() { - return run.id().toString(); - } - - } - - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java deleted file mode 100644 index 0a466b7ffe8..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.filter; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.interfaces.DecodedJWT; -import com.auth0.jwt.interfaces.Payload; -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.jdisc.Response; -import com.yahoo.jdisc.http.filter.DiscFilterRequest; -import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.restapi.Path; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.athenz.client.zms.ZmsClientException; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.TenantController; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; -import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; -import com.yahoo.vespa.hosted.controller.security.Credentials; -import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.yolean.Exceptions; - -import java.net.URI; -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArraySet; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities.SCREWDRIVER_DOMAIN; - -/** - * Enriches the request principal with roles from Athenz, if an AthenzPrincipal is set on the request. - * - * @author jonmv - */ -public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { - - private static final Logger logger = Logger.getLogger(AthenzRoleFilter.class.getName()); - - private final AthenzFacade athenz; - private final TenantController tenants; - private final ExecutorService executor; - private final ZoneRegistry zones; - - @Inject - public AthenzRoleFilter(AthenzClientFactory athenzClientFactory, Controller controller) { - this.athenz = new AthenzFacade(athenzClientFactory); - this.tenants = controller.tenants(); - this.executor = Executors.newCachedThreadPool(); - this.zones = controller.zoneRegistry(); - } - - @Override - protected Optional<ErrorResponse> filter(DiscFilterRequest request) { - try { - if (request.getUserPrincipal() instanceof AthenzPrincipal principal) { - Optional<DecodedJWT> oktaAt = Optional.ofNullable((String) request.getAttribute("okta.access-token")).map(JWT::decode); - Optional<X509Certificate> cert = request.getClientCertificateChain().stream().findFirst(); - Instant issuedAt = cert.map(X509Certificate::getNotBefore) - .or(() -> oktaAt.map(Payload::getIssuedAt)) - .map(Date::toInstant).orElse(Instant.EPOCH); - Instant expireAt = cert.map(X509Certificate::getNotAfter) - .or(() -> oktaAt.map(Payload::getExpiresAt)) - .map(Date::toInstant).orElse(Instant.MAX); - request.setAttribute(SecurityContext.ATTRIBUTE_NAME, - new SecurityContext(principal, roles(principal, request.getUri()), issuedAt, expireAt)); - } - } - catch (Exception e) { - logger.log(Level.INFO, () -> "Exception mapping Athenz principal to roles: " + Exceptions.toMessageString(e)); - return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Access denied")); - } - return Optional.empty(); - } - - Set<Role> roles(AthenzPrincipal principal, URI uri) throws Exception { - Path path = new Path(uri); - - path.matches("/application/v4/tenant/{tenant}/{*}"); - Optional<Tenant> tenant = Optional.ofNullable(path.get("tenant")).map(TenantName::from).flatMap(tenants::get); - - path.matches("/application/v4/tenant/{tenant}/application/{application}/{*}"); - Optional<ApplicationName> application = Optional.ofNullable(path.get("application")).map(ApplicationName::from); - - final Optional<ZoneId> zone; - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/{*}")) { - zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); - } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/{*}")) { - zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); - } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobname}")) { - zone = Optional.of(JobType.fromJobName(path.get("jobname"), zones).zone()); - } else { - zone = Optional.empty(); - } - - AthenzIdentity identity = principal.getIdentity(); - - Set<Role> roleMemberships = new CopyOnWriteArraySet<>(); - List<Future<?>> futures = new ArrayList<>(); - - futures.add(executor.submit(() -> { - if (athenz.hasHostedOperatorAccess(identity)) - roleMemberships.add(Role.hostedOperator()); - })); - - futures.add(executor.submit(() -> { - if (athenz.hasHostedSupporterAccess(identity)) - roleMemberships.add(Role.hostedSupporter()); - })); - - futures.add(executor.submit(() -> { - // Add all tenants that are accessible for this request - athenz.accessibleTenants(tenants.asList(), new Credentials(principal)) - .forEach(accessibleTenant -> roleMemberships.add(Role.athenzTenantAdmin(accessibleTenant.name()))); - })); - - if ( identity.getDomain().equals(SCREWDRIVER_DOMAIN) - && application.isPresent() - && tenant.isPresent()) - futures.add(executor.submit(() -> { - if ( tenant.get().type() == Tenant.Type.athenz - && hasDeployerAccess(identity, ((AthenzTenant) tenant.get()).domain(), application.get(), zone)) - roleMemberships.add(Role.buildService(tenant.get().name(), application.get())); - })); - - if (identity instanceof AthenzUser - && zone.isPresent() - && tenant.isPresent() - && application.isPresent()) { - ZoneId z = zone.get(); - futures.add(executor.submit(() -> { - if (tenant.get().type() == Tenant.Type.athenz - && canDeployToManualZones(identity, ((AthenzTenant) tenant.get()).domain(), application.get(), z)) - roleMemberships.add(Role.hostedDeveloper(tenant.get().name())); - })); - } - - futures.add(executor.submit(() -> { - if (athenz.hasSystemFlagsAccess(identity, /*dryrun*/false)) - roleMemberships.add(Role.systemFlagsDeployer()); - })); - - futures.add(executor.submit(() -> { - if (athenz.hasPaymentCallbackAccess(identity)) - roleMemberships.add(Role.paymentProcessor()); - })); - - futures.add(executor.submit(() -> { - if (athenz.hasAccountingAccess(identity)) - roleMemberships.add(Role.hostedAccountant()); - })); - - // Run last request in handler thread to avoid creating extra thread. - if (athenz.hasSystemFlagsAccess(identity, /*dryrun*/true)) - roleMemberships.add(Role.systemFlagsDryrunner()); - - for (Future<?> future : futures) - future.get(30, TimeUnit.SECONDS); - - logger.log(Level.FINE, () -> "Roles for principal (" + principal.getName() + "): " + - roleMemberships.stream().map(role -> role.definition().name()).collect(Collectors.joining())); - - return roleMemberships.isEmpty() - ? Set.of(Role.everyone()) - : Set.copyOf(roleMemberships); - } - - @Override - public void deconstruct() { - try { - executor.shutdown(); - if ( ! executor.awaitTermination(30, TimeUnit.SECONDS)) { - executor.shutdownNow(); - if ( ! executor.awaitTermination(10, TimeUnit.SECONDS)) - throw new IllegalStateException("Failed to shut down executor 40 seconds"); - } - } - catch (InterruptedException e) { - throw new IllegalStateException("Interrupted while shutting down executor", e); - } - } - - private boolean hasDeployerAccess(AthenzIdentity identity, AthenzDomain tenantDomain, ApplicationName application, Optional<ZoneId> zone) { - try { - return athenz.hasApplicationAccess(identity, - ApplicationAction.deploy, - tenantDomain, - application, - zone); - } catch (ZmsClientException e) { - throw new RuntimeException("Failed to authorize operation: (" + e.getMessage() + ")", e); - } - } - - private boolean canDeployToManualZones(AthenzIdentity identity, AthenzDomain tenantDomain, ApplicationName application, ZoneId zone) { - if (! zone.environment().isManuallyDeployed()) return false; - try { - return athenz.hasApplicationAccess(identity, ApplicationAction.deploy, tenantDomain, application, Optional.of(zone)); - } catch (ZmsClientException e) { - throw new RuntimeException("Failed to authorize operation: (" + e.getMessage() + ")", e); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java deleted file mode 100644 index 115467ac805..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.filter; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.jdisc.Response; -import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.jdisc.http.filter.DiscFilterRequest; -import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.role.Action; -import com.yahoo.vespa.hosted.controller.api.role.Enforcer; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; - -import java.util.Optional; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * A security filter protects all controller apis. - * - * @author bjorncs - */ -public class ControllerAuthorizationFilter extends JsonSecurityRequestFilterBase { - - private static final Logger log = Logger.getLogger(ControllerAuthorizationFilter.class.getName()); - - private final Enforcer enforcer; - - @Inject - public ControllerAuthorizationFilter(Controller controller) { - this.enforcer = new Enforcer(controller.system()); - } - - @Override - public Optional<ErrorResponse> filter(DiscFilterRequest request) { - try { - Optional<SecurityContext> securityContext = Optional.ofNullable((SecurityContext)request.getAttribute(SecurityContext.ATTRIBUTE_NAME)); - - if (securityContext.isEmpty()) - return Optional.of(new ErrorResponse(Response.Status.UNAUTHORIZED, "Access denied - not authenticated")); - - Action action = Action.from(HttpRequest.Method.valueOf(request.getMethod())); - - // Avoid expensive look-ups when request is always legal. - if (enforcer.allows(Role.everyone(), action, request.getUri())) - return Optional.empty(); - - Set<Role> roles = securityContext.get().roles(); - if (roles.stream().anyMatch(role -> enforcer.allows(role, action, request.getUri()))) - return Optional.empty(); - } - catch (Exception e) { - log.log(Level.WARNING, "Exception evaluating access control: ", e); - } - return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Access denied")); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java deleted file mode 100644 index 114dfc8420c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.filter; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.TenantName; -import com.yahoo.jdisc.http.filter.DiscFilterRequest; -import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.TenantController; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; -import com.yahoo.vespa.hosted.controller.api.role.TenantRole; -import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo.UserLevel.administrator; -import static com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo.UserLevel.developer; -import static com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo.UserLevel.user; - -/** - * A security filter protects all controller apis. - * - * @author freva - */ -public class LastLoginUpdateFilter extends JsonSecurityRequestFilterBase { - - private static final Logger log = Logger.getLogger(LastLoginUpdateFilter.class.getName()); - - private final TenantController tenantController; - - @Inject - public LastLoginUpdateFilter(Controller controller) { - this.tenantController = controller.tenants(); - } - - @Override - public Optional<ErrorResponse> filter(DiscFilterRequest request) { - try { - SecurityContext context = (SecurityContext) request.getAttribute(SecurityContext.ATTRIBUTE_NAME); - Map<TenantName, List<LastLoginInfo.UserLevel>> userLevelsByTenant = context.roles().stream() - .flatMap(LastLoginUpdateFilter::filterTenantUserLevels) - .collect(Collectors.groupingBy(Map.Entry::getKey, Collectors.mapping(Map.Entry::getValue, Collectors.toList()))); - - userLevelsByTenant.forEach((tenant, userLevels) -> tenantController.updateLastLogin(tenant, userLevels, context.issuedAt())); - } catch (Exception e) { - log.log(Level.WARNING, "Exception updating last login:", e); - } - return Optional.empty(); - } - - public static Stream<Map.Entry<TenantName, LastLoginInfo.UserLevel>> filterTenantUserLevels(Role role) { - if (!(role instanceof TenantRole)) - return Stream.empty(); - - TenantRole tenantRole = (TenantRole) role; - TenantName name = tenantRole.tenant(); - switch (tenantRole.definition()) { - case athenzTenantAdmin: - return Stream.of(Map.entry(name, user), Map.entry(name, developer), Map.entry(name, administrator)); - case reader: return Stream.of(Map.entry(name, user)); - case developer: return Stream.of(Map.entry(name, developer)); - case administrator: return Stream.of(Map.entry(name, administrator)); - default: return Stream.empty(); - } - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java deleted file mode 100644 index 7173b086b79..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.filter; - -import ai.vespa.hosted.api.Method; -import ai.vespa.hosted.api.RequestVerifier; -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.jdisc.http.filter.DiscFilterRequest; -import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.security.KeyUtils; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; -import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.yolean.Exceptions; - -import java.security.PublicKey; -import java.util.Base64; -import java.util.Optional; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * Assigns the {@link Role#headless(TenantName, ApplicationName)} role or - * {@link Role#developer(TenantName)} to requests with a X-Authorization header signature - * matching the public key of the indicated application. - * Requests which already have a set of roles assigned to them are not modified. - * - * @author jonmv - */ -public class SignatureFilter extends JsonSecurityRequestFilterBase { - - private static final Logger logger = Logger.getLogger(SignatureFilter.class.getName()); - - private final Controller controller; - - @Inject - public SignatureFilter(Controller controller) { - this.controller = controller; - } - - @Override - protected Optional<ErrorResponse> filter(DiscFilterRequest request) { - if ( request.getAttribute(SecurityContext.ATTRIBUTE_NAME) == null - && request.getHeader("X-Authorization") != null) - try { - getSecurityContext(request).ifPresent(securityContext -> { - request.setUserPrincipal(securityContext.principal()); - request.setRemoteUser(securityContext.principal().getName()); - request.setAttribute(SecurityContext.ATTRIBUTE_NAME, securityContext); - }); - } - catch (Exception e) { - logger.log(Level.INFO, () -> "Exception verifying signed request: " + Exceptions.toMessageString(e)); - } - return Optional.empty(); - } - - private boolean keyVerifies(PublicKey key, DiscFilterRequest request) { - /* This method only checks that the content hash has been signed by the provided public key, but - * does not verify the content of the request. jDisc request filters do not allow inspecting the - * request body, so this responsibility falls on the handler consuming the body instead. For the - * deployment cases, the request body is validated in {@link ApplicationApiHandler.parseDataParts}. - */ - return new RequestVerifier(key, controller.clock()).verify(Method.valueOf(request.getMethod()), - request.getUri(), - request.getHeader("X-Timestamp"), - request.getHeader("X-Content-Hash"), - request.getHeader("X-Authorization")); - } - - private Optional<SecurityContext> getSecurityContext(DiscFilterRequest request) { - PublicKey key = KeyUtils.fromPemEncodedPublicKey(new String(Base64.getDecoder().decode(request.getHeader("X-Key")), UTF_8)); - if (keyVerifies(key, request)) { - ApplicationId id = ApplicationId.fromSerializedForm(request.getHeader("X-Key-Id")); - Optional<CloudTenant> tenant = controller.tenants().get(id.tenant()) - .filter(CloudTenant.class::isInstance) - .map(CloudTenant.class::cast); - if (tenant.isPresent() && tenant.get().developerKeys().containsKey(key)) - return Optional.of(new SecurityContext(tenant.get().developerKeys().get(key), - Set.of(Role.reader(id.tenant()), Role.developer(id.tenant())), - controller.clock().instant())); - - Optional <Application> application = controller.applications().getApplication(TenantAndApplicationId.from(id)); - if (application.isPresent() && application.get().deployKeys().contains(key)) - return Optional.of(new SecurityContext(new SimplePrincipal("headless@" + id.tenant() + "." + id.application()), - Set.of(Role.reader(id.tenant()), Role.headless(id.tenant(), id.application())), - controller.clock().instant())); - } - return Optional.empty(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/flags/AuditedFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/flags/AuditedFlagsHandler.java deleted file mode 100644 index 400576abfea..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/flags/AuditedFlagsHandler.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.flags; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.vespa.configserver.flags.FlagsDb; -import com.yahoo.vespa.configserver.flags.http.FlagsHandler; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger; - -/** - * An extension of {@link FlagsHandler} which logs requests to the audit log. - * - * @author mpolden - */ -public class AuditedFlagsHandler extends FlagsHandler { - - private final AuditLogger auditLogger; - - public AuditedFlagsHandler(Context context, Controller controller, FlagsDb flagsDb) { - super(context, flagsDb); - auditLogger = controller.auditLogger(); - } - - @Override - public HttpResponse handle(HttpRequest request) { - return super.handle(auditLogger.log(request)); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java deleted file mode 100644 index 4f12f00eace..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.horizon; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.horizon.HorizonClient; -import com.yahoo.vespa.hosted.controller.api.integration.horizon.HorizonResponse; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; -import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; -import com.yahoo.vespa.hosted.controller.api.role.TenantRole; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.yolean.Exceptions; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.EnumSet; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Proxies metrics requests from Horizon UI - * - * @author valerijf - */ -public class HorizonApiHandler extends ThreadedHttpRequestHandler { - - private final SystemName systemName; - private final HorizonClient client; - private final BooleanFlag enabledHorizonDashboard; - - private static final EnumSet<RoleDefinition> operatorRoleDefinitions = - EnumSet.of(RoleDefinition.hostedOperator, RoleDefinition.hostedSupporter); - - @Inject - public HorizonApiHandler(ThreadedHttpRequestHandler.Context parentCtx, Controller controller, FlagSource flagSource) { - super(parentCtx); - this.systemName = controller.system(); - this.client = controller.serviceRegistry().horizonClient(); - this.enabledHorizonDashboard = Flags.ENABLED_HORIZON_DASHBOARD.bindTo(flagSource); - } - - @Override - public HttpResponse handle(HttpRequest request) { - var roles = getRoles(request); - var operator = roles.stream().map(Role::definition).anyMatch(operatorRoleDefinitions::contains); - var authorizedTenants = getAuthorizedTenants(roles); - - if (!operator && authorizedTenants.isEmpty()) - return ErrorResponse.forbidden("No tenant with enabled metrics view"); - - try { - return switch (request.getMethod()) { - case GET -> get(request); - case POST -> post(request, authorizedTenants, operator); - case PUT -> put(request); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - }; - } - catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } - catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/horizon/v1/config/dashboard/topFolders")) return new JsonInputStreamResponse(client.getTopFolders()); - if (path.matches("/horizon/v1/config/dashboard/file/{id}")) return getDashboard(path.get("id")); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse post(HttpRequest request, Set<TenantName> authorizedTenants, boolean operator) { - Path path = new Path(request.getUri()); - if (path.matches("/horizon/v1/tsdb/api/query/graph")) return metricQuery(request, authorizedTenants, operator); - if (path.matches("/horizon/v1/meta/search/timeseries")) return metaQuery(request, authorizedTenants, operator); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse put(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/horizon/v1/config/user")) return new JsonInputStreamResponse(client.getUser()); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse metricQuery(HttpRequest request, Set<TenantName> authorizedTenants, boolean operator) { - try { - byte[] data = TsdbQueryRewriter.rewrite(request.getData().readAllBytes(), authorizedTenants, operator, systemName); - return new JsonInputStreamResponse(client.getMetrics(data)); - } catch (TsdbQueryRewriter.UnauthorizedException e) { - return ErrorResponse.forbidden("Access denied"); - } catch (IOException e) { - return ErrorResponse.badRequest("Failed to parse request body: " + e.getMessage()); - } - } - - private HttpResponse metaQuery(HttpRequest request, Set<TenantName> authorizedTenants, boolean operator) { - try { - byte[] data = TsdbQueryRewriter.rewrite(request.getData().readAllBytes(), authorizedTenants, operator, systemName); - return new JsonInputStreamResponse(client.getMetaData(data)); - } catch (TsdbQueryRewriter.UnauthorizedException e) { - return ErrorResponse.forbidden("Access denied"); - } catch (IOException e) { - return ErrorResponse.badRequest("Failed to parse request body: " + e.getMessage()); - } - } - - private HttpResponse getDashboard(String id) { - try { - int dashboardId = Integer.parseInt(id); - return new JsonInputStreamResponse(client.getDashboard(dashboardId)); - } catch (NumberFormatException e) { - return ErrorResponse.badRequest("Dashboard ID must be integer, was " + id); - } - } - - private static Set<Role> getRoles(HttpRequest request) { - return Optional.ofNullable(request.getJDiscRequest().context().get(SecurityContext.ATTRIBUTE_NAME)) - .filter(SecurityContext.class::isInstance) - .map(SecurityContext.class::cast) - .map(SecurityContext::roles) - .orElseThrow(() -> new IllegalArgumentException("Attribute '" + SecurityContext.ATTRIBUTE_NAME + "' was not set on request")); - } - - private Set<TenantName> getAuthorizedTenants(Set<Role> roles) { - return roles.stream() - .filter(TenantRole.class::isInstance) - .map(role -> ((TenantRole) role).tenant()) - .filter(tenant -> enabledHorizonDashboard.with(FetchVector.Dimension.TENANT_ID, tenant.value()).value()) - .collect(Collectors.toSet()); - } - - private static class JsonInputStreamResponse extends HttpResponse { - - private final HorizonResponse response; - - public JsonInputStreamResponse(HorizonResponse response) { - super(response.code()); - this.response = response; - } - - @Override - public String getContentType() { - return "application/json"; - } - - @Override - public void render(OutputStream outputStream) throws IOException { - try (InputStream inputStream = response.inputStream()) { - inputStream.transferTo(outputStream); - } - } - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java deleted file mode 100644 index 2f3957af70d..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.horizon; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; - -import java.io.IOException; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -/** - * @author valerijf - */ -public class TsdbQueryRewriter { - - private static final ObjectMapper mapper = new ObjectMapper(); - - public static byte[] rewrite(byte[] data, Set<TenantName> authorizedTenants, boolean operator, SystemName systemName) throws IOException { - JsonNode root = mapper.readTree(data); - requireLegalType(root); - getField(root, "executionGraph", ArrayNode.class) - .ifPresent(graph -> rewriteQueryGraph(root, graph, authorizedTenants, operator, systemName)); - getField(root, "filters", ArrayNode.class) - .ifPresent(filters -> rewriteFilters(filters, authorizedTenants, operator, systemName)); - getField(root, "queries", ArrayNode.class) - .ifPresent(graph -> rewriteQueryGraph(root, graph, authorizedTenants, operator, systemName)); - - return mapper.writeValueAsBytes(root); - } - - private static void rewriteQueryGraph(JsonNode root, ArrayNode executionGraph, Set<TenantName> tenantNames, boolean operator, SystemName systemName) { - for (int i = 0; i < executionGraph.size(); i++) { - JsonNode execution = executionGraph.get(i); - - // Will be handled by rewriteFilters() - if (execution.has("filterId")) { - if (filterExists(root, execution.get("filterId").asText())) - continue; - else - throw new IllegalArgumentException("Invalid filterId: " + execution.get("filterId").asText()); - } - - rewriteFilter((ObjectNode) execution, tenantNames, operator, systemName); - } - } - - private static void rewriteFilters(ArrayNode filters, Set<TenantName> tenantNames, boolean operator, SystemName systemName) { - for (int i = 0; i < filters.size(); i++) - rewriteFilter((ObjectNode) filters.get(i), tenantNames, operator, systemName); - } - - private static void rewriteFilter(ObjectNode parent, Set<TenantName> tenantNames, boolean operator, SystemName systemName) { - ObjectNode prev = ((ObjectNode) parent.get("filter")); - ArrayNode filters; - // If we dont already have a filter object, or the object that we have is not an AND filter - if (prev == null || !"Chain".equals(prev.get("type").asText()) || prev.get("op") != null && !"AND".equals(prev.get("op").asText())) { - // Create new filter object - filters = parent.putObject("filter") - .put("type", "Chain") - .put("op", "AND") - .putArray("filters"); - - // Add the previous filter to the AND expression - if (prev != null) filters.add(prev); - } else filters = (ArrayNode) prev.get("filters"); - - // Make sure we only show metrics in the relevant system - ObjectNode systemFilter = filters.addObject(); - systemFilter.put("type", "TagValueLiteralOr"); - systemFilter.put("filter", systemName.name().toLowerCase()); - systemFilter.put("tagKey", "system"); - - // Make sure non-operators cannot see metrics outside of their tenants - if (!operator) { - ObjectNode appFilter = filters.addObject(); - appFilter.put("type", "TagValueRegex"); - appFilter.put("filter", - tenantNames.stream().map(TenantName::value).sorted().collect(Collectors.joining("|", "^(", ")\\..*"))); - appFilter.put("tagKey", "applicationId"); - } - } - - private static boolean filterExists(JsonNode root, String filterId) { - return getField(root, "filters", ArrayNode.class).stream() - .flatMap(filters -> IntStream.range(0, filters.size()) - .mapToObj(i -> filters.get(i).get("id"))) - .filter(Objects::nonNull) - .filter(JsonNode::isTextual) - .map(JsonNode::asText) - .anyMatch(filterId::equals); - } - - private static void requireLegalType(JsonNode root) { - Optional.ofNullable(root.get("type")) - .map(JsonNode::asText) - .filter(type -> !"TAG_KEYS_AND_VALUES".equals(type)) - .ifPresent(type -> { throw new IllegalArgumentException("Illegal type " + type); }); - } - - private static <T extends JsonNode> Optional<T> getField(JsonNode object, String fieldName, Class<T> clazz) { - return Optional.ofNullable(object.get(fieldName)).filter(clazz::isInstance).map(clazz::cast); - } - - static class UnauthorizedException extends RuntimeException { } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java deleted file mode 100644 index 701761895c3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.os; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.config.provision.zone.ZoneList; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.io.IOUtils; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.slime.Type; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; -import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; -import com.yahoo.vespa.hosted.controller.maintenance.OsUpgradeScheduler; -import com.yahoo.vespa.hosted.controller.maintenance.OsUpgradeScheduler.Change; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.vespa.hosted.controller.versions.CertifiedOsVersion; -import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; -import com.yahoo.yolean.Exceptions; - -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.Scanner; -import java.util.Set; -import java.util.StringJoiner; -import java.util.function.Function; - -/** - * This implements the /os/v1 API which provides operators with information about, and scheduling of OS upgrades for - * nodes in the system. - * - * @author mpolden - */ -@SuppressWarnings("unused") // Injected -public class OsApiHandler extends AuditLoggingRequestHandler { - - private final Controller controller; - private final OsUpgradeScheduler osUpgradeScheduler; - - public OsApiHandler(Context ctx, Controller controller, ControllerMaintenance controllerMaintenance) { - super(ctx, controller.auditLogger()); - this.controller = controller; - this.osUpgradeScheduler = controllerMaintenance.osUpgradeScheduler(); - } - - @Override - public HttpResponse auditAndHandle(HttpRequest request) { - try { - return switch (request.getMethod()) { - case GET -> get(request); - case POST -> post(request); - case DELETE -> delete(request); - case PATCH -> patch(request); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); - }; - } catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse patch(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/os/v1/")) return setOsVersion(request); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/os/v1/")) return new SlimeJsonResponse(osVersions()); - if (path.matches("/os/v1/certify")) return new SlimeJsonResponse(certifiedOsVersions()); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse post(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/os/v1/certify/{cloud}/{version}")) return certifyVersion(request, path.get("version"), path.get("cloud")); - if (path.matches("/os/v1/firmware/")) return requestFirmwareCheckResponse(path); - if (path.matches("/os/v1/firmware/{environment}/")) return requestFirmwareCheckResponse(path); - if (path.matches("/os/v1/firmware/{environment}/{region}/")) return requestFirmwareCheckResponse(path); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse delete(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/os/v1/certify/{cloud}/{version}")) return uncertifyVersion(request, path.get("version"), path.get("cloud")); - if (path.matches("/os/v1/firmware/")) return cancelFirmwareCheckResponse(path); - if (path.matches("/os/v1/firmware/{environment}/")) return cancelFirmwareCheckResponse(path); - if (path.matches("/os/v1/firmware/{environment}/{region}/")) return cancelFirmwareCheckResponse(path); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse certifyVersion(HttpRequest request, String versionString, String cloudName) { - Version version = Version.fromString(versionString); - CloudName cloud = CloudName.from(cloudName); - String vespaVersionString = asString(request.getData()); - if (vespaVersionString.isEmpty()) { - throw new IllegalArgumentException("Missing Vespa version in request body"); - } - Version vespaVersion = Version.fromString(vespaVersionString); - CertifiedOsVersion certified = controller.os().certify(version, cloud, vespaVersion); - if (certified.vespaVersion().equals(vespaVersion)) { - return new MessageResponse("Certified " + version.toFullString() + " in cloud " + cloud + - " as compatible with Vespa version " + vespaVersion.toFullString()); - } - return new MessageResponse(version.toFullString() + " is already certified in cloud " + cloud + - " as compatible with Vespa version " + certified.vespaVersion().toFullString() + - ". Leaving certification unchanged"); - } - - private HttpResponse uncertifyVersion(HttpRequest request, String versionString, String cloudName) { - Version version = Version.fromString(versionString); - CloudName cloud = CloudName.from(cloudName); - controller.os().uncertify(version, cloud); - return new MessageResponse("Removed certification of " + version.toFullString() + " in cloud " + cloud); - } - - private HttpResponse requestFirmwareCheckResponse(Path path) { - List<ZoneId> zones = zonesAt(path); - if (zones.isEmpty()) - return ErrorResponse.notFoundError("No zones at " + path); - - StringJoiner response = new StringJoiner(", ", "Requested firmware checks in ", "."); - for (ZoneId zone : zones) { - controller.serviceRegistry().configServer().nodeRepository().requestFirmwareCheck(zone); - response.add(zone.value()); - } - return new MessageResponse(response.toString()); - } - - private HttpResponse cancelFirmwareCheckResponse(Path path) { - List<ZoneId> zones = zonesAt(path); - if (zones.isEmpty()) - return ErrorResponse.notFoundError("No zones at " + path); - - StringJoiner response = new StringJoiner(", ", "Cancelled firmware checks in ", "."); - for (ZoneId zone : zones) { - controller.serviceRegistry().configServer().nodeRepository().cancelFirmwareCheck(zone); - response.add(zone.value()); - } - return new MessageResponse(response.toString()); - } - - private List<ZoneId> zonesAt(Path path) { - ZoneList zones = controller.zoneRegistry().zones().controllerUpgraded(); - if (path.get("region") != null) zones = zones.in(RegionName.from(path.get("region"))); - if (path.get("environment") != null) zones = zones.in(Environment.from(path.get("environment"))); - return zones.zones().stream().map(ZoneApi::getId).toList(); - } - - private HttpResponse setOsVersion(HttpRequest request) { - Slime requestData = toSlime(request.getData()); - Inspector root = requestData.get(); - CloudName cloud = parseStringField("cloud", root, CloudName::from); - if (requireField("version", root).type() == Type.NIX) { - controller.os().cancelUpgrade(cloud); - return new MessageResponse("Cleared target OS version for cloud '" + cloud.value() + "'"); - } - Version target = parseStringField("version", root, Version::fromString); - boolean force = root.field("force").asBool(); - boolean pin = root.field("pin").asBool(); - controller.os().upgradeTo(target, cloud, force, pin); - return new MessageResponse("Set target OS version for cloud '" + cloud.value() + "' to " + - target.toFullString() + (pin ? " (pinned)" : "")); - } - - private Slime certifiedOsVersions() { - Slime slime = new Slime(); - Cursor array = slime.setArray(); - controller.os().readCertified().stream().sorted().forEach(cv -> { - Cursor object = array.addObject(); - object.setString("version", cv.osVersion().version().toFullString()); - object.setString("cloud", cv.osVersion().cloud().value()); - object.setString("vespaVersion", cv.vespaVersion().toFullString()); - }); - return slime; - } - - private Slime osVersions() { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Set<OsVersionTarget> targets = controller.os().targets(); - - Cursor versions = root.setArray("versions"); - Instant now = controller.clock().instant(); - controller.os().status().versions().forEach((osVersion, nodeVersions) -> { - Cursor currentVersionObject = versions.addObject(); - currentVersionObject.setString("version", osVersion.version().toFullString()); - Optional<OsVersionTarget> target = targets.stream().filter(t -> t.osVersion().equals(osVersion)).findFirst(); - currentVersionObject.setBool("targetVersion", target.isPresent()); - target.ifPresent(t -> { - currentVersionObject.setString("upgradeBudget", Duration.ZERO.toString()); - currentVersionObject.setLong("scheduledAt", t.scheduledAt().toEpochMilli()); - currentVersionObject.setBool("pinned", t.pinned()); - Optional<Change> nextChange = osUpgradeScheduler.changeIn(t.osVersion().cloud(), now, true); - nextChange.ifPresent(c -> { - currentVersionObject.setString("nextVersion", c.osVersion().version().toFullString()); - currentVersionObject.setLong("nextScheduledAt", c.scheduleAt().toEpochMilli()); - currentVersionObject.setBool("certified", c.certified()); - }); - }); - - currentVersionObject.setString("cloud", osVersion.cloud().value()); - Cursor nodesArray = currentVersionObject.setArray("nodes"); - nodeVersions.forEach(nodeVersion -> { - Cursor nodeObject = nodesArray.addObject(); - nodeObject.setString("hostname", nodeVersion.hostname().value()); - nodeObject.setString("environment", nodeVersion.zone().environment().value()); - nodeObject.setString("region", nodeVersion.zone().region().value()); - }); - }); - - return slime; - } - - private static Slime toSlime(InputStream json) { - try { - return SlimeUtils.jsonToSlime(IOUtils.readBytes(json, 1000 * 1000)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static <T> T parseStringField(String name, Inspector root, Function<String, T> parser) { - String fieldValue = requireField(name, root).asString(); - try { - return parser.apply(fieldValue); - } catch (Exception e) { - throw new IllegalArgumentException("Invalid " + name + " '" + fieldValue + "'", e); - } - } - - private static Inspector requireField(String name, Inspector root) { - Inspector field = root.field(name); - if (!field.valid()) throw new IllegalArgumentException("Field '" + name + "' is required"); - return field; - } - - private static String asString(InputStream in) { - Scanner scanner = new Scanner(in).useDelimiter("\\A"); - if (scanner.hasNext()) { - return scanner.next(); - } - return ""; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java deleted file mode 100644 index 2a6778870b1..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ /dev/null @@ -1,374 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.routing; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.ResourceResponse; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; -import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.EndpointList; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; -import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; -import com.yahoo.vespa.hosted.controller.routing.context.RoutingContext; -import com.yahoo.yolean.Exceptions; - -import java.net.URI; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * This implements the /routing/v1 API, which provides operators and tenants routing control at both zone- (operator - * only) and deployment-level. - * - * @author mpolden - */ -public class RoutingApiHandler extends AuditLoggingRequestHandler { - - private final Controller controller; - - public RoutingApiHandler(Context ctx, Controller controller) { - super(ctx, controller.auditLogger()); - this.controller = Objects.requireNonNull(controller, "controller must be non-null"); - } - - @Override - public HttpResponse auditAndHandle(HttpRequest request) { - try { - var path = new Path(request.getUri()); - return switch (request.getMethod()) { - case GET -> get(path, request); - case POST -> post(path, request); - case DELETE -> delete(path, request); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - }; - } catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse delete(Path path, HttpRequest request) { - if (path.matches("/routing/v1/inactive/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return setDeploymentStatus(path, true, request); - if (path.matches("/routing/v1/inactive/environment/{environment}/region/{region}")) return setZoneStatus(path, true); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse post(Path path, HttpRequest request) { - if (path.matches("/routing/v1/inactive/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return setDeploymentStatus(path, false, request); - if (path.matches("/routing/v1/inactive/environment/{environment}/region/{region}")) return setZoneStatus(path, false); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse get(Path path, HttpRequest request) { - if (path.matches("/routing/v1/")) return status(request.getUri()); - if (path.matches("/routing/v1/status/tenant/{tenant}")) return tenant(path, request); - if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}")) return application(path, request); - if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}")) return instance(path, request); - if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path); - if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}/endpoint")) return endpoints(path); - if (path.matches("/routing/v1/status/environment")) return environment(request); - if (path.matches("/routing/v1/status/environment/{environment}/region/{region}")) return zone(path); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse endpoints(Path path) { - ApplicationId instanceId = instanceFrom(path); - List<Endpoint> endpoints = controller.routing().readDeclaredEndpointsOf(instanceId) - .sortedBy(Comparator.comparing(Endpoint::dnsName)) - .asList(); - - List<DeploymentId> deployments = endpoints.stream() - .flatMap(e -> e.deployments().stream()) - .distinct() - .toList(); - - Map<DeploymentId, RoutingStatus> deploymentsStatus = deployments.stream() - .collect(Collectors.toMap( - deploymentId -> deploymentId, - deploymentId -> controller.routing().of(deploymentId).routingStatus()) - ); - - var slime = new Slime(); - var root = slime.setObject(); - var endpointsRoot = root.setArray("endpoints"); - endpoints.forEach(endpoint -> { - var endpointRoot = endpointsRoot.addObject(); - endpointToSlime(endpointRoot, endpoint); - var zonesRoot = endpointRoot.setArray("zones"); - endpoint.deployments().stream().sorted(Comparator.comparing(d -> d.zoneId().value())) - .forEach(deployment -> { - RoutingStatus status = deploymentsStatus.get(deployment); - deploymentStatusToSlime(zonesRoot.addObject(), deployment, status, endpoint.routingMethod()); - }); - }); - - return new SlimeJsonResponse(slime); - } - - private HttpResponse environment(HttpRequest request) { - var zones = controller.zoneRegistry().zones().all().ids(); - if (isRecursive(request)) { - var slime = new Slime(); - var root = slime.setObject(); - var zonesArray = root.setArray("zones"); - for (var zone : zones) { - toSlime(zone, zonesArray.addObject()); - } - return new SlimeJsonResponse(slime); - } - var resources = controller.zoneRegistry().zones().all().ids().stream() - .map(zone -> zone.environment().value() + - "/region/" + zone.region().value()) - .sorted() - .toList(); - return new ResourceResponse(request.getUri(), resources); - } - - private HttpResponse status(URI requestUrl) { - return new ResourceResponse(requestUrl, "status/tenant", "status/environment"); - } - - private HttpResponse tenant(Path path, HttpRequest request) { - var tenantName = tenantFrom(path); - if (isRecursive(request)) { - var slime = new Slime(); - var root = slime.setObject(); - toSlime(controller.applications().asList(tenantName), null, null, root); - return new SlimeJsonResponse(slime); - } - var resources = controller.applications().asList(tenantName).stream() - .map(Application::id) - .map(TenantAndApplicationId::application) - .map(ApplicationName::value) - .map(application -> "application/" + application) - .sorted() - .toList(); - return new ResourceResponse(request.getUri(), resources); - } - - private HttpResponse application(Path path, HttpRequest request) { - var tenantAndApplicationId = tenantAndApplicationIdFrom(path); - if (isRecursive(request)) { - var slime = new Slime(); - var root = slime.setObject(); - toSlime(List.of(controller.applications().requireApplication(tenantAndApplicationId)), null, - null, root); - return new SlimeJsonResponse(slime); - } - var resources = controller.applications().requireApplication(tenantAndApplicationId).instances().keySet().stream() - .map(InstanceName::value) - .map(instance -> "instance/" + instance) - .sorted() - .toList(); - return new ResourceResponse(request.getUri(), resources); - } - - private HttpResponse instance(Path path, HttpRequest request) { - var instanceId = instanceFrom(path); - if (isRecursive(request)) { - var slime = new Slime(); - var root = slime.setObject(); - toSlime(List.of(controller.applications().requireApplication(TenantAndApplicationId.from(instanceId))), - instanceId, null, root); - return new SlimeJsonResponse(slime); - } - var resources = controller.applications().requireInstance(instanceId).deployments().keySet().stream() - .map(zone -> "environment/" + zone.environment().value() + - "/region/" + zone.region().value()) - .sorted() - .toList(); - return new ResourceResponse(request.getUri(), resources); - } - - private HttpResponse setZoneStatus(Path path, boolean in) { - ZoneId zone = zoneFrom(path); - RoutingContext context = controller.routing().of(zone); - RoutingStatus.Value newStatus = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; - context.setRoutingStatus(newStatus, RoutingStatus.Agent.operator); - return new MessageResponse("Set global routing status for deployments in " + zone + " to " + - (in ? "IN" : "OUT")); - } - - private HttpResponse zone(Path path) { - var zone = zoneFrom(path); - var slime = new Slime(); - var root = slime.setObject(); - toSlime(zone, root); - return new SlimeJsonResponse(slime); - } - - private void toSlime(ZoneId zone, Cursor zoneObject) { - RoutingContext context = controller.routing().of(zone); - zoneStatusToSlime(zoneObject, zone, context.routingStatus(), context.routingMethod()); - } - - private HttpResponse setDeploymentStatus(Path path, boolean in, HttpRequest request) { - var deployment = deploymentFrom(path); - var instance = controller.applications().requireInstance(deployment.applicationId()); - var status = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; - var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; - requireDeployment(deployment, instance); - controller.routing().of(deployment).setRoutingStatus(status, agent); - return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT")); - } - - private HttpResponse deployment(Path path) { - var slime = new Slime(); - var root = slime.setObject(); - var deploymentId = deploymentFrom(path); - var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); - toSlime(List.of(application), deploymentId.applicationId(), deploymentId.zoneId(), root); - return new SlimeJsonResponse(slime); - } - - private void toSlime(List<Application> applications, ApplicationId instanceId, ZoneId zoneId, Cursor root) { - var deploymentsArray = root.setArray("deployments"); - for (var application : applications) { - var instances = instanceId == null - ? application.instances().values() - : List.of(application.require(instanceId.instance())); - EndpointList declaredEndpoints = controller.routing().readDeclaredEndpointsOf(application); - for (var instance : instances) { - var zones = zoneId == null - ? instance.deployments().keySet().stream().sorted(Comparator.comparing(ZoneId::value)).toList() - : List.of(zoneId); - for (var zone : zones) { - DeploymentId deploymentId = requireDeployment(new DeploymentId(instance.id(), zone), instance); - DeploymentRoutingContext context = controller.routing().of(deploymentId); - if (declaredEndpoints.targets(deploymentId).isEmpty()) continue; // No declared endpoints point to this deployment - deploymentStatusToSlime(deploymentsArray.addObject(), - deploymentId, - context.routingStatus(), - context.routingMethod()); - } - } - } - - } - - private static void zoneStatusToSlime(Cursor object, ZoneId zone, RoutingStatus routingStatus, RoutingMethod method) { - object.setString("routingMethod", asString(method)); - object.setString("environment", zone.environment().value()); - object.setString("region", zone.region().value()); - object.setString("status", asString(routingStatus.value())); - object.setString("agent", asString(routingStatus.agent())); - object.setLong("changedAt", routingStatus.changedAt().toEpochMilli()); - } - - private static void deploymentStatusToSlime(Cursor object, DeploymentId deployment, RoutingStatus routingStatus, RoutingMethod method) { - object.setString("routingMethod", asString(method)); - object.setString("instance", deployment.applicationId().serializedForm()); - object.setString("environment", deployment.zoneId().environment().value()); - object.setString("region", deployment.zoneId().region().value()); - object.setString("status", asString(routingStatus.value())); - object.setString("agent", asString(routingStatus.agent())); - object.setLong("changedAt", routingStatus.changedAt().toEpochMilli()); - } - - private static void endpointToSlime(Cursor object, Endpoint endpoint) { - object.setString("name", endpoint.name()); - object.setString("dnsName", endpoint.dnsName()); - object.setString("routingMethod", endpoint.routingMethod().name()); - object.setString("cluster", endpoint.cluster().value()); - object.setString("scope", endpoint.scope().name()); - } - - private TenantName tenantFrom(Path path) { - return TenantName.from(path.get("tenant")); - } - - private ApplicationName applicationFrom(Path path) { - return ApplicationName.from(path.get("application")); - } - - private TenantAndApplicationId tenantAndApplicationIdFrom(Path path) { - return TenantAndApplicationId.from(tenantFrom(path), applicationFrom(path)); - } - - private ApplicationId instanceFrom(Path path) { - return ApplicationId.from(tenantFrom(path), applicationFrom(path), InstanceName.from(path.get("instance"))); - } - - private DeploymentId deploymentFrom(Path path) { - return new DeploymentId(instanceFrom(path), zoneFrom(path)); - } - - private ZoneId zoneFrom(Path path) { - var zone = ZoneId.from(path.get("environment"), path.get("region")); - if (!controller.zoneRegistry().hasZone(zone)) { - throw new IllegalArgumentException("No such zone: " + zone); - } - return zone; - } - - private static DeploymentId requireDeployment(DeploymentId deployment, Instance instance) { - if (!instance.deployments().containsKey(deployment.zoneId())) { - throw new IllegalArgumentException("No such deployment: " + deployment); - } - return deployment; - } - - private static boolean isOperator(HttpRequest request) { - SecurityContext securityContext = Optional.ofNullable(request.getJDiscRequest().context().get(SecurityContext.ATTRIBUTE_NAME)) - .filter(SecurityContext.class::isInstance) - .map(SecurityContext.class::cast) - .orElseThrow(() -> new IllegalArgumentException("Attribute '" + SecurityContext.ATTRIBUTE_NAME + "' was not set on request")); - return securityContext.roles().stream() - .map(Role::definition) - .anyMatch(definition -> definition == RoleDefinition.hostedOperator); - } - - private static boolean isRecursive(HttpRequest request) { - return "true".equals(request.getProperty("recursive")); - } - - private static String asString(RoutingStatus.Value value) { - return switch (value) { - case in -> "in"; - case out -> "out"; - }; - } - - private static String asString(RoutingStatus.Agent agent) { - return switch (agent) { - case operator -> "operator"; - case system -> "system"; - case tenant -> "tenant"; - case unknown -> "unknown"; - }; - } - - private static String asString(RoutingMethod method) { - return switch (method) { - case exclusive -> "exclusive"; - case sharedLayer4 -> "sharedLayer4"; - }; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java deleted file mode 100644 index 2b53b1a32f5..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.systemflags; - -import ai.vespa.util.http.hc4.SslConnectionSocketFactory; -import ai.vespa.util.http.hc4.retry.DelayedConnectionLevelRetryHandler; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; -import com.yahoo.vespa.flags.FlagId; -import com.yahoo.vespa.flags.json.FlagData; -import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireErrorResponse; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.client.ResponseHandler; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLSession; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -import static java.util.stream.Collectors.toSet; - -/** - * A client for /flags/v1 rest api on configserver and controller. - * - * @author bjorncs - */ -class FlagsClient { - - private static final String FLAGS_V1_PATH = "/flags/v1"; - - private static final ObjectMapper mapper = new ObjectMapper(); - - private final CloseableHttpClient client; - - FlagsClient(ControllerIdentityProvider identityProvider, Set<FlagsTarget> targets) { - this.client = createClient(identityProvider, targets); - } - - List<FlagData> listFlagData(FlagsTarget target) throws FlagsException, UncheckedIOException { - HttpGet request = new HttpGet(createUri(target, "/data", List.of(new BasicNameValuePair("recursive", "true")))); - return executeRequest(request, response -> { - verifySuccess(response, null); - return FlagData.deserializeList(EntityUtils.toByteArray(response.getEntity())); - }); - } - - List<FlagId> listDefinedFlags(FlagsTarget target) { - HttpGet request = new HttpGet(createUri(target, "/defined", List.of())); - return executeRequest(request, response -> { - verifySuccess(response, null); - JsonNode json = mapper.readTree(response.getEntity().getContent()); - List<FlagId> flagIds = new ArrayList<>(); - json.fieldNames().forEachRemaining(fieldName -> flagIds.add(new FlagId(fieldName))); - return flagIds; - }); - } - - void putFlagData(FlagsTarget target, FlagData flagData) throws FlagsException, UncheckedIOException { - HttpPut request = new HttpPut(createUri(target, "/data/" + flagData.id().toString(), List.of())); - request.setEntity(jsonContent(flagData.serializeToJson())); - executeRequest(request, response -> { - verifySuccess(response, flagData.id()); - return null; - }); - } - - void deleteFlagData(FlagsTarget target, FlagId flagId) throws FlagsException, UncheckedIOException { - HttpDelete request = new HttpDelete(createUri(target, "/data/" + flagId.toString(), List.of(new BasicNameValuePair("force", "true")))); - executeRequest(request, response -> { - verifySuccess(response, flagId); - return null; - }); - } - - private static CloseableHttpClient createClient(ControllerIdentityProvider identityProvider, Set<FlagsTarget> targets) { - DelayedConnectionLevelRetryHandler retryHandler = DelayedConnectionLevelRetryHandler.Builder - .withExponentialBackoff(Duration.ofSeconds(1), Duration.ofSeconds(20), 5) - .build(); - - return HttpClientBuilder.create() - .setUserAgent("controller-flags-v1-client") - .setSSLSocketFactory(SslConnectionSocketFactory.of( - identityProvider.getConfigServerSslSocketFactory(), new FlagTargetsHostnameVerifier(targets))) - .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout((int) Duration.ofSeconds(10).toMillis()) - .setConnectionRequestTimeout((int) Duration.ofSeconds(10).toMillis()) - .setSocketTimeout((int) Duration.ofSeconds(20).toMillis()) - .build()) - .setMaxConnPerRoute(2) - .setMaxConnTotal(100) - .setRetryHandler(retryHandler) - .build(); - } - - private <T> T executeRequest(HttpUriRequest request, ResponseHandler<T> handler) { - try { - return client.execute(request, handler); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static URI createUri(FlagsTarget target, String subPath, List<NameValuePair> queryParams) { - try { - return new URIBuilder(target.endpoint()) - .setPath(FLAGS_V1_PATH + subPath) - .setParameters(queryParams) - .build(); - } catch (URISyntaxException e) { - throw new RuntimeException(e); // should never happen - } - } - - private static void verifySuccess(HttpResponse response, FlagId flagId) throws IOException { - if (!success(response)) { - throw createFlagsException(response, flagId); - } - } - - private static FlagsException createFlagsException(HttpResponse response, FlagId flagId) throws IOException { - HttpEntity entity = response.getEntity(); - String content = EntityUtils.toString(entity); - int statusCode = response.getStatusLine().getStatusCode(); - if (ContentType.get(entity).getMimeType().equals(ContentType.APPLICATION_JSON.getMimeType())) { - WireErrorResponse error = mapper.readValue(content, WireErrorResponse.class); - return new FlagsException(statusCode, flagId, error.errorCode, error.message); - } else { - return new FlagsException(statusCode, flagId, null, content); - } - } - - private static boolean success(HttpResponse response) { - return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK; - } - - private static StringEntity jsonContent(String json) { - return new StringEntity(json, ContentType.APPLICATION_JSON); - } - - private static class FlagTargetsHostnameVerifier implements HostnameVerifier { - - private final AthenzIdentityVerifier athenzVerifier; - - FlagTargetsHostnameVerifier(Set<FlagsTarget> targets) { - this.athenzVerifier = createAthenzIdentityVerifier(targets); - } - - private static AthenzIdentityVerifier createAthenzIdentityVerifier(Set<FlagsTarget> targets) { - Set<AthenzIdentity> identities = targets.stream() - .flatMap(target -> target.athenzHttpsIdentity().stream()) - .collect(toSet()); - return new AthenzIdentityVerifier(identities); - } - - @Override - public boolean verify(String hostname, SSLSession session) { - return "localhost".equals(hostname) /* for controllers */ || athenzVerifier.verify(hostname, session); - } - } - - static class FlagsException extends RuntimeException { - - private FlagsException(int statusCode, FlagId flagId, String errorCode, String errorMessage) { - super(createErrorMessage(statusCode, flagId, errorCode, errorMessage)); - } - - private static String createErrorMessage(int statusCode, FlagId flagId, String errorCode, String errorMessage) { - StringBuilder builder = new StringBuilder().append("Received ").append(statusCode); - if (errorCode != null) { - builder.append('/').append(errorCode); - } - if (flagId != null) { - builder.append(" for flag '").append(flagId).append("'"); - } - return builder.append(": ").append(errorMessage).toString(); - } - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClientException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClientException.java deleted file mode 100644 index e1b3da65e6e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClientException.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.systemflags; - -import java.util.OptionalInt; - -/** - * @author bjorncs - */ -class FlagsClientException extends RuntimeException { - - private final int responseCode; - - FlagsClientException(int responseCode, String message) { - super(message); - this.responseCode = responseCode; - } - - FlagsClientException(String message, Throwable cause) { - super(message, cause); - this.responseCode = -1; - } - - OptionalInt responseCode() { - return responseCode > 0 ? OptionalInt.of(responseCode) : OptionalInt.empty(); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java deleted file mode 100644 index c006fa13223..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java +++ /dev/null @@ -1,431 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.systemflags; - -import com.fasterxml.jackson.databind.JsonNode; -import com.yahoo.vespa.flags.FlagDefinition; -import com.yahoo.vespa.flags.FlagId; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.json.FlagData; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireSystemFlagsDeployResult; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireSystemFlagsDeployResult.WireFlagDataChange; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireSystemFlagsDeployResult.WireOperationFailure; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireSystemFlagsDeployResult.WireWarning; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.BiFunction; -import java.util.function.Function; - -/** - * @author bjorncs - */ -class SystemFlagsDeployResult { - - private final List<FlagDataChange> flagChanges; - private final List<OperationError> errors; - private final List<Warning> warnings; - - SystemFlagsDeployResult(List<FlagDataChange> flagChanges, List<OperationError> errors, List<Warning> warnings) { - this.flagChanges = flagChanges; - this.errors = errors; - this.warnings = warnings; - } - - SystemFlagsDeployResult(List<OperationError> errors) { - this(List.of(), errors, List.of()); - } - - List<FlagDataChange> flagChanges() { - return flagChanges; - } - - List<OperationError> errors() { - return errors; - } - - List<Warning> warnings() { return warnings; } - - static SystemFlagsDeployResult merge(List<SystemFlagsDeployResult> results) { - List<FlagDataChange> mergedChanges = mergeChanges(results); - List<OperationError> mergedErrors = mergeErrors(results); - List<Warning> mergedWarnings = mergeWarnings(results); - return new SystemFlagsDeployResult(mergedChanges, mergedErrors, mergedWarnings); - } - - private static List<OperationError> mergeErrors(List<SystemFlagsDeployResult> results) { - return merge(results, SystemFlagsDeployResult::errors, OperationError::targets, - OperationErrorWithoutTarget::new, OperationErrorWithoutTarget::toOperationError); - } - - private static List<FlagDataChange> mergeChanges(List<SystemFlagsDeployResult> results) { - return merge(results, SystemFlagsDeployResult::flagChanges, FlagDataChange::targets, - FlagDataChangeWithoutTarget::new, FlagDataChangeWithoutTarget::toFlagDataChange); - } - - private static List<Warning> mergeWarnings(List<SystemFlagsDeployResult> results) { - return merge(results, SystemFlagsDeployResult::warnings, Warning::targets, - WarningWithoutTarget::new, WarningWithoutTarget::toWarning); - } - - private static <VALUE, VALUE_WITHOUT_TARGET> List<VALUE> merge( - List<SystemFlagsDeployResult> results, - Function<SystemFlagsDeployResult, List<VALUE>> valuesGetter, - Function<VALUE, Set<FlagsTarget>> targetsGetter, - Function<VALUE, VALUE_WITHOUT_TARGET> transformer, - BiFunction<VALUE_WITHOUT_TARGET, Set<FlagsTarget>, VALUE> reverseTransformer) { - Map<VALUE_WITHOUT_TARGET, Set<FlagsTarget>> targetsForValue = new HashMap<>(); - for (SystemFlagsDeployResult result : results) { - for (VALUE value : valuesGetter.apply(result)) { - VALUE_WITHOUT_TARGET valueWithoutTarget = transformer.apply(value); - targetsForValue.computeIfAbsent(valueWithoutTarget, k -> new HashSet<>()) - .addAll(targetsGetter.apply(value)); - } - } - List<VALUE> mergedValues = new ArrayList<>(); - targetsForValue.forEach( - (value, targets) -> mergedValues.add(reverseTransformer.apply(value, targets))); - return mergedValues; - } - - WireSystemFlagsDeployResult toWire() { - var wireResult = new WireSystemFlagsDeployResult(); - wireResult.changes = new ArrayList<>(); - for (FlagDataChange change : flagChanges) { - var wireChange = new WireFlagDataChange(); - wireChange.flagId = change.flagId().toString(); - wireChange.owners = owners(change.flagId()); - wireChange.operation = change.operation().asString(); - wireChange.targets = change.targets().stream().map(FlagsTarget::asString).toList(); - wireChange.data = change.data().map(FlagData::toWire).orElse(null); - wireChange.previousData = change.previousData().map(FlagData::toWire).orElse(null); - wireResult.changes.add(wireChange); - } - wireResult.errors = new ArrayList<>(); - for (OperationError error : errors) { - var wireError = new WireOperationFailure(); - wireError.message = error.message(); - wireError.operation = error.operation().asString(); - wireError.targets = error.targets().stream().map(FlagsTarget::asString).toList(); - wireError.flagId = error.flagId().map(FlagId::toString).orElse(null); - wireError.owners = error.flagId().map(id -> owners(id)).orElse(List.of()); - wireError.data = error.flagData().map(FlagData::toWire).orElse(null); - wireResult.errors.add(wireError); - } - wireResult.warnings = new ArrayList<>(); - for (Warning warning : warnings) { - var wireWarning = new WireWarning(); - wireWarning.message = warning.message(); - wireWarning.flagId = warning.flagId().toString(); - wireWarning.owners = owners(warning.flagId()); - wireWarning.targets = warning.targets().stream().map(FlagsTarget::asString).toList(); - wireResult.warnings.add(wireWarning); - } - return wireResult; - } - - private static List<String> owners(FlagId id) { - return Flags.getFlag(id).map(FlagDefinition::getOwners).orElse(List.of()); - } - - static class FlagDataChange { - - private final FlagId flagId; - private final Set<FlagsTarget> targets; - private final OperationType operationType; - private final FlagData data; - private final FlagData previousData; - - private FlagDataChange( - FlagId flagId, Set<FlagsTarget> targets, OperationType operationType, FlagData data, FlagData previousData) { - this.flagId = flagId; - this.targets = targets; - this.operationType = operationType; - this.data = data; - this.previousData = previousData; - } - - static FlagDataChange created(FlagId flagId, FlagsTarget target, FlagData data) { - return new FlagDataChange(flagId, Set.of(target), OperationType.CREATE, data, null); - } - - static FlagDataChange deleted(FlagId flagId, FlagsTarget target) { - return new FlagDataChange(flagId, Set.of(target), OperationType.DELETE, null, null); - } - - static FlagDataChange updated(FlagId flagId, FlagsTarget target, FlagData data, FlagData previousData) { - return new FlagDataChange(flagId, Set.of(target), OperationType.UPDATE, data, previousData); - } - - FlagId flagId() { - return flagId; - } - - Set<FlagsTarget> targets() { - return targets; - } - - OperationType operation() { - return operationType; - } - - Optional<FlagData> data() { - return Optional.ofNullable(data); - } - - Optional<FlagData> previousData() { - return Optional.ofNullable(previousData); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - FlagDataChange that = (FlagDataChange) o; - return Objects.equals(flagId, that.flagId) && - Objects.equals(targets, that.targets) && - operationType == that.operationType && - Objects.equals(data, that.data) && - Objects.equals(previousData, that.previousData); - } - - @Override - public int hashCode() { - return Objects.hash(flagId, targets, operationType, data, previousData); - } - - @Override - public String toString() { - return "FlagDataChange{" + - "flagId=" + flagId + - ", targets=" + targets + - ", operationType=" + operationType + - ", data=" + data + - ", previousData=" + previousData + - '}'; - } - } - - static class OperationError { - - final String message; - final Set<FlagsTarget> targets; - final OperationType operation; - final FlagId flagId; - final FlagData flagData; - - private OperationError( - String message, Set<FlagsTarget> targets, OperationType operation, FlagId flagId, FlagData flagData) { - this.message = message; - this.targets = targets; - this.operation = operation; - this.flagId = flagId; - this.flagData = flagData; - } - - static OperationError listFailed(String message, FlagsTarget target) { - return new OperationError(message, Set.of(target), OperationType.LIST, null, null); - } - - static OperationError createFailed(String message, FlagsTarget target, FlagData flagData) { - return new OperationError(message, Set.of(target), OperationType.CREATE, flagData.id(), flagData); - } - - static OperationError updateFailed(String message, FlagsTarget target, FlagData flagData) { - return new OperationError(message, Set.of(target), OperationType.UPDATE, flagData.id(), flagData); - } - - static OperationError deleteFailed(String message, FlagsTarget target, FlagId id) { - return new OperationError(message, Set.of(target), OperationType.DELETE, id, null); - } - - static OperationError archiveValidationFailed(String message) { - return new OperationError(message, Set.of(), OperationType.VALIDATE_ARCHIVE, null, null); - } - - static OperationError dataForUndefinedFlag(FlagsTarget target, FlagId id) { - return new OperationError("Flag data present for undefined flag. Remove flag data files if flag's definition " + - "is already removed from Flags / PermanentFlags. Consult ModelContext.FeatureFlags " + - "for safe removal of flag used by config-model.", - Set.of(), OperationType.DATA_FOR_UNDEFINED_FLAG, id, null); - } - - String message() { return message; } - Set<FlagsTarget> targets() { return targets; } - OperationType operation() { return operation; } - Optional<FlagId> flagId() { return Optional.ofNullable(flagId); } - Optional<FlagData> flagData() { return Optional.ofNullable(flagData); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - OperationError that = (OperationError) o; - return Objects.equals(message, that.message) && - Objects.equals(targets, that.targets) && - operation == that.operation && - Objects.equals(flagId, that.flagId) && - Objects.equals(flagData, that.flagData); - } - - @Override public int hashCode() { return Objects.hash(message, targets, operation, flagId, flagData); } - - @Override - public String toString() { - return "OperationFailure{" + - "message='" + message + '\'' + - ", targets=" + targets + - ", operation=" + operation + - ", flagId=" + flagId + - ", flagData=" + flagData + - '}'; - } - } - - enum OperationType { - CREATE("create"), DELETE("delete"), UPDATE("update"), LIST("list"), VALIDATE_ARCHIVE("validate-archive"), - DATA_FOR_UNDEFINED_FLAG("data-for-undefined-flag"); - - private final String stringValue; - - OperationType(String stringValue) { this.stringValue = stringValue; } - - String asString() { return stringValue; } - } - - static class Warning { - final String message; - final Set<FlagsTarget> targets; - final FlagId flagId; - - private Warning(String message, Set<FlagsTarget> targets, FlagId flagId) { - this.message = message; - this.targets = targets; - this.flagId = flagId; - } - - static Warning dataForUndefinedFlag(FlagsTarget target, FlagId flagId) { - return new Warning( - "Flag data present for undefined flag. Remove flag data files if flag's definition is already removed from Flags/PermanentFlags. " + - "Consult ModelContext.FeatureFlags for safe removal of flag used by config-model.", Set.of(target), flagId); - } - - String message() { return message; } - Set<FlagsTarget> targets() { return targets; } - FlagId flagId() { return flagId; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Warning warning = (Warning) o; - return Objects.equals(message, warning.message) && - Objects.equals(targets, warning.targets) && - Objects.equals(flagId, warning.flagId); - } - - @Override public int hashCode() { return Objects.hash(message, targets, flagId); } - } - - private static class FlagDataChangeWithoutTarget { - final FlagId flagId; - final OperationType operationType; - final FlagData data; - final FlagData previousData; - final JsonNode jsonData; // needed for FlagData equality check - final JsonNode jsonPreviousData; // needed for FlagData equality check - - - FlagDataChangeWithoutTarget(FlagDataChange change) { - this.flagId = change.flagId(); - this.operationType = change.operation(); - this.data = change.data().orElse(null); - this.previousData = change.previousData().orElse(null); - this.jsonData = Optional.ofNullable(data).map(FlagData::toJsonNode).orElse(null); - this.jsonPreviousData = Optional.ofNullable(previousData).map(FlagData::toJsonNode).orElse(null); - } - - FlagDataChange toFlagDataChange(Set<FlagsTarget> targets) { - return new FlagDataChange(flagId, targets, operationType, data, previousData); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - FlagDataChangeWithoutTarget that = (FlagDataChangeWithoutTarget) o; - return Objects.equals(flagId, that.flagId) && - operationType == that.operationType && - Objects.equals(jsonData, that.jsonData) && - Objects.equals(jsonPreviousData, that.jsonPreviousData); - } - - @Override - public int hashCode() { - return Objects.hash(flagId, operationType, jsonData, jsonPreviousData); - } - } - - private static class OperationErrorWithoutTarget { - final String message; - final OperationType operation; - final FlagId flagId; - final FlagData flagData; - - OperationErrorWithoutTarget(OperationError operationError) { - this.message = operationError.message(); - this.operation = operationError.operation(); - this.flagId = operationError.flagId().orElse(null); - this.flagData = operationError.flagData().orElse(null); - } - - OperationError toOperationError(Set<FlagsTarget> targets) { - return new OperationError(message, targets, operation, flagId, flagData); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - OperationErrorWithoutTarget that = (OperationErrorWithoutTarget) o; - return Objects.equals(message, that.message) && - operation == that.operation && - Objects.equals(flagId, that.flagId) && - Objects.equals(flagData, that.flagData); - } - - @Override public int hashCode() { return Objects.hash(message, operation, flagId, flagData); } - } - - private static class WarningWithoutTarget { - final String message; - final FlagId flagId; - - WarningWithoutTarget(Warning warning) { - this.message = warning.message(); - this.flagId = warning.flagId(); - } - - Warning toWarning(Set<FlagsTarget> targets) { return new Warning(message, targets, flagId); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - WarningWithoutTarget that = (WarningWithoutTarget) o; - return Objects.equals(message, that.message) && - Objects.equals(flagId, that.flagId); - } - - @Override - public int hashCode() { - return Objects.hash(message, flagId); - } - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java deleted file mode 100644 index 0fa800e7367..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.systemflags; - -import com.yahoo.concurrent.DaemonThreadFactory; -import com.yahoo.config.provision.SystemName; -import com.yahoo.text.Text; -import com.yahoo.vespa.flags.FlagId; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.json.FlagData; -import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagValidationException; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive; -import com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.OperationError; -import com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.Warning; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.FlagDataChange; - -/** - * Deploy a flags data archive to all targets in a given system - * - * @author bjorncs - */ -class SystemFlagsDeployer { - - private static final Logger log = Logger.getLogger(SystemFlagsDeployer.class.getName()); - - private final FlagsClient client; - private final SystemName system; - private final Set<FlagsTarget> targets; - private final ExecutorService executor = Executors.newCachedThreadPool(new DaemonThreadFactory("system-flags-deployer-")); - - - SystemFlagsDeployer(ControllerIdentityProvider identityProvider, SystemName system, Set<FlagsTarget> targets) { - this(new FlagsClient(identityProvider, targets), system, targets); - } - - SystemFlagsDeployer(FlagsClient client, SystemName system, Set<FlagsTarget> targets) { - this.client = client; - this.system = system; - this.targets = targets; - } - - SystemFlagsDeployResult deployFlags(SystemFlagsDataArchive archive, boolean dryRun) { - try { - archive.validateAllFilesAreForTargets(targets); - } catch (FlagValidationException e) { - return new SystemFlagsDeployResult(List.of(OperationError.archiveValidationFailed(e.getMessage()))); - } - - Map<FlagsTarget, Future<SystemFlagsDeployResult>> futures = new HashMap<>(); - for (FlagsTarget target : targets) { - futures.put(target, executor.submit(() -> deployFlags(target, archive.flagData(target), dryRun))); - } - List<SystemFlagsDeployResult> results = new ArrayList<>(); - futures.forEach((target, future) -> { - try { - results.add(future.get()); - } catch (InterruptedException | ExecutionException e) { - log.log(Level.SEVERE, Text.format("Failed to deploy flags for target '%s': %s", target, e.getMessage()), e); - throw new RuntimeException(e); - } - }); - return SystemFlagsDeployResult.merge(results); - } - - private SystemFlagsDeployResult deployFlags(FlagsTarget target, List<FlagData> flagDataList, boolean dryRun) { - flagDataList = flagDataList.stream() - .map(target::partiallyResolveFlagData) - .filter(flagData -> !flagData.isEmpty()) - .toList(); - Map<FlagId, FlagData> wantedFlagData = lookupTable(flagDataList); - Map<FlagId, FlagData> currentFlagData; - List<FlagId> definedFlags; - try { - currentFlagData = lookupTable(client.listFlagData(target)); - definedFlags = client.listDefinedFlags(target); - } catch (Exception e) { - log.log(Level.WARNING, Text.format("Failed to list flag data for target '%s': %s", target, e.getMessage()), e); - return new SystemFlagsDeployResult(List.of(OperationError.listFailed(e.getMessage(), target))); - } - - List<OperationError> errors = new ArrayList<>(); - List<FlagDataChange> results = new ArrayList<>(); - List<Warning> warnings = new ArrayList<>(); - - createNewFlagData(target, dryRun, wantedFlagData, currentFlagData, results, errors); - updateExistingFlagData(target, dryRun, wantedFlagData, currentFlagData, results, errors); - removeOldFlagData(target, dryRun, wantedFlagData, currentFlagData, results, errors); - failOnNewFlagDataForUndefinedFlags(target, wantedFlagData, currentFlagData, definedFlags, errors); - failOnFlagDataForUndefinedFlags(target, wantedFlagData, currentFlagData, definedFlags, errors); - return new SystemFlagsDeployResult(results, errors, warnings); - } - - private void createNewFlagData(FlagsTarget target, - boolean dryRun, - Map<FlagId, FlagData> wantedFlagData, - Map<FlagId, FlagData> currentFlagData, - List<FlagDataChange> results, - List<OperationError> errors) { - wantedFlagData.forEach((id, data) -> { - FlagData currentData = currentFlagData.get(id); - if (currentData != null) { - return; // not a new flag - } - try { - if (!dryRun) { - client.putFlagData(target, data); - } else { - dryRunFlagDataValidation(data); - } - } catch (Exception e) { - log.log(Level.WARNING, Text.format("Failed to put flag '%s' for target '%s': %s", data.id(), target, e.getMessage()), e); - errors.add(OperationError.createFailed(e.getMessage(), target, data)); - return; - } - results.add(FlagDataChange.created(id, target, data)); - }); - } - - private void updateExistingFlagData(FlagsTarget target, - boolean dryRun, - Map<FlagId, FlagData> wantedFlagData, - Map<FlagId, FlagData> currentFlagData, - List<FlagDataChange> results, - List<OperationError> errors) { - wantedFlagData.forEach((id, wantedData) -> { - FlagData currentData = currentFlagData.get(id); - if (currentData == null || isEqual(currentData, wantedData)) { - return; // not an flag data update - } - try { - if (!dryRun) { - client.putFlagData(target, wantedData); - } else { - dryRunFlagDataValidation(wantedData); - } - } catch (Exception e) { - log.log(Level.WARNING, Text.format("Failed to update flag '%s' for target '%s': %s", wantedData.id(), target, e.getMessage()), e); - errors.add(OperationError.updateFailed(e.getMessage(), target, wantedData)); - return; - } - results.add(FlagDataChange.updated(id, target, wantedData, currentData)); - }); - } - - private void removeOldFlagData(FlagsTarget target, - boolean dryRun, - Map<FlagId, FlagData> wantedFlagData, - Map<FlagId, FlagData> currentFlagData, - List<FlagDataChange> results, - List<OperationError> errors) { - currentFlagData.forEach((id, data) -> { - if (wantedFlagData.containsKey(id)) { - return; // not a removed flag - } - if (!dryRun) { - try { - client.deleteFlagData(target, id); - } catch (Exception e) { - log.log(Level.WARNING, Text.format("Failed to delete flag '%s' for target '%s': %s", id, target, e.getMessage()), e); - errors.add(OperationError.deleteFailed(e.getMessage(), target, id)); - return; - } - } - results.add(FlagDataChange.deleted(id, target)); - }); - } - - private static void failOnNewFlagDataForUndefinedFlags(FlagsTarget target, - Map<FlagId, FlagData> wantedFlagData, - Map<FlagId, FlagData> currentFlagData, - List<FlagId> definedFlags, - List<OperationError> errors) { - String errorMessage = "Flag not defined in target zone. If zone/configserver cluster is new, add an empty flag " + - "data file for this zone as a temporary measure until the stale flag data files are removed."; - for (FlagId flagId : wantedFlagData.keySet()) { - if (!currentFlagData.containsKey(flagId) && !definedFlags.contains(flagId)) { - errors.add(OperationError.createFailed(errorMessage, target, wantedFlagData.get(flagId))); - } - } - } - - private static void failOnFlagDataForUndefinedFlags(FlagsTarget target, - Map<FlagId, FlagData> wantedFlagData, - Map<FlagId, FlagData> currentFlagData, - List<FlagId> definedFlags, - List<OperationError> errors) { - for (FlagId flagId : currentFlagData.keySet()) { - if (wantedFlagData.containsKey(flagId) && !definedFlags.contains(flagId)) { - errors.add(OperationError.dataForUndefinedFlag(target, flagId)); - } - } - } - - private static void dryRunFlagDataValidation(FlagData data) { - Flags.getFlag(data.id()) - .ifPresent(definition -> data.validate(definition.getUnboundFlag().serializer())); - } - - private static Map<FlagId, FlagData> lookupTable(Collection<FlagData> data) { - return data.stream().collect(Collectors.toMap(FlagData::id, Function.identity())); - } - - private static boolean isEqual(FlagData l, FlagData r) { - return Objects.equals(l.toJsonNode(), r.toJsonNode()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java deleted file mode 100644 index 6318dc8c6fa..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.systemflags; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.JacksonJsonResponse; -import com.yahoo.restapi.Path; -import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagValidationException; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget; -import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; - -import java.io.InputStream; -import java.util.List; -import java.util.concurrent.Executor; - -/** - * Handler implementation for '/system-flags/v1', an API for controlling system-wide feature flags - * - * @author bjorncs - */ -@SuppressWarnings("unused") // Request handler listed in controller's services.xml -public class SystemFlagsHandler extends ThreadedHttpRequestHandler { - - private static final String API_PREFIX = "/system-flags/v1"; - - private final SystemFlagsDeployer deployer; - private final ZoneRegistry zoneRegistry; - - @Inject - public SystemFlagsHandler(ZoneRegistry zoneRegistry, - ControllerIdentityProvider identityProvider, - Executor executor) { - super(executor); - this.zoneRegistry = zoneRegistry; - this.deployer = new SystemFlagsDeployer(identityProvider, zoneRegistry.system(), FlagsTarget.getAllTargetsInSystem(zoneRegistry, true)); - } - - @Override - public HttpResponse handle(HttpRequest request) { - return switch (request.getMethod()) { - case PUT -> put(request); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); - }; - } - - private HttpResponse put(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches(API_PREFIX + "/deploy")) return deploy(request, /*dryRun*/false); - if (path.matches(API_PREFIX + "/dryrun")) return deploy(request, /*dryRun*/true); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse deploy(HttpRequest request, boolean dryRun) { - try { - String contentType = request.getHeader("Content-Type"); - if (!contentType.equalsIgnoreCase("application/zip")) { - return ErrorResponse.badRequest("Invalid content type: " + contentType); - } - SystemFlagsDeployResult result = deploy(request.getData(), dryRun); - return new JacksonJsonResponse<>(200, result.toWire()); - } catch (Exception e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private SystemFlagsDeployResult deploy(InputStream zipStream, boolean dryRun) { - SystemFlagsDataArchive archive; - try { - archive = SystemFlagsDataArchive.fromZip(zipStream, zoneRegistry); - } catch (FlagValidationException e) { - return new SystemFlagsDeployResult(List.of(SystemFlagsDeployResult.OperationError.archiveValidationFailed(e.getMessage()))); - } - - return deployer.deployFlags(archive, dryRun); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java deleted file mode 100644 index 11a5e178703..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ /dev/null @@ -1,417 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.user; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.container.jdisc.EmptyResponse; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.io.IOUtils; -import com.yahoo.jdisc.http.filter.security.misc.User; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeStream; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.text.Text; -import com.yahoo.vespa.configserver.flags.FlagsDb; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.IntFlag; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.LockedTenant; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; -import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; -import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; -import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; -import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; -import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; -import com.yahoo.vespa.hosted.controller.api.role.TenantRole; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.yolean.Exceptions; - -import java.security.PublicKey; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * API for user management related to access control. - * - * @author jonmv - */ -@SuppressWarnings("unused") // Handler -public class UserApiHandler extends ThreadedHttpRequestHandler { - - private final static Logger log = Logger.getLogger(UserApiHandler.class.getName()); - - private final UserManagement users; - private final Controller controller; - private final FlagsDb flagsDb; - private final IntFlag maxTrialTenants; - - @Inject - public UserApiHandler(Context parentCtx, UserManagement users, Controller controller, FlagSource flagSource, FlagsDb flagsDb) { - super(parentCtx); - this.users = users; - this.controller = controller; - this.flagsDb = flagsDb; - this.maxTrialTenants = PermanentFlags.MAX_TRIAL_TENANTS.bindTo(flagSource); - } - - @Override - public HttpResponse handle(HttpRequest request) { - try { - Path path = new Path(request.getUri()); - switch (request.getMethod()) { - case GET: return handleGET(path, request); - case POST: return handlePOST(path, request); - case DELETE: return handleDELETE(path, request); - case OPTIONS: return handleOPTIONS(); - default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - } - } - catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } - catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse handleGET(Path path, HttpRequest request) { - if (path.matches("/user/v1/user")) return userMetadata(request); - if (path.matches("/user/v1/find")) return findUser(request); - if (path.matches("/user/v1/tenant/{tenant}")) return listTenantRoleMembers(path.get("tenant")); - if (path.matches("/user/v1/tenant/{tenant}/application/{application}")) return listApplicationRoleMembers(path.get("tenant"), path.get("application")); - - return ErrorResponse.notFoundError(Text.format("No '%s' handler at '%s'", request.getMethod(), - request.getUri().getPath())); - } - - private HttpResponse handlePOST(Path path, HttpRequest request) { - if (path.matches("/user/v1/tenant/{tenant}")) return addTenantRoleMember(path.get("tenant"), request); - if (path.matches("/user/v1/email/verify")) return verifyEmail(request); - - return ErrorResponse.notFoundError(Text.format("No '%s' handler at '%s'", request.getMethod(), - request.getUri().getPath())); - } - - private HttpResponse handleDELETE(Path path, HttpRequest request) { - if (path.matches("/user/v1/tenant/{tenant}")) return removeTenantRoleMember(path.get("tenant"), request); - - return ErrorResponse.notFoundError(Text.format("No '%s' handler at '%s'", request.getMethod(), - request.getUri().getPath())); - } - - private HttpResponse handleOPTIONS() { - EmptyResponse response = new EmptyResponse(); - response.headers().put("Allow", "GET,PUT,POST,PATCH,DELETE,OPTIONS"); - return response; - } - - private static final Set<RoleDefinition> hostedOperators = Set.of( - RoleDefinition.hostedOperator, - RoleDefinition.hostedSupporter, - RoleDefinition.hostedAccountant); - - private HttpResponse findUser(HttpRequest request) { - var email = request.getProperty("email"); - var query = request.getProperty("query"); - if (email != null) return userMetadataFromUserId(email); - if (query != null) return userMetadataQuery(query); - return ErrorResponse.badRequest("Need 'email' or 'query' parameter"); - } - - private HttpResponse userMetadataFromUserId(String email) { - var maybeUser = users.findUser(email); - - var slime = new Slime(); - var root = slime.setObject(); - var usersRoot = root.setArray("users"); - - if (maybeUser.isPresent()) { - var user = maybeUser.get(); - var roles = users.listRoles(new UserId(user.email())); - renderUserMetaData(usersRoot.addObject(), user, Set.copyOf(roles)); - } - - return new SlimeJsonResponse(slime); - } - - private HttpResponse userMetadataQuery(String query) { - var userList = users.findUsers(query); - - var slime = new Slime(); - var root = slime.setObject(); - var userSlime = root.setArray("users"); - - for (var user : userList) { - var roles = users.listRoles(new UserId((user.email()))); - renderUserMetaData(userSlime.addObject(), user, Set.copyOf(roles)); - } - - return new SlimeJsonResponse(slime); - } - - private HttpResponse userMetadata(HttpRequest request) { - User user; - if (request.getJDiscRequest().context().get(User.ATTRIBUTE_NAME) instanceof User) { - user = getAttribute(request, User.ATTRIBUTE_NAME, User.class); - } else { - // Remove this after June 2021 (once all security filters are setting this) - @SuppressWarnings("unchecked") - Map<String, String> attr = (Map<String, String>) getAttribute(request, User.ATTRIBUTE_NAME, Map.class); - user = new User(attr.get("email"), attr.get("name"), attr.get("nickname"), attr.get("picture")); - } - - Set<Role> roles = getAttribute(request, SecurityContext.ATTRIBUTE_NAME, SecurityContext.class).roles(); - - var slime = new Slime(); - renderUserMetaData(slime.setObject(), user, roles); - return new SlimeJsonResponse(slime); - } - - private void renderUserMetaData(Cursor root, User user, Set<Role> roles) { - Map<TenantName, List<TenantRole>> tenantRolesByTenantName = roles.stream() - .flatMap(role -> filterTenantRoles(role).stream()) - .distinct() - .sorted(Comparator.comparing(Role::definition).reversed()) - .collect(Collectors.groupingBy(TenantRole::tenant, Collectors.toList())); - - // List of operator roles as defined in `hostedOperators` above - List<Role> operatorRoles = roles.stream() - .filter(role -> hostedOperators.contains(role.definition())) - .sorted(Comparator.comparing(Role::definition)) - .toList(); - - root.setBool("isPublic", controller.system().isPublic()); - root.setBool("isCd", controller.system().isCd()); - root.setBool("hasTrialCapacity", hasTrialCapacity()); - - toSlime(root.setObject("user"), user); - - Cursor tenants = root.setObject("tenants"); - tenantRolesByTenantName.keySet().stream() - .sorted() - .forEach(tenant -> { - Cursor tenantObject = tenants.setObject(tenant.value()); - tenantObject.setBool("supported", hasSupportedPlan(tenant)); - - Cursor tenantRolesObject = tenantObject.setArray("roles"); - tenantRolesByTenantName.getOrDefault(tenant, List.of()) - .forEach(role -> tenantRolesObject.addString(role.definition().name())); - }); - - if (!operatorRoles.isEmpty()) { - Cursor operator = root.setArray("operator"); - operatorRoles.forEach(role -> operator.addString(role.definition().name())); - } - - UserFlagsSerializer.toSlime(root, flagsDb.getAllFlagData(), tenantRolesByTenantName.keySet(), !operatorRoles.isEmpty(), user.email()); - } - - private HttpResponse listTenantRoleMembers(String tenantName) { - if (controller.tenants().get(tenantName).isPresent()) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("tenant", tenantName); - fillRoles(root, - Roles.tenantRoles(TenantName.from(tenantName)), - Collections.emptyList()); - return new SlimeJsonResponse(slime); - } - return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist"); - } - - private HttpResponse listApplicationRoleMembers(String tenantName, String applicationName) { - var id = TenantAndApplicationId.from(tenantName, applicationName); - if (controller.applications().getApplication(id).isPresent()) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("tenant", tenantName); - root.setString("application", applicationName); - fillRoles(root, - Roles.applicationRoles(TenantName.from(tenantName), ApplicationName.from(applicationName)), - Roles.tenantRoles(TenantName.from(tenantName))); - return new SlimeJsonResponse(slime); - } - return ErrorResponse.notFoundError("Application '" + id + "' does not exist"); - } - - private void fillRoles(Cursor root, List<? extends Role> roles, List<? extends Role> superRoles) { - Cursor rolesArray = root.setArray("roleNames"); - for (Role role : roles) - rolesArray.addString(valueOf(role)); - - Map<User, List<Role>> memberships = new LinkedHashMap<>(); - List<Role> allRoles = new ArrayList<>(superRoles); // Membership in a super role may imply membership in a role. - allRoles.addAll(roles); - for (Role role : allRoles) - for (User user : users.listUsers(role)) { - memberships.putIfAbsent(user, new ArrayList<>()); - memberships.get(user).add(role); - } - - Cursor usersArray = root.setArray("users"); - memberships.forEach((user, userRoles) -> { - Cursor userObject = usersArray.addObject(); - toSlime(userObject, user); - - Cursor rolesObject = userObject.setObject("roles"); - for (Role role : roles) { - Cursor roleObject = rolesObject.setObject(valueOf(role)); - roleObject.setBool("explicit", userRoles.contains(role)); - roleObject.setBool("implied", userRoles.stream().anyMatch(userRole -> userRole.implies(role))); - } - }); - } - - private static void toSlime(Cursor userObject, User user) { - if (user.name() != null) userObject.setString("name", user.name()); - userObject.setString("email", user.email()); - if (user.nickname() != null) userObject.setString("nickname", user.nickname()); - if (user.picture() != null) userObject.setString("picture", user.picture()); - userObject.setBool("verified", user.isVerified()); - if (!user.lastLogin().equals(User.NO_DATE)) - userObject.setString("lastLogin", user.lastLogin().format(DateTimeFormatter.ISO_DATE)); - if (user.loginCount() > -1) - userObject.setLong("loginCount", user.loginCount()); - } - - private HttpResponse addTenantRoleMember(String tenantName, HttpRequest request) { - Inspector requestObject = bodyInspector(request); - var tenant = TenantName.from(tenantName); - var user = new UserId(require("user", Inspector::asString, requestObject)); - var roles = SlimeStream.fromArray(requestObject.field("roles"), Inspector::asString) - .map(roleName -> Roles.toRole(tenant, roleName)) - .toList(); - - users.addToRoles(user, roles); - return new MessageResponse(user + " is now a member of " + roles.stream().map(Role::toString).collect(Collectors.joining(", "))); - } - - private HttpResponse verifyEmail(HttpRequest request) { - var inspector = bodyInspector(request); - var verificationCode = require("verificationCode", Inspector::asString, inspector); - var verified = controller.mailVerifier().verifyMail(verificationCode); - - if (verified) - return new MessageResponse("Email with verification code " + verificationCode + " has been verified"); - return ErrorResponse.notFoundError("No pending email verification with code " + verificationCode + " found"); - } - - private HttpResponse removeTenantRoleMember(String tenantName, HttpRequest request) { - Inspector requestObject = bodyInspector(request); - var tenant = TenantName.from(tenantName); - var user = new UserId(require("user", Inspector::asString, requestObject)); - var roles = SlimeStream.fromArray(requestObject.field("roles"), Inspector::asString) - .map(roleName -> Roles.toRole(tenant, roleName)) - .toList(); - - enforceLastAdminOfTenant(tenant, user, roles); - removeDeveloperKey(tenant, user, roles); - users.removeFromRoles(user, roles); - - controller.tenants().lockIfPresent(tenant, LockedTenant.class, lockedTenant -> { - if (lockedTenant instanceof LockedTenant.Cloud cloudTenant) - controller.tenants().store(cloudTenant.withInvalidateUserSessionsBefore(controller.clock().instant())); - }); - - return new MessageResponse(user + " is no longer a member of " + roles.stream().map(Role::toString).collect(Collectors.joining(", "))); - } - - private void enforceLastAdminOfTenant(TenantName tenantName, UserId user, List<Role> roles) { - for (Role role : roles) { - if (role.definition().equals(RoleDefinition.administrator)) { - if (Set.of(user.value()).equals(users.listUsers(role).stream().map(User::email).collect(Collectors.toSet()))) { - throw new IllegalArgumentException("Can't remove the last administrator of a tenant."); - } - break; - } - } - } - - private void removeDeveloperKey(TenantName tenantName, UserId user, List<Role> roles) { - for (Role role : roles) { - if (role.definition().equals(RoleDefinition.developer)) { - controller.tenants().lockIfPresent(tenantName, LockedTenant.Cloud.class, tenant -> { - PublicKey key = tenant.get().developerKeys().inverse().get(new SimplePrincipal(user.value())); - if (key != null) - controller.tenants().store(tenant.withoutDeveloperKey(key)); - }); - break; - } - } - } - - private boolean hasTrialCapacity() { - if (! controller.system().isPublic()) return true; - var plan = controller.serviceRegistry().planRegistry().plan("trial"); - return controller.serviceRegistry().billingController().tenantsWithPlanUnderLimit(plan.get(), maxTrialTenants.value()); - } - - private static Inspector bodyInspector(HttpRequest request) { - return Exceptions.uncheck(() -> SlimeUtils.jsonToSlime(IOUtils.readBytes(request.getData(), 1 << 10)).get()); - } - - private static <Type> Type require(String name, Function<Inspector, Type> mapper, Inspector object) { - if ( ! object.field(name).valid()) throw new IllegalArgumentException("Missing field '" + name + "'."); - return mapper.apply(object.field(name)); - } - - private static String valueOf(Role role) { - switch (role.definition()) { - case administrator: return "administrator"; - case developer: return "developer"; - case reader: return "reader"; - case headless: return "headless"; - default: throw new IllegalArgumentException("Unexpected role type '" + role.definition() + "'."); - } - } - - private static Collection<TenantRole> filterTenantRoles(Role role) { - if (role instanceof TenantRole tenantRole) { - switch (tenantRole.definition()) { - case administrator, developer, reader, hostedDeveloper: return Set.of(tenantRole); - case athenzTenantAdmin: return Roles.tenantRoles(tenantRole.tenant()); - } - } - return Set.of(); - } - - private static <T> T getAttribute(HttpRequest request, String attributeName, Class<T> clazz) { - return Optional.ofNullable(request.getJDiscRequest().context().get(attributeName)) - .filter(clazz::isInstance) - .map(clazz::cast) - .orElseThrow(() -> new IllegalArgumentException("Attribute '" + attributeName + "' was not set on request")); - } - - private boolean hasSupportedPlan(TenantName tenantName) { - var planId = controller.serviceRegistry().billingController().getPlan(tenantName); - return controller.serviceRegistry().planRegistry().plan(planId) - .map(Plan::isSupported) - .orElse(false); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializer.java deleted file mode 100644 index 46de4b7a348..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializer.java +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.user; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.lang.MutableBoolean; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.FlagDefinition; -import com.yahoo.vespa.flags.FlagId; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.RawFlag; -import com.yahoo.vespa.flags.UnboundFlag; -import com.yahoo.vespa.flags.json.Condition; -import com.yahoo.vespa.flags.json.FlagData; -import com.yahoo.vespa.flags.json.Rule; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Stream; - -/** - * @author freva - */ -public class UserFlagsSerializer { - static void toSlime(Cursor cursor, Map<FlagId, FlagData> rawFlagData, - Set<TenantName> authorizedForTenantNames, boolean isOperator, String userEmail) { - FetchVector resolveVector = FetchVector.fromMap(Map.of(FetchVector.Dimension.CONSOLE_USER_EMAIL, userEmail)); - List<FlagData> filteredFlagData = Flags.getAllFlags().stream() - // Only include flags that have CONSOLE_USER_EMAIL dimension, this should be replaced with more explicit - // 'target' annotation if/when that is added to flag definition - .filter(fd -> fd.getDimensions().contains(FetchVector.Dimension.CONSOLE_USER_EMAIL)) - .map(FlagDefinition::getUnboundFlag) - .map(flag -> filteredFlagData(flag, Optional.ofNullable(rawFlagData.get(flag.id())), authorizedForTenantNames, isOperator, resolveVector)) - .toList(); - - byte[] bytes = FlagData.serializeListToUtf8Json(filteredFlagData); - SlimeUtils.copyObject(SlimeUtils.jsonToSlime(bytes).get(), cursor); - } - - private static <T> FlagData filteredFlagData(UnboundFlag<T, ?, ?> definition, Optional<FlagData> original, - Set<TenantName> authorizedForTenantNames, boolean isOperator, FetchVector resolveVector) { - MutableBoolean encounteredEmpty = new MutableBoolean(false); - Optional<RawFlag> defaultValue = Optional.of(definition.serializer().serialize(definition.defaultValue())); - // Include the original rules from flag DB and the default value from code if there is no default rule in DB - List<Rule> rules = Stream.concat(original.stream().flatMap(fd -> fd.rules().stream()), Stream.of(new Rule(defaultValue))) - // Exclude rules that do not match the resolveVector - .filter(rule -> rule.partialMatch(resolveVector)) - // Re-create each rule with value explicitly set, either from DB or default from code and - // a filtered set of conditions - .map(rule -> new Rule(rule.getValueToApply().or(() -> defaultValue), - rule.conditions().stream() - .flatMap(condition -> filteredCondition(condition, authorizedForTenantNames, isOperator, resolveVector).stream()) - .toList())) - // We can stop as soon as we hit the first rule that has no conditions - .takeWhile(rule -> !encounteredEmpty.getAndSet(rule.conditions().isEmpty())) - .toList(); - - return new FlagData(definition.id(), new FetchVector(), rules); - } - - private static Optional<Condition> filteredCondition(Condition condition, Set<TenantName> authorizedForTenantNames, - boolean isOperator, FetchVector resolveVector) { - // If the condition is one of the conditions that we resolve on the server, e.g. email, we do not need to - // propagate it back to the user - if (resolveVector.hasDimension(condition.dimension())) return Optional.empty(); - - // For the other dimensions, filter the values down to an allowed subset - switch (condition.dimension()) { - case TENANT_ID: return valueSubset(condition, tenant -> isOperator || authorizedForTenantNames.contains(TenantName.from(tenant))); - case INSTANCE_ID: return valueSubset(condition, appId -> isOperator || authorizedForTenantNames.stream().anyMatch(tenant -> appId.startsWith(tenant.value() + ":"))); - default: throw new IllegalArgumentException("Dimension " + condition.dimension() + " is not supported for user flags"); - } - } - - private static Optional<Condition> valueSubset(Condition condition, Predicate<String> predicate) { - Condition.CreateParams createParams = condition.toCreateParams(); - return Optional.of(createParams - .withValues(createParams.values().stream().filter(predicate).toList()) - .createAs(condition.type())); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java deleted file mode 100644 index 90792e9febe..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.zone.v1; - -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.yolean.Exceptions; - -import java.util.Comparator; -import java.util.List; - -/** - * Read-only REST API that provides information about zones in hosted Vespa (version 1) - * - * @author mpolden - */ -@SuppressWarnings("unused") -public class ZoneApiHandler extends ThreadedHttpRequestHandler { - - private final ZoneRegistry zoneRegistry; - - public ZoneApiHandler(ThreadedHttpRequestHandler.Context parentCtx, ServiceRegistry serviceRegistry) { - super(parentCtx); - this.zoneRegistry = serviceRegistry.zoneRegistry(); - } - - @Override - public HttpResponse handle(HttpRequest request) { - try { - return switch (request.getMethod()) { - case GET -> get(request); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); - }; - } catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/zone/v1")) { - return root(request); - } - if (path.matches("/zone/v1/environment/{environment}")) { - return environment(request, Environment.from(path.get("environment"))); - } - if (path.matches("/zone/v1/environment/{environment}/default")) { - return defaultRegion(request, Environment.from(path.get("environment"))); - } - return notFound(path); - } - - private HttpResponse root(HttpRequest request) { - List<Environment> environments = zoneRegistry.zones().publiclyVisible().zones().stream() - .map(ZoneApi::getEnvironment) - .distinct() - .sorted(Comparator.comparing(Environment::value)) - .toList(); - Slime slime = new Slime(); - Cursor root = slime.setArray(); - environments.forEach(environment -> { - Cursor object = root.addObject(); - object.setString("name", environment.value()); - object.setString("url", request.getUri() - .resolve("/zone/v1/environment/") - .resolve(environment.value()) - .toString()); - }); - return new SlimeJsonResponse(slime); - } - - private HttpResponse environment(HttpRequest request, Environment environment) { - Slime slime = new Slime(); - Cursor root = slime.setArray(); - zoneRegistry.zones().publiclyVisible().all().in(environment).zones().forEach(zone -> { - Cursor object = root.addObject(); - object.setString("name", zone.getRegionName().value()); - object.setString("url", request.getUri() - .resolve("/zone/v2/") - .resolve(environment.value() + "/") - .resolve(zone.getRegionName().value()) - .toString()); - }); - return new SlimeJsonResponse(slime); - } - - private HttpResponse defaultRegion(HttpRequest request, Environment environment) { - RegionName region = zoneRegistry.getDefaultRegion(environment) - .orElseThrow(() -> new IllegalArgumentException("No default region for environment: " + environment)); - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("name", region.value()); - root.setString("url", request.getUri() - .resolve("/zone/v2/") - .resolve(environment.value() + "/") - .resolve(region.value()) - .toString()); - return new SlimeJsonResponse(slime); - } - - private HttpResponse notFound(Path path) { - return ErrorResponse.notFoundError("Nothing at " + path); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/package-info.java deleted file mode 100644 index c5b29dad8b9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author mpolden - */ -package com.yahoo.vespa.hosted.controller.restapi.zone.v1; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java deleted file mode 100644 index f29845d2476..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.zone.v2; - -import ai.vespa.http.HttpURL; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.config.provision.zone.ZoneList; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; -import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; -import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.yolean.Exceptions; - -/** - * REST API for proxying requests to config servers in a given zone (version 2). - * - * This API does something completely different from /zone/v1, but such is the world. - * - * @author mpolden - */ -@SuppressWarnings("unused") -public class ZoneApiHandler extends AuditLoggingRequestHandler { - - private final ZoneRegistry zoneRegistry; - private final ConfigServerRestExecutor proxy; - - public ZoneApiHandler(ThreadedHttpRequestHandler.Context parentCtx, ServiceRegistry serviceRegistry, - ConfigServerRestExecutor proxy, Controller controller) { - super(parentCtx, controller.auditLogger()); - this.zoneRegistry = serviceRegistry.zoneRegistry(); - this.proxy = proxy; - } - - @Override - public HttpResponse auditAndHandle(HttpRequest request) { - try { - return switch (request.getMethod()) { - case GET -> get(request); - case POST, PUT, DELETE, PATCH -> proxy(request); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); - }; - } catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/zone/v2")) { - return root(request); - } - return proxy(request); - } - - private HttpResponse proxy(HttpRequest request) { - Path path = new Path(request.getUri()); - if ( ! path.matches("/zone/v2/{environment}/{region}/{*}")) { - return notFound(path); - } - ZoneId zoneId = ZoneId.from(path.get("environment"), path.get("region")); - if ( ! zoneRegistry.hasZone(zoneId)) { - throw new IllegalArgumentException("No such zone: " + zoneId.value()); - } - return proxy.handle(proxyRequest(zoneId, path.getRest(), request)); - } - - private HttpResponse root(HttpRequest request) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor uris = root.setArray("uris"); - ZoneList zoneList = zoneRegistry.zones().reachable(); - zoneList.zones().forEach(zone -> uris.addString(request.getUri() - .resolve("/zone/v2/") - .resolve(zone.getEnvironment().value() + "/") - .resolve(zone.getRegionName().value()) - .toString())); - Cursor zones = root.setArray("zones"); - zoneList.zones().forEach(zone -> { - Cursor object = zones.addObject(); - object.setString("environment", zone.getEnvironment().value()); - object.setString("region", zone.getRegionName().value()); - }); - return new SlimeJsonResponse(slime); - } - - private HttpResponse notFound(Path path) { - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private ProxyRequest proxyRequest(ZoneId zoneId, HttpURL.Path path, HttpRequest request) { - return ProxyRequest.tryOne(zoneRegistry.getConfigServerVipUri(zoneId), path, request); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/package-info.java deleted file mode 100644 index 7902c38982c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author mpolden - */ -package com.yahoo.vespa.hosted.controller.restapi.zone.v2; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java deleted file mode 100644 index 1d5bf5e6aa2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing; - -/** - * Endpoint configurations supported for an application. - * - * @author mpolden - */ -public enum EndpointConfig { - - /** Only legacy endpoints will be published in DNS. Certificate will contain both legacy and generated names, and is never assigned from a pool */ - legacy, - - /** Legacy and generated endpoints will be published in DNS. Certificate will contain both legacy and generated names, and is never assigned from a pool */ - combined, - - /** Only generated endpoints will be published in DNS. Certificate will contain generated names only. Certificate is assigned from a pool */ - generated; - - /** Returns whether this config supports legacy endpoints */ - public boolean supportsLegacy() { - return this == legacy || this == combined; - } - - /** Returns whether this config supports generated endpoints */ - public boolean supportsGenerated() { - return this == combined || this == generated; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpointList.java deleted file mode 100644 index af1abff142b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpointList.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing; - -import com.yahoo.collections.AbstractFilteringList; -import com.yahoo.config.provision.zone.AuthMethod; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; - -import java.util.Collection; -import java.util.List; - -/** - * An immutable, filterable list of {@link GeneratedEndpoint}. - * - * @author mpolden - */ -public class GeneratedEndpointList extends AbstractFilteringList<GeneratedEndpoint, GeneratedEndpointList> { - - public static final GeneratedEndpointList EMPTY = new GeneratedEndpointList(List.of(), false); - - private GeneratedEndpointList(Collection<? extends GeneratedEndpoint> items, boolean negate) { - super(items, negate, GeneratedEndpointList::new); - } - - /** Returns the subset of endpoints which are generated for given endpoint ID */ - public GeneratedEndpointList declared(EndpointId endpoint) { - return matching(e -> e.endpoint().isPresent() && e.endpoint().get().equals(endpoint)); - } - - /** Returns the subset of endpoints which are generated for endpoints declared in {@link com.yahoo.config.application.api.DeploymentSpec} */ - public GeneratedEndpointList declared() { - return matching(GeneratedEndpoint::declared); - } - - /** Returns the subset endpoints which are generated for clusters declared in {@link com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml} */ - public GeneratedEndpointList cluster() { - return not().declared(); - } - - /** Returns the subset of endpoints matching given auth method */ - public GeneratedEndpointList authMethod(AuthMethod authMethod) { - return matching(ge -> ge.authMethod() == authMethod); - } - - public static GeneratedEndpointList of(GeneratedEndpoint... generatedEndpoint) { - return copyOf(List.of(generatedEndpoint)); - } - - public static GeneratedEndpointList copyOf(Collection<GeneratedEndpoint> generatedEndpoints) { - return generatedEndpoints.isEmpty() ? EMPTY : new GeneratedEndpointList(generatedEndpoints, false); - } - - @Override - public String toString() { - return asList().toString(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java deleted file mode 100644 index 63b17a087f2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing; - -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; -import com.yahoo.vespa.hosted.controller.application.AssignedRotation; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.EndpointList; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * This represents the endpoints, and associated resources, that have been prepared for a deployment. - * - * @author mpolden - */ -public record PreparedEndpoints(DeploymentId deployment, - EndpointList endpoints, - List<AssignedRotation> rotations, - EndpointCertificate certificate) { - - public PreparedEndpoints(DeploymentId deployment, EndpointList endpoints, List<AssignedRotation> rotations, EndpointCertificate certificate) { - this.deployment = Objects.requireNonNull(deployment); - this.endpoints = Objects.requireNonNull(endpoints); - this.rotations = List.copyOf(Objects.requireNonNull(rotations)); - this.certificate = requireMatchingSans(certificate, endpoints); - } - - /** Returns the endpoints contained in this as {@link com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint} */ - public Set<ContainerEndpoint> containerEndpoints() { - Map<EndpointId, AssignedRotation> rotationsByEndpointId = rotations.stream() - .collect(Collectors.toMap(AssignedRotation::endpointId, - Function.identity())); - Set<ContainerEndpoint> containerEndpoints = new HashSet<>(); - endpoints.scope(Endpoint.Scope.zone).groupingBy(Endpoint::cluster).forEach((clusterId, clusterEndpoints) -> { - clusterEndpoints.groupingBy(Endpoint::authMethod).forEach((authMethod, endpointsByAuthMethod) -> { - containerEndpoints.add(new ContainerEndpoint(clusterId.value(), - asString(Endpoint.Scope.zone), - endpointsByAuthMethod.mapToList(Endpoint::dnsName), - OptionalInt.empty(), - endpointsByAuthMethod.first().get().routingMethod(), - authMethod)); - }); - }); - endpoints.scope(Endpoint.Scope.global).groupingBy(Endpoint::cluster).forEach((clusterId, clusterEndpoints) -> { - for (var endpoint : clusterEndpoints) { - List<String> names = new ArrayList<>(2); - names.add(endpoint.dnsName()); - if (endpoint.requiresRotation()) { - EndpointId endpointId = EndpointId.of(endpoint.name()); - AssignedRotation rotation = rotationsByEndpointId.get(endpointId); - if (rotation == null) { - throw new IllegalStateException(endpoint + " requires a rotation, but no rotation has been assigned to " + endpointId); - } - // Include the rotation ID as a valid name of this container endpoint - // (required by global routing health checks) - names.add(rotation.rotationId().asString()); - } - containerEndpoints.add(new ContainerEndpoint(clusterId.value(), - asString(Endpoint.Scope.global), - names, - OptionalInt.empty(), - endpoint.routingMethod(), - endpoint.authMethod())); - } - }); - endpoints.scope(Endpoint.Scope.application).groupingBy(Endpoint::cluster).forEach((clusterId, clusterEndpoints) -> { - for (var endpoint : clusterEndpoints) { - Optional<Endpoint.Target> matchingTarget = endpoint.targets().stream() - .filter(t -> t.routesTo(deployment)) - .findFirst(); - if (matchingTarget.isEmpty()) throw new IllegalStateException("No target found routing to " + deployment + " in " + endpoint); - containerEndpoints.add(new ContainerEndpoint(clusterId.value(), - asString(Endpoint.Scope.application), - List.of(endpoint.dnsName()), - OptionalInt.of(matchingTarget.get().weight()), - endpoint.routingMethod(), - endpoint.authMethod())); - } - }); - return containerEndpoints; - } - - private static String asString(Endpoint.Scope scope) { - return switch (scope) { - case application -> "application"; - case global -> "global"; - case weighted -> "weighted"; - case zone -> "zone"; - }; - } - - private static EndpointCertificate requireMatchingSans(EndpointCertificate certificate, EndpointList endpoints) { - Objects.requireNonNull(certificate); - for (var endpoint : endpoints.not().scope(Endpoint.Scope.weighted)) { // Weighted endpoints are not present in certificate - if (!certificate.sanMatches(endpoint.dnsName())) { - throw new IllegalArgumentException(endpoint + " has no matching SAN. Certificate contains " + - certificate.requestedDnsSans()); - } - } - return certificate; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java deleted file mode 100644 index 50e54423f9a..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; - -import java.util.Objects; - -/** - * Unique identifier for a instance routing table entry (instance x endpoint ID). - * - * @author mpolden - */ -public record RoutingId(ApplicationId instance, - EndpointId endpointId, - TenantAndApplicationId application) { - - public RoutingId { - Objects.requireNonNull(instance, "application must be non-null"); - Objects.requireNonNull(endpointId, "endpointId must be non-null"); - Objects.requireNonNull(application, "application must be non-null"); - } - - @Override - public String toString() { - return "routing id for " + endpointId + " of " + instance; - } - - public static RoutingId of(ApplicationId instance, EndpointId endpoint) { - return new RoutingId(instance, endpoint, TenantAndApplicationId.from(instance)); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java deleted file mode 100644 index e93bc637a6b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ /dev/null @@ -1,781 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing; - -import ai.vespa.http.DomainName; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.zone.AuthMethod; -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.DirectTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.LatencyAliasTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.ChallengeState; -import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; -import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedDirectTarget; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.EndpointList; -import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.dns.NameServiceRequest; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Updates routing policies and their associated DNS records based on a deployment's load balancers. - * - * @author mortent - * @author mpolden - */ -public class RoutingPolicies { - - private static final Logger LOG = Logger.getLogger(RoutingPolicies.class.getName()); - - private final Controller controller; - private final CuratorDb db; - - public RoutingPolicies(Controller controller) { - this.controller = Objects.requireNonNull(controller, "controller must be non-null"); - this.db = controller.curator(); - try (var lock = db.lockRoutingPolicies()) { // Update serialized format - for (var policy : db.readRoutingPolicies().entrySet()) { - db.writeRoutingPolicies(policy.getKey(), policy.getValue()); - } - } - } - - /** Read all routing policies for given deployment */ - public RoutingPolicyList read(DeploymentId deployment) { - return read(deployment.applicationId()).deployment(deployment); - } - - /** Read all routing policies for given instance */ - public RoutingPolicyList read(ApplicationId instance) { - return RoutingPolicyList.copyOf(db.readRoutingPolicies(instance)); - } - - /** Read all routing policies for given application */ - public RoutingPolicyList read(TenantAndApplicationId application) { - return db.readRoutingPolicies((instance) -> TenantAndApplicationId.from(instance).equals(application)) - .values() - .stream() - .flatMap(Collection::stream) - .collect(Collectors.collectingAndThen(Collectors.toList(), RoutingPolicyList::copyOf)); - } - - /** Read all routing policies */ - private RoutingPolicyList readAll() { - return db.readRoutingPolicies() - .values() - .stream() - .flatMap(Collection::stream) - .collect(Collectors.collectingAndThen(Collectors.toList(), RoutingPolicyList::copyOf)); - } - - /** Read routing policy for given zone */ - public ZoneRoutingPolicy read(ZoneId zone) { - return db.readZoneRoutingPolicy(zone); - } - - /** - * Refresh routing policies for instance in given zone. This is idempotent and changes will only be performed if - * routing configuration affecting given deployment has changed. - */ - public void refresh(DeploymentId deployment, DeploymentSpec deploymentSpec, EndpointList generatedEndpoints) { - if (!generatedEndpoints.not().generated().isEmpty()) { - throw new IllegalStateException("Generated endpoints contains non-generated, got " + generatedEndpoints); - } - ApplicationId instance = deployment.applicationId(); - List<LoadBalancer> loadBalancers = controller.serviceRegistry().configServer() - .getLoadBalancers(instance, deployment.zoneId()); - LoadBalancerAllocation allocation = new LoadBalancerAllocation(deployment, deploymentSpec, loadBalancers); - Optional<TenantAndApplicationId> owner = ownerOf(allocation); - try (var lock = db.lockRoutingPolicies()) { - RoutingPolicyList applicationPolicies = read(TenantAndApplicationId.from(instance)); - RoutingPolicyList deploymentPolicies = applicationPolicies.deployment(allocation.deployment); - - removeGlobalDnsUnreferencedBy(allocation, deploymentPolicies, lock); - removeApplicationDnsUnreferencedBy(allocation, deploymentPolicies, lock); - - RoutingPolicyList instancePolicies = storePoliciesOf(allocation, applicationPolicies, generatedEndpoints, lock); - instancePolicies = removePoliciesUnreferencedBy(allocation, instancePolicies, lock); - - RoutingPolicyList updatedApplicationPolicies = applicationPolicies.replace(instance, instancePolicies); - updateGlobalDnsOf(instancePolicies, Optional.of(deployment), owner, lock); - updateApplicationDnsOf(updatedApplicationPolicies, deployment, owner, lock); - } - } - - /** Set the status of all global endpoints in given zone */ - public void setRoutingStatus(ZoneId zone, RoutingStatus.Value value) { - try (var lock = db.lockRoutingPolicies()) { - db.writeZoneRoutingPolicy(new ZoneRoutingPolicy(zone, RoutingStatus.create(value, RoutingStatus.Agent.operator, - controller.clock().instant()))); - Map<ApplicationId, RoutingPolicyList> allPolicies = readAll().groupingBy(policy -> policy.id().owner()); - allPolicies.forEach((instance, policies) -> { - updateGlobalDnsOf(policies, Optional.empty(), Optional.of(TenantAndApplicationId.from(instance)), lock); - }); - } - } - - /** Set the status of all global endpoints for given deployment */ - public void setRoutingStatus(DeploymentId deployment, RoutingStatus.Value value, RoutingStatus.Agent agent) { - ApplicationId instance = deployment.applicationId(); - try (var lock = db.lockRoutingPolicies()) { - RoutingPolicyList applicationPolicies = read(TenantAndApplicationId.from(instance)); - RoutingPolicyList deploymentPolicies = applicationPolicies.deployment(deployment); - Map<RoutingPolicyId, RoutingPolicy> updatedPolicies = new LinkedHashMap<>(applicationPolicies.asMap()); - for (var policy : deploymentPolicies) { - var newPolicy = policy.with(RoutingStatus.create(value, agent, controller.clock().instant())); - updatedPolicies.put(policy.id(), newPolicy); - } - RoutingPolicyList effectivePolicies = RoutingPolicyList.copyOf(updatedPolicies.values()); - Map<ApplicationId, RoutingPolicyList> policiesByInstance = effectivePolicies.groupingBy(policy -> policy.id().owner()); - policiesByInstance.forEach((ignored, instancePolicies) -> updateGlobalDnsOf(instancePolicies, - Optional.of(deployment), - ownerOf(deployment), - lock)); - updateApplicationDnsOf(effectivePolicies, deployment, ownerOf(deployment), lock); - policiesByInstance.forEach((owner, instancePolicies) -> db.writeRoutingPolicies(owner, instancePolicies.asList())); - } - } - - /** Update global DNS records for given policies */ - private void updateGlobalDnsOf(RoutingPolicyList instancePolicies, Optional<DeploymentId> deployment, - Optional<TenantAndApplicationId> owner, - @SuppressWarnings("unused") Mutex lock) { - Map<RoutingId, List<RoutingPolicy>> routingTable = instancePolicies.asInstanceRoutingTable(); - for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { - RoutingId routingId = routeEntry.getKey(); - controller.routing().readDeclaredEndpointsOf(routingId.instance()) - .named(routingId.endpointId(), Endpoint.Scope.global) - .not().requiresRotation() - .forEach(endpoint -> updateGlobalDnsOf(endpoint, routeEntry.getValue(), deployment, owner)); - } - } - - /** Update global DNS records for given global endpoint */ - private void updateGlobalDnsOf(Endpoint endpoint, List<RoutingPolicy> policies, - Optional<DeploymentId> deployment, Optional<TenantAndApplicationId> owner) { - if (endpoint.scope() != Endpoint.Scope.global) throw new IllegalStateException("Endpoint " + endpoint + " is not global"); - if (deployment.isPresent() && !endpoint.deployments().contains(deployment.get())) return; - - Collection<RegionEndpoint> regionEndpoints = computeRegionEndpoints(endpoint, policies); - Set<AliasTarget> latencyTargets = new LinkedHashSet<>(); - Set<AliasTarget> inactiveLatencyTargets = new LinkedHashSet<>(); - for (var regionEndpoint : regionEndpoints) { - if (regionEndpoint.active()) { - latencyTargets.add(regionEndpoint.target()); - } else { - inactiveLatencyTargets.add(regionEndpoint.target()); - } - } - - // Refuse removal of last target in an endpoint. We do this because removing 100% of the ALIAS records would - // cause the application endpoint to stop resolving entirely (NXDOMAIN). - if (latencyTargets.isEmpty() && !inactiveLatencyTargets.isEmpty()) { - if (deployment.isPresent()) { - throw new IllegalArgumentException("Cannot deactivate routing for " + deployment.get() + - " as it's the last remaining active deployment in " + endpoint); - } else { - // Operator is deactivating routing for entire zone, but this endpoint only has one target - LOG.log(Level.WARNING, "Cannot deactivate routing for " + endpoint + " because it has only one " + - "active zone. Leaving it in"); - return; - } - } - - // Create a weighted ALIAS per region, pointing to all zones within the same region - regionEndpoints.forEach(regionEndpoint -> { - if ( ! regionEndpoint.zoneAliasTargets().isEmpty()) { - controller.nameServiceForwarder().createAlias(RecordName.from(regionEndpoint.target().name().value()), - regionEndpoint.zoneAliasTargets(), - Priority.normal, - owner); - } - if ( ! regionEndpoint.zoneDirectTargets().isEmpty()) { - controller.nameServiceForwarder().createDirect(RecordName.from(regionEndpoint.target().name().value()), - regionEndpoint.zoneDirectTargets(), - Priority.normal, - owner); - } - }); - - // Create global latency-based ALIAS pointing to each per-region weighted ALIAS - controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), latencyTargets, Priority.normal, owner); - inactiveLatencyTargets.forEach(t -> controller.nameServiceForwarder() - .removeRecords(Record.Type.ALIAS, - RecordName.from(endpoint.dnsName()), - RecordData.from(t.name().value()), - Priority.normal, - owner)); - } - - /** Compute region endpoints and their targets from given policies */ - private Collection<RegionEndpoint> computeRegionEndpoints(Endpoint parent, List<RoutingPolicy> policies) { - if (!parent.scope().multiDeployment()) { - throw new IllegalStateException(parent + " has unexpected scope, got " + parent.scope()); - } - Map<Endpoint, RegionEndpoint> endpoints = new LinkedHashMap<>(); - for (var policy : policies) { - if (policy.dnsZone().isEmpty() && policy.canonicalName().isPresent()) continue; - if (controller.zoneRegistry().routingMethod(policy.id().zone()) != RoutingMethod.exclusive) continue; - var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); - // A record with 0 weight will not receive traffic. If all records within a group have 0 - // weight, traffic is routed to all records with equal probability - long weight = isConfiguredOut(zonePolicy, policy) ? 0 : 1; - boolean generated = parent.generated().isPresent(); - EndpointList weightedEndpoints = controller.routing() - .endpointsOf(policy.id().deployment(), - policy.id().cluster(), - policy.generatedEndpoints().cluster()) - .scope(Endpoint.Scope.weighted); - if (generated) { - weightedEndpoints = weightedEndpoints.generated(); - } else { - weightedEndpoints = weightedEndpoints.not().generated(); - } - if (generated && weightedEndpoints.isEmpty()) { - // Ignore this policy. If an instance has a global endpoint, and is switching from non-generated to - // generated endpoints we cannot update global DNS record for a deployment until it has been deployed at - // least once (which assigns a generated endpoint). - continue; - } - if (weightedEndpoints.size() != 1) { - throw new IllegalStateException("Expected to compute exactly one region endpoint for " + policy.id() + " with parent " + parent + ", got " + weightedEndpoints); - } - Endpoint endpoint = weightedEndpoints.first().get(); - RegionEndpoint regionEndpoint = endpoints.computeIfAbsent(endpoint, (k) -> new RegionEndpoint( - new LatencyAliasTarget(DomainName.of(endpoint.dnsName()), policy.dnsZone().get(), policy.id().zone()))); - - if (policy.canonicalName().isPresent()) { - var weightedTarget = new WeightedAliasTarget( - policy.canonicalName().get(), policy.dnsZone().get(), policy.id().zone().value(), weight); - regionEndpoint.add(weightedTarget); - } else { - var weightedTarget = new WeightedDirectTarget( - RecordData.from(policy.ipAddress().get()), policy.id().zone(), weight); - regionEndpoint.add(weightedTarget); - } - } - return endpoints.values(); - } - - - private void updateApplicationDnsOf(RoutingPolicyList routingPolicies, DeploymentId deployment, - Optional<TenantAndApplicationId> owner, @SuppressWarnings("unused") Mutex lock) { - // In the context of single deployment (which this is) there is only one routing policy per routing ID. I.e. - // there is no scenario where more than one deployment within an instance can be a member the same - // application-level endpoint. However, to allow this in the future the routing table remains - // Map<RoutingId, List<RoutingPolicy>> instead of Map<RoutingId, RoutingPolicy>. - Map<RoutingId, List<RoutingPolicy>> routingTable = routingPolicies.asApplicationRoutingTable(); - if (routingTable.isEmpty()) return; - - Application application = controller.applications().requireApplication(routingTable.keySet().iterator().next().application()); - Map<Endpoint, Set<Target>> targetsByEndpoint = new LinkedHashMap<>(); - Map<Endpoint, Set<Target>> inactiveTargetsByEndpoint = new LinkedHashMap<>(); - for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { - RoutingId routingId = routeEntry.getKey(); - EndpointList endpoints = controller.routing().readDeclaredEndpointsOf(application) - .named(routingId.endpointId(), Endpoint.Scope.application); - for (Endpoint endpoint : endpoints) { - for (var policy : routeEntry.getValue()) { - for (var target : endpoint.targets()) { - if (!policy.appliesTo(target.deployment())) continue; - if (policy.dnsZone().isEmpty() && policy.canonicalName().isPresent()) - continue; // Does not support ALIAS records - ZoneRoutingPolicy zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); - - Set<Target> activeTargets = targetsByEndpoint.computeIfAbsent(endpoint, (k) -> new LinkedHashSet<>()); - Set<Target> inactiveTargets = inactiveTargetsByEndpoint.computeIfAbsent(endpoint, (k) -> new LinkedHashSet<>()); - if (isConfiguredOut(zonePolicy, policy)) { - inactiveTargets.add(Target.weighted(policy, target)); - } else { - activeTargets.add(Target.weighted(policy, target)); - } - } - } - } - } - - // Refuse removal of last target in an endpoint. We do this because removing 100% of the ALIAS records would - // cause the application endpoint to stop resolving entirely (NXDOMAIN). - targetsByEndpoint.forEach((endpoint, targets) -> { - if (targets.isEmpty()) { - throw new IllegalArgumentException("Cannot deactivate routing for " + deployment + - " as it's the last remaining active deployment in " + endpoint); - } - }); - - // Create DNS records for active targets - targetsByEndpoint.forEach((applicationEndpoint, targets) -> { - // Where multiple zones are permitted, they all have the same routing policy, and nameServiceForwarder (below). - ZoneId targetZone = applicationEndpoint.targets().iterator().next().deployment().zoneId(); - Set<AliasTarget> aliasTargets = new LinkedHashSet<>(); - Set<DirectTarget> directTargets = new LinkedHashSet<>(); - for (Target target : targets) { - if (!target.deployment().equals(deployment)) continue; // Do not update target not matching this deployment - if (target.aliasOrDirectTarget() instanceof AliasTarget at) { - aliasTargets.add(at); - } else { - directTargets.add((DirectTarget) target.aliasOrDirectTarget()); - } - } - if (!aliasTargets.isEmpty()) { - nameServiceForwarder(applicationEndpoint).createAlias( - RecordName.from(applicationEndpoint.dnsName()), aliasTargets, Priority.normal, owner); - } - if (!directTargets.isEmpty()) { - nameServiceForwarder(applicationEndpoint).createDirect( - RecordName.from(applicationEndpoint.dnsName()), directTargets, Priority.normal, owner); - } - }); - - // Remove DNS records for inactive targets - inactiveTargetsByEndpoint.forEach((applicationEndpoint, targets) -> { - targets.forEach(target -> { - if (!target.deployment().equals(deployment)) return; // Do not update target not matching this deployment - nameServiceForwarder(applicationEndpoint).removeRecords(target.type(), - RecordName.from(applicationEndpoint.dnsName()), - target.data(), - Priority.normal, - owner); - }); - }); - } - - /** - * Store routing policies for given load balancers - * - * @return the updated policies - */ - private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList applicationPolicies, EndpointList generatedEndpoints, @SuppressWarnings("unused") Mutex lock) { - Map<RoutingPolicyId, RoutingPolicy> policies = new LinkedHashMap<>(applicationPolicies.instance(allocation.deployment.applicationId()).asMap()); - for (LoadBalancer loadBalancer : allocation.loadBalancers) { - if (loadBalancer.hostname().isEmpty() && loadBalancer.ipAddress().isEmpty()) continue; - RoutingPolicyId policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), allocation.deployment.zoneId()); - RoutingPolicy existingPolicy = policies.get(policyId); - Optional<String> dnsZone = loadBalancer.ipAddress().isPresent() ? Optional.of("ignored") : loadBalancer.dnsZone(); - List<GeneratedEndpoint> clusterGeneratedEndpoints = generatedEndpoints.cluster(loadBalancer.cluster()) - .mapToList(e -> e.generated().get()); - clusterGeneratedEndpoints.forEach(ge -> requireNonClashing(ge, applicationPolicies.without(existingPolicy))); - var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.ipAddress(), dnsZone, - allocation.instanceEndpointsOf(loadBalancer), - allocation.applicationEndpointsOf(loadBalancer), - RoutingStatus.DEFAULT, - loadBalancer.isPublic(), - GeneratedEndpointList.copyOf(clusterGeneratedEndpoints)); - if (existingPolicy != null) { - newPolicy = newPolicy.with(existingPolicy.routingStatus()); // Always preserve routing status - } - updateZoneDnsOf(newPolicy, loadBalancer, allocation.deployment); - policies.put(newPolicy.id(), newPolicy); - } - RoutingPolicyList updated = RoutingPolicyList.copyOf(policies.values()); - db.writeRoutingPolicies(allocation.deployment.applicationId(), updated.asList()); - return updated; - } - - /** Update zone DNS record for given policy */ - private void updateZoneDnsOf(RoutingPolicy policy, LoadBalancer loadBalancer, DeploymentId deploymentId) { - EndpointList zoneEndpoints = controller.routing().endpointsOf(deploymentId, - policy.id().cluster(), - policy.generatedEndpoints().cluster()) - .scope(Endpoint.Scope.zone); - for (var endpoint : zoneEndpoints) { - RecordName name = RecordName.from(endpoint.dnsName()); - Record record = policy.canonicalName().isPresent() ? - new Record(Record.Type.CNAME, name, RecordData.fqdn(policy.canonicalName().get().value())) : - new Record(Record.Type.A, name, RecordData.from(policy.ipAddress().orElseThrow())); - nameServiceForwarder(endpoint).createRecord(record, Priority.normal, ownerOf(deploymentId)); - } - setPrivateDns(zoneEndpoints, loadBalancer, deploymentId); - } - - private void setPrivateDns(EndpointList endpoints, LoadBalancer loadBalancer, DeploymentId deploymentId) { - if (loadBalancer.service().isEmpty()) return; - // TODO(mpolden): Model one service for each endpoint (type), to allow private endpoints with tokens. - EndpointList mtlsEndpoints = endpoints.authMethod(AuthMethod.mtls); - if (mtlsEndpoints.isEmpty()) return; - Endpoint endpoint = mtlsEndpoints.generated().first().orElse(mtlsEndpoints.first().get()); - if (endpoint.routingMethod() != RoutingMethod.exclusive) return; // Not supported for this routing method - controller.serviceRegistry().vpcEndpointService() - .setPrivateDns(DomainName.of(endpoint.dnsName()), - new ClusterId(deploymentId, endpoint.cluster()), - loadBalancer.cloudAccount(), - endpoint.generated().isPresent()) - .ifPresent(challenge -> { - try (Mutex lock = db.lockNameServiceQueue()) { - controller.nameServiceForwarder().createTxt(challenge.name(), List.of(challenge.data()), Priority.high, ownerOf(deploymentId)); - db.writeDnsChallenge(challenge); - } - }); - } - - /** Deletes all DNS challenges, and corresponding TXT records, for the given deployment. */ - public void removeDnsChallenges(DeploymentId deploymentId) { - try (Mutex lock = db.lockNameServiceQueue()) { - db.readDnsChallenges(deploymentId).forEach(this::removeDnsChallenge); - } - } - - /** Returns true iff. the given deployment has no incomplete DNS challenges, or throws (and cleans up) on errors. */ - public boolean processDnsChallenges(DeploymentId deploymentId) { - try (Mutex lock = db.lockNameServiceQueue()) { - List<DnsChallenge> challenges = new ArrayList<>(db.readDnsChallenges(deploymentId)); - challenges.removeIf(challenge -> challenge.state() == ChallengeState.done); - Set<RecordName> pendingRequests = controller.curator().readNameServiceQueue().requests().stream() - .map(NameServiceRequest::name) - .collect(Collectors.toSet()); - try { - challenges.removeIf(challenge -> { - if (challenge.state() == ChallengeState.pending) { - if (pendingRequests.contains(challenge.name())) return false; - challenge = challenge.withState(ChallengeState.ready); - } - ChallengeState state = controller.serviceRegistry().vpcEndpointService().process(challenge); - db.writeDnsChallenge(challenge.withState(state)); - return state == ChallengeState.done; - }); - return challenges.isEmpty(); - } - catch (RuntimeException e) { - challenges.forEach(this::removeDnsChallenge); - throw e; - } - } - } - - private void removeDnsChallenge(DnsChallenge challenge) { - controller.nameServiceForwarder().removeRecords(Record.Type.TXT, challenge.name(), Priority.normal, ownerOf(challenge.clusterId().deploymentId())); - db.deleteDnsChallenge(challenge.clusterId()); - } - - /** - * Remove policies and zone DNS records unreferenced by given load balancers - * - * @return the updated policies - */ - private RoutingPolicyList removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) { - Map<RoutingPolicyId, RoutingPolicy> newPolicies = new LinkedHashMap<>(instancePolicies.asMap()); - Set<RoutingPolicyId> activeIds = allocation.asPolicyIds(); - RoutingPolicyList removable = instancePolicies.deployment(allocation.deployment) - .not().matching(policy -> activeIds.contains(policy.id())); - for (var policy : removable) { - EndpointList zoneEndpoints = controller.routing().endpointsOf(allocation.deployment, - policy.id().cluster(), - policy.generatedEndpoints().cluster()) - .scope(Endpoint.Scope.zone); - for (var endpoint : zoneEndpoints) { - Record.Type type = policy.canonicalName().isPresent() ? Record.Type.CNAME : Record.Type.A; - nameServiceForwarder(endpoint).removeRecords(type, - RecordName.from(endpoint.dnsName()), - Priority.normal, - ownerOf(allocation)); - } - newPolicies.remove(policy.id()); - } - RoutingPolicyList updated = RoutingPolicyList.copyOf(newPolicies.values()); - db.writeRoutingPolicies(allocation.deployment.applicationId(), updated.asList()); - return updated; - } - - /** Remove unreferenced instance endpoints from DNS */ - private void removeGlobalDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Mutex lock) { - Map<RoutingId, List<RoutingPolicy>> routingTable = deploymentPolicies.asInstanceRoutingTable(); - Set<RoutingId> removalCandidates = new HashSet<>(routingTable.keySet()); - Set<RoutingId> activeRoutingIds = instanceRoutingIds(allocation); - removalCandidates.removeAll(activeRoutingIds); - for (var id : removalCandidates) { - List<RoutingPolicy> policies = routingTable.get(id); - Map<ClusterSpec.Id, List<RoutingPolicy>> policyByCluster = policies.stream().collect(Collectors.groupingBy(p -> p.id().cluster())); - Set<Endpoint> endpoints = new LinkedHashSet<>(); - policyByCluster.forEach((cluster, clusterPolicies) -> { - List<DeploymentId> deployments = clusterPolicies.stream().map(p -> p.id().deployment()).toList(); - GeneratedEndpointList generated = declaredGeneratedEndpoints(id.endpointId(), clusterPolicies); - endpoints.addAll(controller.routing().declaredEndpointsOf(id, cluster, deployments, generated) - .not().requiresRotation() - .named(id.endpointId(), Endpoint.Scope.global).asList()); - }); - // This removes all ALIAS records having this DNS name. There is no attempt to delete only the entry for the - // affected zone. Instead, the correct set of records is (re)created by updateGlobalDnsOf - for (var endpoint : endpoints) { - for (var regionEndpoint : computeRegionEndpoints(endpoint, deploymentPolicies.asList())) { - Record.Type type = regionEndpoint.zoneDirectTargets().isEmpty() ? Record.Type.ALIAS : Record.Type.DIRECT; - controller.nameServiceForwarder().removeRecords(type, - RecordName.from(regionEndpoint.target().name().value()), - Priority.normal, - ownerOf(allocation)); - } - nameServiceForwarder(endpoint).removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), - Priority.normal, - ownerOf(allocation)); - } - } - } - - /** Remove unreferenced application endpoints in given allocation from DNS */ - private void removeApplicationDnsUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList deploymentPolicies, @SuppressWarnings("unused") Mutex lock) { - Map<RoutingId, List<RoutingPolicy>> routingTable = deploymentPolicies.asApplicationRoutingTable(); - Set<RoutingId> removalCandidates = new HashSet<>(routingTable.keySet()); - Set<RoutingId> activeRoutingIds = applicationRoutingIds(allocation); - removalCandidates.removeAll(activeRoutingIds); - for (var id : removalCandidates) { - TenantAndApplicationId application = TenantAndApplicationId.from(id.instance()); - List<RoutingPolicy> policies = routingTable.get(id); - Map<ClusterSpec.Id, List<RoutingPolicy>> policyByCluster = policies.stream().collect(Collectors.groupingBy(p -> p.id().cluster())); - Set<Endpoint> endpoints = new LinkedHashSet<>(); - policyByCluster.forEach((cluster, clusterPolicies) -> { - // Weights are not available in this context, but they're not used for anything when removing records - Map<DeploymentId, Integer> deployments = clusterPolicies.stream() - .map(p -> p.id().deployment()) - .collect(Collectors.toMap(Function.identity(), (ignored) -> 1)); - GeneratedEndpointList generated = declaredGeneratedEndpoints(id.endpointId(), clusterPolicies); - endpoints.addAll(controller.routing().declaredEndpointsOf(application, id.endpointId(), cluster, - deployments, generated).asList()); - }); - for (var policy : policies) { - if (!policy.appliesTo(allocation.deployment)) continue; - for (Endpoint endpoint : endpoints) { - NameServiceForwarder forwarder = nameServiceForwarder(endpoint); - if (policy.canonicalName().isPresent()) { - forwarder.removeRecords(Record.Type.ALIAS, - RecordName.from(endpoint.dnsName()), - RecordData.fqdn(policy.canonicalName().get().value()), - Priority.normal, - ownerOf(allocation)); - } else { - forwarder.removeRecords(Record.Type.DIRECT, - RecordName.from(endpoint.dnsName()), - RecordData.from(policy.ipAddress().get()), - Priority.normal, - ownerOf(allocation)); - } - } - } - } - } - - private Set<RoutingId> instanceRoutingIds(LoadBalancerAllocation allocation) { - return routingIdsFrom(allocation, false); - } - - private Set<RoutingId> applicationRoutingIds(LoadBalancerAllocation allocation) { - return routingIdsFrom(allocation, true); - } - - private static GeneratedEndpointList declaredGeneratedEndpoints(EndpointId endpoint, List<RoutingPolicy> clusterPolicies) { - return GeneratedEndpointList.copyOf(clusterPolicies.stream() - .flatMap(p -> p.generatedEndpoints().declared(endpoint).asList().stream()) - .distinct() - .toList()); - } - - /** Compute routing IDs from given load balancers */ - private static Set<RoutingId> routingIdsFrom(LoadBalancerAllocation allocation, boolean applicationLevel) { - Set<RoutingId> routingIds = new LinkedHashSet<>(); - for (var loadBalancer : allocation.loadBalancers) { - Set<EndpointId> endpoints = applicationLevel - ? allocation.applicationEndpointsOf(loadBalancer) - : allocation.instanceEndpointsOf(loadBalancer); - for (var endpointId : endpoints) { - routingIds.add(RoutingId.of(loadBalancer.application(), endpointId)); - } - } - return Collections.unmodifiableSet(routingIds); - } - - /** Returns whether the endpoints of given policy are configured {@link RoutingStatus.Value#out} */ - private static boolean isConfiguredOut(ZoneRoutingPolicy zonePolicy, RoutingPolicy policy) { - // A deployment can be configured out from endpoints at any of the following levels: - // - zone level (ZoneRoutingPolicy) - // - deployment level (RoutingPolicy) - return zonePolicy.routingStatus().value() == RoutingStatus.Value.out || - policy.routingStatus().value() == RoutingStatus.Value.out; - } - - /** Represents records for a region-wide endpoint */ - private static class RegionEndpoint { - - private final LatencyAliasTarget target; - private final Set<WeightedAliasTarget> zoneAliasTargets = new LinkedHashSet<>(); - private final Set<WeightedDirectTarget> zoneDirectTargets = new LinkedHashSet<>(); - - public RegionEndpoint(LatencyAliasTarget target) { - this.target = Objects.requireNonNull(target); - } - - public LatencyAliasTarget target() { return target; } - public Set<AliasTarget> zoneAliasTargets() { return Collections.unmodifiableSet(zoneAliasTargets); } - public Set<DirectTarget> zoneDirectTargets() { return Collections.unmodifiableSet(zoneDirectTargets); } - - public void add(WeightedAliasTarget target) { zoneAliasTargets.add(target); } - public void add(WeightedDirectTarget target) { zoneDirectTargets.add(target); } - - public boolean active() { - return zoneAliasTargets.stream().anyMatch(target -> target.weight() > 0) || - zoneDirectTargets.stream().anyMatch(target -> target.weight() > 0); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RegionEndpoint that = (RegionEndpoint) o; - return target.name().equals(that.target.name()); - } - - @Override - public int hashCode() { - return Objects.hash(target.name()); - } - - } - - /** Active load balancers allocated to a deployment */ - record LoadBalancerAllocation(DeploymentId deployment, - DeploymentSpec deploymentSpec, - List<LoadBalancer> loadBalancers) { - - public LoadBalancerAllocation(DeploymentId deployment, - DeploymentSpec deploymentSpec, - List<LoadBalancer> loadBalancers) { - this.deployment = deployment; - this.loadBalancers = loadBalancers.stream().filter(LoadBalancerAllocation::isActive).toList(); - this.deploymentSpec = deploymentSpec; - } - - private static boolean isActive(LoadBalancer loadBalancer) { - return switch (loadBalancer.state()) { - // Count reserved as active as we want to do DNS updates as early as possible - case reserved, active -> true; - default -> false; - }; - } - - /** Returns the policy IDs of the load balancers contained in this */ - private Set<RoutingPolicyId> asPolicyIds() { - return loadBalancers.stream() - .map(lb -> new RoutingPolicyId(lb.application(), - lb.cluster(), - deployment.zoneId())) - .collect(Collectors.toUnmodifiableSet()); - } - - /** Returns all instance endpoint IDs served by given load balancer */ - private Set<EndpointId> instanceEndpointsOf(LoadBalancer loadBalancer) { - if (!deployment.zoneId().environment().isProduction()) { // Only production deployments have configurable endpoints - return Set.of(); - } - var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance()); - if (instanceSpec.isEmpty()) { - return Set.of(); - } - return instanceSpec.get().endpoints().stream() - .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value())) - .filter(endpoint -> endpoint.regions().contains(deployment.zoneId().region())) - .map(com.yahoo.config.application.api.Endpoint::endpointId) - .map(EndpointId::of) - .collect(Collectors.toUnmodifiableSet()); - } - - /** Returns all application endpoint IDs served by given load balancer */ - private Set<EndpointId> applicationEndpointsOf(LoadBalancer loadBalancer) { - if (!deployment.zoneId().environment().isProduction()) { // Only production deployments have configurable endpoints - return Set.of(); - } - return deploymentSpec.endpoints().stream() - .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value())) - .filter(endpoint -> endpoint.targets().stream() - .anyMatch(target -> target.region().equals(deployment.zoneId().region()) && - target.instance().equals(deployment.applicationId().instance()))) - .map(com.yahoo.config.application.api.Endpoint::endpointId) - .map(EndpointId::of) - .collect(Collectors.toUnmodifiableSet()); - } - - } - - /** Returns the name updater to use for given endpoint */ - private NameServiceForwarder nameServiceForwarder(Endpoint endpoint) { - return switch (endpoint.routingMethod()) { - case exclusive -> controller.nameServiceForwarder(); - case sharedLayer4 -> endpoint.generated().isPresent() ? controller.nameServiceForwarder() : new NameServiceDiscarder(controller.curator()); - }; - } - - /** Denotes record data (record rhs) of either an ALIAS or a DIRECT target */ - private record Target(Record.Type type, RecordData data, DeploymentId deployment, Object aliasOrDirectTarget) { - static Target weighted(RoutingPolicy policy, Endpoint.Target endpointTarget) { - if (policy.ipAddress().isPresent()) { - var wt = new WeightedDirectTarget(RecordData.from(policy.ipAddress().get()), - endpointTarget.deployment().zoneId(), endpointTarget.weight()); - return new Target(Record.Type.DIRECT, wt.recordData(), endpointTarget.deployment(), wt); - } - var wt = new WeightedAliasTarget(policy.canonicalName().get(), policy.dnsZone().get(), - endpointTarget.deployment().zoneId().value(), endpointTarget.weight()); - return new Target(Record.Type.ALIAS, RecordData.fqdn(wt.name().value()), endpointTarget.deployment(), wt); - } - } - - /** A {@link NameServiceForwarder} that does nothing. Used in zones where no explicit DNS updates are needed */ - private static class NameServiceDiscarder extends NameServiceForwarder { - - public NameServiceDiscarder(CuratorDb db) { - super(db); - } - - @Override - protected void forward(NameServiceRequest request, Priority priority) { - // Ignored - } - } - - private static Optional<TenantAndApplicationId> ownerOf(DeploymentId deploymentId) { - return Optional.of(TenantAndApplicationId.from(deploymentId.applicationId())); - } - - private static Optional<TenantAndApplicationId> ownerOf(LoadBalancerAllocation allocation) { - return ownerOf(allocation.deployment); - } - - private static void requireNonClashing(GeneratedEndpoint generatedEndpoint, RoutingPolicyList applicationPolicies) { - for (var policy : applicationPolicies) { - for (var other : policy.generatedEndpoints()) { - if (other.clusterPart().equals(generatedEndpoint.clusterPart()) && !other.endpoint().equals(generatedEndpoint.endpoint())) { - throw new IllegalStateException(generatedEndpoint + " clashes with " + other + " in " + policy.id()); - } - } - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java deleted file mode 100644 index fc72f3ed663..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing; - -import ai.vespa.http.DomainName; -import com.google.common.collect.ImmutableSortedSet; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.application.EndpointId; - -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * Represents the DNS routing policy for a {@link com.yahoo.vespa.hosted.controller.application.Deployment}. - * - * @author mortent - * @author mpolden - */ -public record RoutingPolicy(RoutingPolicyId id, - Optional<DomainName> canonicalName, - Optional<String> ipAddress, - Optional<String> dnsZone, - Set<EndpointId> instanceEndpoints, - Set<EndpointId> applicationEndpoints, - RoutingStatus routingStatus, - boolean isPublic, - GeneratedEndpointList generatedEndpoints) { - - /** DO NOT USE. Public for serialization purposes */ - public RoutingPolicy(RoutingPolicyId id, Optional<DomainName> canonicalName, Optional<String> ipAddress, Optional<String> dnsZone, - Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, RoutingStatus routingStatus, boolean isPublic, - GeneratedEndpointList generatedEndpoints) { - this.id = Objects.requireNonNull(id, "id must be non-null"); - this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); - this.ipAddress = Objects.requireNonNull(ipAddress, "ipAddress must be non-null"); - this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); - this.instanceEndpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(instanceEndpoints, "instanceEndpoints must be non-null")); - this.applicationEndpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(applicationEndpoints, "applicationEndpoints must be non-null")); - this.routingStatus = Objects.requireNonNull(routingStatus, "status must be non-null"); - this.isPublic = isPublic; - this.generatedEndpoints = Objects.requireNonNull(generatedEndpoints, "generatedEndpoints must be non-null"); - - if (canonicalName.isEmpty() == ipAddress.isEmpty()) - throw new IllegalArgumentException("Exactly 1 of canonicalName=%s and ipAddress=%s must be set".formatted( - canonicalName.map(DomainName::value).orElse("<empty>"), ipAddress.orElse("<empty>"))); - if ( ! instanceEndpoints.isEmpty() && ! isPublic) - throw new IllegalArgumentException("Non-public zone endpoint cannot be part of any global endpoint, but was in: " + instanceEndpoints); - if ( ! applicationEndpoints.isEmpty() && ! isPublic) - throw new IllegalArgumentException("Non-public zone endpoint cannot be part of any application endpoint, but was in: " + applicationEndpoints); - } - - /** The ID of this */ - public RoutingPolicyId id() { - return id; - } - - /** The canonical name for the load balancer this applies to (rhs of a CNAME or ALIAS record) */ - public Optional<DomainName> canonicalName() { - return canonicalName; - } - - /** The IP address for the load balancer this applies to (rhs of an A or DIRECT record) */ - public Optional<String> ipAddress() { - return ipAddress; - } - - /** DNS zone for the load balancer this applies to, if any. Used when creating ALIAS records. */ - public Optional<String> dnsZone() { - return dnsZone; - } - - /** The instance-level endpoints this participates in */ - public Set<EndpointId> instanceEndpoints() { - return instanceEndpoints; - } - - /** The application-level endpoints this participates in */ - public Set<EndpointId> applicationEndpoints() { - return applicationEndpoints; - } - - /** The endpoints generated for this policy, if any */ - public GeneratedEndpointList generatedEndpoints() { - return generatedEndpoints; - } - - /** Return status of routing */ - public RoutingStatus routingStatus() { - return routingStatus; - } - - /** Returns whether this has a load balancer which is available from public internet. */ - public boolean isPublic() { - return isPublic; - } - - /** Returns whether this policy applies to given deployment */ - public boolean appliesTo(DeploymentId deployment) { - return id.owner().equals(deployment.applicationId()) && - id.zone().equals(deployment.zoneId()); - } - - /** Returns a copy of this with routing status set to given status */ - public RoutingPolicy with(RoutingStatus routingStatus) { - return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, routingStatus, isPublic, generatedEndpoints); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RoutingPolicy that = (RoutingPolicy) o; - return id.equals(that.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java deleted file mode 100644 index ea8ae6820c9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; - -import java.util.Objects; - -/** - * Unique identifier for a {@link RoutingPolicy}. - * - * @author mpolden - */ -public record RoutingPolicyId(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone) { - - public RoutingPolicyId { - Objects.requireNonNull(owner, "owner must be non-null"); - Objects.requireNonNull(cluster, "cluster must be non-null"); - Objects.requireNonNull(zone, "zone must be non-null"); - } - - /** The deployment this applies to */ - public DeploymentId deployment() { - return new DeploymentId(owner, zone); - } - - @Override - public String toString() { - return "routing policy for " + cluster + ", in " + zone + ", owned by " + owner; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyList.java deleted file mode 100644 index f96275a0d5a..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyList.java +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing; - -import com.yahoo.collections.AbstractFilteringList; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.application.EndpointId; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * A filterable list of {@link RoutingPolicy}'s. - * - * This is immutable. - * - * @author mpolden - */ -public class RoutingPolicyList extends AbstractFilteringList<RoutingPolicy, RoutingPolicyList> { - - private final Map<RoutingPolicyId, RoutingPolicy> policiesById; - - protected RoutingPolicyList(Collection<RoutingPolicy> items, boolean negate) { - super(items, negate, RoutingPolicyList::new); - this.policiesById = items.stream().collect(Collectors.collectingAndThen( - Collectors.toMap(RoutingPolicy::id, - Function.identity(), - (p1, p2) -> { - throw new IllegalArgumentException("Duplicate key " + p1.id()); - }, - LinkedHashMap::new), - Collections::unmodifiableMap) - ); - } - - /** Returns the subset of policies owned by given instance */ - public RoutingPolicyList instance(ApplicationId instance) { - return matching(policy -> policy.id().owner().equals(instance)); - } - - /** Returns the subset of policies applying to given cluster */ - public RoutingPolicyList cluster(ClusterSpec.Id cluster) { - return matching(policy -> policy.id().cluster().equals(cluster)); - } - - /** Returns the subset of policies applying to given deployment */ - public RoutingPolicyList deployment(DeploymentId deployment) { - return matching(policy -> policy.appliesTo(deployment)); - } - - /** Returns the policy with given ID, if any */ - public Optional<RoutingPolicy> of(RoutingPolicyId id) { - return Optional.ofNullable(policiesById.get(id)); - } - - /** Returns this grouped by policy ID */ - public Map<RoutingPolicyId, RoutingPolicy> asMap() { - return policiesById; - } - - /** Returns a copy of this with all policies for instance replaced with given policies */ - public RoutingPolicyList replace(ApplicationId instance, RoutingPolicyList policies) { - List<RoutingPolicy> copy = new ArrayList<>(asList()); - copy.removeIf(policy -> policy.id().owner().equals(instance)); - policies.forEach(copy::add); - return copyOf(copy); - } - - /** Returns a copy of this excluding the given policy */ - public RoutingPolicyList without(RoutingPolicy policy) { - List<RoutingPolicy> copy = new ArrayList<>(asList()); - copy.remove(policy); - return copyOf(copy); - } - - /** Create a routing table for instance-level endpoints backed by routing policies in this */ - Map<RoutingId, List<RoutingPolicy>> asInstanceRoutingTable() { - return asRoutingTable(false); - } - - /** Create a routing table for application-level endpoints backed by routing policies in this */ - Map<RoutingId, List<RoutingPolicy>> asApplicationRoutingTable() { - return asRoutingTable(true); - } - - private Map<RoutingId, List<RoutingPolicy>> asRoutingTable(boolean applicationLevel) { - Map<RoutingId, List<RoutingPolicy>> routingTable = new LinkedHashMap<>(); - for (var policy : this) { - Set<EndpointId> endpoints = applicationLevel ? policy.applicationEndpoints() : policy.instanceEndpoints(); - for (var endpoint : endpoints) { - RoutingId id = RoutingId.of(policy.id().owner(), endpoint); - routingTable.computeIfAbsent(id, k -> new ArrayList<>()) - .add(policy); - } - } - return Collections.unmodifiableMap(routingTable); - } - - public static RoutingPolicyList copyOf(Collection<RoutingPolicy> policies) { - return new RoutingPolicyList(policies, false); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingStatus.java deleted file mode 100644 index bd46760cc3e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingStatus.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing; - -import java.time.Instant; -import java.util.Objects; - -/** - * Represents the routing status of a {@link RoutingPolicy} or {@link ZoneRoutingPolicy}. - * - * This describes which agent last changed the routing status and at which time. - * - * This is immutable. - * - * @author mpolden - */ -public record RoutingStatus(Value value, Agent agent, Instant changedAt) { - - public static final RoutingStatus DEFAULT = new RoutingStatus(Value.in, Agent.system, Instant.EPOCH); - - /** DO NOT USE. Public for serialization purposes */ - public RoutingStatus { - Objects.requireNonNull(value, "value must be non-null"); - Objects.requireNonNull(agent, "agent must be non-null"); - Objects.requireNonNull(changedAt, "changedAt must be non-null"); - } - - /** - * The wanted value of this. The system will try to set this value, but there are constraints that may lead to - * the effective value not matching this. See {@link RoutingPolicies}. - */ - public Value value() { - return value; - } - - /** The agent who last changed this */ - public Agent agent() { - return agent; - } - - /** The time this was last changed */ - public Instant changedAt() { - return changedAt; - } - - @Override - public String toString() { - return "status " + value + ", changed by " + agent + " @ " + changedAt; - } - - public static RoutingStatus create(Value value, Agent agent, Instant instant) { - return new RoutingStatus(value, agent, instant); - } - - // Used in serialization. Do not change. - public enum Value { - /** Status is determined by health checks **/ - in, - - /** Status is explicitly set to out */ - out, - } - - /** Agents that can change the state of global routing */ - public enum Agent { - operator, - tenant, - system, - unknown, // For compatibility old values from /routing/v1 on config server, which may contain a specific username. - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java deleted file mode 100644 index 3ca72a7dd67..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing; - -import com.yahoo.config.provision.zone.ZoneId; - -import java.util.Objects; - -/** - * Represents the DNS routing policy for a zone. This takes precedence over of a deployment-specific - * {@link RoutingPolicy}. - * - * This is immutable. - * - * @author mpolden - */ -public record ZoneRoutingPolicy(ZoneId zone, RoutingStatus routingStatus) { - - public ZoneRoutingPolicy { - Objects.requireNonNull(zone, "zone must be non-null"); - Objects.requireNonNull(routingStatus, "globalRouting must be non-null"); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java deleted file mode 100644 index 50e65187835..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing.context; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.vespa.hosted.controller.LockedApplication; -import com.yahoo.vespa.hosted.controller.RoutingController; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.EndpointList; -import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml; -import com.yahoo.vespa.hosted.controller.routing.PreparedEndpoints; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; -import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; - -import java.time.Clock; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * A deployment routing context. This extends {@link RoutingContext} to support configuration of routing for a deployment. - * - * @author mpolden - */ -public abstract class DeploymentRoutingContext implements RoutingContext { - - final DeploymentId deployment; - final RoutingController routing; - final RoutingMethod method; - - public DeploymentRoutingContext(DeploymentId deployment, RoutingMethod method, RoutingController routing) { - this.deployment = Objects.requireNonNull(deployment); - this.routing = Objects.requireNonNull(routing); - this.method = Objects.requireNonNull(method); - } - - /** - * Prepare routing configuration for the deployment in this context - * - * @return the container endpoints relevant for this deployment, as declared in deployment spec - */ - public final PreparedEndpoints prepare(BasicServicesXml services, EndpointCertificate certificate, LockedApplication application) { - return routing.prepare(deployment, services, certificate, application); - } - - /** Finalize routing configuration for the deployment in this context, using given deployment spec */ - public final void activate(DeploymentSpec deploymentSpec, EndpointList generatedEndpoints) { - routing.policies().refresh(deployment, deploymentSpec, generatedEndpoints); - } - - /** Deactivate routing configuration for the deployment in this context, using given deployment spec */ - public final void deactivate(DeploymentSpec deploymentSpec) { - routing.policies().refresh(deployment, deploymentSpec, EndpointList.EMPTY); - routing.policies().removeDnsChallenges(deployment); - } - - /** Routing method of this context */ - public final RoutingMethod routingMethod() { - return method; - } - - /** Read the routing policy for given cluster in this deployment */ - public final Optional<RoutingPolicy> routingPolicy(ClusterSpec.Id cluster) { - RoutingPolicyId id = new RoutingPolicyId(deployment.applicationId(), cluster, deployment.zoneId()); - return routing.policies().read(deployment).of(id); - } - - /** Extension of a {@link DeploymentRoutingContext} for deployments using {@link RoutingMethod#sharedLayer4} routing */ - public static class SharedDeploymentRoutingContext extends DeploymentRoutingContext { - - private final Clock clock; - private final ConfigServer configServer; - - public SharedDeploymentRoutingContext(DeploymentId deployment, RoutingController controller, ConfigServer configServer, Clock clock) { - super(deployment, RoutingMethod.sharedLayer4, controller); - this.clock = Objects.requireNonNull(clock); - this.configServer = Objects.requireNonNull(configServer); - } - - @Override - public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { - EndpointStatus newStatus = new EndpointStatus(value == RoutingStatus.Value.in - ? EndpointStatus.Status.in - : EndpointStatus.Status.out, - agent.name(), - clock.instant()); - try { - configServer.setGlobalRotationStatus(deployment, upstreamNames(), newStatus); - } catch (Exception e) { - throw new RuntimeException("Failed to change rotation status of " + deployment, e); - } - } - - @Override - public RoutingStatus routingStatus() { - // In a given deployment, all upstreams (clusters) share the same status, so we can query using any - // upstream name - String upstreamName = upstreamNames().get(0); - EndpointStatus status = configServer.getGlobalRotationStatus(deployment, upstreamName); - RoutingStatus.Agent agent; - try { - agent = RoutingStatus.Agent.valueOf(status.agent().toLowerCase()); - } catch (IllegalArgumentException e) { - agent = RoutingStatus.Agent.unknown; - } - return new RoutingStatus(status.status() == EndpointStatus.Status.in - ? RoutingStatus.Value.in - : RoutingStatus.Value.out, - agent, - status.changedAt()); - } - - private List<String> upstreamNames() { - List<String> upstreamNames = routing.readEndpointsOf(deployment) - .scope(Endpoint.Scope.zone) - .shared() - .asList().stream() - .map(endpoint -> endpoint.upstreamName(deployment)) - .distinct() - .toList(); - if (upstreamNames.isEmpty()) { - throw new IllegalArgumentException("No upstream names found for " + deployment); - } - return upstreamNames; - } - - } - - /** - * Implementation of a {@link DeploymentRoutingContext} for deployments using {@link RoutingMethod#exclusive} - * routing. - */ - public static class ExclusiveDeploymentRoutingContext extends DeploymentRoutingContext { - - public ExclusiveDeploymentRoutingContext(DeploymentId deployment, RoutingController controller) { - super(deployment, RoutingMethod.exclusive, controller); - } - - @Override - public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { - routing.policies().setRoutingStatus(deployment, value, agent); - } - - @Override - public RoutingStatus routingStatus() { - // Status for a deployment applies to all clusters within the deployment, so we use the status from the - // first matching policy here - return routing.policies().read(deployment) - .first() - .map(RoutingPolicy::routingStatus) - .orElse(RoutingStatus.DEFAULT); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveZoneRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveZoneRoutingContext.java deleted file mode 100644 index 201baa78437..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveZoneRoutingContext.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing.context; - -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; -import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; - -import java.util.Objects; - -/** - * An implementation of {@link RoutingContext} for a zone using {@link RoutingMethod#exclusive} routing. - * - * @author mpolden - */ -public class ExclusiveZoneRoutingContext implements RoutingContext { - - private final RoutingPolicies policies; - private final ZoneId zone; - - public ExclusiveZoneRoutingContext(ZoneId zone, RoutingPolicies policies) { - this.policies = Objects.requireNonNull(policies); - this.zone = Objects.requireNonNull(zone); - } - - @Override - public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { - policies.setRoutingStatus(zone, value); - } - - @Override - public RoutingStatus routingStatus() { - return policies.read(zone).routingStatus(); - } - - @Override - public RoutingMethod routingMethod() { - return RoutingMethod.exclusive; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java deleted file mode 100644 index 84315e319ec..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing.context; - -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; - -/** - * Top-level interface for a routing context, which provides control of routing status for a deployment or zone. - * - * @author mpolden - */ -public interface RoutingContext { - - /** Change the routing status for the zone or deployment represented by this context */ - void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent); - - /** Get the current routing status for the zone or deployment represented by this context */ - RoutingStatus routingStatus(); - - /** Routing method used in this context */ - RoutingMethod routingMethod(); - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedZoneRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedZoneRoutingContext.java deleted file mode 100644 index 00ab41fc61c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedZoneRoutingContext.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing.context; - -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; -import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; - -import java.time.Instant; -import java.util.Objects; - -/** - * An implementation of {@link RoutingContext} for a zone, using {@link RoutingMethod#sharedLayer4} routing. - * - * @author mpolden - */ -public class SharedZoneRoutingContext implements RoutingContext { - - private final ConfigServer configServer; - private final ZoneId zone; - - public SharedZoneRoutingContext(ZoneId zone, ConfigServer configServer) { - this.configServer = Objects.requireNonNull(configServer); - this.zone = Objects.requireNonNull(zone); - } - - @Override - public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { - boolean in = value == RoutingStatus.Value.in; - configServer.setGlobalRotationStatus(zone, in); - } - - @Override - public RoutingStatus routingStatus() { - boolean in = configServer.getGlobalRotationStatus(zone); - RoutingStatus.Value newValue = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; - return new RoutingStatus(newValue, - RoutingStatus.Agent.operator, - Instant.EPOCH); // API does not support time of change - } - - @Override - public RoutingMethod routingMethod() { - return RoutingMethod.sharedLayer4; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java deleted file mode 100644 index d94124709f7..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing.rotation; - -import com.yahoo.text.Text; - -import java.util.Objects; - -/** - * Represents a global routing rotation. - * - * @author mpolden - */ -public record Rotation(RotationId id, String name) { - - public Rotation { - Objects.requireNonNull(id); - Objects.requireNonNull(name); - } - - /** The ID of the allocated rotation. This value is generated by global routing system */ - public RotationId id() { - return id; - } - - /** The global rotation FQDN */ - public String name() { - return name; - } - - @Override - public String toString() { - return Text.format("rotation %s -> %s", id().asString(), name()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java deleted file mode 100644 index a99c9ada0f9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing.rotation; - -/** - * ID of a global rotation. - * - * @author mpolden - */ -public record RotationId(String id) { - - /** Rotation ID, e.g. rotation-42.vespa.global.routing */ - public String asString() { - return id; - } - - @Override - public String toString() { - return "rotation ID " + id; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java deleted file mode 100644 index 3043ec146a6..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing.rotation; - -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.curator.Lock; - -import java.util.Objects; - -/** - * A lock for the rotation repository. This is a type-safe wrapper for a curator lock. - * - * @author mpolden - */ -public class RotationLock implements AutoCloseable { - - private final Mutex lock; - - RotationLock(Mutex lock) { - this.lock = Objects.requireNonNull(lock, "lock cannot be null"); - } - - @Override - public void close() { - lock.close(); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java deleted file mode 100644 index c70826161da..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing.rotation; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.Endpoint; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.controller.ApplicationController; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.application.AssignedRotation; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.collectingAndThen; - -/** - * The rotation repository offers global rotations to Vespa applications. - * - * The list of rotations comes from RotationsConfig, which is set in the controller's services.xml. - * - * @author Oyvind Gronnesby - * @author mpolden - */ -public class RotationRepository { - - private final Map<RotationId, Rotation> allRotations; - private final ApplicationController applications; - private final CuratorDb curator; - - public RotationRepository(RotationsConfig rotationsConfig, ApplicationController applications, CuratorDb curator) { - this.allRotations = from(rotationsConfig); - this.applications = applications; - this.curator = curator; - } - - /** Acquire a exclusive lock for this */ - public RotationLock lock() { - return new RotationLock(curator.lockRotations()); - } - - /** Get rotation with given id */ - public Rotation requireRotation(RotationId id) { - Rotation rotation = allRotations.get(id); - if (rotation == null) throw new IllegalArgumentException("No such rotation: '" + id.asString() + "'"); - return rotation; - } - - /** - * Returns rotation assignments for all endpoints in application. - * - * If rotations are already assigned, these will be returned. - * If rotations are not assigned, a new assignment will be created taking new rotations from the repository. - * - * @param deploymentSpec The deployment spec of the application - * @param instance The application requesting rotations - * @param lock Lock which by acquired by the caller - * @return List of rotation assignments - either new or existing - */ - public List<AssignedRotation> getOrAssignRotations(DeploymentSpec deploymentSpec, Instance instance, RotationLock lock) { - // Skip assignment if no rotations are configured in this system - if (allRotations.isEmpty()) { - return List.of(); - } - var instanceSpec = deploymentSpec.requireInstance(instance.name()); - return assignRotationsTo(instanceSpec.endpoints(), instance, lock); - } - - private List<AssignedRotation> assignRotationsTo(List<Endpoint> endpoints, Instance instance, RotationLock lock) { - if (endpoints.isEmpty()) return List.of(); // No endpoints declared, nothing to assign. - var availableRotations = new ArrayList<>(availableRotations(lock).values()); - var assignedRotationsByEndpointId = instance.rotations().stream() - .collect(Collectors.toMap(AssignedRotation::endpointId, - Function.identity())); - var assignments = new ArrayList<AssignedRotation>(); - for (var endpoint : endpoints) { - var endpointId = EndpointId.of(endpoint.endpointId()); - var assignedRotation = assignedRotationsByEndpointId.get(endpointId); - RotationId rotationId; - if (assignedRotation == null) { // No rotation is assigned to this endpoint, assign from available - rotationId = requireNonEmpty(availableRotations).remove(0).id(); - } else { // Rotation already assigned to this endpoint, reuse it - rotationId = assignedRotation.rotationId(); - } - assignments.add(new AssignedRotation(ClusterSpec.Id.from(endpoint.containerId()), endpointId, rotationId, Set.copyOf(endpoint.regions()))); - } - return Collections.unmodifiableList(assignments); - } - - /** - * Returns all unassigned rotations - * @param lock Lock which must be acquired by the caller - */ - public Map<RotationId, Rotation> availableRotations(@SuppressWarnings("unused") RotationLock lock) { - List<RotationId> assignedRotations = applications.asList().stream() - .flatMap(application -> application.instances().values().stream()) - .flatMap(instance -> instance.rotations().stream()) - .map(AssignedRotation::rotationId) - .toList(); - Map<RotationId, Rotation> unassignedRotations = new LinkedHashMap<>(this.allRotations); - assignedRotations.forEach(unassignedRotations::remove); - return Collections.unmodifiableMap(unassignedRotations); - } - - /** Returns a immutable map of rotation ID to rotation sorted by rotation ID */ - private static Map<RotationId, Rotation> from(RotationsConfig rotationConfig) { - return rotationConfig.rotations().entrySet().stream() - .map(entry -> new Rotation(new RotationId(entry.getKey()), entry.getValue().trim())) - .sorted(Comparator.comparing(rotation -> rotation.id().asString())) - .collect(collectingAndThen(Collectors.toMap(Rotation::id, - rotation -> rotation, - (k, v) -> v, - LinkedHashMap::new), - Collections::unmodifiableMap)); - } - - private static <T extends Collection<?>> T requireNonEmpty(T rotations) { - if (rotations.isEmpty()) throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation"); - return rotations; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java deleted file mode 100644 index 53ebbd1e95e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing.rotation; - -/** - * The possible states of a global rotation. - * - * @author mpolden - */ -public enum RotationState { - - /** Rotation has status 'in' and is receiving traffic */ - in, - - /** Rotation has status 'out' and is *NOT* receiving traffic */ - out, - - /** Rotation status is currently unknown, or no global rotation has been assigned */ - unknown - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java deleted file mode 100644 index 7ad841c96f9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing.rotation; - -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.Deployment; - -import java.time.Instant; -import java.util.Map; -import java.util.Objects; - -/** - * The status of all rotations assigned to an application. - * - * @author mpolden - */ -public record RotationStatus(Map<RotationId, Targets> status) { - - public static final RotationStatus EMPTY = new RotationStatus(Map.of()); - - public RotationStatus(Map<RotationId, Targets> status) { - this.status = Map.copyOf(Objects.requireNonNull(status)); - } - - public Map<RotationId, Targets> asMap() { - return status; - } - - /** Get targets of given rotation, if any */ - public Targets of(RotationId rotation) { - return status.getOrDefault(rotation, Targets.NONE); - } - - /** Get status of deployment in given rotation, if any */ - public RotationState of(RotationId rotation, Deployment deployment) { - return of(rotation).asMap().entrySet().stream() - .filter(kv -> kv.getKey().equals(deployment.zone())) - .map(Map.Entry::getValue) - .findFirst() - .orElse(RotationState.unknown); - } - - @Override - public String toString() { - return "rotation status " + status; - } - - public static RotationStatus from(Map<RotationId, Targets> targets) { - return targets.isEmpty() ? EMPTY : new RotationStatus(targets); - } - - /** Targets of a rotation */ - public record Targets(Map<ZoneId, RotationState> targets, Instant lastUpdated) { - - public static final Targets NONE = new Targets(Map.of(), Instant.EPOCH); - - public Targets(Map<ZoneId, RotationState> targets, Instant lastUpdated) { - this.targets = Map.copyOf(Objects.requireNonNull(targets, "states must be non-null")); - this.lastUpdated = Objects.requireNonNull(lastUpdated, "lastUpdated must be non-null"); - } - - public Map<ZoneId, RotationState> asMap() { - return targets; - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java deleted file mode 100644 index 4c2f2627026..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.security; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.time.Instant; -import java.util.List; - -/** - * Stores permissions for tenant and application resources. - * - * The signatures use vague types, and the exact types is a contract between this and the - * {@link AccessControlRequests} generating data consumed by this. - * - * @author jonmv - */ -public interface AccessControl { - - /** - * Sets up access control based on the given credentials, and returns a tenant, based on the given specification. - * - * @param tenantSpec specification for the tenant to create - * @param createdAt instant when the tenant was created - * @param credentials the credentials for the entity requesting the creation - * @param existing list of existing tenants, to check for conflicts - * @return the created tenant, for keeping - */ - Tenant createTenant(TenantSpec tenantSpec, Instant createdAt, Credentials credentials, List<Tenant> existing); - - /** - * Modifies access control based on the given credentials, and returns a modified tenant, based on the given specification. - * - * @param tenantSpec specification for the tenant to update - * @param credentials the credentials for the entity requesting the update - * @param existing list of existing tenants, to check for conflicts - * @param applications list of applications this tenant already owns - * @return the updated tenant, for keeping - */ - Tenant updateTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing, List<Application> applications); - - /** - * Deletes access control for the given tenant. - * - * @param tenant the tenant to delete - * @param credentials the credentials for the entity requesting the deletion - */ - void deleteTenant(TenantName tenant, Credentials credentials); - - /** - * Sets up access control for the given application, based on the given credentials. - * - * @param id the ID of the application to create - * @param credentials the credentials for the entity requesting the creation - */ - void createApplication(TenantAndApplicationId id, Credentials credentials); - - /** - * Deletes access control for the given tenant. - * - * @param id the ID of the application to delete - * @param credentials the credentials for the entity requesting the deletion - */ - void deleteApplication(TenantAndApplicationId id, Credentials credentials); - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControlRequests.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControlRequests.java deleted file mode 100644 index 081c72f7e25..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControlRequests.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.security; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.slime.Inspector; - -/** - * Extracts {@link TenantSpec}s and {@link Credentials}s from HTTP requests, to be stored in an {@link AccessControl}. - * - * @author jonmv - */ -public interface AccessControlRequests { - - /** Extracts claim data for a tenant, from the given request. */ - TenantSpec specification(TenantName tenant, Inspector requestObject); - - /** Extracts credentials required for an access control modification for the given tenant, from the given request. */ - Credentials credentials(TenantName tenant, Inspector requestObject, HttpRequest jDiscRequest); - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java deleted file mode 100644 index ccf3db5d204..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.security; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.TenantName; -import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.slime.Inspector; -import com.yahoo.text.Text; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.OAuthCredentials; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.TenantController; -import com.yahoo.vespa.hosted.controller.api.identifiers.Property; -import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; -import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; - -import java.security.Principal; -import java.util.Objects; -import java.util.Optional; - -/** - * Extracts access control data for Athenz or user tenants from HTTP requests. - * - * @author jonmv - */ -public class AthenzAccessControlRequests implements AccessControlRequests { - - private final TenantController tenants; - - @Inject - public AthenzAccessControlRequests(Controller controller) { - this.tenants = controller.tenants(); - } - - @Override - public TenantSpec specification(TenantName tenant, Inspector requestObject) { - return new AthenzTenantSpec(tenant, - new AthenzDomain(required("athensDomain", requestObject)), - new Property(required("property", requestObject)), - optional("propertyId", requestObject).map(PropertyId::new)); - } - - @Override - public Credentials credentials(TenantName tenant, Inspector requestObject, HttpRequest request) { - return new AthenzCredentials(requireAthenzPrincipal(request), - tenants.get(tenant).map(AthenzTenant.class::cast).map(AthenzTenant::domain) - .orElseGet(() -> new AthenzDomain(required("athensDomain", requestObject))), - OAuthCredentials.fromOktaRequestContext(request.context())); - } - - private static String required(String fieldName, Inspector object) { - return optional(fieldName, object) .orElseThrow(() -> new IllegalArgumentException("Missing required field '" + fieldName + "'.")); - } - - private static Optional<String> optional(String fieldName, Inspector object) { - return object.field(fieldName).valid() ? Optional.of(object.field(fieldName).asString()) : Optional.empty(); - } - - private static AthenzPrincipal requireAthenzPrincipal(HttpRequest request) { - Principal principal = request.getUserPrincipal(); - Objects.requireNonNull(principal, "Expected a user principal"); - if ( ! (principal instanceof AthenzPrincipal)) - throw new RuntimeException(Text.format("Expected principal of type %s, got %s", - AthenzPrincipal.class.getSimpleName(), principal.getClass().getName())); - return (AthenzPrincipal) principal; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzCredentials.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzCredentials.java deleted file mode 100644 index aa8ab8375b0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzCredentials.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.security; - -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.OAuthCredentials; - -import static java.util.Objects.requireNonNull; - -/** - * Like {@link Credentials}, but the entity is rather an Athenz domain, and thus contains also a - * token which can be used to validate the user's role memberships under this domain. - * <em>This validation is done by Athenz, not by us.</em> - * - * @author jonmv - */ -public class AthenzCredentials extends Credentials { - - private final AthenzDomain domain; - private final OAuthCredentials oAuthCredentials; - - public AthenzCredentials(AthenzPrincipal user, AthenzDomain domain, OAuthCredentials oAuthCredentials) { - super(user); - this.domain = requireNonNull(domain); - this.oAuthCredentials = requireNonNull(oAuthCredentials); - } - - @Override - public AthenzPrincipal user() { return (AthenzPrincipal) super.user(); } - - /** Returns the Athenz domain of the tenant on whose behalf this request is made. */ - public AthenzDomain domain() { return domain; } - - /** Returns the OAuth credentials required for Athenz tenancy operation */ - public OAuthCredentials oAuthCredentials() { return oAuthCredentials; } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzTenantSpec.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzTenantSpec.java deleted file mode 100644 index 70799250773..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzTenantSpec.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.security; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.hosted.controller.api.identifiers.Property; -import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; - -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -/** - * Extends the specification for creating an Athenz tenant. - * - * @author jonmv - */ -public class AthenzTenantSpec extends TenantSpec { - - private final AthenzDomain domain; - private final Property property; - private final Optional<PropertyId> propertyId; - - public AthenzTenantSpec(TenantName tenant, AthenzDomain domain, Property property, Optional<PropertyId> propertyId) { - super(tenant); - this.domain = domain; - this.property = requireNonNull(property); - this.propertyId = requireNonNull(propertyId); - } - - /** The domain to create this tenant under. */ - public AthenzDomain domain() { return domain; } - - /** The property name of the tenant. */ - public Property property() { return property; } - - /** The ID of the property of the tenant. */ - public Optional<PropertyId> propertyId() { return propertyId; } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java deleted file mode 100644 index aaf2b5a9367..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.security; - -import com.yahoo.vespa.hosted.controller.api.role.Role; - -import java.security.Principal; -import java.util.Collections; -import java.util.Set; - -/** - * Like {@link Credentials}, but we know the principal is authenticated by Auth0. - * Also includes the set of roles for which the principal is a member. - * - * @author andreer - */ -public class Auth0Credentials extends Credentials { - - private final Set<Role> roles; - - public Auth0Credentials(Principal user, Set<Role> roles) { - super(user); - this.roles = Collections.unmodifiableSet(roles); - } - - /** The set of roles set in the auth0 cookie, extracted by CloudAccessControlRequests. */ - public Set<Role> getRolesFromCookie() { - return roles; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java deleted file mode 100644 index 051298d4f8b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.security; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.TenantName; -import com.yahoo.restapi.RestApiException; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.IntFlag; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; -import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; -import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; -import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; -import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; -import com.yahoo.vespa.hosted.controller.api.role.ApplicationRole; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.TenantRole; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import java.time.Instant; -import java.util.List; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.controller.api.role.RoleDefinition.administrator; -import static com.yahoo.vespa.hosted.controller.api.role.RoleDefinition.hostedOperator; -import static com.yahoo.vespa.hosted.controller.api.role.RoleDefinition.hostedSupporter; - -/** - * @author jonmv - * @author andreer - */ -public class CloudAccessControl implements AccessControl { - - private final UserManagement userManagement; - private final BooleanFlag enablePublicSignup; - private final IntFlag maxTrialTenants; - private final BillingController billingController; - - @Inject - public CloudAccessControl(UserManagement userManagement, FlagSource flagSource, ServiceRegistry serviceRegistry) { - this.userManagement = userManagement; - this.enablePublicSignup = PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource); - this.maxTrialTenants = PermanentFlags.MAX_TRIAL_TENANTS.bindTo(flagSource); - billingController = serviceRegistry.billingController(); - } - - @Override - public CloudTenant createTenant(TenantSpec tenantSpec, Instant createdAt, Credentials credentials, List<Tenant> existing) { - requireTenantCreationAllowed((Auth0Credentials) credentials); - requireTenantTrialLimitNotReached(existing); - - CloudTenantSpec spec = (CloudTenantSpec) tenantSpec; - CloudTenant tenant = CloudTenant.create(spec.tenant(), createdAt, credentials.user()); - - for (Role role : Roles.tenantRoles(spec.tenant())) { - userManagement.createRole(role); - } - - var userId = List.of(new UserId(credentials.user().getName())); - userManagement.addUsers(Role.administrator(spec.tenant()), userId); - userManagement.addUsers(Role.developer(spec.tenant()), userId); - userManagement.addUsers(Role.reader(spec.tenant()), userId); - - return tenant; - } - - private void requireTenantTrialLimitNotReached(List<Tenant> existing) { - var trialPlanId = PlanId.from("trial"); - var tenantNames = existing.stream().filter(tenant -> tenant.type() == Tenant.Type.cloud).map(Tenant::name).toList(); - var trialTenants = billingController.tenantsWithPlan(tenantNames, trialPlanId).size(); - - if (maxTrialTenants.value() >= 0 && maxTrialTenants.value() <= trialTenants) { - throw new RestApiException.Forbidden("Too many tenants with trial plans, please contact the Vespa support team"); - } - } - - private void requireTenantCreationAllowed(Auth0Credentials auth0Credentials) { - if (allowedByPrivilegedRole(auth0Credentials)) return; - - if (!allowedByFeatureFlag(auth0Credentials)) { - throw new RestApiException.Forbidden("You are not currently permitted to create tenants. Please contact the Vespa team to request access."); - } - - if(administeredTenants(auth0Credentials) >= 3) { - throw new RestApiException.Forbidden("You are already administering 3 tenants. If you need more, please contact the Vespa team."); - } - } - - private boolean allowedByPrivilegedRole(Auth0Credentials auth0Credentials) { - return auth0Credentials.getRolesFromCookie().stream() - .map(Role::definition) - .anyMatch(rd -> rd == hostedOperator || rd == hostedSupporter); - } - - private boolean allowedByFeatureFlag(Auth0Credentials auth0Credentials) { - return enablePublicSignup.with(FetchVector.Dimension.CONSOLE_USER_EMAIL, auth0Credentials.user().getName()).value(); - } - - private long administeredTenants(Auth0Credentials auth0Credentials) { - // We have to verify the roles with auth0 to ensure the user is not using an "old" cookie to make too many tenants. - return userManagement.listRoles(new UserId(auth0Credentials.user().getName())).stream() - .map(Role::definition) - .filter(rd -> rd == administrator) - .count(); - } - - @Override - public Tenant updateTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing, List<Application> applications) { - throw new UnsupportedOperationException("Update is not supported here, as it would entail changing the tenant name."); - } - - @Override - public void deleteTenant(TenantName tenant, Credentials credentials) { - for (TenantRole role : Roles.tenantRoles(tenant)) - userManagement.deleteRole(role); - } - - @Override - public void createApplication(TenantAndApplicationId id, Credentials credentials) { - for (Role role : Roles.applicationRoles(id.tenant(), id.application())) - userManagement.createRole(role); - } - - @Override - public void deleteApplication(TenantAndApplicationId id, Credentials credentials) { - for (ApplicationRole role : Roles.applicationRoles(id.tenant(), id.application())) - userManagement.deleteRole(role); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java deleted file mode 100644 index 697b324dc3e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.security; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.slime.Inspector; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; - -import java.util.Optional; -import java.util.Set; - -/** - * Extracts access control data for {@link CloudTenant}s from HTTP requests. - * - * @author jonmv - * @author andreer - */ -public class CloudAccessControlRequests implements AccessControlRequests { - - @Override - public CloudTenantSpec specification(TenantName tenant, Inspector requestObject) { - return new CloudTenantSpec(tenant, "token"); // TODO: remove token - } - - @Override - public Credentials credentials(TenantName tenant, Inspector requestObject, HttpRequest request) { - return new Auth0Credentials(request.getUserPrincipal(), getUserRoles(request)); - } - - private static Set<Role> getUserRoles(HttpRequest request) { - var securityContext = Optional.ofNullable(request.context().get(SecurityContext.ATTRIBUTE_NAME)) - .filter(SecurityContext.class::isInstance) - .map(SecurityContext.class::cast) - .orElseThrow(() -> new IllegalArgumentException("Attribute '" + SecurityContext.ATTRIBUTE_NAME + "' was not set on request")); - return securityContext.roles(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudTenantSpec.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudTenantSpec.java deleted file mode 100644 index f746df2b71e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudTenantSpec.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.security; - -import com.yahoo.config.provision.TenantName; - -import static java.util.Objects.requireNonNull; - -/** - * Extends the specification for creating a cloud tenant. - * - * @author jonmv - */ -public class CloudTenantSpec extends TenantSpec { - - private final String registrationToken; - - public CloudTenantSpec(TenantName tenant, String registrationToken) { - super(tenant); - this.registrationToken = requireNonNull(registrationToken); - } - - /** The cloud issued token proving the user intends to register the given tenant. */ - public String getRegistrationToken() { return registrationToken; } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudUserSessionManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudUserSessionManager.java deleted file mode 100644 index c2a505fc185..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudUserSessionManager.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.security; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.flags.LongFlag; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.TenantController; -import com.yahoo.vespa.hosted.controller.api.integration.user.UserSessionManager; -import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; -import com.yahoo.vespa.hosted.controller.api.role.TenantRole; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; - -import java.time.Instant; - -/** - * @author freva - */ -public class CloudUserSessionManager implements UserSessionManager { - - private final TenantController tenantController; - private final LongFlag invalidateConsoleSessions; - - public CloudUserSessionManager(Controller controller) { - this.tenantController = controller.tenants(); - this.invalidateConsoleSessions = PermanentFlags.INVALIDATE_CONSOLE_SESSIONS.bindTo(controller.flagSource()); - } - - @Override - public boolean shouldExpireSessionFor(SecurityContext context) { - if (context.issuedAt().isBefore(Instant.ofEpochSecond(invalidateConsoleSessions.value()))) - return true; - - return context.roles().stream() - .filter(TenantRole.class::isInstance) - .map(TenantRole.class::cast) - .map(TenantRole::tenant) - .distinct() - .anyMatch(tenantName -> shouldExpireSessionFor(tenantName, context.issuedAt())); - } - - private boolean shouldExpireSessionFor(TenantName tenantName, Instant contextIssuedAt) { - return tenantController.get(tenantName) - .filter(CloudTenant.class::isInstance) - .map(CloudTenant.class::cast) - .flatMap(CloudTenant::invalidateUserSessionsBefore) - .map(contextIssuedAt::isBefore) - .orElse(false); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Credentials.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Credentials.java deleted file mode 100644 index d2ad1433413..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Credentials.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.security; - -import java.security.Principal; - -import static java.util.Objects.requireNonNull; - -/** - * Credentials representing an entity for which to modify access control rules. - * - * @author jonmv - */ -public class Credentials { - - private final Principal user; - - public Credentials(Principal user) { - this.user = requireNonNull(user); - } - - /** Returns the user which makes the request. */ - public Principal user() { return user; } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/TenantSpec.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/TenantSpec.java deleted file mode 100644 index 8f74c59941d..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/TenantSpec.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.security; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; - -import static java.util.Objects.requireNonNull; - -/** - * A specification of a tenant, typically to create or modify one. - * - * @author jonmv - */ -public abstract class TenantSpec { - - private final TenantName tenant; - - protected TenantSpec(TenantName tenant) { - this.tenant = Tenant.requireName(requireNonNull(tenant)); - } - - /** The name of the tenant. */ - public TenantName tenant() { return tenant; } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccess.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccess.java deleted file mode 100644 index ae1231fa450..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccess.java +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.support.access; - -import java.time.Instant; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** Immutable state of support access, keeping history of all changes/grants. */ -public class SupportAccess { - - public static final SupportAccess DISALLOWED_NO_HISTORY = new SupportAccess(List.of(), List.of()); - - private final List<SupportAccessChange> changeHistory; - private final List<SupportAccessGrant> grantHistory; - - /** public for serializer - do not use */ - public SupportAccess(List<SupportAccessChange> changeHistory, List<SupportAccessGrant> grantHistory) { - this.changeHistory = Collections.unmodifiableList(changeHistory); - this.grantHistory = Collections.unmodifiableList(grantHistory); - } - - public List<SupportAccessChange> changeHistory() { - return changeHistory; - } - - public List<SupportAccessGrant> grantHistory() { - return grantHistory; - } - - public CurrentStatus currentStatus(Instant now) { - Optional<SupportAccessChange> latestChange = changeHistory.stream().findFirst(); - - if (latestChange.isEmpty() || latestChange.get().accessAllowedUntil().isEmpty() || now.isAfter(latestChange.get().accessAllowedUntil().get())) - return new CurrentStatus(State.NOT_ALLOWED, Optional.empty(), Optional.empty()); - - return new CurrentStatus(State.ALLOWED, latestChange.get().accessAllowedUntil(), Optional.of(latestChange.get().madeBy())); - } - - public SupportAccess withAllowedUntil(Instant until, String changedBy, Instant changeTime) { - if (!until.isAfter(changeTime)) - throw new IllegalArgumentException("Support access cannot be allowed for the past"); - - verifyChangeOrdering(changeTime); - return new SupportAccess( - prepend(new SupportAccessChange(Optional.of(until), changeTime, changedBy), changeHistory), - grantHistory); - } - - public SupportAccess withDisallowed(String changedBy, Instant changeTime) { - verifyChangeOrdering(changeTime); - return new SupportAccess( - prepend(new SupportAccessChange(Optional.empty(), changeTime, changedBy), changeHistory), - grantHistory); - } - - public SupportAccess withGrant(SupportAccessGrant supportAccessGrant) { - return new SupportAccess(changeHistory, prepend(supportAccessGrant, grantHistory)); - } - - private void verifyChangeOrdering(Instant changeTime) { - changeHistory.stream().findFirst().ifPresent(lastChange -> { - if (changeTime.isBefore(lastChange.changeTime())) { - throw new IllegalArgumentException("Support access change cannot be dated before previous change"); - } - }); - } - - private <T> List<T> prepend(T newEntry, List<T> existingEntries) { - return Stream.concat(Stream.of(newEntry), existingEntries.stream()) // latest change first - .toList(); - } - - public static class CurrentStatus { - private final State state; - private final Optional<Instant> allowedUntil; - private final Optional<String> allowedBy; - - private CurrentStatus(State state, Optional<Instant> allowedUntil, Optional<String> allowedBy) { - this.state = state; - this.allowedUntil = allowedUntil; - this.allowedBy = allowedBy; - } - - public State state() { - return state; - } - - public Optional<Instant> allowedUntil() { - return allowedUntil; - } - - public Optional<String> allowedBy() { - return allowedBy; - } - } - - public enum State { - NOT_ALLOWED, - ALLOWED - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SupportAccess that = (SupportAccess) o; - return changeHistory.equals(that.changeHistory) && grantHistory.equals(that.grantHistory); - } - - @Override - public int hashCode() { - return Objects.hash(changeHistory, grantHistory); - } - - @Override - public String toString() { - return "SupportAccess{" + - "changeHistory=" + changeHistory + - ", grantHistory=" + grantHistory + - '}'; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessChange.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessChange.java deleted file mode 100644 index 6b6c869d400..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessChange.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.support.access; - -import java.time.Instant; -import java.util.Objects; -import java.util.Optional; - -/** An (immutable) change in support access, recording what change was made, when, and by whom. */ -public class SupportAccessChange { - private final Instant madeAt; - private final Optional<Instant> accessAllowedUntil; - private final String changedBy; - - public SupportAccessChange(Optional<Instant> accessAllowedUntil, Instant changeTime, String changedBy) { - this.madeAt = changeTime; - this.accessAllowedUntil = accessAllowedUntil; - this.changedBy = changedBy; - } - - public Instant changeTime() { - return madeAt; - } - - public Optional<Instant> accessAllowedUntil() { - return accessAllowedUntil; - } - - public String madeBy() { - return changedBy; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SupportAccessChange that = (SupportAccessChange) o; - return madeAt.equals(that.madeAt) && accessAllowedUntil.equals(that.accessAllowedUntil) && changedBy.equals(that.changedBy); - } - - @Override - public int hashCode() { - return Objects.hash(madeAt, accessAllowedUntil, changedBy); - } - - @Override - public String toString() { - return "SupportAccessChange{" + - "madeAt=" + madeAt + - ", accessAllowedUntil=" + accessAllowedUntil + - ", changedBy='" + changedBy + '\'' + - '}'; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java deleted file mode 100644 index 7e3dc77822f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessControl.java +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.support.access; - -import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; - -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.time.Period; -import java.util.List; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.controller.support.access.SupportAccess.State.ALLOWED; -import static com.yahoo.vespa.hosted.controller.support.access.SupportAccess.State.NOT_ALLOWED; - -/** - * Which application endpoints should Vespa support be allowed to access for debugging? - * - * @author andreer - */ -public class SupportAccessControl { - - private final Controller controller; - - private final java.time.Period MAX_SUPPORT_ACCESS_TIME = Period.ofDays(10); - - public SupportAccessControl(Controller controller) { - this.controller = controller; - } - - public SupportAccess forDeployment(DeploymentId deploymentId) { - return controller.curator().readSupportAccess(deploymentId); - } - - public SupportAccess disallow(DeploymentId deployment, String by) { - try (Mutex lock = controller.curator().lockSupportAccess(deployment)) { - var now = controller.clock().instant(); - SupportAccess supportAccess = forDeployment(deployment); - if (supportAccess.currentStatus(now).state() == NOT_ALLOWED) { - throw new IllegalArgumentException("Support access is no longer allowed"); - } else { - var disallowed = supportAccess.withDisallowed(by, now); - controller.curator().writeSupportAccess(deployment, disallowed); - return disallowed; - } - } - } - - public SupportAccess allow(DeploymentId deployment, Instant until, String by) { - try (Mutex lock = controller.curator().lockSupportAccess(deployment)) { - var now = controller.clock().instant(); - if (until.isAfter(now.plus(MAX_SUPPORT_ACCESS_TIME))) { - throw new IllegalArgumentException("Support access cannot be allowed for more than 10 days"); - } - SupportAccess allowed = forDeployment(deployment).withAllowedUntil(until, by, now); - controller.curator().writeSupportAccess(deployment, allowed); - return allowed; - } - } - - public SupportAccess registerGrant(DeploymentId deployment, String by, X509Certificate certificate) { - try (Mutex lock = controller.curator().lockSupportAccess(deployment)) { - var now = controller.clock().instant(); - SupportAccess supportAccess = forDeployment(deployment); - if (certificate.getNotAfter().toInstant().isBefore(now)) { - throw new IllegalArgumentException("Support access certificate has already expired!"); - } - if (certificate.getNotAfter().toInstant().isAfter(now.plus(MAX_SUPPORT_ACCESS_TIME))) { - throw new IllegalArgumentException("Support access certificate validity time is limited to " + MAX_SUPPORT_ACCESS_TIME); - } - if (supportAccess.currentStatus(now).state() == NOT_ALLOWED) { - throw new IllegalArgumentException("Support access is not currently allowed by " + deployment.toUserFriendlyString()); - } - SupportAccess granted = supportAccess.withGrant(new SupportAccessGrant(by, certificate)); - controller.curator().writeSupportAccess(deployment, granted); - return granted; - } - } - - public List<SupportAccessGrant> activeGrantsFor(DeploymentId deployment) { - var now = controller.clock().instant(); - SupportAccess supportAccess = forDeployment(deployment); - if (supportAccess.currentStatus(now).state() == NOT_ALLOWED) return List.of(); - - return supportAccess.grantHistory().stream() - .filter(grant -> now.isAfter(grant.certificate().getNotBefore().toInstant())) - .filter(grant -> now.isBefore(grant.certificate().getNotAfter().toInstant())) - .toList(); - } - - public boolean allowDataplaneMembership(AthenzUser identity, DeploymentId deploymentId) { - Instant instant = controller.clock().instant(); - SupportAccess supportAccess = forDeployment(deploymentId); - SupportAccess.CurrentStatus currentStatus = supportAccess.currentStatus(instant); - if(currentStatus.state() == ALLOWED) { - return controller.serviceRegistry().accessControlService().approveDataPlaneAccess(identity, currentStatus.allowedUntil().orElse(instant.plus(MAX_SUPPORT_ACCESS_TIME))); - } else { - return false; - } - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessGrant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessGrant.java deleted file mode 100644 index ee57f14c71b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/support/access/SupportAccessGrant.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.support.access; - -import java.security.cert.X509Certificate; -import java.util.Objects; - -public class SupportAccessGrant { - private final String requestor; - private final X509Certificate certificate; - - public SupportAccessGrant(String requestor, X509Certificate certificate) { - this.requestor = Objects.requireNonNull(requestor); - this.certificate = Objects.requireNonNull(certificate); - } - - public String requestor() { - return requestor; - } - - public X509Certificate certificate() { - return certificate; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SupportAccessGrant that = (SupportAccessGrant) o; - return requestor.equals(that.requestor) && certificate.equals(that.certificate); - } - - @Override - public int hashCode() { - return Objects.hash(requestor, certificate); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java deleted file mode 100644 index a91b5ad72ed..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.tls; - -import com.google.common.collect.Sets; -import com.yahoo.component.annotation.Inject; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SslContextBuilder; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.security.tls.DefaultTlsContext; -import com.yahoo.security.tls.PeerAuthentication; -import com.yahoo.security.tls.TlsContext; -import com.yahoo.vespa.hosted.controller.tls.config.TlsConfig; - -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.KeyStore; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Configures the controller's HTTPS connector with certificate and private key from a secret store. - * - * @author mpolden - * @author bjorncs - */ -@SuppressWarnings("unused") // Injected -public class ControllerSslContextFactoryProvider extends TlsContextBasedProvider { - - private final KeyStore truststore; - private final KeyStore keystore; - private final Map<Integer, TlsContext> tlsContextMap = new ConcurrentHashMap<>(); - - @Inject - public ControllerSslContextFactoryProvider(SecretStore secretStore, TlsConfig config) { - if (!Files.isReadable(Paths.get(config.caTrustStore()))) { - throw new IllegalArgumentException("CA trust store file is not readable: " + config.caTrustStore()); - } - // Trust store containing CA trust store from file - this.truststore = KeyStoreBuilder.withType(KeyStoreType.JKS) - .fromFile(Paths.get(config.caTrustStore())) - .build(); - - TlsCredentials tlsCredentials = latestValidCredentials(secretStore, config); - - // Key store containing key pair from secret store - this.keystore = KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry(getClass().getSimpleName(), tlsCredentials.privateKey, tlsCredentials.certificates) - .build(); - } - - @Override - protected TlsContext getTlsContext(String containerId, int port) { - return tlsContextMap.computeIfAbsent(port, this::createTlsContext); - } - - private TlsContext createTlsContext(int port) { - return new DefaultTlsContext( - new SslContextBuilder() - .withKeyStore(keystore, new char[0]) - .withTrustStore(truststore) - .build(), - port != 443 ? PeerAuthentication.WANT : PeerAuthentication.DISABLED); - } - - record TlsCredentials(List<X509Certificate> certificates, PrivateKey privateKey){} - - private static TlsCredentials latestValidCredentials(SecretStore secretStore, TlsConfig tlsConfig) { - int version = latestVersionInSecretStore(secretStore, tlsConfig); - return new TlsCredentials(certificates(secretStore, tlsConfig, version), privateKey(secretStore, tlsConfig, version)); - } - - private static int latestVersionInSecretStore(SecretStore secretStore, TlsConfig tlsConfig) { - var certVersions = new HashSet<>(secretStore.listSecretVersions(tlsConfig.certificateSecret())); - var keyVersions = new HashSet<>(secretStore.listSecretVersions(tlsConfig.privateKeySecret())); - return Sets.intersection(certVersions, keyVersions).stream().mapToInt(Integer::intValue).max().orElseThrow( - () -> new RuntimeException("No valid certificate versions found in secret store!") - ); - } - - /** Get private key from secret store **/ - private static PrivateKey privateKey(SecretStore secretStore, TlsConfig config, int version) { - return KeyUtils.fromPemEncodedPrivateKey(secretStore.getSecret(config.privateKeySecret(), version)); - } - - /** - * Get certificate from secret store. If certificate secret contains multiple certificates, e.g. intermediate - * certificates, the entire chain will be read - */ - private static List<X509Certificate> certificates(SecretStore secretStore, TlsConfig config, int version) { - return X509CertificateUtils.certificateListFromPem(secretStore.getSecret(config.certificateSecret(), version)); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/package-info.java deleted file mode 100644 index be84cfdfca0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author mpolden - */ -@ExportPackage -package com.yahoo.vespa.hosted.controller.tls; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/CertifiedOsVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/CertifiedOsVersion.java deleted file mode 100644 index 0a790be1ab8..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/CertifiedOsVersion.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.component.Version; - -import java.util.Comparator; -import java.util.Objects; - -/** - * An OS version that has been certified to work on a specific Vespa version. - * - * @author mpolden - */ -public record CertifiedOsVersion(OsVersion osVersion, Version vespaVersion) implements Comparable<CertifiedOsVersion> { - - private static final Comparator<CertifiedOsVersion> comparator = Comparator.comparing(CertifiedOsVersion::osVersion) - .thenComparing(CertifiedOsVersion::vespaVersion); - - public CertifiedOsVersion { - Objects.requireNonNull(osVersion); - Objects.requireNonNull(vespaVersion); - } - - @Override - public int compareTo(CertifiedOsVersion that) { - return comparator.compare(this, that); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java deleted file mode 100644 index 4e4f00e6d4b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; -import com.yahoo.vespa.hosted.controller.deployment.JobList; -import com.yahoo.vespa.hosted.controller.deployment.JobStatus; -import com.yahoo.vespa.hosted.controller.deployment.Run; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Stream; - -import static java.util.Comparator.naturalOrder; -import static java.util.function.Function.identity; - -/** - * Statistics about deployments on a platform version. - * - * @param version the version these statistics are for - * @param failingUpgrades the runs on the version of this, for currently failing instances, where the failure may be because of the upgrade - * @param otherFailing all other failing runs on the version of this, for currently failing instances - * @param productionSuccesses the production runs where the last success was on the version of this - * @param runningUpgrade the currently running runs on the version of this, where an upgrade is attempted - * @param otherRunning all other currently running runs on the version on this - * - * @author jonmv - */ -public record DeploymentStatistics(Version version, - List<Run> failingUpgrades, - List<Run> otherFailing, - List<Run> productionSuccesses, - List<Run> runningUpgrade, - List<Run> otherRunning) { - - public DeploymentStatistics(Version version, List<Run> failingUpgrades, List<Run> otherFailing, - List<Run> productionSuccesses, List<Run> runningUpgrade, List<Run> otherRunning) { - this.version = Objects.requireNonNull(version); - this.failingUpgrades = List.copyOf(failingUpgrades); - this.otherFailing = List.copyOf(otherFailing); - this.productionSuccesses = List.copyOf(productionSuccesses); - this.runningUpgrade = List.copyOf(runningUpgrade); - this.otherRunning = List.copyOf(otherRunning); - } - - public static List<DeploymentStatistics> compute(Collection<Version> infrastructureVersions, DeploymentStatusList statuses) { - Set<Version> allVersions = new HashSet<>(infrastructureVersions); - Map<Version, List<Run>> failingUpgrade = new HashMap<>(); - Map<Version, List<Run>> otherFailing = new HashMap<>(); - Map<Version, List<Run>> productionSuccesses = new HashMap<>(); - Map<Version, List<Run>> runningUpgrade = new HashMap<>(); - Map<Version, List<Run>> otherRunning = new HashMap<>(); - - for (DeploymentStatus status : statuses.asList()) { - if (status.application().projectId().isEmpty()) - continue; - - for (Instance instance : status.application().instances().values()) - for (Deployment deployment : instance.productionDeployments().values()) - allVersions.add(deployment.version()); - - JobList failing = status.jobs().failingHard(); - - // Add all unsuccessful runs for failing production jobs as any run may have resulted in an incomplete deployment - // where a subset of nodes has upgraded. - failing.not().failingApplicationChange() - .production() - .mapToList(JobStatus::runs) - .forEach(runs -> runs.descendingMap().values().stream() - .dropWhile(run -> ! run.hasEnded()) - .takeWhile(run -> run.hasFailed()) - .forEach(run -> { - failingUpgrade.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>()); - if (failingUpgrade.get(run.versions().targetPlatform()).stream().noneMatch(existing -> existing.id().job().equals(run.id().job()))) - failingUpgrade.get(run.versions().targetPlatform()).add(run); - })); - - // Add only the last failing run for test jobs. - failing.not().failingApplicationChange() - .not().production() - .lastCompleted().asList() - .forEach(run -> { - failingUpgrade.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>()); - failingUpgrade.get(run.versions().targetPlatform()).add(run); - }); - - // Add only the last failing for instances failing only an application change, i.e., no upgrade. - failing.failingApplicationChange() - .lastCompleted().asList() - .forEach(run -> { - otherFailing.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>()); - otherFailing.get(run.versions().targetPlatform()).add(run); - }); - - status.jobs().production() - .lastSuccess().asList() - .forEach(run -> { - productionSuccesses.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>()); - productionSuccesses.get(run.versions().targetPlatform()).add(run); - }); - - JobList running = status.jobs().running(); - running.upgrading() - .lastTriggered().asList() - .forEach(run -> { - runningUpgrade.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>()); - runningUpgrade.get(run.versions().targetPlatform()).add(run); - }); - - running.not().upgrading() - .lastTriggered().asList() - .forEach(run -> { - otherRunning.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>()); - otherRunning.get(run.versions().targetPlatform()).add(run); - }); - } - - return Stream.of(allVersions.stream(), - failingUpgrade.keySet().stream(), - otherFailing.keySet().stream(), - productionSuccesses.keySet().stream(), - runningUpgrade.keySet().stream(), - otherRunning.keySet().stream()) - .flatMap(identity()) // Lol. - .distinct() - .sorted(naturalOrder()) - .map(version -> new DeploymentStatistics(version, - failingUpgrade.getOrDefault(version, List.of()), - otherFailing.getOrDefault(version, List.of()), - productionSuccesses.getOrDefault(version, List.of()), - runningUpgrade.getOrDefault(version, List.of()), - otherRunning.getOrDefault(version, List.of()))) - .toList(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java deleted file mode 100644 index e0d6dcfe36e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; -import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; -import com.yahoo.vespa.hosted.controller.api.integration.maven.Metadata; -import com.yahoo.vespa.hosted.controller.maven.repository.config.MavenRepositoryConfig; - -import java.io.IOException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; - -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * Http client implementation of a {@link MavenRepository}, which uses a configured repository and artifact ID. - * - * @author jonmv - */ -public class MavenRepositoryClient implements MavenRepository { - - private final HttpClient client; - private final URI apiUrl; - private final ArtifactId id; - - public MavenRepositoryClient(MavenRepositoryConfig config) { - this.client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); - this.apiUrl = URI.create(config.apiUrl() + "/").normalize(); - this.id = new ArtifactId(config.groupId(), config.artifactId()); - } - - @Override - public Metadata metadata() { - try { - HttpRequest request = HttpRequest.newBuilder(withArtifactPath(apiUrl, id)).build(); - HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString(UTF_8)); - if (response.statusCode() != 200) - throw new RuntimeException("Status code '" + response.statusCode() + "' and body\n'''\n" + - response.body() + "\n'''\nfor request " + request); - - return Metadata.fromXml(response.body()); - } - catch (IOException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - @Override - public ArtifactId artifactId() { - return id; - } - - static URI withArtifactPath(URI baseUrl, ArtifactId id) { - List<String> parts = new ArrayList<>(List.of(id.groupId().split("\\."))); - parts.add(id.artifactId()); - parts.add("maven-metadata.xml"); - return baseUrl.resolve(String.join("/", parts)); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java deleted file mode 100644 index c3b8a825cb8..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.zone.ZoneId; - -import java.time.Duration; -import java.time.Instant; -import java.util.Objects; -import java.util.Optional; - -/** - * Version information for a node allocated to a {@link com.yahoo.vespa.hosted.controller.application.SystemApplication}. - * - * @author mpolden - */ -public record NodeVersion(HostName hostname, - ZoneId zone, - Version currentVersion, - Version wantedVersion, - Optional<Instant> suspendedAt) { - - public NodeVersion { - Objects.requireNonNull(hostname, "hostname must be non-null"); - Objects.requireNonNull(zone, "zone must be non-null"); - Objects.requireNonNull(currentVersion, "version must be non-null"); - Objects.requireNonNull(wantedVersion, "wantedVersion must be non-null"); - Objects.requireNonNull(suspendedAt, "suspendedAt must be non-null"); - } - - /** Returns the duration of the change in this, measured relative to instant */ - public Duration changeDuration(Instant instant) { - if (!upgrading()) return Duration.ZERO; - if (suspendedAt.isEmpty()) return Duration.ZERO; // Node hasn't suspended to apply the change yet - return Duration.between(suspendedAt.get(), instant).abs(); - } - - @Override - public String toString() { - return hostname + ": " + currentVersion.toFullString() + " -> " + wantedVersion.toFullString() + - " [zone=" + zone + ", suspendedAt=" + suspendedAt.map(Instant::toString) - .orElse("<not suspended>") + "]"; - } - - /** Returns whether this is upgrading */ - private boolean upgrading() { - return currentVersion.isBefore(wantedVersion); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java deleted file mode 100644 index 68b3b01f75a..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.CloudName; - -import java.util.Comparator; -import java.util.Objects; - -/** - * An OS version for a cloud in this system. - * - * @author mpolden - */ -public record OsVersion(Version version, CloudName cloud) implements Comparable<OsVersion> { - - private static final Comparator<OsVersion> comparator = Comparator.comparing(OsVersion::cloud) - .thenComparing(OsVersion::version); - - public OsVersion { - Objects.requireNonNull(version, "version must be non-null"); - Objects.requireNonNull(cloud, "cloud must be non-null"); - } - - @Override - public String toString() { - return "version " + version.toFullString() + " for " + cloud + " cloud"; - } - - @Override - public int compareTo(OsVersion that) { - return comparator.compare(this, that); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java deleted file mode 100644 index f031b906dc0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.google.common.collect.ImmutableMap; -import com.yahoo.component.Version; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.zone.UpgradePolicy; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.maintenance.OsUpgrader; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Information about OS versions in this system. - * - * @author mpolden - */ -public record OsVersionStatus(Map<OsVersion, List<NodeVersion>> versions) { - - public static final OsVersionStatus empty = new OsVersionStatus(ImmutableMap.of()); - - /** Public for serialization purpose only. Use {@link OsVersionStatus#compute(Controller)} for an up-to-date status */ - public OsVersionStatus(Map<OsVersion, List<NodeVersion>> versions) { - this.versions = ImmutableMap.copyOf(Objects.requireNonNull(versions, "versions must be non-null")); - } - - /** Returns all node versions that exist in given cloud */ - public List<NodeVersion> nodesIn(CloudName cloud) { - List<NodeVersion> nodeVersions = new ArrayList<>(); - versions.forEach((osVersion, nodesOnVersion) -> { - if (osVersion.cloud().equals(cloud)) { - nodeVersions.addAll(nodesOnVersion); - } - }); - return Collections.unmodifiableList(nodeVersions); - } - - /** Returns versions that exist in given cloud */ - public Set<Version> versionsIn(CloudName cloud) { - return versions.keySet().stream() - .filter(osVersion -> osVersion.cloud().equals(cloud)) - .map(OsVersion::version) - .collect(Collectors.toUnmodifiableSet()); - } - - /** Compute the current OS versions in this system. This is expensive and should be called infrequently */ - public static OsVersionStatus compute(Controller controller) { - Map<OsVersion, List<NodeVersion>> osVersions = new HashMap<>(); - controller.os().targets().forEach(target -> osVersions.put(target.osVersion(), new ArrayList<>())); - - for (var application : SystemApplication.all()) { - for (var zone : zonesToUpgrade(controller)) { - if (!application.shouldUpgradeOs()) continue; - Version targetOsVersion = controller.serviceRegistry().configServer().nodeRepository() - .targetVersionsOf(zone.getVirtualId()) - .osVersion(application.nodeType()) - .orElse(Version.emptyVersion); - - for (var node : controller.serviceRegistry().configServer().nodeRepository().list(zone.getVirtualId(), NodeFilter.all().applications(application.id()))) { - if (!OsUpgrader.canUpgrade(node, true)) continue; - Optional<Instant> suspendedAt = node.suspendedSince(); - NodeVersion nodeVersion = new NodeVersion(node.hostname(), zone.getVirtualId(), node.currentOsVersion(), - targetOsVersion, suspendedAt); - OsVersion osVersion = new OsVersion(nodeVersion.currentVersion(), zone.getCloudName()); - osVersions.computeIfAbsent(osVersion, (k) -> new ArrayList<>()) - .add(nodeVersion); - } - } - } - - return new OsVersionStatus(osVersions); - } - - private static List<ZoneApi> zonesToUpgrade(Controller controller) { - return controller.zoneRegistry().osUpgradePolicies().stream() - .flatMap(upgradePolicy -> upgradePolicy.steps().stream()) - .map(UpgradePolicy.Step::zones) - .flatMap(Collection::stream) - .toList(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java deleted file mode 100644 index ea9322b5fab..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.component.Version; - -import java.time.Instant; -import java.util.Objects; - -/** - * The OS version target for a cloud and the time it was scheduled. - * - * @author mpolden - */ -public record OsVersionTarget(OsVersion osVersion, Instant scheduledAt, boolean pinned, boolean downgrade) implements VersionTarget, Comparable<OsVersionTarget> { - - public OsVersionTarget { - Objects.requireNonNull(osVersion); - Objects.requireNonNull(scheduledAt); - } - - @Override - public int compareTo(OsVersionTarget o) { - return osVersion.compareTo(o.osVersion); - } - - @Override - public Version version() { - return osVersion.version(); - } - - @Override - public boolean downgrade() { - return downgrade; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java deleted file mode 100644 index 28938577876..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.maintenance.SystemUpgrader; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Information about the current platform versions in use. - * The versions in use are the set of all versions running in current applications, versions - * of config servers in all zones, and the version of this controller itself. - * - * @author bratseth - * @author mpolden - */ -public record VersionStatus(List<VespaVersion> versions, int currentMajor) { - - private static final Logger log = Logger.getLogger(VersionStatus.class.getName()); - - /** Create a version status. DO NOT USE: Public for testing and serialization only */ - public VersionStatus(List<VespaVersion> versions, int currentMajor) { - this.versions = List.copyOf(versions); - this.currentMajor = currentMajor; - } - - /** Returns the current version of controllers in this system */ - public Optional<VespaVersion> controllerVersion() { - return versions().stream().filter(VespaVersion::isControllerVersion).findFirst(); - } - - /** - * Returns the current Vespa version of the system controlled by this, - * or empty if we have not currently determined what the system version is in this status. - */ - public Optional<VespaVersion> systemVersion() { - return versions().stream().filter(VespaVersion::isSystemVersion).findFirst(); - } - - /** Returns whether the system is currently upgrading */ - public boolean isUpgrading() { - return systemVersion().map(VespaVersion::versionNumber).orElse(Version.emptyVersion) - .isBefore(controllerVersion().map(VespaVersion::versionNumber) - .orElse(Version.emptyVersion)); - } - - /** - * Lists all currently active Vespa versions, with deployment statistics, - * sorted from lowest to highest version number. - * The returned list is immutable. - * Calling this is free, but the returned status is slightly out of date. - */ - public List<VespaVersion> versions() { return versions; } - - /** Lists all currently active Vespa versions, from lowest to highest number, which are not newer than the system version. */ - public List<VespaVersion> deployableVersions() { - List<VespaVersion> deployable = new ArrayList<>(); - for (VespaVersion version : versions) { - deployable.add(version); - if (version.isSystemVersion()) - return deployable; - } - return List.of(); - } - - /** Returns the given version, or null if it is not present */ - public VespaVersion version(Version version) { - return versions.stream().filter(v -> v.versionNumber().equals(version)).findFirst().orElse(null); - } - - /** Returns whether given version is active in this system */ - public boolean isActive(Version version) { - if (version(version) != null) return true; - // Occasionally we may deploy unofficial versions of a given Vespa version, i.e. given the version 8.42.1, - // an unofficial version 8.42.1.a may exist. Count such versions as active if their root version is active - Version rootVersion = new Version(version.getMajor(), version.getMinor(), version.getMicro()); - return version(rootVersion) != null; - } - - /** Create the empty version status */ - public static VersionStatus empty() { return new VersionStatus(List.of(), -1); } - - /** Create a full, updated version status. This is expensive and should be done infrequently */ - public static VersionStatus compute(Controller controller) { - VersionStatus versionStatus = controller.readVersionStatus(); - int currentMajor = versionStatus.currentMajor(); - List<NodeVersion> systemApplicationVersions = findSystemApplicationVersions(controller, versionStatus); - Map<ControllerVersion, List<HostName>> controllerVersions = findControllerVersions(controller); - - Map<Version, List<HostName>> infrastructureVersions = new HashMap<>(); - for (var kv : controllerVersions.entrySet()) { - infrastructureVersions.computeIfAbsent(kv.getKey().version(), (k) -> new ArrayList<>()) - .addAll(kv.getValue()); - } - for (var nodeVersion : systemApplicationVersions) { - infrastructureVersions.computeIfAbsent(nodeVersion.currentVersion(), (k) -> new ArrayList<>()) - .add(nodeVersion.hostname()); - } - - // The system version is the oldest infrastructure version, if that version is newer than the current system - // version - Version newSystemVersion = infrastructureVersions.keySet().stream().min(Comparator.naturalOrder()).get(); - Version systemVersion = versionStatus.systemVersion() - .map(VespaVersion::versionNumber) - .orElse(newSystemVersion); - if (newSystemVersion.isBefore(systemVersion)) { - log.warning("Refusing to lower system version from " + - systemVersion.toFullString() + - " to " + - newSystemVersion.toFullString() + - ", nodes on " + newSystemVersion.toFullString() + ": " + - infrastructureVersions.get(newSystemVersion).stream() - .map(HostName::value) - .collect(Collectors.joining(", "))); - } else { - systemVersion = newSystemVersion; - } - - Set<Version> allVersions = new HashSet<>(infrastructureVersions.keySet()); - for (Application application : controller.applications().asList()) - for (Instance instance : application.instances().values()) - for (Deployment deployment : instance.deployments().values()) - allVersions.add(deployment.version()); - - List<DeploymentStatistics> deploymentStatistics = DeploymentStatistics.compute(allVersions, - controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()) - .withProjectId() - .withJobs())); - List<VespaVersion> versions = new ArrayList<>(); - List<Version> releasedVersions = controller.mavenRepository().metadata().versions(controller.clock().instant()); - - for (DeploymentStatistics statistics : deploymentStatistics) { - if (statistics.version().isEmpty()) continue; - - try { - boolean isReleased = Collections.binarySearch(releasedVersions, statistics.version()) >= 0; - List<NodeVersion> nodeVersions = systemApplicationVersions.stream() - .filter(nodeVersion -> nodeVersion.currentVersion().equals(statistics.version())) - .toList(); - VespaVersion vespaVersion = createVersion(statistics, - controllerVersions.keySet(), - systemVersion, - isReleased, - nodeVersions, - controller, - versionStatus); - versions.add(vespaVersion); - if (vespaVersion.confidence().equalOrHigherThan(Confidence.high)) - currentMajor = Math.max(currentMajor, vespaVersion.versionNumber().getMajor()); - } catch (IllegalArgumentException e) { - log.log(Level.WARNING, "Unable to create VespaVersion for version " + - statistics.version().toFullString(), e); - } - } - - Collections.sort(versions); - - return new VersionStatus(versions, currentMajor); - } - - private static List<NodeVersion> findSystemApplicationVersions(Controller controller, VersionStatus versionStatus) { - List<NodeVersion> nodeVersions = new ArrayList<>(); - for (var zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) { - for (var application : SystemApplication.notController()) { - var nodes = controller.serviceRegistry().configServer().nodeRepository() - .list(zone.getId(), NodeFilter.all().applications(application.id())).stream() - .filter(SystemUpgrader::eligibleForUpgrade) - .toList(); - if (nodes.isEmpty()) continue; - boolean configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty()); - if (!configConverged) { - log.log(Level.WARNING, "Config for " + application.id() + " in " + zone.getId() + - " has not converged"); - } - for (var node : nodes) { - // Only use current node version if config has converged - var version = configConverged ? node.currentVersion() : controller.systemVersion(versionStatus); - var nodeVersion = new NodeVersion(node.hostname(), zone.getId(), version, node.wantedVersion(), - node.suspendedSince()); - nodeVersions.add(nodeVersion); - } - } - } - return nodeVersions; - } - - private static Map<ControllerVersion, List<HostName>> findControllerVersions(Controller controller) { - Map<ControllerVersion, List<HostName>> versions = new HashMap<>(); - if (controller.curator().cluster().isEmpty()) { // Use vtag if we do not have cluster - versions.computeIfAbsent(ControllerVersion.CURRENT, (k) -> new ArrayList<>()) - .add(controller.hostname()); - } else { - for (String host : controller.curator().cluster()) { - HostName hostname = HostName.of(host); - versions.computeIfAbsent(controller.curator().readControllerVersion(hostname), (k) -> new ArrayList<>()) - .add(hostname); - } - } - return versions; - } - - private static VespaVersion createVersion(DeploymentStatistics statistics, - Set<ControllerVersion> controllerVersions, - Version systemVersion, - boolean isReleased, - List<NodeVersion> nodeVersions, - Controller controller, - VersionStatus versionStatus) { - ControllerVersion latestVersion = controllerVersions.stream().max(Comparator.naturalOrder()).get(); - boolean isSystemVersion = statistics.version().equals(systemVersion); - boolean isControllerVersion = controllerVersions.size() == 1 && - statistics.version().equals(controllerVersions.iterator().next().version()); - VespaVersion.Confidence confidence = controller.curator().readConfidenceOverrides().get(statistics.version()); - boolean confidenceIsOverridden = confidence != null; - VespaVersion existingVespaVersion = versionStatus.version(statistics.version()); - - // Compute confidence - if (!confidenceIsOverridden) { - Confidence newConfidence = VespaVersion.confidenceFrom(statistics, controller, versionStatus); - Confidence oldConfidence = Optional.ofNullable(versionStatus.version(statistics.version())) - .map(VespaVersion::confidence) - .orElse(newConfidence); - // Always update confidence for system and controller - // Also allow older versions to transition from normal to high confidence - if (isSystemVersion || isControllerVersion || oldConfidence == Confidence.normal && newConfidence == Confidence.high) { - confidence = newConfidence; - } else { - // Otherwise, this is an older version, so we preserve the existing confidence, if any - confidence = oldConfidence; - } - } - - // Preserve existing commit details if we've previously computed status for this version - var commitSha = latestVersion.commitSha(); - var commitDate = latestVersion.commitDate(); - if (existingVespaVersion != null) { - commitSha = existingVespaVersion.releaseCommit(); - commitDate = existingVespaVersion.committedAt(); - - // Keep existing confidence if we cannot raise it at this moment in time - if (!confidenceIsOverridden && - !existingVespaVersion.confidence().canChangeTo(confidence, - controller.serviceRegistry().zoneRegistry().system(), - controller.clock().instant())) { - confidence = existingVespaVersion.confidence(); - } - } - - return new VespaVersion(statistics.version(), - commitSha, - commitDate, - isControllerVersion, - isSystemVersion, - isReleased, - nodeVersions, - confidence); - } - - /** Whether no version on a newer major, with high confidence, can be deployed. */ - public boolean isOnCurrentMajor(Version version) { - return version.getMajor() >= currentMajor; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionTarget.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionTarget.java deleted file mode 100644 index 6d3aac9475e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionTarget.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.component.Version; - -/** - * Interface for a version target of some kind of upgrade. - * - * @author mpolden - */ -public interface VersionTarget { - - /** The version of this target */ - Version version(); - - /** Returns whether this target is potentially a downgrade */ - boolean downgrade(); - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java deleted file mode 100644 index 9921102d460..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.InstanceList; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; - -import java.time.Instant; -import java.time.ZoneOffset; -import java.util.List; - -import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; - -/** - * Information about a particular Vespa version. - * - * Vespa versions are identified by their version number and ordered by increasing version numbers. - * - * @author bratseth - */ -public record VespaVersion(Version version, - String releaseCommit, - Instant committedAt, - boolean isControllerVersion, - boolean isSystemVersion, - boolean isReleased, - List<NodeVersion> nodeVersions, - Confidence confidence) implements Comparable<VespaVersion> { - - public static Confidence confidenceFrom(DeploymentStatistics statistics, Controller controller, VersionStatus versionStatus) { - int thisMajorVersion = statistics.version().getMajor(); - InstanceList all = InstanceList.from(controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()) - .withProductionDeployment())) - .allowingMajorVersion(thisMajorVersion, versionStatus); - // 'production on this': All production deployment jobs upgrading to this version have completed without failure - InstanceList productionOnThis = all.matching(instance -> statistics.productionSuccesses().stream().anyMatch(run -> run.id().application().equals(instance))) - .not().failingUpgrade() - .not().upgradingTo(statistics.version()); - InstanceList failingOnThis = all.matching(instance -> statistics.failingUpgrades().stream().anyMatch(run -> run.id().application().equals(instance))); - - // 'broken' if any canary fails, and no non-canary is upgraded - if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty() && productionOnThis.not().with(UpgradePolicy.canary).isEmpty()) - return Confidence.broken; - - // 'broken' if 6 non-canary was broken by this, and that is at least 5% of all - if (nonCanaryApplicationsBroken(statistics.version(), failingOnThis, productionOnThis)) - return Confidence.broken; - - // 'low' unless all unpinned canary applications are upgraded - if (productionOnThis.with(UpgradePolicy.canary).unpinned().size() < all.withProductionDeployment().with(UpgradePolicy.canary).unpinned().size()) - return Confidence.low; - - // 'low' unless at least half of all canary applications are upgraded - if (productionOnThis.with(UpgradePolicy.canary).size() < all.withProductionDeployment().with(UpgradePolicy.canary).size() * 0.5) - return Confidence.low; - - // 'high' if 90% of all unpinned default upgrade applications, and 50% of all of them, have upgraded - if ( productionOnThis.with(UpgradePolicy.defaultPolicy).unpinned().groupingBy(TenantAndApplicationId::from).size() >= - all.withProductionDeployment().with(UpgradePolicy.defaultPolicy).unpinned().groupingBy(TenantAndApplicationId::from).size() * 0.9 - && productionOnThis.with(UpgradePolicy.defaultPolicy).groupingBy(TenantAndApplicationId::from).size() >= - all.withProductionDeployment().with(UpgradePolicy.defaultPolicy).groupingBy(TenantAndApplicationId::from).size() * 0.5) - return Confidence.high; - - return Confidence.normal; - } - - /** Returns the version number of this Vespa version */ - public Version versionNumber() { return version; } - - /** Returns the sha of the release tag commit for this version in git */ - public String releaseCommit() { return releaseCommit; } - - /** Returns the time of the release commit */ - public Instant committedAt() { return committedAt; } - - /** Returns whether this is the current version of controllers in this system (the lowest version across all - * controllers) */ - public boolean isControllerVersion() { - return isControllerVersion; - } - - /** - * Returns whether this is the current version of the infrastructure of the system - * (i.e the lowest version across all controllers and all config servers in all zones). - * A goal of the controllers is to eventually (limited by safety and upgrade capacity) drive - * all applications to this version. - * - * Note that the self version may be higher than the current system version if - * all config servers are not yet upgraded to the version of the controllers. - */ - public boolean isSystemVersion() { return isSystemVersion; } - - /** Returns whether the artifacts of this release are available in the configured maven repository. */ - public boolean isReleased() { return isReleased; } - - /** Returns the versions of nodes allocated to system applications (across all zones) */ - public List<NodeVersion> nodeVersions() { - return nodeVersions; - } - - /** Returns the confidence we have in this versions suitability for production */ - public Confidence confidence() { return confidence; } - - @Override - public int compareTo(VespaVersion other) { - return this.versionNumber().compareTo(other.versionNumber()); - } - - @Override - public int hashCode() { return versionNumber().hashCode(); } - - @Override - public boolean equals(Object other) { - if (other == this) return true; - if ( ! (other instanceof VespaVersion)) return false; - return ((VespaVersion)other).versionNumber().equals(this.versionNumber()); - } - - /** The confidence of a version. */ - public enum Confidence { - - /** Rollout was aborted. The system infrastructure should stay on, or roll back to, its current version */ - aborted, - - /** This version has been proven defective */ - broken, - - /** We don't have sufficient evidence that this version is working */ - low, - - /** We have sufficient evidence that this version is working */ - normal, - - /** This version works, but we want users to stop using it */ - legacy, - - /** We have overwhelming evidence that this version is working */ - high; - - /** Returns true if this confidence is at least as high as the given confidence */ - public boolean equalOrHigherThan(Confidence other) { - return this.compareTo(other) >= 0; - } - - /** Returns true if this can be changed to target at given instant */ - public boolean canChangeTo(Confidence target, SystemName system, Instant instant) { - if (this.equalOrHigherThan(normal)) return true; // Confidence can always change from >= normal - if (!target.equalOrHigherThan(normal)) return true; // Confidence can always change to < normal - - var hourOfDay = instant.atZone(ZoneOffset.UTC).getHour(); - var dayOfWeek = instant.atZone(ZoneOffset.UTC).getDayOfWeek(); - var hourEnd = system == SystemName.Public ? 13 : 11; - // Confidence can only be raised between 05:00:00 and 11:59:59Z (13:59:59Z for public), and not during weekends or Friday. - return hourOfDay >= 5 && hourOfDay <= hourEnd - && dayOfWeek.getValue() < 5; - } - - } - - private static boolean nonCanaryApplicationsBroken(Version version, - InstanceList failingOnThis, - InstanceList productionOnThis) { - int failingNonCanaries = failingOnThis.startedFailingOn(version) - .not().with(UpgradePolicy.canary) - .groupingBy(TenantAndApplicationId::from).size(); - int productionNonCanaries = productionOnThis.not().with(UpgradePolicy.canary) - .groupingBy(TenantAndApplicationId::from).size(); - - if (productionNonCanaries + failingNonCanaries == 0) return false; - - // 'broken' if 6 non-canary was broken by this, and that is at least 5% of all - return failingNonCanaries >= 6 && failingNonCanaries >= productionOnThis.groupingBy(TenantAndApplicationId::from).size() * 0.05; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersionTarget.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersionTarget.java deleted file mode 100644 index bf66425fe81..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersionTarget.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.component.Version; - -import java.util.Objects; - -/** - * The target Vespa version for a system. - * - * @author mpolden - */ -public record VespaVersionTarget(Version version, boolean downgrade) implements VersionTarget { - - public VespaVersionTarget { - Objects.requireNonNull(version); - } - - @Override - public String toString() { - return "vespa version target " + version.toFullString() + (downgrade ? " (downgrade)" : ""); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java deleted file mode 100644 index 73ca7d2b42f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.versions; - -import com.yahoo.osgi.annotation.ExportPackage; |